From 8d630a9d08751e3d05d9cf22905b3203e803d98f Mon Sep 17 00:00:00 2001 From: Magnus Andersson Date: Sat, 19 Oct 2024 22:11:41 +0200 Subject: [PATCH] Prepare sto3 worker nodes. --- IaC-test/images.tf | 6 ++ IaC-test/k8snodes-sto3.tf | 75 ++++++++++++++ IaC-test/network.tf | 5 + IaC-test/securitygroups-k8s-dco.tf | 31 ++++++ IaC-test/securitygroups-k8s-sto3.tf | 148 ++++++++++++++++++++++++++++ IaC-test/securitygroups-k8s-sto4.tf | 27 +++++ IaC-test/servergroups-sto3.tf | 11 +++ 7 files changed, 303 insertions(+) create mode 100644 IaC-test/k8snodes-sto3.tf create mode 100644 IaC-test/securitygroups-k8s-sto3.tf create mode 100644 IaC-test/servergroups-sto3.tf diff --git a/IaC-test/images.tf b/IaC-test/images.tf index 25f570d..c0a39b9 100644 --- a/IaC-test/images.tf +++ b/IaC-test/images.tf @@ -10,3 +10,9 @@ data "openstack_images_image_v2" "debian12image-sto4" { provider = openstack.sto4 } +data "openstack_images_image_v2" "debian12image-sto3" { + name = "debian-12" # Name of image to be used + most_recent = true + provider = openstack.sto3 +} + diff --git a/IaC-test/k8snodes-sto3.tf b/IaC-test/k8snodes-sto3.tf new file mode 100644 index 0000000..7c27af8 --- /dev/null +++ b/IaC-test/k8snodes-sto3.tf @@ -0,0 +1,75 @@ +# +# Controller node resources +# + +locals { + sto3dc = "sto3" + sto3nodenrbase = index(var.datacenters, "sto3") + sto3indexjump = length(var.datacenters) +} + +resource "openstack_networking_port_v2" "kubewport-sto3" { + name = "${var.worker_name}${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}-${replace(var.dns_suffix,".","-")}-${local.sto3dc}-port" + # We create as many ports as there are instances created + count = var.workerdcreplicas + network_id = data.openstack_networking_network_v2.public-sto3.id + # A list of security group ID + security_group_ids = [ + resource.openstack_networking_secgroup_v2.ssh-from-jump-hosts-sto3.id, + resource.openstack_networking_secgroup_v2.microk8s-sto3.id + ] + admin_state_up = "true" + provider = openstack.sto3 +} + +# Boot volume for node +resource "openstack_blockstorage_volume_v3" "kubewvolumeboot-sto3" { + count = var.workerdcreplicas # Replicas per datacenter + name = "${var.controller_name}${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}-${replace(var.dns_suffix,".","-")}-${local.sto3dc}-vol" + description = "OS volume for kubernetes worker node ${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}" + size = 100 + image_id = data.openstack_images_image_v2.debian12image-sto3.id + enable_online_resize = true # Allow us to resize volume while attached. + provider = openstack.sto3 +} + +resource "openstack_blockstorage_volume_v3" "kubewvolumerook-sto3" { + count = var.workerdcreplicas # Replicas per datacenter + name = "${var.controller_name}${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}-${replace(var.dns_suffix,".","-")}-${local.sto3dc}-rook-vol" + description = "Rook storage volume for kubernetes worker node ${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}" + size = 100 + enable_online_resize = true # Allow us to resize volume while attached. + provider = openstack.sto3 +} + +resource "openstack_compute_instance_v2" "worker-nodes-sto3" { + count = var.workerdcreplicas # Replicas per datacenter + name = "${var.worker_name}${count.index * local.sto3indexjump + 1 + local.sto3nodenrbase}.${var.dns_suffix}" + flavor_name = "${var.worker_instance_type}" + key_pair = "${var.keynameworkers}" + provider = openstack.sto3 + security_groups = [ + resource.openstack_networking_secgroup_v2.microk8s-sto3.name, + resource.openstack_networking_secgroup_v2.ssh-from-jump-hosts-sto3.name + ] + + block_device { + uuid = resource.openstack_blockstorage_volume_v3.kubewvolumeboot-sto3[count.index].id + source_type = "volume" + destination_type = "volume" + boot_index = 0 + } + block_device { + uuid = resource.openstack_blockstorage_volume_v3.kubewvolumerook-sto3[count.index].id + source_type = "volume" + destination_type = "volume" + boot_index = 1 + } + + scheduler_hints { + group = openstack_compute_servergroup_v2.workers-sto3.id + } + network { + port = resource.openstack_networking_port_v2.kubewport-sto3[count.index].id + } +} diff --git a/IaC-test/network.tf b/IaC-test/network.tf index fccaea3..f55effe 100644 --- a/IaC-test/network.tf +++ b/IaC-test/network.tf @@ -6,3 +6,8 @@ data "openstack_networking_network_v2" "public-sto4" { name = "public" # Name of network to use. provider = openstack.sto4 } + +data "openstack_networking_network_v2" "public-sto3" { + name = "public" # Name of network to use. + provider = openstack.sto3 +} diff --git a/IaC-test/securitygroups-k8s-dco.tf b/IaC-test/securitygroups-k8s-dco.tf index 41a3f66..eac5e20 100644 --- a/IaC-test/securitygroups-k8s-dco.tf +++ b/IaC-test/securitygroups-k8s-dco.tf @@ -1,3 +1,6 @@ +# +# From STO4 to DCO +# resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v4_sto4_to_dco" { count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto4) @@ -22,3 +25,31 @@ resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_sto4_t remote_ip_prefix = join("/",[ replace(resource.openstack_compute_instance_v2.worker-nodes-sto4[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto4)].access_ip_v6, "/[\\[\\]']/",""), "128"]) security_group_id = openstack_networking_secgroup_v2.microk8s.id } + +# +# From STO3 to DCO +# + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v4_sto3_to_dco" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto3) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + provider = openstack.dco + remote_ip_prefix = join("/", [ resource.openstack_compute_instance_v2.worker-nodes-sto3[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto3)].access_ip_v4, "32" ]) + security_group_id = openstack_networking_secgroup_v2.microk8s.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_sto3_to_dco" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto3) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + provider = openstack.dco + remote_ip_prefix = join("/",[ replace(resource.openstack_compute_instance_v2.worker-nodes-sto3[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto3)].access_ip_v6, "/[\\[\\]']/",""), "128"]) + security_group_id = openstack_networking_secgroup_v2.microk8s.id +} diff --git a/IaC-test/securitygroups-k8s-sto3.tf b/IaC-test/securitygroups-k8s-sto3.tf new file mode 100644 index 0000000..bcd8e3b --- /dev/null +++ b/IaC-test/securitygroups-k8s-sto3.tf @@ -0,0 +1,148 @@ + +# Security groups sto3 +resource "openstack_networking_secgroup_v2" "microk8s-sto3" { + name = "microk8s" + description = "Traffic to allow between microk8s hosts" + provider=openstack.sto3 +} + +resource "openstack_networking_secgroup_v2" "ssh-from-jump-hosts-sto3" { + name = "ssh-from-jumphosts" + description = "Allow ssh traffic from sunet jumphosts." + provider=openstack.sto3 +} + +# +# Security group rules for microk8s +# +resource "openstack_networking_secgroup_rule_v2" "microk8s_rule_v4_sto3" { + count = length(var.k8sports) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[count.index][keys(var.k8sports[count.index])[0]] + port_range_min = keys(var.k8sports[count.index])[0] + port_range_max = keys(var.k8sports[count.index])[0] + provider = openstack.sto3 + remote_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_rule_v6_sto3" { + count = length(var.k8sports) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[count.index][keys(var.k8sports[count.index])[0]] + port_range_min = keys(var.k8sports[count.index])[0] + port_range_max = keys(var.k8sports[count.index])[0] + provider = openstack.sto3 + remote_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +# +# From DCO to STO3 +# +resource "openstack_networking_secgroup_rule_v2" "microk8s_controller_rule_v4_dco_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.controller-nodes) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/", [ resource.openstack_compute_instance_v2.controller-nodes[count.index % length(resource.openstack_compute_instance_v2.controller-nodes)].access_ip_v4, "32"]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_controller_rule_v6_dco_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.controller-nodes) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.controller-nodes))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/", [ replace(resource.openstack_compute_instance_v2.controller-nodes[count.index % length(resource.openstack_compute_instance_v2.controller-nodes)].access_ip_v6, "/[\\[\\]']/",""),"128"]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v4_dco_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/", [ resource.openstack_compute_instance_v2.worker-nodes[count.index % length(resource.openstack_compute_instance_v2.worker-nodes)].access_ip_v4, "32" ]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_dco_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/",[ replace(resource.openstack_compute_instance_v2.worker-nodes[count.index % length(resource.openstack_compute_instance_v2.worker-nodes)].access_ip_v6, "/[\\[\\]']/",""), "128"]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +# +# From STO4 to STO3 +# + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v4_sto4_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto4) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/", [ resource.openstack_compute_instance_v2.worker-nodes-sto4[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto4)].access_ip_v4, "32" ]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_sto4_to_sto3" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto4) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto4))])[0] + provider = openstack.sto3 + remote_ip_prefix = join("/",[ replace(resource.openstack_compute_instance_v2.worker-nodes-sto4[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto4)].access_ip_v6, "/[\\[\\]']/",""), "128"]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto3.id +} + +# +# Security group rules for ssh-from-jump-hosts +# + + +resource "openstack_networking_secgroup_rule_v2" "ssh-from-jumphosts-v4rules-sto3" { + count = length(var.jumphostv4-ips) + direction = "ingress" + ethertype = "IPv4" + protocol = "tcp" + port_range_min = "22" + port_range_max = "22" + provider = openstack.sto3 + remote_ip_prefix = "${var.jumphostv4-ips[count.index]}/32" + security_group_id = openstack_networking_secgroup_v2.ssh-from-jump-hosts-sto3.id +} + +resource "openstack_networking_secgroup_rule_v2" "ssh-from-jumphosts-v6rules-sto3" { + count = length(var.jumphostv6-ips) + direction = "ingress" + ethertype = "IPv6" + protocol = "tcp" + port_range_min = "22" + port_range_max = "22" + provider = openstack.sto3 + remote_ip_prefix = "${var.jumphostv6-ips[count.index]}/128" + security_group_id = openstack_networking_secgroup_v2.ssh-from-jump-hosts-sto3.id +} diff --git a/IaC-test/securitygroups-k8s-sto4.tf b/IaC-test/securitygroups-k8s-sto4.tf index 6013a46..bb776bc 100644 --- a/IaC-test/securitygroups-k8s-sto4.tf +++ b/IaC-test/securitygroups-k8s-sto4.tf @@ -87,6 +87,33 @@ resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_dco" { security_group_id = openstack_networking_secgroup_v2.microk8s-sto4.id } +# +# From STO3 to STO4 +# + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v4_sto3_to_sto4" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto3) + direction = "ingress" + ethertype = "IPv4" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + provider = openstack.sto4 + remote_ip_prefix = join("/", [ resource.openstack_compute_instance_v2.worker-nodes-sto3[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto3)].access_ip_v4, "32" ]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto4.id +} + +resource "openstack_networking_secgroup_rule_v2" "microk8s_worker_rule_v6_sto3_to_sto4" { + count = length(var.k8sports) * length(resource.openstack_compute_instance_v2.worker-nodes-sto3) + direction = "ingress" + ethertype = "IPv6" + protocol = var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))][keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0]] + port_range_min = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + port_range_max = keys(var.k8sports[floor(count.index/length(resource.openstack_compute_instance_v2.worker-nodes-sto3))])[0] + provider = openstack.sto4 + remote_ip_prefix = join("/",[ replace(resource.openstack_compute_instance_v2.worker-nodes-sto3[count.index % length(resource.openstack_compute_instance_v2.worker-nodes-sto3)].access_ip_v6, "/[\\[\\]']/",""), "128"]) + security_group_id = openstack_networking_secgroup_v2.microk8s-sto4.id +} # # Security group rules for ssh-from-jump-hosts diff --git a/IaC-test/servergroups-sto3.tf b/IaC-test/servergroups-sto3.tf new file mode 100644 index 0000000..e58864b --- /dev/null +++ b/IaC-test/servergroups-sto3.tf @@ -0,0 +1,11 @@ +resource "openstack_compute_servergroup_v2" "workers-sto3" { + name = "workers" + policies = ["anti-affinity"] + provider = openstack.sto3 +} +resource "openstack_compute_servergroup_v2" "controllers-sto3" { + name = "controllers" + policies = ["anti-affinity"] + provider = openstack.sto3 +} +