locals { public_domain = "${var.cluster.api.public.subdomain}.${var.cluster.api.public.domain}" internal_domain = "${var.cluster.api.internal.subdomain}.${var.cluster.api.internal.domain}" cluster_internal_endpoint = "https://${local.internal_domain}:${var.cluster.api.internal.port}" cluster_public_endpoint = "https://${local.public_domain}:${var.cluster.api.public.port}" talos_controlplane_config = { machine = { kubelet = { extraArgs = { rotate-server-certificates = true } } features = { rbac = true apidCheckExtKeyUsage = true kubernetesTalosAPIAccess = { enabled = true allowedRoles = [ "os:reader" ] allowedKubernetesNamespaces = [ "kube-system" ] } } } cluster = { allowSchedulingOnControlPlanes = true, apiServer = { certSANs = [ local.public_domain, local.internal_domain, "localhost:7445", ] extraArgs = { "oidc-issuer-url" = "https://id.tjo.space/application/o/k8stjocloud/", "oidc-client-id" = "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg", "oidc-username-claim" = "sub", "oidc-username-prefix" = "oidc:", "oidc-groups-claim" = "groups", "oidc-groups-prefix" = "oidc:groups:", } } inlineManifests = concat([ { name = "proxmox-cloud-controller-manager" contents = data.helm_template.proxmox-ccm.manifest }, { name = "talos-cloud-controller-manager" contents = data.helm_template.talos-ccm.manifest }, { name = "promxmox-csi-plugin" contents = data.helm_template.proxmox-csi.manifest }, { name = "gateway-api-crds" contents = file("${path.module}/manifests/gateway-api.crds.yaml") }, { name = "cilium" contents = data.helm_template.cilium.manifest }, { name = "cilium-bgp-advertisement" contents = <<-EOF apiVersion: cilium.io/v2alpha1 kind: CiliumBGPAdvertisement metadata: name: pods-and-services labels: k8s.tjo.cloud/default: "true" spec: advertisements: - advertisementType: "PodCIDR" - advertisementType: "Service" service: addresses: - ClusterIP - ExternalIP - LoadBalancerIP EOF }, { name = "cilium-bgp-peer-config" contents = <<-EOF apiVersion: cilium.io/v2alpha1 kind: CiliumBGPPeerConfig metadata: name: default spec: families: - afi: ipv4 safi: unicast advertisements: matchLabels: k8s.tjo.cloud/default: "true" - afi: ipv6 safi: unicast advertisements: matchLabels: k8s.tjo.cloud/default: "true" EOF }, { name = "oidc-admins" contents = <<-EOF apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: id-tjo-space:admins subjects: - kind: Group name: oidc:groups:k8s.tjo.cloud admin apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io EOF }, ], [for name, attributes in var.hosts : { name = "cilium-bgp-node-config-override-${name}" contents = <<-EOF apiVersion: cilium.io/v2alpha1 kind: CiliumBGPClusterConfig metadata: name: ${name} spec: gracefulRestart: enabled: true restartTimeSeconds: 15 nodeSelector: matchLabels: k8s.tjo.cloud/bgp: "true" k8s.tjo.cloud/host: ${name} k8s.tjo.cloud/proxmox: ${var.proxmox.name} bgpInstances: - name: "${name}" localASN: ${attributes.asn} peers: - name: "local-router-vip" peerASN: ${attributes.asn} peerAddress: "10.0.0.1" peerConfigRef: name: "default" EOF } ] ) } } talos_worker_config = { cluster = { network = { cni = { name = "none" } podSubnets = [ var.cluster.pod_cidr.ipv4, var.cluster.pod_cidr.ipv6 ] serviceSubnets = [ var.cluster.service_cidr.ipv4, var.cluster.service_cidr.ipv6 ] } proxy = { disabled = true } } machine = { kubelet = { extraArgs = { rotate-server-certificates = true cloud-provider = "external" } } install = { image = "factory.talos.dev/installer/${var.talos.schematic_id}:${var.talos.version}" disk = "/dev/vda" } features = { hostDNS = { enabled = true forwardKubeDNSToHost = false } } } } talos_node_config = { for k, node in local.nodes_with_address : k => [ yamlencode({ machine = { network = { hostname = node.name } nodeLabels = { "k8s.tjo.cloud/bgp" = "true" "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } } }), ] } } resource "talos_machine_secrets" "this" { talos_version = var.talos.version } data "talos_machine_configuration" "controlplane" { cluster_name = var.cluster.name machine_type = "controlplane" cluster_endpoint = local.cluster_internal_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes } data "talos_machine_configuration" "worker" { cluster_name = var.cluster.name machine_type = "worker" cluster_endpoint = local.cluster_internal_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes } resource "talos_machine_configuration_apply" "controlplane" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration node = each.value.name endpoint = each.value.ipv4 config_patches = sensitive(concat( [ yamlencode(local.talos_worker_config), yamlencode(local.talos_controlplane_config) ], local.talos_node_config[each.key] )) } resource "talos_machine_configuration_apply" "worker" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration node = each.value.name endpoint = each.value.ipv4 config_patches = sensitive(concat( [ yamlencode(local.talos_worker_config) ], local.talos_node_config[each.key] )) } resource "talos_machine_bootstrap" "this" { depends_on = [ talos_machine_configuration_apply.controlplane, talos_machine_configuration_apply.worker ] node = local.first_controlplane_node.name endpoint = local.first_controlplane_node.ipv4 client_configuration = talos_machine_secrets.this.client_configuration } resource "talos_cluster_kubeconfig" "this" { depends_on = [ talos_machine_bootstrap.this ] client_configuration = talos_machine_secrets.this.client_configuration node = local.first_controlplane_node.ipv4 } resource "local_file" "kubeconfig" { content = talos_cluster_kubeconfig.this.kubeconfig_raw filename = "${path.root}/admin.kubeconfig" lifecycle { ignore_changes = [content] } } data "talos_client_configuration" "this" { count = length(values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 cluster_name = var.cluster.name client_configuration = talos_machine_secrets.this.client_configuration endpoints = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[*].ipv4 } resource "local_file" "talosconfig" { count = length(values({ for k, v in local.nodes : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 content = nonsensitive(data.talos_client_configuration.this[0].talos_config) filename = "${path.root}/admin.talosconfig" } resource "digitalocean_record" "api-internal-ipv4" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } domain = var.cluster.api.internal.domain type = "A" name = var.cluster.api.internal.subdomain value = each.value.ipv4 ttl = 30 } resource "digitalocean_record" "api-internal-ipv6" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } domain = var.cluster.api.internal.domain type = "AAAA" name = var.cluster.api.internal.subdomain value = each.value.ipv6 ttl = 30 }