locals { cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}" cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}" podSubnets = [ "10.200.0.0/16", "fd9b:5314:fc70::/56", ] serviceSubnets = [ "10.201.0.0/16", "fd9b:5314:fc71::/112", ] # Nodes will use IPs from this subnets # for communication between each other. tailscaleSubnets = [ "100.64.0.0/10", "fd7a:115c:a1e0::/96" ] talos_controlplane_config = { machine = { features = { rbac = true apidCheckExtKeyUsage = true kubernetesTalosAPIAccess = { enabled = true allowedRoles = [ "os:reader" ] allowedKubernetesNamespaces = [ "kube-system" ] } } } cluster = { etcd = { advertisedSubnets = local.tailscaleSubnets listenSubnets = local.tailscaleSubnets } allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes, apiServer = { extraArgs = { "oidc-issuer-url" = "https://id.tjo.space/application/o/k8stjocloud/", "oidc-client-id" = "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg", "oidc-username-claim" = "sub", "oidc-username-prefix" = "oidc:", "oidc-groups-claim" = "groups", "oidc-groups-prefix" = "oidc:groups:", } } inlineManifests = [ { name = "proxmox-cloud-controller-manager" contents = data.helm_template.proxmox-ccm.manifest }, { name = "talos-cloud-controller-manager" contents = data.helm_template.talos-ccm.manifest }, { name = "promxmox-csi-plugin" contents = data.helm_template.proxmox-csi.manifest }, { name = "gateway-api-crds" contents = file("${path.module}/manifests/gateway-api.crds.yaml") }, { name = "cilium" contents = data.helm_template.cilium.manifest }, { name = "oidc-admins" contents = <<-EOF apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: id-tjo-space:admins subjects: - kind: Group name: oidc:groups:k8s.tjo.cloud admin apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io EOF }, ] } } talos_worker_config = { cluster = { externalCloudProvider = { enabled = true } controlPlane = { endpoint = local.cluster_endpoint localAPIServerPort = var.cluster.api.port } network = { cni = { name = "none" } podSubnets = local.podSubnets serviceSubnets = local.serviceSubnets } proxy = { disabled = true } } machine = { kubelet = { nodeIP = { validSubnets = local.tailscaleSubnets } extraArgs = { rotate-server-certificates = true cloud-provider = "external" } } install = { image = "factory.talos.dev/installer/${var.talos.schematic_id}:${var.talos.version}" disk = "/dev/vda" } } } talos_node_config = { for k, node in local.nodes_with_address : k => [ yamlencode({ machine = { network = { hostname = node.name kubespan = { enabled = false } } nodeLabels = { "k8s.tjo.cloud/public" = node.public ? "true" : "false" "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } sysctls = { "net.ipv4.ip_forward" = "1" "net.ipv6.conf.all.forwarding" = "1" } } }), yamlencode( { apiVersion = "v1alpha1" kind = "ExtensionServiceConfig" name = "tailscale" environment = [ "TS_AUTHKEY=${var.tailscale_authkey}", "TS_HOSTNAME=${node.name}", "TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}", #"TS_EXTRA_ARGS=--accept-routes --snat-subnet-routes", ] }) ] } } resource "digitalocean_record" "controlplane-A" { for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } domain = var.cluster.domain type = "A" name = var.cluster.api.subdomain value = each.value.ipv4 ttl = 30 } resource "digitalocean_record" "controlplane-AAAA" { for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } domain = var.cluster.domain type = "AAAA" name = var.cluster.api.subdomain value = each.value.ipv6 ttl = 30 } resource "talos_machine_secrets" "this" { talos_version = var.talos.version } data "talos_machine_configuration" "controlplane" { cluster_name = var.cluster.name machine_type = "controlplane" cluster_endpoint = local.cluster_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes depends_on = [ digitalocean_record.controlplane-A, digitalocean_record.controlplane-AAAA, ] } data "talos_machine_configuration" "worker" { cluster_name = var.cluster.name machine_type = "worker" cluster_endpoint = local.cluster_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes depends_on = [ digitalocean_record.controlplane-A, digitalocean_record.controlplane-AAAA ] } resource "talos_machine_configuration_apply" "controlplane" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration node = each.value.name endpoint = each.value.ipv4 apply_mode = "reboot" config_patches = concat( [ yamlencode(local.talos_worker_config), yamlencode(local.talos_controlplane_config) ], local.talos_node_config[each.key] ) } resource "talos_machine_configuration_apply" "worker" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration node = each.value.name endpoint = each.value.ipv4 apply_mode = "reboot" config_patches = concat( [ yamlencode(local.talos_worker_config) ], local.talos_node_config[each.key] ) } resource "talos_machine_bootstrap" "this" { depends_on = [ talos_machine_configuration_apply.controlplane, talos_machine_configuration_apply.worker ] node = local.first_controlplane_node.name endpoint = local.first_controlplane_node.ipv4 client_configuration = talos_machine_secrets.this.client_configuration } data "talos_cluster_kubeconfig" "this" { depends_on = [ talos_machine_bootstrap.this ] client_configuration = talos_machine_secrets.this.client_configuration node = local.first_controlplane_node.ipv4 } resource "local_file" "kubeconfig" { content = data.talos_cluster_kubeconfig.this.kubeconfig_raw filename = "${path.root}/admin.kubeconfig" lifecycle { ignore_changes = [content] } } data "talos_client_configuration" "this" { count = length(values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 cluster_name = var.cluster.name client_configuration = talos_machine_secrets.this.client_configuration endpoints = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[*].ipv4 } resource "local_file" "talosconfig" { count = length(values({ for k, v in local.nodes : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 content = nonsensitive(data.talos_client_configuration.this[0].talos_config) filename = "${path.root}/admin.talosconfig" }