locals { cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}" cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}" podSubnets = [ "10.200.0.0/16", #"fd9b:5314:fc70::/48", ] serviceSubnets = [ "10.201.0.0/16", #"fd9b:5314:fc71::/48", ] # Nodes will use IPs from this subnets # for communication between each other. tailscaleSubnets = [ "100.64.0.0/10", "fd7a:115c:a1e0::/96" ] talos_controlplane_config = { machine : { features : { rbac : true apidCheckExtKeyUsage : true kubernetesTalosAPIAccess : { enabled : true allowedRoles : [ "os:reader" ] allowedKubernetesNamespaces : [ "kube-system" ] } } } cluster : { etcd : { advertisedSubnets : local.tailscaleSubnets listenSubnets : local.tailscaleSubnets } allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes, apiServer : { extraArgs : { "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/", "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg", "oidc-username-claim" : "sub", "oidc-username-prefix" : "oidc:", "oidc-groups-claim" : "groups", "oidc-groups-prefix" : "oidc:groups:", } } inlineManifests : [ { name : "proxmox-cloud-controller-manager" contents : data.helm_template.proxmox-ccm.manifest }, { name : "talos-cloud-controller-manager" contents : data.helm_template.talos-ccm.manifest }, { name : "promxmox-csi-plugin" contents : data.helm_template.proxmox-csi.manifest }, { name : "gateway-api-crds" contents : file("${path.module}/manifests/gateway-api.crds.yaml") }, { name : "metrics-server" contents : file("${path.module}/manifests/metrics-server.yaml") }, { name : "cilium" contents : data.helm_template.cilium.manifest }, { name : "envoy" contents : data.helm_template.envoy.manifest }, { name : "cert-manager" contents : data.helm_template.cert-manager.manifest }, { name : "oidc-admins" contents : <<-EOF apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: id-tjo-space:admins subjects: - kind: Group name: oidc:groups:k8s.tjo.cloud admin apiGroup: rbac.authorization.k8s.io roleRef: kind: ClusterRole name: cluster-admin apiGroup: rbac.authorization.k8s.io EOF }, ] } } talos_worker_config = { cluster : { externalCloudProvider : { enabled : true } controlPlane : { endpoint : local.cluster_endpoint localAPIServerPort : var.cluster.api.port } network : { cni : { name : "none" } podSubnets : local.podSubnets serviceSubnets : local.serviceSubnets } proxy : { disabled : true } } machine = { kubelet = { nodeIP : { validSubnets : local.tailscaleSubnets } extraArgs : { rotate-server-certificates : true cloud-provider : "external" } } install = { image = "factory.talos.dev/installer/${var.talos.schematic_id}:${var.talos.version}" disk = "/dev/vda" } } } talos_node_config = { for k, node in local.nodes_with_address : k => [ yamlencode({ machine = { network = { hostname = node.name } nodeLabels = { "k8s.tjo.cloud/public" = node.public ? "true" : "false" #"k8s.tjo.cloud/ipv4" = node.ipv4 #"k8s.tjo.cloud/ipv6" = node.ipv6 "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } kubelet = { extraConfig = { podCIDR = "" } } } }), yamlencode( { apiVersion : "v1alpha1" kind : "ExtensionServiceConfig" name : "tailscale" environment : [ "TS_AUTHKEY=${var.tailscale_authkey}", "TS_HOSTNAME=${node.name}", ] }) ] } } resource "digitalocean_record" "controlplane-A" { for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } domain = var.cluster.domain type = "A" name = var.cluster.api.subdomain value = each.value.ipv4 ttl = 30 } resource "digitalocean_record" "controlplane-AAAA" { for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } domain = var.cluster.domain type = "AAAA" name = var.cluster.api.subdomain value = each.value.ipv6 ttl = 30 } resource "talos_machine_secrets" "this" { talos_version = var.talos.version } data "talos_machine_configuration" "controlplane" { cluster_name = var.cluster.name machine_type = "controlplane" cluster_endpoint = local.cluster_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes depends_on = [ digitalocean_record.controlplane-A, digitalocean_record.controlplane-AAAA, ] } data "talos_machine_configuration" "worker" { cluster_name = var.cluster.name machine_type = "worker" cluster_endpoint = local.cluster_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version kubernetes_version = var.talos.kubernetes depends_on = [ digitalocean_record.controlplane-A, digitalocean_record.controlplane-AAAA ] } resource "talos_machine_configuration_apply" "controlplane" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration node = each.value.name endpoint = each.value.ipv4 apply_mode = "reboot" config_patches = concat( [ yamlencode(local.talos_worker_config), yamlencode(local.talos_controlplane_config) ], local.talos_node_config[each.key] ) } resource "talos_machine_configuration_apply" "worker" { for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" } client_configuration = talos_machine_secrets.this.client_configuration machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration node = each.value.name endpoint = each.value.ipv4 apply_mode = "reboot" config_patches = concat( [ yamlencode(local.talos_worker_config) ], local.talos_node_config[each.key] ) } resource "talos_machine_bootstrap" "this" { depends_on = [ talos_machine_configuration_apply.controlplane, talos_machine_configuration_apply.worker ] node = local.first_controlplane_node.name endpoint = local.first_controlplane_node.ipv4 client_configuration = talos_machine_secrets.this.client_configuration } data "talos_cluster_kubeconfig" "this" { depends_on = [ talos_machine_bootstrap.this ] client_configuration = talos_machine_secrets.this.client_configuration node = local.first_controlplane_node.ipv4 } resource "local_file" "kubeconfig" { content = data.talos_cluster_kubeconfig.this.kubeconfig_raw filename = "${path.root}/admin.kubeconfig" } data "talos_client_configuration" "this" { count = length(values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 cluster_name = var.cluster.name client_configuration = talos_machine_secrets.this.client_configuration endpoints = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[*].ipv4 } resource "local_file" "talosconfig" { count = length(values({ for k, v in local.nodes : k => v if v.type == "controlplane" })) > 0 ? 1 : 0 content = nonsensitive(data.talos_client_configuration.this[0].talos_config) filename = "${path.root}/admin.talosconfig" }