infrastructure/modules/cluster/main.tf

locals {
  cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}"
  cluster_endpoint   = "https://${local.cluster_api_domain}:${var.cluster.api.port}"

  podSubnets = [
    "10.200.0.0/16",
    #"fd9b:5314:fc70::/48",
  ]
  serviceSubnets = [
    "10.201.0.0/16",
    #"fd9b:5314:fc71::/48",
  ]

  # Nodes will use IPs from this subnets
  # for communication between each other.
  tailscaleSubnets = [
    "100.64.0.0/10",
    "fd7a:115c:a1e0::/96"
  ]

  talos_controlplane_config = {
    cluster : {
      etcd : {
        #advertisedSubnets : [
        #  local.tailscaleSubnet
        #]
      }
      allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes,
      apiServer : {
        extraArgs : {
          "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
          "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
          "oidc-username-claim" : "sub",
          "oidc-username-prefix" : "oidc:",
          "oidc-groups-claim" : "groups",
          "oidc-groups-prefix" : "oidc:groups:",
        }
      }
      inlineManifests : [
        {
          name : "oidc-groups"
          contents : <<-EOF
            apiVersion: rbac.authorization.k8s.io/v1
            kind: ClusterRoleBinding
            metadata:
              name: id-tjo-space:admins
            subjects:
            - kind: Group
              name: oidc:groups:k8s.tjo.cloud admin
              apiGroup: rbac.authorization.k8s.io
            roleRef:
              kind: ClusterRole
              name: cluster-admin
              apiGroup: rbac.authorization.k8s.io
            EOF
        },
        {
          name : "cilium"
          contents : data.helm_template.cilium.manifest
        },
        {
          name : "promxmox-csi-plugin"
          contents : data.helm_template.csi.manifest
        },
        {
          name : "proxmox-cloud-controller-manager"
          contents : data.helm_template.ccm.manifest
        }
      ]
      externalCloudProvider : {
        enabled : true
      }
      extraManifests : [
        #"https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/v0.8.5/deploy/standalone-install.yaml",
        "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml",
      ]
    }
  }

  talos_worker_config = {
    cluster : {
      controlPlane : {
        endpoint : local.cluster_endpoint
        localAPIServerPort : var.cluster.api.port
      }
      network : {
        cni : {
          name : "none"
        }
        podSubnets : local.podSubnets
        serviceSubnets : local.serviceSubnets
      }
      proxy : {
        disabled : true
      }
    }
    machine = {
      kubelet = {
        nodeIP : {
          validSubnets : local.tailscaleSubnets
        }
        extraArgs : {
          rotate-server-certificates : true
        }
      }
      install = {
        image = "factory.talos.dev/installer/${var.talos.schematic_id}:${var.talos.version}"
        disk  = "/dev/vda"
      }
    }
  }
}

resource "digitalocean_record" "controlplane-A" {
  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }

  domain = var.cluster.domain
  type   = "A"
  name   = var.cluster.api.subdomain
  value  = each.value.ipv4
  ttl    = 30
}

resource "digitalocean_record" "controlplane-AAAA" {
  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }

  domain = var.cluster.domain
  type   = "AAAA"
  name   = var.cluster.api.subdomain
  value  = each.value.ipv6
  ttl    = 30
}

resource "talos_machine_secrets" "this" {}

data "talos_machine_configuration" "controlplane" {
  cluster_name     = var.cluster.name
  machine_type     = "controlplane"
  cluster_endpoint = local.cluster_endpoint
  machine_secrets  = talos_machine_secrets.this.machine_secrets

  talos_version      = var.talos.version
  kubernetes_version = var.talos.kubernetes

  depends_on = [
    digitalocean_record.controlplane-A,
    digitalocean_record.controlplane-AAAA,
  ]
}

data "talos_machine_configuration" "worker" {
  cluster_name     = var.cluster.name
  machine_type     = "worker"
  cluster_endpoint = local.cluster_endpoint
  machine_secrets  = talos_machine_secrets.this.machine_secrets

  talos_version      = var.talos.version
  kubernetes_version = var.talos.kubernetes

  depends_on = [
    digitalocean_record.controlplane-A,
    digitalocean_record.controlplane-AAAA
  ]
}

data "helm_template" "cilium" {
  provider = helm.template

  name       = "cilium"
  chart      = "cilium"
  repository = "https://helm.cilium.io/"
  version    = "1.15.6"
  namespace  = "kube-system"

  kube_version = var.talos.kubernetes
  api_versions = [
    "gateway.networking.k8s.io/v1/GatewayClass",
  ]

  values = [yamlencode({
    ipam : {
      mode : "kubernetes"
    },
    nodeIPAM : {
      enabled : true
    },
    kubeProxyReplacement : "true"
    securityContext : {
      capabilities : {
        ciliumAgent : [
          "CHOWN",
          "KILL",
          "NET_ADMIN",
          "NET_RAW",
          "IPC_LOCK",
          "SYS_ADMIN",
          "SYS_RESOURCE",
          "DAC_OVERRIDE",
          "FOWNER",
          "SETGID",
          "SETUID"
        ],
        cleanCiliumState : [
          "NET_ADMIN",
          "SYS_ADMIN",
          "SYS_RESOURCE"
        ]
      }
    },
    cgroup : {
      autoMount : {
        enabled : false
      },
      hostRoot : "/sys/fs/cgroup"
    },
    k8sServiceHost : local.cluster_api_domain
    k8sServicePort : var.cluster.api.port
    ipv4 : {
      enabled : true
    },
    #ipv6 : {
    #  enabled : true
    #},
    hubble : {
      tls : {
        auto : {
          enabled : true
          method : "cronJob"
          schedule : "0 0 1 */4 *"
        }
      }
      ui : {
        enabled : true
      }
      relay : {
        enabled : true
      }
    },
    gatewayAPI : {
      enabled : false
    }
    envoy : {
      enabled : false
    }
  })]
}

data "helm_template" "csi" {
  provider = helm.template

  name       = "proxmox-csi-plugin"
  chart      = "proxmox-csi-plugin"
  repository = "oci://ghcr.io/sergelogvinov/charts"
  version    = "0.2.5"
  namespace  = "kube-system"

  kube_version = var.talos.kubernetes

  values = [<<-EOF
    config:
      clusters:
        - url: ${var.proxmox.url}
          insecure: ${var.proxmox.insecure}
          token_id: "${proxmox_virtual_environment_user_token.csi.id}"
          token_secret: "${proxmox_virtual_environment_user_token.csi.value}"
          region: "${var.proxmox.name}"

    storageClass:
      - name: default
        storage: local-storage
        reclaimPolicy: Delete
        fstype: xfs
  EOF
  ]
}

data "helm_template" "ccm" {
  provider   = helm.template
  name       = "proxmox-cloud-controller-manager"
  chart      = "proxmox-cloud-controller-manager"
  repository = "oci://ghcr.io/sergelogvinov/charts"
  version    = "0.2.3"
  namespace  = "kube-system"

  kube_version = var.talos.kubernetes

  values = [<<-EOF
    config:
      clusters:
        - url: ${var.proxmox.url}
          insecure: ${var.proxmox.insecure}
          token_id: "${proxmox_virtual_environment_user_token.ccm.id}"
          token_secret: "${proxmox_virtual_environment_user_token.ccm.value}"
          region: "${var.proxmox.name}"
  EOF
  ]
}

resource "talos_machine_configuration_apply" "controlplane" {
  for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }

  client_configuration        = talos_machine_secrets.this.client_configuration
  machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration

  node     = each.value.name
  endpoint = each.value.ipv4

  apply_mode = "reboot"

  config_patches = [
    yamlencode(merge(local.talos_worker_config, local.talos_controlplane_config, {
      machine = {
        network = {
          hostname = each.value.name
        }
        nodeLabels = {
          "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
          "k8s.tjo.cloud/host"    = each.value.host
          "k8s.tjo.cloud/proxmox" = var.proxmox.name
        }
      }

    })),
    yamlencode(
      {
        apiVersion : "v1alpha1"
        kind : "ExtensionServiceConfig"
        name : "tailscale"
        environment : [
          "TS_AUTHKEY=${var.tailscale_authkey}",
          "TS_HOSTNAME=${each.value.name}",
        ]
    })
  ]
}

resource "talos_machine_configuration_apply" "worker" {
  for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" }

  client_configuration        = talos_machine_secrets.this.client_configuration
  machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration

  node     = each.value.name
  endpoint = each.value.ipv4

  apply_mode = "reboot"

  config_patches = [
    yamlencode(merge(local.talos_worker_config, {
      machine = {
        network = {
          hostname = each.value.name
        }
        nodeLabels = {
          "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
          "k8s.tjo.cloud/host"    = each.value.host
          "k8s.tjo.cloud/proxmox" = var.proxmox.name
        }
      }
    })),
    yamlencode(
      {
        apiVersion : "v1alpha1"
        kind : "ExtensionServiceConfig"
        name : "tailscale"
        environment : [
          "TS_AUTHKEY=${var.tailscale_authkey}",
          "TS_HOSTNAME=${each.value.name}",
        ]
    })
  ]
}

resource "talos_machine_bootstrap" "this" {
  depends_on = [
    talos_machine_configuration_apply.controlplane,
    talos_machine_configuration_apply.worker
  ]

  node                 = local.first_controlplane_node.name
  endpoint             = local.first_controlplane_node.ipv4
  client_configuration = talos_machine_secrets.this.client_configuration
}

data "talos_cluster_kubeconfig" "this" {
  depends_on = [
    talos_machine_bootstrap.this
  ]

  client_configuration = talos_machine_secrets.this.client_configuration
  node                 = local.first_controlplane_node.ipv4
}