refactor: use different proxmox provider, figure out nocloud bootstrap

k8s.tjo.cloud/.terraform.lock.hcl

@@ -1,6 +1,29 @@
 # This file is maintained automatically by "tofu init".
 # Manual edits may be lost in future updates.
 
+provider "registry.opentofu.org/bpg/proxmox" {
+  version     = "0.61.1"
+  constraints = "0.61.1"
+  hashes = [
+    "h1:6kz2Rdjc8+TVq2aUxEQXLOwbb9OdhJJei0L1fC4K2R4=",
+    "zh:27d8b589a2dc1e0a5b0f8ab299b9f3704a2f0b69799d1d4d8845c68056986d1f",
+    "zh:46dfa6b33ddd7007a2144f38090457604eb56a59a303b37bb0ad1be5c84ddaca",
+    "zh:47a1b14a759393c5ecc76f2feb950677c418c910b8c677fde0dd3e4675c41579",
+    "zh:582e49d109d1c2b1f3b1268a7cbc43548f3c6d96a87c92a5428767097a5e383e",
+    "zh:5e98ad6afae5969a4c3ffb14c0484936550c66c8313d7686551c29b633ff32f2",
+    "zh:7b9e24b76f947ab8f1e571cf61beefc983b7d2aa1b85df35c4f015728fe37a38",
+    "zh:8255ca210f279a0f7b8ca2762df26d2ea1a01704298c5e3d5cf601bd39a743f0",
+    "zh:85d7655fdc95dedced9cf8105a0beeb0d7bc8f668c55f62019a7215a76d60300",
+    "zh:8aeea5a1d001b06baaf923b754e1a14d06c75eb8c8b87a7f65a3c8205fc8b079",
+    "zh:a9cfab6c06f613658c5fdd83742cd22c0eb7563778924b1407965ef8c36c1ce0",
+    "zh:ceaab67801d49a92eb5858b1ddae6df2569462e5ffbe31f9dbd79dcb684ea142",
+    "zh:dc25b506d5c55d1d78a335d3ebd03213c99b4b2a5859812349a955c2f746ff7e",
+    "zh:e04b477fd77a0d37a0bdb76a7cf69184dad9e7fbba9b4f3a378a8901b82b75e5",
+    "zh:f1e6838d9141557f73340df9b21fce5a82b41cc16ae36f063a920ccc36bc0758",
+    "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
+  ]
+}
+
 provider "registry.opentofu.org/digitalocean/digitalocean" {
   version     = "2.39.2"
   constraints = "~> 2.0"
@@ -136,25 +159,3 @@ provider "registry.opentofu.org/siderolabs/talos" {
     "zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581",
   ]
 }
-
-provider "registry.opentofu.org/telmate/proxmox" {
-  version     = "3.0.1-rc3"
-  constraints = "3.0.1-rc3"
-  hashes = [
-    "h1:x7TfUaW+RpBtGov4DBuSJ5YPYBozapWuLyyZs0qjsKY=",
-    "zh:3699c41289c6fbe0f33b6c54360d43dcfba429de5fbf49506df9276d03aea915",
-    "zh:486c9ddda427d3fecdc6dfa189fce85c4a2aa1f490b024d636c0ac6a4dd3c692",
-    "zh:6091e141a0b8dcb1632c31e0f9555117bb023176c5d083f0e03441bbcf673a4e",
-    "zh:63d312c2c2994ed39dcb47b4d43c89990bd5fff20dbda63cddfb11c9202270f4",
-    "zh:6e69c70a85cfa720f543090ee3ce7d2eb2902df19657121b8b7ae64d44875d9f",
-    "zh:897b9f6075262fc9533f87d470217b14ae82614c6818a26b578a6d41c403d4eb",
-    "zh:91c24bd374fb8ee0c9e4e1c213d157139c047be78b0cafac3c4c9724db8083b0",
-    "zh:a224b58759314dc045fdbfc88b63b036b8ca6f75ad32606e94b553f150077c13",
-    "zh:a56e940c71b45e222c69a2a45388b58ed319836b922f84f62bded5b063662f4a",
-    "zh:b2e0a83aa535cd3493fbc7485d05d1a823c48bf487e313703f01a17edc631908",
-    "zh:ba0ad4fea8ba3b01c67fb164ed92fa927ac70d2d898378d192a01e818fcf6bee",
-    "zh:c49ebe13e7011d35d72e8e6a720df83f21c106444ef4383c5d6c0015aee55db6",
-    "zh:c53e2775040e103aedcce06b9acb79ca5fccdb4c578a4b6e32489c89e9c652dc",
-    "zh:c9002cc470ccfd8cd298d5655cf76af84b1d8a200207973d9ad80235818e89e3",
-  ]
-}

k8s.tjo.cloud/kubeconfig

@@ -1,26 +0,0 @@
-apiVersion: v1
-kind: Config
-clusters:
-- name: tjo-cloud
-  cluster:
-    server: https://api.k8s.tjo.cloud:6443
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRVHV4SURwTXNNUXNYWitLb2k1RkdsakFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN4TmpFNE1qWXhPVm9YRFRNME1EY3hOREU0TWpZeApPVm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCQ01zTVdqWjU1alRzaWd6aVcrT1BUSm1uS1VLQ2VTbktnTmptOFNRVE0ydm1tQklaM1I1UDlKcTY4ZmEKUU9zZ0RXSE96aFF6MjRNRUw1aFhpUzRjU3B1allUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVEVG41Sy8raXROdEgyczM0MTR3TEFjV0V0Wm93Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQU56bVV6OVEKZVhVVVJzei9KWlRlb29ibzF0Y1BHN2p3cGFlSDQ0VFVwVEE4QWlFQTFZcTdhUWdkb2lVTFNac2IrT3JLeDJyWQpGRTdSVkpxR0lyM1U4c3daY3k4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
-contexts:
-- name: oidc@tjo-cloud
-  context:
-    cluster: tjo-cloud
-    namespace: default
-    user: oidc
-current-context: oidc@tjo-cloud
-users:
-- name: oidc
-  user:
-    exec:
-      apiVersion: client.authentication.k8s.io/v1beta1
-      command: kubectl
-      args:
-      - oidc-login
-      - get-token
-      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
-      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
-      - --oidc-extra-scope=profile

k8s.tjo.cloud/main.tf

@@ -5,13 +5,11 @@ module "cluster" {
     helm.template = helm.template
   }
 
-  versions = {
+  talos = {
-    talos      = "v1.7.5"
+    version    = "v1.7.5"
     kubernetes = "v1.30.0"
   }
 
-  iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
-
   cluster = {
     name   = "tjo-cloud"
     domain = "k8s.tjo.cloud"
@@ -22,16 +20,9 @@ module "cluster" {
   }
 
   proxmox = {
-    name = "tjo-cloud"
+    name           = "tjo-cloud"
-    url  = "https://proxmox.tjo.cloud/api2/json"
+    url            = "https://proxmox.tjo.cloud/api2/json"
-    ccm = {
+    iso_storage_id = "proxmox-backup-tjo-cloud"
-      username = var.proxmox_ccm_username
-      token    = var.proxmox_ccm_token
-    }
-    csi = {
-      username = var.proxmox_csi_username
-      token    = var.proxmox_csi_token
-    }
   }
 
   tailscale_authkey = var.tailscale_authkey

k8s.tjo.cloud/providers.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "Telmate/proxmox"
+      source  = "bpg/proxmox"
-      version = "3.0.1-rc3"
+      version = "0.61.1"
     }
     talos = {
       source  = "siderolabs/talos"
@@ -37,8 +37,13 @@ terraform {
 
 provider "proxmox" {
   # FIXME: Traefik/NGINX breaks this! 500 ERROR
-  pm_api_url      = "https://178.63.49.225:8006/api2/json"
+  endpoint  = "https://178.63.49.225:8006/api2/json"
-  pm_tls_insecure = true
+  insecure  = true
+  api_token = var.proxmox_token
+  ssh {
+    agent    = true
+    username = "root"
+  }
 }
 
 provider "digitalocean" {

k8s.tjo.cloud/variables.tf

@@ -22,18 +22,7 @@ variable "digitalocean_token" {
   sensitive = true
 }
 
-variable "proxmox_csi_username" {
+variable "proxmox_token" {
-  type = string
-}
-variable "proxmox_csi_token" {
-  type      = string
-  sensitive = true
-}
-
-variable "proxmox_ccm_username" {
-  type = string
-}
-variable "proxmox_ccm_token" {
   type      = string
   sensitive = true
 }

modules/cluster/main.tf

@@ -2,12 +2,6 @@ locals {
   cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}"
   cluster_endpoint   = "https://${local.cluster_api_domain}:${var.cluster.api.port}"
 
-  nodes = { for k, v in var.nodes : k => merge(v, { name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-") }) }
-
-  nodes_with_address = { for k, v in local.nodes : k => merge(v, { ipv4 = try(proxmox_vm_qemu.this[k].default_ipv4_address, false), ipv6 = try(proxmox_vm_qemu.this[k].default_ipv6_address, false) }) }
-
-  first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0]
-
   podSubnets = [
     "10.200.0.0/16",
     #"fd9b:5314:fc70::/48",
@@ -24,85 +18,116 @@ locals {
     "fd7a:115c:a1e0::/96"
   ]
 
-  proxmox-cloud-controller-manager = {
+  talos_controlplane_config = {
-    clusters : [{
+    cluster : {
-      region : var.proxmox.name,
+      etcd : {
-      url : var.proxmox.url,
+        #advertisedSubnets : [
-      insecure : var.proxmox.insecure,
+        #  local.tailscaleSubnet
-      token_id : var.proxmox.ccm.username,
+        #]
-      token_secret : var.proxmox.ccm.token,
+      }
-    }]
+      allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes,
-  }
+      apiServer : {
-}
+        extraArgs : {
-
+          "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
-resource "macaddress" "private" {
+          "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
-  for_each = local.nodes
+          "oidc-username-claim" : "sub",
-}
+          "oidc-username-prefix" : "oidc:",
-resource "macaddress" "public" {
+          "oidc-groups-claim" : "groups",
-  for_each = local.nodes
+          "oidc-groups-prefix" : "oidc:groups:",
-}
-
-resource "proxmox_vm_qemu" "this" {
-  for_each = local.nodes
-
-  name        = each.value.name
-  target_node = each.value.host
-  tags = join(";", concat(
-    ["kubernetes", "terraform"],
-    each.value.public ? ["public"] : ["private"],
-    [each.value.type]
-  ))
-
-  cores  = each.value.cores
-  memory = each.value.memory
-
-  scsihw  = "virtio-scsi-pci"
-  qemu_os = "l26"
-
-  agent = 1
-
-  network {
-    model   = "virtio"
-    bridge  = each.value.public ? "vmpublic0" : "vmprivate0"
-    macaddr = macaddress.private[each.key].address
-  }
-
-  disks {
-    scsi {
-      scsi0 {
-        cdrom {
-          iso = var.iso
         }
       }
-    }
+      inlineManifests : [
-    virtio {
+        {
-      virtio0 {
+          name : "oidc-groups"
-        disk {
+          contents : <<-EOF
-          size    = each.value.boot_size
+            apiVersion: rbac.authorization.k8s.io/v1
-          storage = each.value.boot_pool
+            kind: ClusterRoleBinding
+            metadata:
+              name: id-tjo-space:admins
+            subjects:
+            - kind: Group
+              name: oidc:groups:k8s.tjo.cloud admin
+              apiGroup: rbac.authorization.k8s.io
+            roleRef:
+              kind: ClusterRole
+              name: cluster-admin
+              apiGroup: rbac.authorization.k8s.io
+            EOF
+        },
+        {
+          name : "cilium"
+          contents : data.helm_template.cilium.manifest
+        },
+        {
+          name : "promxmox-csi-plugin"
+          contents : data.helm_template.csi.manifest
+        },
+        {
+          name : "proxmox-cloud-controller-manager"
+          contents : data.helm_template.ccm.manifest
         }
+      ]
+      externalCloudProvider : {
+        enabled : true
+      }
+      extraManifests : [
+        #"https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/v0.8.5/deploy/standalone-install.yaml",
+        "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml",
+      ]
+    }
+  }
+
+  talos_worker_config = {
+    cluster : {
+      controlPlane : {
+        endpoint : local.cluster_endpoint
+        localAPIServerPort : var.cluster.api.port
+      }
+      network : {
+        cni : {
+          name : "none"
+        }
+        podSubnets : local.podSubnets
+        serviceSubnets : local.serviceSubnets
+      }
+      proxy : {
+        disabled : true
+      }
+    }
+    machine = {
+      kubelet = {
+        nodeIP : {
+          validSubnets : local.tailscaleSubnets
+        }
+        extraArgs : {
+          rotate-server-certificates : true
+        }
+      }
+      install = {
+        image = "factory.talos.dev/installer/${var.talos.schematic_id}:${var.talos.version}"
+        disk  = "/dev/vda"
       }
     }
   }
 }
 
 resource "digitalocean_record" "controlplane-A" {
-  for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) }
+  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }
 
   domain = var.cluster.domain
   type   = "A"
   name   = var.cluster.api.subdomain
-  value  = each.value.default_ipv4_address
+  value  = each.value.ipv4
   ttl    = 30
 }
 
 resource "digitalocean_record" "controlplane-AAAA" {
-  for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) }
+  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }
 
   domain = var.cluster.domain
   type   = "AAAA"
   name   = var.cluster.api.subdomain
-  value  = each.value.default_ipv6_address
+  value  = each.value.ipv6
   ttl    = 30
 }
 
@@ -114,8 +139,8 @@ data "talos_machine_configuration" "controlplane" {
   cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
-  talos_version      = var.versions.talos
+  talos_version      = var.talos.version
-  kubernetes_version = var.versions.kubernetes
+  kubernetes_version = var.talos.kubernetes
 
   depends_on = [
     digitalocean_record.controlplane-A,
@@ -129,8 +154,8 @@ data "talos_machine_configuration" "worker" {
   cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
-  talos_version      = var.versions.talos
+  talos_version      = var.talos.version
-  kubernetes_version = var.versions.kubernetes
+  kubernetes_version = var.talos.kubernetes
 
   depends_on = [
     digitalocean_record.controlplane-A,
@@ -147,7 +172,7 @@ data "helm_template" "cilium" {
   version    = "1.15.6"
   namespace  = "kube-system"
 
-  kube_version = var.versions.kubernetes
+  kube_version = var.talos.kubernetes
   api_versions = [
     "gateway.networking.k8s.io/v1/GatewayClass",
   ]
@@ -220,6 +245,57 @@ data "helm_template" "cilium" {
   })]
 }
 
+data "helm_template" "csi" {
+  provider = helm.template
+
+  name       = "proxmox-csi-plugin"
+  chart      = "proxmox-csi-plugin"
+  repository = "oci://ghcr.io/sergelogvinov/charts"
+  version    = "0.2.5"
+  namespace  = "kube-system"
+
+  kube_version = var.talos.kubernetes
+
+  values = [<<-EOF
+    config:
+      clusters:
+        - url: ${var.proxmox.url}
+          insecure: ${var.proxmox.insecure}
+          token_id: "${proxmox_virtual_environment_user_token.csi.id}"
+          token_secret: "${proxmox_virtual_environment_user_token.csi.value}"
+          region: "${var.proxmox.name}"
+
+    storageClass:
+      - name: default
+        storage: local-storage
+        reclaimPolicy: Delete
+        fstype: xfs
+  EOF
+  ]
+}
+
+data "helm_template" "ccm" {
+  provider   = helm.template
+  name       = "proxmox-cloud-controller-manager"
+  chart      = "proxmox-cloud-controller-manager"
+  repository = "oci://ghcr.io/sergelogvinov/charts"
+  version    = "0.2.3"
+  namespace  = "kube-system"
+
+  kube_version = var.talos.kubernetes
+
+  values = [<<-EOF
+    config:
+      clusters:
+        - url: ${var.proxmox.url}
+          insecure: ${var.proxmox.insecure}
+          token_id: "${proxmox_virtual_environment_user_token.ccm.id}"
+          token_secret: "${proxmox_virtual_environment_user_token.ccm.value}"
+          region: "${var.proxmox.name}"
+  EOF
+  ]
+}
+
 resource "talos_machine_configuration_apply" "controlplane" {
   for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
 
@@ -232,117 +308,28 @@ resource "talos_machine_configuration_apply" "controlplane" {
   apply_mode = "reboot"
 
   config_patches = [
-    yamlencode({
+    yamlencode(merge(local.talos_worker_config, local.talos_controlplane_config, {
-      cluster : {
-        controlPlane : {
-          endpoint : local.cluster_endpoint
-          localAPIServerPort : var.cluster.api.port
-        }
-        etcd : {
-          #advertisedSubnets : [
-          #  local.tailscaleSubnet
-          #]
-        }
-        network : {
-          cni : {
-            name : "none"
-          }
-          podSubnets : local.podSubnets
-          serviceSubnets : local.serviceSubnets
-        }
-        proxy : {
-          disabled : true
-        }
-        allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes,
-        apiServer : {
-          extraArgs : {
-            "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
-            "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
-            "oidc-username-claim" : "sub",
-            "oidc-username-prefix" : "oidc:",
-            "oidc-groups-claim" : "groups",
-            "oidc-groups-prefix" : "oidc:groups:",
-          }
-        }
-        inlineManifests : [
-          {
-            name : "oidc-groups"
-            contents : <<-EOF
-            apiVersion: rbac.authorization.k8s.io/v1
-            kind: ClusterRoleBinding
-            metadata:
-              name: id-tjo-space:admins
-            subjects:
-            - kind: Group
-              name: oidc:groups:k8s.tjo.cloud admin
-              apiGroup: rbac.authorization.k8s.io
-            roleRef:
-              kind: ClusterRole
-              name: cluster-admin
-              apiGroup: rbac.authorization.k8s.io
-            EOF
-          },
-          {
-            name : "cilium"
-            contents : data.helm_template.cilium.manifest
-          },
-          {
-            name : "proxmox-cloud-controller-manager"
-            contents : <<-EOF
-            apiVersion: v1
-            kind: Secret
-            metadata:
-              name: proxmox-cloud-controller-manager
-              namespace: kube-system
-            data:
-              config.yaml: ${base64encode(yamlencode(local.proxmox-cloud-controller-manager))}
-            EOF
-          }
-        ]
-        externalCloudProvider : {
-          enabled : true
-          manifests : [
-            "https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/v0.7.0/docs/deploy/proxmox-csi-plugin-talos.yml",
-            "https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/v0.4.2/docs/deploy/cloud-controller-manager-talos.yml"
-          ]
-        }
-        extraManifests : [
-          #"https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/v0.8.5/deploy/standalone-install.yaml",
-          "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml",
-        ]
-      }
       machine = {
-        kubelet = {
-          nodeIP : {
-            validSubnets : local.tailscaleSubnets
-          }
-          extraArgs : {
-            rotate-server-certificates : true
-            cloud-provider : "external"
-          }
-        }
         network = {
           hostname = each.value.name
         }
-        install = {
-          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}"
-          disk  = "/dev/vda"
-        }
         nodeLabels = {
           "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
           "k8s.tjo.cloud/host"    = each.value.host
           "k8s.tjo.cloud/proxmox" = var.proxmox.name
         }
       }
-    }),
+
-    yamlencode({
+    })),
-      apiVersion : "v1alpha1"
+    yamlencode(
-      kind : "ExtensionServiceConfig"
+      {
-      name : "tailscale"
+        apiVersion : "v1alpha1"
-      environment : [
+        kind : "ExtensionServiceConfig"
-        "TS_AUTHKEY=${var.tailscale_authkey}",
+        name : "tailscale"
-        "TS_HOSTNAME=${replace(each.value.name, ".", "-")}",
+        environment : [
-      ]
+          "TS_AUTHKEY=${var.tailscale_authkey}",
+          "TS_HOSTNAME=${each.value.name}",
+        ]
     })
   ]
 }
@@ -359,55 +346,27 @@ resource "talos_machine_configuration_apply" "worker" {
   apply_mode = "reboot"
 
   config_patches = [
-    yamlencode({
+    yamlencode(merge(local.talos_worker_config, {
-      cluster : {
-        controlPlane : {
-          endpoint : local.cluster_endpoint
-          localAPIServerPort : var.cluster.api.port
-        }
-        network : {
-          cni : {
-            name : "none"
-          }
-          podSubnets : local.podSubnets
-          serviceSubnets : local.serviceSubnets
-        }
-        proxy : {
-          disabled : true
-        }
-      }
       machine = {
-        kubelet = {
-          nodeIP : {
-            validSubnets : local.tailscaleSubnets
-          }
-          extraArgs : {
-            rotate-server-certificates : true
-            cloud-provider : "external"
-          }
-        }
         network = {
           hostname = each.value.name
         }
-        install = {
-          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}"
-          disk  = "/dev/vda"
-        }
         nodeLabels = {
           "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
           "k8s.tjo.cloud/host"    = each.value.host
           "k8s.tjo.cloud/proxmox" = var.proxmox.name
         }
       }
-    }),
+    })),
-    yamlencode({
+    yamlencode(
-      apiVersion : "v1alpha1"
+      {
-      kind : "ExtensionServiceConfig"
+        apiVersion : "v1alpha1"
-      name : "tailscale"
+        kind : "ExtensionServiceConfig"
-      environment : [
+        name : "tailscale"
-        "TS_AUTHKEY=${var.tailscale_authkey}",
+        environment : [
-        "TS_HOSTNAME=${replace(each.value.name, ".", "-")}",
+          "TS_AUTHKEY=${var.tailscale_authkey}",
-      ]
+          "TS_HOSTNAME=${each.value.name}",
+        ]
     })
   ]
 }

modules/cluster/proxmox.tf

@@ -0,0 +1,162 @@
+locals {
+  nodes = { for k, v in var.nodes : k => merge(v, { name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-") }) }
+
+
+
+  first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0]
+
+  ipv4_addresses = {
+    for key, node in local.nodes : key => {
+      for k, v in proxmox_virtual_environment_vm.nodes[key].ipv4_addresses :
+      proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v
+    }
+  }
+  ipv6_addresses = {
+    for key, node in local.nodes : key => {
+      for k, v in proxmox_virtual_environment_vm.nodes[key].ipv6_addresses :
+      proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v
+    }
+  }
+
+  nodes_with_address = {
+    for k, v in local.nodes :
+    k => merge(v, {
+      ipv4 = try(local.ipv4_addresses[k]["eth0"][0], null)
+      ipv6 = try(local.ipv6_addresses[k]["eth0"][0], null)
+    })
+  }
+}
+
+resource "proxmox_virtual_environment_download_file" "talos" {
+  content_type = "iso"
+  datastore_id = var.proxmox.iso_storage_id
+  node_name    = values(var.nodes)[0].host
+  file_name    = "talos-${var.talos.version}-amd64.iso"
+  url          = "https://factory.talos.dev/image/${var.talos.schematic_id}/${var.talos.version}/nocloud-amd64.iso"
+}
+
+resource "proxmox_virtual_environment_file" "controlplane" {
+  for_each     = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
+  node_name    = each.value.host
+  content_type = "snippets"
+  datastore_id = each.value.boot_pool
+
+  source_raw {
+    data      = <<-EOF
+      hostname: ${each.value.name}
+      instance-id: 1000
+      instance-type: ${each.value.cpu}VCPU-${floor(each.value.memory / 1024)}GB
+      provider-id: "proxmox://${var.proxmox.name}/1000"
+      region: ${var.proxmox.name}
+      zone: ${each.value.host}
+    EOF
+    file_name = "${each.value.name}.metadata.yaml"
+  }
+}
+
+resource "macaddress" "private" {
+  for_each = local.nodes
+}
+resource "macaddress" "public" {
+  for_each = local.nodes
+}
+
+resource "proxmox_virtual_environment_vm" "nodes" {
+  for_each = local.nodes
+
+  name      = each.value.name
+  node_name = each.value.host
+
+  description = "Node ${each.value.name} for cluster ${var.cluster.name}."
+  tags = concat(
+    ["kubernetes", "terraform"],
+    each.value.public ? ["public"] : ["private"],
+    [each.value.type]
+  )
+
+  stop_on_destroy     = true
+  timeout_stop_vm     = 60
+  timeout_shutdown_vm = 60
+
+  cpu {
+    cores = each.value.cores
+    type  = "host"
+  }
+  memory {
+    dedicated = each.value.memory
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  agent {
+    enabled = true
+    timeout = "1m"
+  }
+
+  network_device {
+    bridge      = each.value.public ? "vmpublic0" : "vmprivate0"
+    mac_address = macaddress.private[each.key].address
+  }
+
+  disk {
+    file_format  = "raw"
+    interface    = "scsi0"
+    datastore_id = each.value.boot_pool
+    file_id      = proxmox_virtual_environment_download_file.talos.id
+    backup       = false
+  }
+
+  disk {
+    file_format  = "raw"
+    interface    = "virtio0"
+    datastore_id = each.value.boot_pool
+    size         = each.value.boot_size
+    backup       = true
+  }
+
+  initialization {
+    meta_data_file_id = proxmox_virtual_environment_file.controlplane[each.key].id
+  }
+}
+
+resource "proxmox_virtual_environment_role" "csi" {
+  role_id = "kubernetes-csi"
+
+  privileges = [
+    "VM.Audit",
+    "VM.Config.Disk",
+    "Datastore.Allocate",
+    "Datastore.AllocateSpace",
+    "Datastore.Audit",
+  ]
+}
+resource "proxmox_virtual_environment_user" "csi" {
+  comment = "Managed by Terraform"
+  user_id = "kubernetes-csi@pve"
+  enabled = true
+}
+resource "proxmox_virtual_environment_user_token" "csi" {
+  comment    = "Managed by Terraform"
+  token_name = "terraform"
+  user_id    = proxmox_virtual_environment_user.csi.user_id
+}
+
+resource "proxmox_virtual_environment_role" "ccm" {
+  role_id = "kubernetes-ccm"
+
+  privileges = [
+    "VM.Audit",
+  ]
+}
+resource "proxmox_virtual_environment_user" "ccm" {
+  comment = "Managed by Terraform"
+  user_id = "kubernetes-ccm@pve"
+  enabled = true
+}
+resource "proxmox_virtual_environment_user_token" "ccm" {
+  comment    = "Managed by Terraform"
+  token_name = "terraform"
+  user_id    = proxmox_virtual_environment_user.ccm.user_id
+}

modules/cluster/variables.tf

@@ -5,24 +5,29 @@ variable "nodes" {
     host   = string
 
     cores  = optional(number, 4)
-    memory = optional(string, 4096)
+    memory = optional(number, 4096)
 
     boot_pool = string
-    boot_size = optional(string, "32G")
+    boot_size = optional(number, 32)
   }))
 }
 
-variable "versions" {
+variable "talos" {
   type = object({
-    talos      = optional(string, "v1.7.5")
+    version    = optional(string, "v1.7.5")
     kubernetes = optional(string, "v1.30.0")
+
+    # Default is:
+    #  customization:
+    #    systemExtensions:
+    #      officialExtensions:
+    #          - siderolabs/kata-containers
+    #          - siderolabs/qemu-guest-agent
+    #          - siderolabs/tailscale
+    schematic_id = optional(string, "a3f29a65dfd32b73c76f14eef96ef7588cf08c7d737d24fae9b8216d1ffa5c3d")
   })
 }
 
-variable "iso" {
-  type        = string
-  description = "Downloaded from factory.talos.dev, select quemu agent and tailscale extensions."
-}
 
 variable "allow_scheduling_on_control_planes" {
   default     = false
@@ -52,17 +57,10 @@ variable "tailscale_authkey" {
 
 variable "proxmox" {
   type = object({
-    name     = string
+    name           = string
-    url      = string
+    url            = string
-    insecure = optional(bool, false)
+    insecure       = optional(bool, false)
-    csi = object({
+    iso_storage_id = string
-      username = string
-      token    = string
-    })
-    ccm = object({
-      username = string
-      token    = string
-    })
   })
   sensitive = true
 }

modules/cluster/versions.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     proxmox = {
-      source  = "Telmate/proxmox"
+      source  = "bpg/proxmox"
-      version = "3.0.1-rc3"
+      version = "0.61.1"
     }
     talos = {
       source  = "siderolabs/talos"