feat: work done, but using wrong talos image, should use nocloud version not metal

clusters/pink/kubeconfig

@@ -1,26 +0,0 @@
-apiVersion: v1
-kind: Config
-clusters:
-- name: pink
-  cluster:
-    server: https://api.pink.k8s.tjo.cloud:6443
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVRDZ0F3SUJBZ0lSQU9vTSthdnlEZkN1THc4TVptRnFnNll3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOREEzTVRReE1UQTFNRGRhRncwek5EQTNNVEl4TVRBMQpNRGRhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVNEU3JVdUk1RFRxMHR4MkM4SmJHWmsvZXJlU0dlQU1tUzgzcE9RZklMbEFaMWE1NjJ4MnZobi9lMHYKWHQ0NmJFbzQ3QWttTlh3eEFNTExSRmkrYjF5V28yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGSGNLNFBEZHNSMmxnbHdRZmUwN2c3L0gralpGTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFERm1qeTkKcm4yVXpUN1hTL1pUZW5nbHpIZC9OOFZHU1JIQlQzV0xCbkpZendJZ0hKMFdEd1NoOGJQTHpvSzNxSkNjVDlmQgp1ZmZ0U0ZMVHdhdkNvdmF6Y09BPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
-contexts:
-- name: oidc@pink
-  context:
-    cluster: pink
-    namespace: default
-    user: oidc
-current-context: oidc@pink
-users:
-- name: oidc
-  user:
-    exec:
-      apiVersion: client.authentication.k8s.io/v1beta1
-      command: kubectl
-      args:
-      - oidc-login
-      - get-token
-      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
-      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
-      - --oidc-extra-scope=profile

clusters/pink/kubernetes.tf

@@ -1,271 +0,0 @@
-module "cluster" {
-  source = "../../modules/cluster"
-
-  providers = {
-    helm.template = helm.template
-  }
-
-  nodes = {
-    one = {
-      public    = true
-      type      = "controlplane"
-      host      = "hetzner"
-      boot_pool = "hetzner-main-data"
-    },
-    two = {
-      public    = true
-      type      = "controlplane"
-      host      = "hetzner"
-      boot_pool = "hetzner-main-data"
-    },
-    three = {
-      public    = true
-      type      = "controlplane"
-      host      = "hetzner"
-      boot_pool = "hetzner-main-data"
-    },
-    four = {
-      public    = false
-      type      = "worker"
-      host      = "hetzner"
-      boot_pool = "hetzner-main-data"
-    }
-    five = {
-      public    = false
-      type      = "worker"
-      host      = "hetzner"
-      boot_pool = "hetzner-main-data"
-    }
-  }
-
-  versions = {
-    talos      = "v1.7.5"
-    kubernetes = "v1.30.0"
-  }
-
-  iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
-
-  cluster = {
-    name   = "pink"
-    domain = "k8s.tjo.cloud"
-    oidc = {
-      client_id  = var.oidc_client_id
-      issuer_url = var.oidc_issuer_url
-    }
-  }
-
-  tailscale_authkey = var.tailscale_authkey
-}
-
-resource "local_file" "kubeconfig" {
-  content  = module.cluster.kubeconfig
-  filename = "${path.module}/kubeconfig"
-}
-
-resource "kubernetes_manifest" "hetzner-nodes-as-loadbalancers" {
-  manifest = {
-    apiVersion = "cilium.io/v2alpha1"
-    kind       = "CiliumLoadBalancerIPPool"
-    metadata = {
-      name = "hetzner-nodes"
-    }
-    spec = {
-      blocks = concat(
-        [for k, node in module.cluster.nodes : { start : node.address_ipv4 } if node.public],
-        # [for k, node in module.cluster.nodes : { start : node.address_ipv6 } if node.public],
-      )
-    }
-  }
-}
-
-resource "kubernetes_namespace" "tjo-cloud" {
-  metadata {
-    name = "tjo-cloud"
-  }
-}
-
-resource "kubernetes_secret" "digitalocean-token" {
-  metadata {
-    name      = "digitalocean-token"
-    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-  }
-  data = {
-    token = var.digitalocean_token
-  }
-}
-
-resource "helm_release" "external-dns" {
-  name       = "external-dns"
-  chart      = "external-dns"
-  repository = "https://kubernetes-sigs.github.io/external-dns/"
-  version    = "v1.14.5"
-  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
-
-  set {
-    name  = "namespaced"
-    value = "true"
-  }
-
-  set {
-    name  = "provider"
-    value = "digitalocean"
-  }
-
-  set {
-    name  = "env[0].name"
-    value = "DO_TOKEN"
-  }
-  set {
-    name  = "env[0].valueFrom.secretKeyRef.name"
-    value = kubernetes_secret.digitalocean-token.metadata[0].name
-  }
-  set {
-    name  = "env[0].valueFrom.secretKeyRef.key"
-    value = "token"
-  }
-
-  set_list {
-    name  = "sources"
-    value = ["gateway-httproute", "gateway-tlsroute", "gateway-tcproute", "gateway-udproute", "ingress", "service"]
-  }
-}
-
-resource "helm_release" "cert-manager" {
-  name       = "cert-manager"
-  chart      = "cert-manager"
-  repository = "https://charts.jetstack.io"
-  version    = "v1.15.1"
-  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
-
-  set {
-    name  = "crds.enabled"
-    value = true
-  }
-
-  set_list {
-    name  = "extraArgs"
-    value = ["--enable-gateway-api"]
-  }
-}
-
-
-resource "kubernetes_manifest" "tjo-cloud-issuer" {
-  manifest = {
-    apiVersion = "cert-manager.io/v1"
-    kind       = "Issuer"
-    metadata = {
-      name      = "tjo-cloud"
-      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-    }
-    spec = {
-      acme = {
-        email  = "tine@tjo.space"
-        server = "https://acme-staging-v02.api.letsencrypt.org/directory"
-        privateKeySecretRef = {
-          name = "tjo-cloud-acme-account"
-        }
-        solvers = [
-          {
-            dns01 = {
-              digitalocean = {
-                tokenSecretRef = {
-                  name = kubernetes_secret.digitalocean-token.metadata[0].name
-                  key  = "token"
-                }
-              }
-            }
-          }
-        ]
-      }
-    }
-  }
-}
-
-resource "kubernetes_manifest" "gateway" {
-  manifest = {
-    apiVersion = "gateway.networking.k8s.io/v1"
-    kind       = "Gateway"
-    metadata = {
-      name      = "gateway"
-      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-      annotations = {
-        "cert-manager.io/issuer" : "tjo-cloud"
-      }
-    }
-    spec = {
-      gatewayClassName = "cilium"
-      listeners = [
-        {
-          name : "http"
-          hostname : "*.${module.cluster.name}.${module.cluster.domain}"
-          protocol : "HTTPS"
-          port : 443
-          allowedRoutes : {
-            namespaces : {
-              from : "Same"
-            }
-          }
-          tls : {
-            mode : "Terminate"
-            certificateRefs : [
-              {
-                name : "tjo-cloud-tls"
-              }
-            ]
-          }
-        }
-      ]
-    }
-  }
-}
-
-resource "helm_release" "dashboard" {
-  name       = "kubernetes-dashboard"
-  repository = "https://kubernetes.github.io/dashboard"
-  chart      = "kubernetes-dashboard"
-  version    = "7.5.0"
-  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
-}
-
-resource "kubernetes_manifest" "dashoard-http-route" {
-  depends_on = [
-    kubernetes_manifest.gateway,
-  ]
-
-  manifest = {
-    apiVersion = "gateway.networking.k8s.io/v1"
-    kind       = "HTTPRoute"
-    metadata = {
-      name      = "dashboard"
-      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-    }
-    spec = {
-      parentRefs = [
-        {
-          name : "gateway"
-        }
-      ]
-      hostnames = [
-        "dashboard.${module.cluster.name}.${module.cluster.domain}"
-      ]
-      rules = [
-        {
-          matches = [
-            {
-              path : {
-                value : "/"
-                type : "PathPrefix"
-              }
-            }
-          ]
-          backendRefs = [
-            {
-              name : "kubernetes-dashboard-web"
-              port : 8000
-            }
-          ]
-        }
-      ]
-    }
-  }
-}

k8s.tjo.cloud/dashboard.tf

@@ -0,0 +1,140 @@
+resource "helm_release" "dashboard" {
+  name       = "kubernetes-dashboard"
+  repository = "https://kubernetes.github.io/dashboard"
+  chart      = "kubernetes-dashboard"
+  version    = "7.5.0"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  set {
+    name  = "kong.enabled"
+    value = false
+  }
+}
+
+resource "kubernetes_manifest" "dashoard-http-route" {
+  depends_on = [
+    kubernetes_manifest.gateway,
+  ]
+
+  manifest = {
+    apiVersion = "gateway.networking.k8s.io/v1"
+    kind       = "HTTPRoute"
+    metadata = {
+      name      = "dashboard"
+      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+    }
+    spec = {
+      parentRefs = [
+        {
+          name : "gateway"
+        }
+      ]
+      hostnames = [
+        "dashboard.${module.cluster.domain}"
+      ]
+      rules = [
+        {
+          matches = [
+            {
+              path : {
+                value : "/"
+                type : "PathPrefix"
+              }
+            }
+          ]
+          backendRefs = [
+            {
+              name : "kubernetes-dashboard-web"
+              port : 8000
+            }
+          ]
+        },
+        {
+          matches = [
+            {
+              path : {
+                value : "/api/v1/login"
+                type : "PathPrefix"
+              }
+            },
+            {
+              path : {
+                value : "/api/v1/csrftoken/login"
+                type : "PathPrefix"
+              }
+            },
+            {
+              path : {
+                value : "/api/v1/me"
+                type : "PathPrefix"
+              }
+            },
+          ]
+          backendRefs = [
+            {
+              name : "kubernetes-dashboard-auth"
+              port : 8000
+            }
+          ]
+        },
+        {
+          matches = [
+            {
+              path : {
+                value : "/api"
+                type : "PathPrefix"
+              }
+            }
+          ]
+          backendRefs = [
+            {
+              name : "kubernetes-dashboard-api"
+              port : 8000
+            }
+          ]
+        },
+      ]
+    }
+  }
+}
+
+resource "kubernetes_secret" "dashboard-oidc" {
+  metadata {
+    name      = "dashboard-oidc"
+    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+  }
+  data = {
+    client-secret = "null"
+  }
+}
+
+resource "kubernetes_manifest" "dashboard-oidc" {
+  manifest = {
+    apiVersion = "gateway.envoyproxy.io/v1alpha1"
+    kind       = "SecurityPolicy"
+    metadata = {
+      name      = "dashboard-oidc"
+      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+    }
+    spec = {
+      targetRef = {
+        group : "gateway.networking.k8s.io"
+        kind : "HTTPRoute"
+        name : "dashboard"
+      }
+      oidc = {
+        provider = {
+          issuer : var.oidc_issuer_url
+        }
+        clientID : var.oidc_client_id
+        clientSecret : {
+          name : "dashboard-oidc"
+        }
+        scopes : ["openid", "email", "profile"]
+        forwardAccessToken : true
+
+        redirectURL : "https://dashboard.${module.cluster.domain}/login"
+      }
+    }
+  }
+}

k8s.tjo.cloud/gateway.tf

@@ -0,0 +1,142 @@
+resource "kubernetes_secret" "digitalocean-token" {
+  metadata {
+    name      = "digitalocean-token"
+    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+  }
+  data = {
+    token = var.digitalocean_token
+  }
+}
+
+resource "helm_release" "cert-manager" {
+  name       = "cert-manager"
+  chart      = "cert-manager"
+  repository = "https://charts.jetstack.io"
+  version    = "v1.15.1"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  set {
+    name  = "crds.enabled"
+    value = true
+  }
+
+  set_list {
+    name  = "extraArgs"
+    value = ["--enable-gateway-api"]
+  }
+}
+
+resource "kubernetes_manifest" "tjo-cloud-issuer" {
+  manifest = {
+    apiVersion = "cert-manager.io/v1"
+    kind       = "Issuer"
+    metadata = {
+      name      = "tjo-cloud"
+      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+    }
+    spec = {
+      acme = {
+        email  = "tine@tjo.space"
+        server = "https://acme-staging-v02.api.letsencrypt.org/directory"
+        privateKeySecretRef = {
+          name = "tjo-cloud-acme-account"
+        }
+        solvers = [
+          {
+            dns01 = {
+              digitalocean = {
+                tokenSecretRef = {
+                  name = kubernetes_secret.digitalocean-token.metadata[0].name
+                  key  = "token"
+                }
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+
+resource "helm_release" "envoy" {
+  name       = "envoy"
+  chart      = "gateway-helm"
+  repository = "oci://docker.io/envoyproxy"
+  version    = "v1.1.0-rc.1"
+  namespace  = "kube-system"
+
+  values = [
+    yamlencode({
+      config = {
+        envoyGateway = {
+          provider = {
+            type = "Kubernetes"
+            kubernetes = {
+              envoyDaemonSet  = {}
+              envoyDeployment = null
+            }
+          }
+          gateway = {
+            controllerName = "gateway.envoyproxy.io/gatewayclass-controller"
+          }
+          logging = {
+            level = {
+              default = "info"
+            }
+          }
+        }
+      }
+    })
+  ]
+}
+
+resource "kubernetes_manifest" "gateway-class" {
+  manifest = {
+    apiVersion = "gateway.networking.k8s.io/v1"
+    kind       = "GatewayClass"
+    metadata = {
+      name = "envoy"
+    }
+    spec = {
+      controllerName : "gateway.envoyproxy.io/gatewayclass-controller"
+    }
+  }
+}
+
+resource "kubernetes_manifest" "gateway" {
+  manifest = {
+    apiVersion = "gateway.networking.k8s.io/v1"
+    kind       = "Gateway"
+    metadata = {
+      name      = "gateway"
+      namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+      annotations = {
+        "cert-manager.io/issuer" : "tjo-cloud"
+      }
+    }
+    spec = {
+      gatewayClassName = "envoy"
+      listeners = [
+        {
+          name : "http"
+          hostname : "*.${module.cluster.name}.${module.cluster.domain}"
+          protocol : "HTTPS"
+          port : 443
+          allowedRoutes : {
+            namespaces : {
+              from : "Same"
+            }
+          }
+          tls : {
+            mode : "Terminate"
+            certificateRefs : [
+              {
+                name : "tjo-cloud-tls"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

k8s.tjo.cloud/kubeconfig

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: tjo-cloud
+  cluster:
+    server: https://api.k8s.tjo.cloud:6443
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRVHV4SURwTXNNUXNYWitLb2k1RkdsakFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN4TmpFNE1qWXhPVm9YRFRNME1EY3hOREU0TWpZeApPVm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCQ01zTVdqWjU1alRzaWd6aVcrT1BUSm1uS1VLQ2VTbktnTmptOFNRVE0ydm1tQklaM1I1UDlKcTY4ZmEKUU9zZ0RXSE96aFF6MjRNRUw1aFhpUzRjU3B1allUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVEVG41Sy8raXROdEgyczM0MTR3TEFjV0V0Wm93Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQU56bVV6OVEKZVhVVVJzei9KWlRlb29ibzF0Y1BHN2p3cGFlSDQ0VFVwVEE4QWlFQTFZcTdhUWdkb2lVTFNac2IrT3JLeDJyWQpGRTdSVkpxR0lyM1U4c3daY3k4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+contexts:
+- name: oidc@tjo-cloud
+  context:
+    cluster: tjo-cloud
+    namespace: default
+    user: oidc
+current-context: oidc@tjo-cloud
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
+      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
+      - --oidc-extra-scope=profile

k8s.tjo.cloud/main.tf

@@ -0,0 +1,117 @@
+module "cluster" {
+  source = "../modules/cluster"
+
+  providers = {
+    helm.template = helm.template
+  }
+
+  versions = {
+    talos      = "v1.7.5"
+    kubernetes = "v1.30.0"
+  }
+
+  iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
+
+  cluster = {
+    name   = "tjo-cloud"
+    domain = "k8s.tjo.cloud"
+    oidc = {
+      client_id  = var.oidc_client_id
+      issuer_url = var.oidc_issuer_url
+    }
+  }
+
+  proxmox = {
+    name = "tjo-cloud"
+    url  = "https://proxmox.tjo.cloud/api2/json"
+    ccm = {
+      username = var.proxmox_ccm_username
+      token    = var.proxmox_ccm_token
+    }
+    csi = {
+      username = var.proxmox_csi_username
+      token    = var.proxmox_csi_token
+    }
+  }
+
+  tailscale_authkey = var.tailscale_authkey
+
+  allow_scheduling_on_control_planes = true
+  nodes = {
+    pink = {
+      public    = true
+      type      = "controlplane"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 4096
+    }
+    purple = {
+      public    = true
+      type      = "controlplane"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 4096
+    }
+    violet = {
+      public    = true
+      type      = "controlplane"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 4096
+    }
+    blue = {
+      public    = false
+      type      = "worker"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 16384
+    }
+    cyan = {
+      public    = false
+      type      = "worker"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 16384
+    }
+    green = {
+      public    = false
+      type      = "worker"
+      host      = "hetzner"
+      boot_pool = "hetzner-main-data"
+      cores     = 4
+      memory    = 16384
+    }
+  }
+}
+
+resource "local_file" "kubeconfig" {
+  content  = module.cluster.kubeconfig
+  filename = "${path.module}/kubeconfig"
+}
+
+resource "kubernetes_manifest" "hetzner-nodes-as-loadbalancers" {
+  manifest = {
+    apiVersion = "cilium.io/v2alpha1"
+    kind       = "CiliumLoadBalancerIPPool"
+    metadata = {
+      name = "hetzner-nodes"
+    }
+    spec = {
+      blocks = concat(
+        [for k, node in module.cluster.nodes : { start : node.ipv4 } if node.public],
+        # [for k, node in module.cluster.nodes : { start : node.ipv6 } if node.public],
+      )
+    }
+  }
+}
+
+resource "kubernetes_namespace" "tjo-cloud" {
+  metadata {
+    name = "tjo-cloud"
+  }
+}

k8s.tjo.cloud/providers.tf

@@ -42,6 +42,7 @@ provider "proxmox" {
 }
 
 provider "digitalocean" {
+  token = var.digitalocean_token
 }
 
 provider "helm" {

k8s.tjo.cloud/variables.tf

@@ -6,16 +6,13 @@ variable "tailscale_authkey" {
 variable "oidc_username" {
   type = string
 }
-
 variable "oidc_password" {
   type      = string
   sensitive = true
 }
-
 variable "oidc_client_id" {
   type = string
 }
-
 variable "oidc_issuer_url" {
   type = string
 }
@@ -24,3 +21,19 @@ variable "digitalocean_token" {
   type      = string
   sensitive = true
 }
+
+variable "proxmox_csi_username" {
+  type = string
+}
+variable "proxmox_csi_token" {
+  type      = string
+  sensitive = true
+}
+
+variable "proxmox_ccm_username" {
+  type = string
+}
+variable "proxmox_ccm_token" {
+  type      = string
+  sensitive = true
+}

modules/cluster/main.tf

@@ -1,12 +1,10 @@
 locals {
-  cluster_domain     = "${var.cluster.name}.${var.cluster.domain}"
-  cluster_api_domain = "${var.cluster.api.subdomain}.${local.cluster_domain}"
+  cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}"
+  cluster_endpoint   = "https://${local.cluster_api_domain}:${var.cluster.api.port}"
 
-  cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}"
+  nodes = { for k, v in var.nodes : k => merge(v, { name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-") }) }
 
-  nodes = { for k, v in var.nodes : k => merge(v, { name = "${k}.node.${local.cluster_domain}" }) }
-
-  nodes_with_address = { for k, v in local.nodes : k => merge(v, { address_ipv4 = proxmox_vm_qemu.this[k].default_ipv4_address, address_ipv6 = proxmox_vm_qemu.this[k].default_ipv6_address }) }
+  nodes_with_address = { for k, v in local.nodes : k => merge(v, { ipv4 = try(proxmox_vm_qemu.this[k].default_ipv4_address, false), ipv6 = try(proxmox_vm_qemu.this[k].default_ipv6_address, false) }) }
 
   first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0]
 
@@ -25,9 +23,22 @@ locals {
     "100.64.0.0/10",
     "fd7a:115c:a1e0::/96"
   ]
+
+  proxmox-cloud-controller-manager = {
+    clusters : [{
+      region : var.proxmox.name,
+      url : var.proxmox.url,
+      insecure : var.proxmox.insecure,
+      token_id : var.proxmox.ccm.username,
+      token_secret : var.proxmox.ccm.token,
+    }]
+  }
 }
 
-resource "macaddress" "this" {
+resource "macaddress" "private" {
+  for_each = local.nodes
+}
+resource "macaddress" "public" {
   for_each = local.nodes
 }
 
@@ -39,6 +50,7 @@ resource "proxmox_vm_qemu" "this" {
   tags = join(";", concat(
     ["kubernetes", "terraform"],
     each.value.public ? ["public"] : ["private"],
+    [each.value.type]
   ))
 
   cores  = each.value.cores
@@ -52,7 +64,7 @@ resource "proxmox_vm_qemu" "this" {
   network {
     model   = "virtio"
     bridge  = each.value.public ? "vmpublic0" : "vmprivate0"
-    macaddr = macaddress.this[each.key].address
+    macaddr = macaddress.private[each.key].address
   }
 
   disks {
@@ -75,21 +87,21 @@ resource "proxmox_vm_qemu" "this" {
 }
 
 resource "digitalocean_record" "controlplane-A" {
-  for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
+  for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) }
 
   domain = var.cluster.domain
   type   = "A"
-  name   = "${var.cluster.api.subdomain}.${var.cluster.name}"
+  name   = var.cluster.api.subdomain
   value  = each.value.default_ipv4_address
   ttl    = 30
 }
 
 resource "digitalocean_record" "controlplane-AAAA" {
-  for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
+  for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) }
 
   domain = var.cluster.domain
   type   = "AAAA"
-  name   = "${var.cluster.api.subdomain}.${var.cluster.name}"
+  name   = var.cluster.api.subdomain
   value  = each.value.default_ipv6_address
   ttl    = 30
 }
@@ -126,18 +138,6 @@ data "talos_machine_configuration" "worker" {
   ]
 }
 
-data "talos_machine_disks" "boot" {
-  for_each = local.nodes_with_address
-
-  client_configuration = talos_machine_secrets.this.client_configuration
-  node                 = each.value.name
-  endpoint             = each.value.address_ipv4
-
-  filters = {
-    size = "< 60GB"
-  }
-}
-
 data "helm_template" "cilium" {
   provider = helm.template
 
@@ -211,25 +211,11 @@ data "helm_template" "cilium" {
         enabled : true
       }
     },
-    # Ingress gateway
     gatewayAPI : {
-      enabled : true
-      hostNetwork : {
-        enabled : true
-      }
+      enabled : false
     }
     envoy : {
-      enabled : true
-      securityContext : {
-        capabilities : {
-          keepCapNetBindService : true
-          envoy : [
-            "NET_ADMIN",
-            "SYS_ADMIN",
-            "NET_BIND_SERVICE"
-          ]
-        }
-      }
+      enabled : false
     }
   })]
 }
@@ -241,7 +227,7 @@ resource "talos_machine_configuration_apply" "controlplane" {
   machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration
 
   node     = each.value.name
-  endpoint = each.value.address_ipv4
+  endpoint = each.value.ipv4
 
   apply_mode = "reboot"
 
@@ -267,7 +253,7 @@ resource "talos_machine_configuration_apply" "controlplane" {
         proxy : {
           disabled : true
         }
-        allowSchedulingOnControlPlanes : true,
+        allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes,
         apiServer : {
           extraArgs : {
             "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
@@ -296,32 +282,56 @@ resource "talos_machine_configuration_apply" "controlplane" {
               apiGroup: rbac.authorization.k8s.io
             EOF
           },
-          {
-            name : "gateway-api-crds"
-            contents : file("${path.module}/gateway-api-crds.yaml")
-          },
           {
             name : "cilium"
             contents : data.helm_template.cilium.manifest
+          },
+          {
+            name : "proxmox-cloud-controller-manager"
+            contents : <<-EOF
+            apiVersion: v1
+            kind: Secret
+            metadata:
+              name: proxmox-cloud-controller-manager
+              namespace: kube-system
+            data:
+              config.yaml: ${base64encode(yamlencode(local.proxmox-cloud-controller-manager))}
+            EOF
           }
-        ],
+        ]
+        externalCloudProvider : {
+          enabled : true
+          manifests : [
+            "https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/v0.7.0/docs/deploy/proxmox-csi-plugin-talos.yml",
+            "https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/v0.4.2/docs/deploy/cloud-controller-manager-talos.yml"
+          ]
+        }
+        extraManifests : [
+          #"https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/v0.8.5/deploy/standalone-install.yaml",
+          "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml",
+        ]
       }
       machine = {
         kubelet = {
           nodeIP : {
             validSubnets : local.tailscaleSubnets
           }
+          extraArgs : {
+            rotate-server-certificates : true
+            cloud-provider : "external"
+          }
         }
         network = {
           hostname = each.value.name
         }
         install = {
           image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}"
-          disk  = data.talos_machine_disks.boot[each.key].disks[0].name
+          disk  = "/dev/vda"
         }
         nodeLabels = {
-          "k8s.tjo.cloud/public" = each.value.public ? "true" : "false"
-          "k8s.tjo.cloud/host"   = each.value.host
+          "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
+          "k8s.tjo.cloud/host"    = each.value.host
+          "k8s.tjo.cloud/proxmox" = var.proxmox.name
         }
       }
     }),
@@ -330,7 +340,8 @@ resource "talos_machine_configuration_apply" "controlplane" {
       kind : "ExtensionServiceConfig"
       name : "tailscale"
       environment : [
-        "TS_AUTHKEY=${var.tailscale_authkey}"
+        "TS_AUTHKEY=${var.tailscale_authkey}",
+        "TS_HOSTNAME=${replace(each.value.name, ".", "-")}",
       ]
     })
   ]
@@ -343,7 +354,7 @@ resource "talos_machine_configuration_apply" "worker" {
   machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration
 
   node     = each.value.name
-  endpoint = each.value.address_ipv4
+  endpoint = each.value.ipv4
 
   apply_mode = "reboot"
 
@@ -370,17 +381,22 @@ resource "talos_machine_configuration_apply" "worker" {
           nodeIP : {
             validSubnets : local.tailscaleSubnets
           }
+          extraArgs : {
+            rotate-server-certificates : true
+            cloud-provider : "external"
+          }
         }
         network = {
           hostname = each.value.name
         }
         install = {
           image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}"
-          disk  = data.talos_machine_disks.boot[each.key].disks[0].name
+          disk  = "/dev/vda"
         }
         nodeLabels = {
-          "k8s.tjo.cloud/public" = each.value.public ? "true" : "false"
-          "k8s.tjo.cloud/host"   = each.value.host
+          "k8s.tjo.cloud/public"  = each.value.public ? "true" : "false"
+          "k8s.tjo.cloud/host"    = each.value.host
+          "k8s.tjo.cloud/proxmox" = var.proxmox.name
         }
       }
     }),
@@ -389,7 +405,8 @@ resource "talos_machine_configuration_apply" "worker" {
       kind : "ExtensionServiceConfig"
       name : "tailscale"
       environment : [
-        "TS_AUTHKEY=${var.tailscale_authkey}"
+        "TS_AUTHKEY=${var.tailscale_authkey}",
+        "TS_HOSTNAME=${replace(each.value.name, ".", "-")}",
       ]
     })
   ]
@@ -402,7 +419,7 @@ resource "talos_machine_bootstrap" "this" {
   ]
 
   node                 = local.first_controlplane_node.name
-  endpoint             = local.first_controlplane_node.address_ipv4
+  endpoint             = local.first_controlplane_node.ipv4
   client_configuration = talos_machine_secrets.this.client_configuration
 }
 
@@ -412,5 +429,5 @@ data "talos_cluster_kubeconfig" "this" {
   ]
 
   client_configuration = talos_machine_secrets.this.client_configuration
-  node                 = local.first_controlplane_node.address_ipv4
+  node                 = local.first_controlplane_node.ipv4
 }

modules/cluster/variables.tf

@@ -24,6 +24,12 @@ variable "iso" {
   description = "Downloaded from factory.talos.dev, select quemu agent and tailscale extensions."
 }
 
+variable "allow_scheduling_on_control_planes" {
+  default     = false
+  type        = bool
+  description = "Allow scheduling on control plane nodes"
+}
+
 variable "cluster" {
   type = object({
     name   = string
@@ -43,3 +49,20 @@ variable "tailscale_authkey" {
   type      = string
   sensitive = true
 }
+
+variable "proxmox" {
+  type = object({
+    name     = string
+    url      = string
+    insecure = optional(bool, false)
+    csi = object({
+      username = string
+      token    = string
+    })
+    ccm = object({
+      username = string
+      token    = string
+    })
+  })
+  sensitive = true
+}