feat: progress...

terraform/.gitignore

@@ -1,2 +1,3 @@
 kubeconfig
+!templates/kubeconfig
 talosconfig

terraform/kubernetes.tf

@@ -1,17 +1,36 @@
 locals {
   # Downloaded from factory.talos.dev
   # https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
-  iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
+  iso = "proxmox-backup-tjo-cloud:iso/talos-${var.talos_version}-tailscale-metal-amd64.iso"
 
   boot_pool = "hetzner-main-data"
 
-  cluster_endpoint = "https://api.${var.cluster_name}.${var.domain}:6443"
+  cluster_domain   = "api.${var.cluster_name}.${var.domain}"
+  cluster_port     = 6443
+  cluster_endpoint = "https://${local.cluster_domain}:${local.cluster_port}"
 
   nodes                     = { for k, v in var.nodes : k => merge(v, { name = "${k}.node.${var.cluster_name}.${var.domain}" }) }
   nodes_with_address        = { for k, v in local.nodes : k => merge(v, { address_ipv4 = proxmox_vm_qemu.this[k].default_ipv4_address, address_ipv6 = proxmox_vm_qemu.this[k].default_ipv6_address }) }
   first_controlplane_node   = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0]
   nodes_public_controlplane = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
 
+  podSubnets = [
+    "10.200.0.0/16",
+    #"fd9b:5314:fc70::/48",
+  ]
+  serviceSubnets = [
+    "10.201.0.0/16",
+    #"fd9b:5314:fc71::/48",
+  ]
+
+  # Nodes will use IPs from this subnets
+  # for communication between each other.
+  tailscaleSubnets = [
+    "100.64.0.0/10",
+    "fd7a:115c:a1e0::/96"
+  ]
+
+  apply_mode = "reboot"
 }
 
 resource "macaddress" "this" {
@@ -89,6 +108,9 @@ data "talos_machine_configuration" "controlplane" {
   cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
+  talos_version      = var.talos_version
+  kubernetes_version = var.kubernetes_version
+
   depends_on = [
     digitalocean_record.controlplane-A,
     digitalocean_record.controlplane-AAAA,
@@ -101,6 +123,9 @@ data "talos_machine_configuration" "worker" {
   cluster_endpoint = local.cluster_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
+  talos_version      = var.talos_version
+  kubernetes_version = var.kubernetes_version
+
   depends_on = [
     digitalocean_record.controlplane-A,
     digitalocean_record.controlplane-AAAA
@@ -120,18 +145,27 @@ data "talos_machine_disks" "boot" {
 }
 
 data "helm_template" "cilium" {
+  provider = helm.template
+
   name       = "cilium"
-  repository   = "https://helm.cilium.io/"
   chart      = "cilium"
+  repository = "https://helm.cilium.io/"
   version    = "1.15.6"
   namespace  = "kube-system"
-  include_crds = true
+
+  kube_version = var.kubernetes_version
+  api_versions = [
+    "gateway.networking.k8s.io/v1/GatewayClass",
+  ]
 
   values = [yamlencode({
     ipam : {
       mode : "kubernetes"
     },
-    kubeProxyReplacement : true
+    nodeIPAM : {
+      enabled : true
+    },
+    kubeProxyReplacement : "true"
     securityContext : {
       capabilities : {
         ciliumAgent : [
@@ -160,15 +194,22 @@ data "helm_template" "cilium" {
       },
       hostRoot : "/sys/fs/cgroup"
     },
-    k8sServiceHost : "localhost"
+    k8sServiceHost : local.cluster_domain
-    k8sServicePort : "7445"
+    k8sServicePort : local.cluster_port
     ipv4 : {
       enabled : true
     },
-    #ipv6: {
+    #ipv6 : {
-    #  enabled: true
+    #  enabled : true
     #},
     hubble : {
+      tls : {
+        auto : {
+          enabled : true
+          method : "cronJob"
+          schedule : "0 0 1 */4 *"
+        }
+      }
       ui : {
         enabled : true
       }
@@ -179,7 +220,6 @@ data "helm_template" "cilium" {
     # Ingress gateway
     gatewayAPI : {
       enabled : true
-      default : true
       hostNetwork : {
         enabled : true
         nodes : {
@@ -214,15 +254,26 @@ resource "talos_machine_configuration_apply" "controlplane" {
   node     = each.value.name
   endpoint = each.value.address_ipv4
 
-  apply_mode = "reboot"
+  apply_mode = local.apply_mode
 
   config_patches = [
     yamlencode({
       cluster : {
+        controlPlane : {
+          endpoint : local.cluster_endpoint
+          localAPIServerPort : local.cluster_port
+        }
+        etcd : {
+          #advertisedSubnets : [
+          #  local.tailscaleSubnet
+          #]
+        }
         network : {
           cni : {
             name : "none"
           }
+          podSubnets : local.podSubnets
+          serviceSubnets : local.serviceSubnets
         }
         proxy : {
           disabled : true
@@ -268,15 +319,15 @@ resource "talos_machine_configuration_apply" "controlplane" {
       }
       machine = {
         kubelet = {
-          extraArgs = {
+          nodeIP : {
-            rotate-server-certificates : "true"
+            validSubnets : local.tailscaleSubnets
           }
         }
         network = {
           hostname = each.value.name
         }
         install = {
-          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5"
+          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.talos_version}"
           disk  = data.talos_machine_disks.boot[each.key].disks[0].name
         }
         nodeLabels = {
@@ -292,7 +343,6 @@ resource "talos_machine_configuration_apply" "controlplane" {
         "TS_AUTHKEY=${var.tailscale_authkey}"
       ]
     })
-
   ]
 }
 
@@ -305,42 +355,48 @@ resource "talos_machine_configuration_apply" "worker" {
   node     = each.value.name
   endpoint = each.value.address_ipv4
 
-  apply_mode = "reboot"
+  apply_mode = local.apply_mode
 
   config_patches = [
     yamlencode({
       cluster : {
+        controlPlane : {
+          endpoint : local.cluster_endpoint
+          localAPIServerPort : local.cluster_port
+        }
         network : {
           cni : {
             name : "none"
           }
+          podSubnets : local.podSubnets
+          serviceSubnets : local.serviceSubnets
         }
         proxy : {
           disabled : true
         }
-        allowSchedulingOnControlPlanes : true,
+        allowSchedulingOnControlPlanes : true
         apiServer : {
           extraArgs : {
-            "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
+            "oidc-issuer-url" : var.oidc_issuer_url
-            "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
+            "oidc-client-id" : var.oidc_client_id
-            "oidc-username-claim" : "sub",
+            "oidc-username-claim" : "sub"
-            "oidc-username-prefix" : "oidc:",
+            "oidc-username-prefix" : "oidc:"
-            "oidc-groups-claim" : "groups",
+            "oidc-groups-claim" : "groups"
-            "oidc-groups-prefix" : "oidc:groups:",
+            "oidc-groups-prefix" : "oidc:groups:"
           }
         }
       }
       machine = {
         kubelet = {
-          extraArgs = {
+          nodeIP : {
-            rotate-server-certificates : "true"
+            validSubnets : local.tailscaleSubnets
           }
         }
         network = {
           hostname = each.value.name
         }
         install = {
-          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5"
+          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.talos_version}"
           disk  = data.talos_machine_disks.boot[each.key].disks[0].name
         }
       }
@@ -353,7 +409,6 @@ resource "talos_machine_configuration_apply" "worker" {
         "TS_AUTHKEY=${var.tailscale_authkey}"
       ]
     })
-
   ]
 }
 
@@ -369,8 +424,6 @@ resource "talos_machine_bootstrap" "this" {
 }
 
 data "talos_cluster_kubeconfig" "this" {
-  count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0
-
   depends_on = [
     talos_machine_bootstrap.this
   ]
@@ -380,25 +433,78 @@ data "talos_cluster_kubeconfig" "this" {
 }
 
 resource "local_file" "kubeconfig" {
-  count = length(local.nodes_public_controlplane) > 0 ? 1 : 0
+  content = templatefile("${path.module}/templates/kubeconfig.tftpl", {
-
+    cluster : {
-  content  = data.talos_cluster_kubeconfig.this[0].kubeconfig_raw
+      name : var.cluster_name,
+      endpoint : local.cluster_endpoint,
+      ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
+    }
+    oidc : {
+      issuer : var.oidc_issuer_url,
+      id : var.oidc_client_id,
+    }
+  })
   filename = "${path.module}/kubeconfig"
 }
 
-data "talos_client_configuration" "this" {
+resource "kubernetes_manifest" "hetzner-nodes-as-loadbalancers" {
-  count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0
+  depends_on = [
+    talos_machine_bootstrap.this
+  ]
 
-  cluster_name         = var.cluster_name
+  manifest = {
-  client_configuration = talos_machine_secrets.this.client_configuration
+    apiVersion = "cilium.io/v2alpha1"
-  endpoints            = values(local.nodes_public_controlplane)[*].default_ipv4_address
+    kind       = "CiliumLoadBalancerIPPool"
+    metadata = {
+      name = "hetzner-nodes"
+    }
+    spec = {
+      blocks = concat(
+        [for k, v in proxmox_vm_qemu.this : { start : v.default_ipv4_address } if var.nodes[k].public],
+        #[for k, v in proxmox_vm_qemu.this : { start : v.default_ipv6_address } if var.nodes[k].public],
+      )
+    }
+  }
 }
 
-resource "local_file" "talosconfig" {
+# TODO: Certmanager, externaldns...
-  count = length(local.nodes_public_controlplane) > 0 ? 1 : 0
+resource "helm_release" "cert-manager" {
+  depends_on = [
+    talos_machine_bootstrap.this
+  ]
 
-  content  = nonsensitive(data.talos_client_configuration.this[0].talos_config)
+  name       = "cert-manager"
-  filename = "${path.module}/talosconfig"
+  chart      = "cert-manager"
+  repository = "https://charts.jetstack.io"
+  version    = "v1.15.1"
+  namespace  = "kube-system"
+
+  set {
+    name  = "crds.enabled"
+    value = true
+  }
+}
+
+resource "kubernetes_manifest" "gateway" {
+  depends_on = [
+    talos_machine_bootstrap.this
+  ]
+
+  manifest = {
+    apiVersion = "gateway.networking.k8s.io/v1"
+    kind       = "Gateway"
+    metadata = {
+      name      = "gateway"
+      namespace = "kube-system"
+    }
+    spec = {
+      gatewayClassName = "cilium"
+      listeners = [
+        { name : "http", protocol : "HTTP", port : 80 },
+        { name : "https", protocol : "HTTPS", port : 443 },
+      ]
+    }
+  }
 }
 
 resource "helm_release" "dashboard" {
@@ -411,13 +517,47 @@ resource "helm_release" "dashboard" {
   chart      = "kubernetes-dashboard"
   version    = "7.5.0"
   namespace  = "kube-system"
+}
+
+resource "kubernetes_manifest" "dashoard-http-route" {
+  depends_on = [
+    talos_machine_bootstrap.this,
+    kubernetes_manifest.gateway,
+  ]
+
+  manifest = {
+    apiVersion = "gateway.networking.k8s.io/v1"
+    kind       = "HTTPRoute"
+    metadata = {
+      name      = "dashboard"
+      namespace = "kube-system"
+    }
+    spec = {
+      parentRefs = [
+        { name : "gateway" }
+      ]
+      hostnames = [
+        "dashboard.${var.cluster_name}.${var.domain}"
+      ]
+      rules = [
+        {
+          matches = [
+            {
+              path : {
+                value : "/"
+                type : "PathPrefix"
+              }
+            }
+          ]
+          backendRefs = [
+            {
+              name : "kubernetes-dashboard-kong-proxy"
+              port : 443
+            }
+          ]
+        }
+      ]
 
-  set {
-    name  = "ingress.enabled"
-    value = true
     }
-  set {
-    name  = "useDefaultIngressClass"
-    value = true
   }
 }

terraform/providers.tf

@@ -44,8 +44,46 @@ provider "proxmox" {
 provider "digitalocean" {
 }
 
+provider "helm" {
+  alias = "template"
+}
+
 provider "helm" {
   kubernetes {
-    config_path = "./kubeconfig"
+    host                   = local.cluster_endpoint
+    cluster_ca_certificate = base64decode(data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate)
+    exec {
+      api_version = "client.authentication.k8s.io/v1beta1"
+      command     = "kubectl"
+      args = [
+        "oidc-login",
+        "get-token",
+        "--oidc-issuer-url", var.oidc_issuer_url,
+        "--oidc-client-id", var.oidc_client_id,
+        "--oidc-extra-scope", "profile",
+        "--grant-type", "password",
+        "--username", var.oidc_username,
+        "--password", var.oidc_password,
+      ]
+    }
+  }
+}
+
+provider "kubernetes" {
+  host                   = local.cluster_endpoint
+  cluster_ca_certificate = base64decode(data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate)
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "kubectl"
+    args = [
+      "oidc-login",
+      "get-token",
+      "--oidc-issuer-url", var.oidc_issuer_url,
+      "--oidc-client-id", var.oidc_client_id,
+      "--oidc-extra-scope", "profile",
+      "--grant-type", "password",
+      "--username", var.oidc_username,
+      "--password", var.oidc_password,
+    ]
   }
 }

terraform/templates/kubeconfig.tftpl

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: ${cluster.name}
+  cluster:
+    server: ${cluster.endpoint}
+    certificate-authority-data: ${cluster.ca}
+contexts:
+- name: oidc@${cluster.name}
+  context:
+    cluster: ${cluster.name}
+    namespace: default
+    user: oidc
+current-context: oidc@${cluster.name}
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=${oidc.issuer}
+      - --oidc-client-id=${oidc.id}
+      - --oidc-extra-scope=profile

terraform/variables.tf

@@ -6,6 +6,16 @@ variable "nodes" {
   }))
 }
 
+variable "talos_version" {
+  type    = string
+  default = "v1.7.5"
+}
+
+variable "kubernetes_version" {
+  type    = string
+  default = "v1.30.0"
+}
+
 variable "cluster_name" {
   type = string
 }
@@ -18,3 +28,20 @@ variable "tailscale_authkey" {
   type      = string
   sensitive = true
 }
+
+variable "oidc_username" {
+  type = string
+}
+
+variable "oidc_password" {
+  type      = string
+  sensitive = true
+}
+
+variable "oidc_client_id" {
+  type = string
+}
+
+variable "oidc_issuer_url" {
+  type = string
+}