From dd21e589d65ec5f558ffa9264e6602547889b2d2 Mon Sep 17 00:00:00 2001
From: Tine <tine@tjo.space>
Date: Mon, 22 Jul 2024 22:31:48 +0200
Subject: [PATCH] feat: better?

---
 k8s.tjo.cloud/kubeconfig      |   2 +-
 modules/cluster/components.tf | 142 ++++++++++++++++------------------
 modules/cluster/main.tf       |  31 ++++----
 modules/cluster/proxmox.tf    |   8 +-
 4 files changed, 86 insertions(+), 97 deletions(-)

diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig
index c56d7d6..6343db2 100755
--- a/k8s.tjo.cloud/kubeconfig
+++ b/k8s.tjo.cloud/kubeconfig
@@ -4,7 +4,7 @@ clusters:
 - name: tjo-cloud
   cluster:
     server: https://api.k8s.tjo.cloud:6443
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVRDZ0F3SUJBZ0lSQU5MekZhUjV4SENpZHFZQjB5K0Z2ZVF3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOREEzTWpBeE56STNNRFZhRncwek5EQTNNVGd4TnpJMwpNRFZhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVN3T2xEMnBYeXRyQjdFUkFNZFQ5VVkwaDZNN1BtU1k1SjVBa2F4bk8vYm1FOW84bjNMTU9JbVZFaFYKdEdScFhvNTl6MHQ5dnJJRkd3VGNLeGlpaG5STW8yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGRHg1bDd1aDdtNUlvNWc5b285NUwvOGpVVXZJTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSUhOblhLV2cKcUZPbkQvbXVyU0Y4S3BlNHJyN3FkSzFLWGQ2dm03aXN2ZThOQWlFQW5TTHM3dy9ySUFQM09GWTJmTnJsbFd2cgpJZlRpRG1IV3AzWVU5UDFVM2ljPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRZXNQaHZSS20xbHlhdTU2RndIbDZMekFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN5TWpJd01UVXhOMW9YRFRNME1EY3lNREl3TVRVeApOMW93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCTnVNUnl0K1lQUncxN094TFNRUDlJdngzZVk1am1pS1FSL2tEeTFENFI2ZVI4WUpqTlVDOXZGNmxzZFcKaWV3M09wekZybFl4eHl3Ym9vZVdDN3R1dlkyallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVU2TUhBdEhTZEJuUTZBbTRjeFVMOVc3b1Y2UFl3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQUpobVdzRXgKVjVnRW5na25uMURndjBBaVNjZTBHVUtrZWdBNStDK1VyOXlWQWlFQTVzQituQmFGVUl3R2JsYkcrSWEvOXFsZApFZEh0dXNkbDRRaHVmT0R5K1d3PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
 contexts:
 - name: oidc@tjo-cloud
   context:
diff --git a/modules/cluster/components.tf b/modules/cluster/components.tf
index 5622aec..d32d48a 100644
--- a/modules/cluster/components.tf
+++ b/modules/cluster/components.tf
@@ -9,72 +9,67 @@ data "helm_template" "cilium" {
 
   kube_version = var.talos.kubernetes
 
-  values = [yamlencode({
-    ipam : {
-      mode : "kubernetes"
-    },
-    nodeIPAM : {
-      enabled : true
-    },
-    kubeProxyReplacement : "true"
-    securityContext : {
-      capabilities : {
-        ciliumAgent : [
-          "CHOWN",
-          "KILL",
-          "NET_ADMIN",
-          "NET_RAW",
-          "IPC_LOCK",
-          "SYS_ADMIN",
-          "SYS_RESOURCE",
-          "DAC_OVERRIDE",
-          "FOWNER",
-          "SETGID",
-          "SETUID"
-        ],
-        cleanCiliumState : [
-          "NET_ADMIN",
-          "SYS_ADMIN",
-          "SYS_RESOURCE"
-        ]
-      }
-    },
-    cgroup : {
-      autoMount : {
-        enabled : false
-      },
-      hostRoot : "/sys/fs/cgroup"
-    },
-    k8sServiceHost : local.cluster_api_domain
-    k8sServicePort : var.cluster.api.port
-    ipv4 : {
-      enabled : true
-    },
-    #ipv6 : {
-    #  enabled : true
-    #},
-    hubble : {
-      tls : {
-        auto : {
-          enabled : true
-          method : "cronJob"
-          schedule : "0 0 1 */4 *"
-        }
-      }
-      ui : {
-        enabled : true
-      }
-      relay : {
-        enabled : true
-      }
-    },
-    gatewayAPI : {
-      enabled : false
-    }
-    envoy : {
-      enabled : false
-    }
-  })]
+  values = [<<-EOF
+    ipam:
+      mode: "kubernetes"
+
+    #routingMode: native
+    #ipv4NativeRoutingCIDR: pod and service cidrs?
+    enableIPv4Masquerade: true
+    ipv4:
+      enabled: true
+
+    #enableIPv6Masquerade: true
+    ipv6:
+      enabled: false
+
+    nodeIPAM:
+      enabled: true
+
+
+    kubeProxyReplacement: "true"
+    securityContext:
+      capabilities:
+        ciliumAgent:
+          - "CHOWN"
+          - "KILL"
+          - "NET_ADMIN"
+          - "NET_RAW"
+          - "IPC_LOCK"
+          - "SYS_ADMIN"
+          - "SYS_RESOURCE"
+          - "DAC_OVERRIDE"
+          - "FOWNER"
+          - "SETGID"
+          - "SETUID"
+        cleanCiliumState:
+          - "NET_ADMIN"
+          - "SYS_ADMIN"
+          - "SYS_RESOURCE"
+    cgroup:
+      hostRoot: "/sys/fs/cgroup"
+      autoMount:
+        enabled: false
+
+    k8sServiceHost: ${local.cluster_api_domain}
+    k8sServicePort: ${var.cluster.api.port}
+
+    hubble:
+      ui:
+        enabled: true
+      relay:
+        enabled: true
+      tls:
+        auto:
+          enabled: true
+          method: "cronJob"
+          schedule: "0 0 1 */4 *"
+    gatewayAPI:
+      enabled: false
+    envoy:
+      enabled: false
+    EOF
+  ]
 }
 
 data "helm_template" "proxmox-csi" {
@@ -173,15 +168,14 @@ data "helm_template" "cert-manager" {
 
   include_crds = true
 
-  set {
-    name  = "crds.enabled"
-    value = true
-  }
+  values = [<<-EOF
+    crds:
+      enabled: true
 
-  set_list {
-    name  = "extraArgs"
-    value = ["--enable-gateway-api"]
-  }
+    extraArgs:
+      - --enable-gateway-api
+    EOF
+  ]
 }
 
 data "helm_template" "envoy" {
diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf
index cde4ac7..ba11396 100644
--- a/modules/cluster/main.tf
+++ b/modules/cluster/main.tf
@@ -4,11 +4,11 @@ locals {
 
   podSubnets = [
     "10.200.0.0/16",
-    #"fd9b:5314:fc70::/48",
+    #"fd9b:5314:fc70::/64",
   ]
   serviceSubnets = [
     "10.201.0.0/16",
-    #"fd9b:5314:fc71::/48",
+    #"fd9b:5314:fc71::/108",
   ]
 
   # Nodes will use IPs from this subnets
@@ -75,14 +75,14 @@ locals {
           name : "cilium"
           contents : data.helm_template.cilium.manifest
         },
-        {
-          name : "envoy"
-          contents : data.helm_template.envoy.manifest
-        },
-        {
-          name : "cert-manager"
-          contents : data.helm_template.cert-manager.manifest
-        },
+        #{
+        #  name : "envoy"
+        #  contents : data.helm_template.envoy.manifest
+        #},
+        #{
+        #  name : "cert-manager"
+        #  contents : data.helm_template.cert-manager.manifest
+        #},
         {
           name : "oidc-admins"
           contents : <<-EOF
@@ -149,17 +149,10 @@ locals {
             hostname = node.name
           }
           nodeLabels = {
-            "k8s.tjo.cloud/public" = node.public ? "true" : "false"
-            #"k8s.tjo.cloud/ipv4"    = node.ipv4
-            #"k8s.tjo.cloud/ipv6"    = node.ipv6
+            "k8s.tjo.cloud/public"  = node.public ? "true" : "false"
             "k8s.tjo.cloud/host"    = node.host
             "k8s.tjo.cloud/proxmox" = var.proxmox.name
           }
-          kubelet = {
-            extraConfig = {
-              podCIDR = ""
-            }
-          }
         }
       }),
       yamlencode(
@@ -170,6 +163,8 @@ locals {
           environment : [
             "TS_AUTHKEY=${var.tailscale_authkey}",
             "TS_HOSTNAME=${node.name}",
+            # IPV6: https://github.com/siderolabs/extensions/issues/432
+            "TS_ROUTES=${local.podSubnets[0]},${local.serviceSubnets[0]}"
           ]
       })
     ]
diff --git a/modules/cluster/proxmox.tf b/modules/cluster/proxmox.tf
index 09a73bb..dfdea35 100644
--- a/modules/cluster/proxmox.tf
+++ b/modules/cluster/proxmox.tf
@@ -126,10 +126,10 @@ resource "proxmox_virtual_environment_vm" "nodes" {
     iothread     = true
   }
 
-  #initialization {
-  #  datastore_id      = each.value.storage
-  #  meta_data_file_id = proxmox_virtual_environment_file.metadata[each.key].id
-  #}
+  initialization {
+    datastore_id      = each.value.storage
+    meta_data_file_id = proxmox_virtual_environment_file.metadata[each.key].id
+  }
 }
 
 resource "proxmox_virtual_environment_role" "csi" {