feat: trying to make cilium work

terraform/.terraform.lock.hcl

@@ -43,6 +43,24 @@ provider "registry.opentofu.org/hashicorp/helm" {
   ]
 }
 
+provider "registry.opentofu.org/hashicorp/kubernetes" {
+  version     = "2.31.0"
+  constraints = "2.31.0"
+  hashes = [
+    "h1:z2qlqn6WbrjbezwQo4vvlwAgVUGz59klzDU4rlYhYi8=",
+    "zh:0dd25babf78a88a61dd329b8c18538a295ea63630f1b69575e7898c89307da39",
+    "zh:3138753e4b2ce6e9ffa5d65d73e9236169ff077c10089c7dc71031a0a139ff6d",
+    "zh:644f94692dc33de0bb1183c307ae373efbf4ef4cb92654ccc646a5716edf9593",
+    "zh:6cc630e43193220b1599e3227286cc4e3ca195910e8c56b6bacb50c5b5176dbf",
+    "zh:764173875e77aa482da4dca9fec5f77c455d028848edfc394aa7dac5dfed6afd",
+    "zh:7b1d380362d50ffbb3697483036ae351b0571e93b33754255cde6968e62b839f",
+    "zh:a1d93ca3d8d1ecdd3b69242d16ff21c91b34e2e98f02a3b2d02c908aeb45189b",
+    "zh:b471d0ab56dbf19c95fba68d2ef127bdb353be96a2be4c4a3dcd4d0db4b4180a",
+    "zh:d610f725ded4acd3d31a240472bb283aa5e657ed020395bdefea18d094b8c2bf",
+    "zh:d7f3ddd636ad5af6049922f212feb24830b7158410819c32073bf81c359cd2fa",
+  ]
+}
+
 provider "registry.opentofu.org/hashicorp/local" {
   version     = "1.4.0"
   constraints = "1.4.0"

terraform/kubernetes.tf

@@ -1,6 +1,6 @@
 locals {
-  // Downloaded from factory.talos.dev
+  # Downloaded from factory.talos.dev
-  // https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
+  # https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
   iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
 
   boot_pool = "hetzner-main-data"
@@ -119,16 +119,106 @@ data "talos_machine_disks" "boot" {
   }
 }
 
+data "helm_template" "cilium" {
+  name         = "cilium"
+  repository   = "https://helm.cilium.io/"
+  chart        = "cilium"
+  version      = "1.15.6"
+  namespace    = "kube-system"
+  include_crds = true
 
-resource "talos_machine_configuration_apply" "this" {
+  values = [yamlencode({
-  for_each = local.nodes_with_address
+    ipam : {
+      mode : "kubernetes"
+    },
+    kubeProxyReplacement : true
+    securityContext : {
+      capabilities : {
+        ciliumAgent : [
+          "CHOWN",
+          "KILL",
+          "NET_ADMIN",
+          "NET_RAW",
+          "IPC_LOCK",
+          "PERFMON",
+          "BPF",
+          "SYS_RESOURCE",
+          "DAC_OVERRIDE",
+          "FOWNER",
+          "SETGID",
+          "SETUID"
+        ],
+        cleanCiliumState : [
+          "NET_ADMIN",
+          "PERFMON",
+          "BPF",
+          "SYS_RESOURCE"
+        ]
+      }
+    },
+    cgroup : {
+      autoMount : {
+        enabled : false
+      },
+      hostRoot : "/sys/fs/cgroup"
+    },
+    k8sServiceHost : "localhost"
+    k8sServicePort : "7445"
+    ipv4 : {
+      enabled : true
+    },
+    #ipv6: {
+    #  enabled: true
+    #},
+    hubble : {
+      ui : {
+        enabled : true
+      }
+      relay : {
+        enabled : true
+      }
+    },
+    # Ingress gateway
+    gatewayAPI : {
+      enabled : true
+      default : true
+      hostNetwork : {
+        enabled : true
+        nodes : {
+          matchLabels : {
+            "k8s.tjo.cloud/gateway" : "true"
+          }
+        }
+      }
+    }
+    envoy : {
+      enabled : true
+      securityContext : {
+        capabilities : {
+          keepCapNetBindService : true
+          envoy : [
+            "NET_ADMIN",
+            "BPF",
+            "PERFMON",
+            "NET_BIND_SERVICE"
+          ]
+        }
+      }
+    }
+  })]
+}
+
+resource "talos_machine_configuration_apply" "controlplane" {
+  for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
 
   client_configuration        = talos_machine_secrets.this.client_configuration
-  machine_configuration_input = each.value.type == "controlplane" ? data.talos_machine_configuration.controlplane.machine_configuration : data.talos_machine_configuration.worker.machine_configuration
+  machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration
 
   node     = each.value.name
   endpoint = each.value.address_ipv4
 
+  apply_mode = "reboot"
+
   config_patches = [
     yamlencode({
       cluster : {
@@ -137,6 +227,9 @@ resource "talos_machine_configuration_apply" "this" {
             name : "none"
           }
         }
+        proxy : {
+          disabled : true
+        }
         allowSchedulingOnControlPlanes : true,
         apiServer : {
           extraArgs : {
@@ -148,7 +241,8 @@ resource "talos_machine_configuration_apply" "this" {
             "oidc-groups-prefix" : "oidc:groups:",
           }
         }
-        inlineManifests : [{
+        inlineManifests : [
+          {
             name : "oidc-groups"
             contents : <<-EOF
             apiVersion: rbac.authorization.k8s.io/v1
@@ -164,7 +258,80 @@ resource "talos_machine_configuration_apply" "this" {
               name: cluster-admin
               apiGroup: rbac.authorization.k8s.io
             EOF
-        }]
+          },
+          {
+            name : "gateway-api-crds"
+            contents : file("${path.module}/manifests/gateway-api-crds.yaml")
+          },
+          {
+            name : "cilium"
+            contents : data.helm_template.cilium.manifest
+          }
+        ],
+      }
+      machine = {
+        kubelet = {
+          extraArgs = {
+            rotate-server-certificates : "true"
+          }
+        }
+        network = {
+          hostname = each.value.name
+        }
+        install = {
+          image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5"
+          disk  = data.talos_machine_disks.boot[each.key].disks[0].name
+        }
+        nodeLabels = {
+          "k8s.tjo.cloud/gateway" = "true"
+        }
+      }
+    }),
+    yamlencode({
+      apiVersion : "v1alpha1"
+      kind : "ExtensionServiceConfig"
+      name : "tailscale"
+      environment : [
+        "TS_AUTHKEY=${var.tailscale_authkey}"
+      ]
+    })
+
+  ]
+}
+
+resource "talos_machine_configuration_apply" "worker" {
+  for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" }
+
+  client_configuration        = talos_machine_secrets.this.client_configuration
+  machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration
+
+  node     = each.value.name
+  endpoint = each.value.address_ipv4
+
+  apply_mode = "reboot"
+
+  config_patches = [
+    yamlencode({
+      cluster : {
+        network : {
+          cni : {
+            name : "none"
+          }
+        }
+        proxy : {
+          disabled : true
+        }
+        allowSchedulingOnControlPlanes : true,
+        apiServer : {
+          extraArgs : {
+            "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
+            "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
+            "oidc-username-claim" : "sub",
+            "oidc-username-prefix" : "oidc:",
+            "oidc-groups-claim" : "groups",
+            "oidc-groups-prefix" : "oidc:groups:",
+          }
+        }
       }
       machine = {
         kubelet = {
@@ -195,7 +362,8 @@ resource "talos_machine_configuration_apply" "this" {
 
 resource "talos_machine_bootstrap" "this" {
   depends_on = [
-    talos_machine_configuration_apply.this
+    talos_machine_configuration_apply.controlplane,
+    talos_machine_configuration_apply.worker
   ]
 
   node                 = local.first_controlplane_node.name
@@ -236,44 +404,6 @@ resource "local_file" "talosconfig" {
   filename = "${path.module}/talosconfig"
 }
 
-
-resource "helm_release" "cilium" {
-  depends_on = [
-    talos_machine_bootstrap.this
-  ]
-
-  name       = "cilium"
-  repository = "https://helm.cilium.io/"
-  chart      = "cilium"
-  version    = "1.15.6"
-  namespace  = "kube-system"
-
-  set {
-    name  = "ipam.mode"
-    value = "kubernetes"
-  }
-  set {
-    name  = "kubeProxyReplacement"
-    value = "disabled"
-  }
-  set {
-    name  = "securityContext.capabilities.ciliumAgent"
-    value = "{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}"
-  }
-  set {
-    name  = "securityContext.capabilities.cleanCiliumState"
-    value = "{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}"
-  }
-  set {
-    name  = "cgroup.autoMount.enabled"
-    value = false
-  }
-  set {
-    name  = "cgroup.hostRoot"
-    value = "/sys/fs/cgroup"
-  }
-}
-
 resource "helm_release" "dashboard" {
   depends_on = [
     talos_machine_bootstrap.this

terraform/providers.tf

@@ -28,6 +28,10 @@ terraform {
       source  = "hashicorp/helm"
       version = "2.14.0"
     }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "2.31.0"
+    }
   }
 }