feat: wip

ingress.tjo.cloud/.terraform.lock.hcl

@@ -24,3 +24,21 @@ provider "registry.opentofu.org/digitalocean/digitalocean" {
     "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2",
   ]
 }
+
+provider "registry.opentofu.org/hashicorp/dns" {
+  version     = "3.4.1"
+  constraints = "~> 3.4.1"
+  hashes = [
+    "h1:6Tb2wZRxfKunvjLw47ihfkaoARWIQrJd+WCOXkx0hg4=",
+    "zh:53f46f16fc3b25d9bdce61d7cc9a67cea9c67ea5347fcde35833451d5011f1c4",
+    "zh:662669802c99c7e698ca52e7a32afc19acf66d97bbc05f5effacdf0fe471848f",
+    "zh:8289aee49df8a0cb2fcdfded575e68df1721fc3e6ce75555def499d31b7e910d",
+    "zh:90aa9b049d19137ed15ea3f88238824e1fde7d1a806378cb97520391e169a96d",
+    "zh:9e5795e10d65927ba689cfabc06b72085055543342d1db26ef5963ae5ab5d64a",
+    "zh:9f39b6e0ca35cf2405fe4e13af8a405cf9b0b0c34b6b06ad36f319631405a3aa",
+    "zh:c37a2c140c95b3eaa9d0aeb79496b220ba1a0456d01a77fcc68bd565bee7edd5",
+    "zh:cd40d356c5382f0b868df2d8539f95031e3cf0c210053674ec4c7cd20c73b82a",
+    "zh:fde59f09e4e62fc84012fff8493bc64eac287a0245e1c8abf29b0138d3018dfb",
+    "zh:ffee5b54936bd43247e9316de0ecc0dde36556a2cd50f090d7f6443a3f689b95",
+  ]
+}

ingress.tjo.cloud/nodes.tf

@@ -1,14 +1,8 @@
 
 locals {
-  nodes = {
+  locations = {
-    hetzner = {
+    DE = ["46.4.88.62", "2a01:4f8:202:2395::"]
-      ipv4 = "46.4.88.62"
+    SI = ["93.103.125.118", "2a01:261:455:6c00:21e:6ff:fe45:c34"]
-      ipv6 = "2a01:4f8:202:2395::"
-    }
-    odroid = {
-      ipv4 = "93.103.125.118"
-      ipv6 = "2a01:261:455:6c00:21e:6ff:fe45:c34"
-    }
   }
 }
 
@@ -16,22 +10,19 @@ data "digitalocean_domain" "ingress" {
   name = "ingress.tjo.cloud"
 }
 
-resource "digitalocean_record" "nodes-a" {
+resource "digitalocean_record" "locations" {
-  for_each = local.nodes
+  for_each = merge([
+    for location, ips in local.locations : {
+      for ip in ips : "${location} at ${ip}" => {
+        location = location,
+        ip       = ip,
+      }
+    }
+  ]...)
 
   domain = data.digitalocean_domain.ingress.id
-  type   = "A"
+  type   = strcontains(each.value.ip, ":") ? "AAAA" : "A"
-  name   = each.key
+  name   = lower(each.value.location)
-  value  = each.value.ipv4
+  value  = each.value.ip
-}
+  ttl    = 60
-
-resource "digitalocean_record" "nodes-aaaa" {
-  for_each = local.nodes
-
-  domain = data.digitalocean_domain.ingress.id
-  type   = "AAAA"
-  name   = each.key
-  value  = each.value.ipv6
-
-  ttl = 60
 }

ingress.tjo.cloud/records.tf

@@ -1,39 +1,35 @@
 locals {
   listeners = [
     {
-      domain = "k8s.tjo.cloud"
+      domain    = "k8s.tjo.cloud"
-      name   = "api"
+      name      = "api"
-      nodes  = ["hetzner", "odroid"]
+      locations = ["SI", "DE"]
+    },
+    {
+      domain    = "k8s.tjo.cloud"
+      name      = "dashboard"
+      locations = ["SI", "DE"]
     }
   ]
 }
 
 resource "digitalocean_record" "listeners" {
-  for_each = merge([
+  for_each = merge(flatten([
-    for listener in local.listeners : merge(
+    for listener in local.listeners :
-      {
+    [
-        for node in listener.nodes : "ipv4 ${listener.name}.${listener.domain} at ${node}" => {
+      for location in listener.locations : {
-          ip     = local.nodes[node].ipv4
+        for ip in local.locations[location] : "${ip} for ${listener.name}.${listener.domain} at ${location}" => {
+          ip     = ip
           domain = listener.domain
           name   = listener.name
-          type   = "A"
-        }
-      },
-      {
-        for node in listener.nodes : "ipv6 ${listener.name}.${listener.domain} at ${node}" => {
-          ip     = local.nodes[node].ipv6
-          domain = listener.domain
-          name   = listener.name
-          type   = "AAAA"
         }
       }
-    )
+    ]
-  ]...)
+  ])...)
 
   domain = each.value.domain
-  type   = each.value.type
+  type   = strcontains(each.value.ip, ":") ? "AAAA" : "A"
   name   = each.value.name
   value  = each.value.ip
-
+  ttl    = 60
-  ttl = 60
 }

ingress.tjo.cloud/terraform.tf

@@ -4,6 +4,10 @@ terraform {
       source  = "digitalocean/digitalocean"
       version = "~> 2.0"
     }
+    dns = {
+      source  = "hashicorp/dns"
+      version = "~> 3.4.1"
+    }
   }
 
   required_version = "~> 1.7.3"

k8s.tjo.cloud/README.md

@@ -0,0 +1,8 @@
+# `k8s.tjo.cloud`
+
+## Kubernetes JWT
+For federation to work, the OIDC JWKS needs to be manually added in to id.tjo.space.
+
+```
+kubectl get --raw /openid/v1/jwks
+````

k8s.tjo.cloud/kubeconfig

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: k8s-tjo-cloud
+  cluster:
+    server: https://api.k8s.tjo.cloud:443
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRWXFNSG94UFhiTFMvZ21oTFhUOWd0akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGd3TXpFeU16TXpORm9YRFRNME1EZ3dNVEV5TXpNegpORm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCRzcvQVVjZ3VsVFhpZ1JpT2hkeWk0a1BycmJNenZqZXZYdXhKMDJkN1pEYVlkMmtzeStQVkZTcDM0a0YKSmE4ZEdWN0VnZHlWQlVZeGZVU0Z0UGpleFBHallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVUycmptRW1vbTJPYnVJYUhzTFNrQTFnamFYUHN3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU9CZWVyWlUKTXhHN1ErczV2OTR4QjUvVE9XaUd5MXRCOVk0aTUxZERFZkNYQWlBLzg1VEs3VDRlWGtDakFqaW1QU0Y3ZUZWago2c1NJbGd0WGhZWHpUUUVSZWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+contexts:
+- name: oidc@k8s-tjo-cloud
+  context:
+    cluster: k8s-tjo-cloud
+    namespace: default
+    user: oidc
+current-context: oidc@k8s-tjo-cloud
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
+      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
+      - --oidc-extra-scope=profile

k8s.tjo.cloud/kubeconfig.tftpl

@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: ${cluster.name}
+  cluster:
+    server: ${cluster.endpoint}
+    certificate-authority-data: ${cluster.ca}
+contexts:
+- name: oidc@${cluster.name}
+  context:
+    cluster: ${cluster.name}
+    namespace: default
+    user: oidc
+current-context: oidc@${cluster.name}
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=${oidc.issuer}
+      - --oidc-client-id=${oidc.id}
+      - --oidc-extra-scope=profile

k8s.tjo.cloud/main.tf

@@ -71,18 +71,28 @@ data "tailscale_device" "controlpane" {
   for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" }
   hostname = each.value.name
 }
-resource "digitalocean_record" "internal-api" {
+resource "digitalocean_record" "api-internal" {
   for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses]))
 
   domain = local.cluster_domain
   type   = strcontains(each.value, ":") ? "AAAA" : "A"
-  name   = "internal.api"
+  name   = trimsuffix(module.cluster.api.internal.domain, ".${local.cluster_domain}")
   value  = each.value
   ttl    = 30
 }
 
 resource "local_file" "kubeconfig" {
-  content  = module.cluster.kubeconfig
+  content = templatefile("${path.module}/kubeconfig.tftpl", {
+    cluster : {
+      name : module.cluster.name,
+      endpoint : module.cluster.api.public.endpoint,
+      ca : module.cluster.api.ca,
+    }
+    oidc : {
+      issuer : var.oidc_issuer_url,
+      id : var.oidc_client_id,
+    }
+  })
   filename = "${path.module}/kubeconfig"
 }
 

k8s.tjo.cloud/modules/cluster-components/external-dns.tf

@@ -0,0 +1,48 @@
+resource "helm_release" "external-dns-privileged" {
+  name       = "external-dns-privileged"
+  chart      = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  version    = "v1.14.5"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  values = [<<-EOF
+    provider: digitalocean
+    env:
+      - name: DO_TOKEN
+        valueFrom:
+          secretKeyRef:
+            name: ${kubernetes_secret.digitalocean-token.metadata[0].name}
+            key: token
+    sources:
+      - ingress
+      - service
+    domainFilters:
+      - k8s.tjo.cloud
+      - internal.k8s.tjo.cloud
+  EOF
+  ]
+}
+
+resource "helm_release" "external-dns-user-content" {
+  name       = "external-dns-user-content"
+  chart      = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  version    = "v1.14.5"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  values = [<<-EOF
+    provider: digitalocean
+    env:
+      - name: DO_TOKEN
+        valueFrom:
+          secretKeyRef:
+            name: ${kubernetes_secret.digitalocean-token.metadata[0].name}
+            key: token
+    sources:
+      - ingress
+      - service
+    domainFilters:
+      - user-content.tjo.cloud
+  EOF
+  ]
+}

k8s.tjo.cloud/modules/cluster-components/gateway.tf

@@ -1,13 +1,3 @@
-resource "kubernetes_secret" "digitalocean-token" {
-  metadata {
-    name      = "digitalocean-token"
-    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-  }
-  data = {
-    token = var.digitalocean_token
-  }
-}
-
 resource "kubernetes_manifest" "tjo-cloud-issuer" {
   manifest = {
     apiVersion = "cert-manager.io/v1"
@@ -33,6 +23,11 @@ resource "kubernetes_manifest" "tjo-cloud-issuer" {
                 }
               }
             }
+            selector : {
+              dnsZones : [
+                "tjo.cloud"
+              ]
+            }
           }
         ]
       }
@@ -57,24 +52,10 @@ resource "kubernetes_manifest" "gateway_class_config" {
             type                  = "ClusterIP"
             externalTrafficPolicy = "Local"
             annotations = {
-              "io.cilium.nodeipam/match-node-labels" = "k8s.tjo.cloud/public=true"
+              "external-dns.alpha.kubernetes.io/internal-hostname" = "envoy.internal.k8s.tjo.cloud"
             }
-            loadBalancerClass = "io.cilium/node"
           }
           envoyDaemonSet = {
-            patch = {
-              type = "StrategicMerge"
-              value = {
-                spec = {
-                  template = {
-                    spec = {
-                      hostNetwork = true
-                      dnsPolicy   = "ClusterFirstWithHostNet"
-                    }
-                  }
-                }
-              }
-            }
             pod = {
               nodeSelector = {
                 "node-role.kubernetes.io/control-plane" = ""

k8s.tjo.cloud/modules/cluster-components/main.tf

@@ -3,3 +3,13 @@ resource "kubernetes_namespace" "tjo-cloud" {
     name = "tjo-cloud"
   }
 }
+
+resource "kubernetes_secret" "digitalocean-token" {
+  metadata {
+    name      = "digitalocean-token"
+    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+  }
+  data = {
+    token = var.digitalocean_token
+  }
+}

k8s.tjo.cloud/modules/cluster-core/monitoring.tf

@@ -203,6 +203,7 @@ resource "helm_release" "grafana-alloy" {
                 client_id = "o6Tz2215HLvhvZ4RCZCR8oMmCapTu30iwkoMkz6m"
                 client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
                 endpoint_params = {
+                  grant_type = "client_credentials",
                   client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
                 }
               }
@@ -282,6 +283,7 @@ resource "helm_release" "grafana-alloy" {
                 client_id = "56TYXtgg7QwLjh4lPl1PTu3C4iExOvO1d6b15WuC"
                 client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
                 endpoint_params = {
+                  grant_type = "client_credentials",
                   client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
                 }
               }

k8s.tjo.cloud/modules/cluster/components.tf

@@ -50,8 +50,8 @@ data "helm_template" "cilium" {
       autoMount:
         enabled: false
 
-    k8sServiceHost: ${var.cluster.api.domain}
+    k8sServiceHost: ${var.cluster.api.internal.domain}
-    k8sServicePort: ${var.cluster.api.port}
+    k8sServicePort: ${var.cluster.api.internal.port}
 
     hubble:
       ui:

k8s.tjo.cloud/modules/cluster/main.tf

@@ -1,5 +1,6 @@
 locals {
-  cluster_endpoint = "https://${var.cluster.api.domain}:${var.cluster.api.port}"
+  cluster_internal_endpoint = "https://${var.cluster.api.internal.domain}:${var.cluster.api.internal.port}"
+  cluster_public_endpoint   = "https://${var.cluster.api.public.domain}:${var.cluster.api.public.port}"
 
   podSubnets = [
     "10.200.0.0/16",
@@ -40,6 +41,10 @@ locals {
       }
       allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes,
       apiServer = {
+        certSANs = [
+          var.cluster.api.internal.domain,
+          var.cluster.api.public.domain,
+        ]
         extraArgs = {
           "oidc-issuer-url"      = "https://id.tjo.space/application/o/k8stjocloud/",
           "oidc-client-id"       = "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
@@ -97,8 +102,8 @@ locals {
         enabled = true
       }
       controlPlane = {
-        endpoint           = local.cluster_endpoint
+        endpoint           = local.cluster_internal_endpoint
-        localAPIServerPort = var.cluster.api.port
+        localAPIServerPort = var.cluster.api.internal.port
       }
       network = {
         cni = {
@@ -172,7 +177,7 @@ resource "talos_machine_secrets" "this" {
 data "talos_machine_configuration" "controlplane" {
   cluster_name     = var.cluster.name
   machine_type     = "controlplane"
-  cluster_endpoint = local.cluster_endpoint
+  cluster_endpoint = local.cluster_internal_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
   talos_version      = var.talos.version
@@ -182,7 +187,7 @@ data "talos_machine_configuration" "controlplane" {
 data "talos_machine_configuration" "worker" {
   cluster_name     = var.cluster.name
   machine_type     = "worker"
-  cluster_endpoint = local.cluster_endpoint
+  cluster_endpoint = local.cluster_internal_endpoint
   machine_secrets  = talos_machine_secrets.this.machine_secrets
 
   talos_version      = var.talos.version

k8s.tjo.cloud/modules/cluster/outputs.tf

@@ -1,24 +1,15 @@
-output "kubeconfig" {
-  value = templatefile("${path.module}/kubeconfig.tftpl", {
-    cluster : {
-      name : var.cluster.name,
-      endpoint : local.cluster_endpoint,
-      ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
-    }
-    oidc : {
-      issuer : var.cluster.oidc.issuer_url,
-      id : var.cluster.oidc.client_id,
-    }
-  })
-}
-
 output "name" {
   value = var.cluster.name
 }
 
 output "api" {
   value = merge(var.cluster.api, {
-    endpoint : local.cluster_endpoint,
+    internal : merge(var.cluster.api.internal, {
+      endpoint : local.cluster_internal_endpoint,
+    }),
+    public : merge(var.cluster.api.public, {
+      endpoint : local.cluster_public_endpoint,
+    }),
     ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
   })
 }

k8s.tjo.cloud/modules/cluster/variables.tf

@@ -40,8 +40,14 @@ variable "cluster" {
   type = object({
     name = string
     api = optional(object({
-      domain = optional(string, "internal.api.k8s.tjo.cloud")
+      internal = optional(object({
-      port   = optional(number, 6443)
+        domain = optional(string, "api.internal.k8s.tjo.cloud")
+        port   = optional(number, 6443)
+      }), {})
+      public = optional(object({
+        domain = optional(string, "api.k8s.tjo.cloud")
+        port   = optional(number, 443)
+      }), {})
     }), {})
     oidc = object({
       client_id  = string

k8s.tjo.cloud/terraform.tf

@@ -62,7 +62,7 @@ provider "helm" {
 
 provider "helm" {
   kubernetes {
-    host                   = module.cluster.api.endpoint
+    host                   = module.cluster.api.internal.endpoint
     cluster_ca_certificate = base64decode(module.cluster.api.ca)
     exec {
       api_version = "client.authentication.k8s.io/v1beta1"
@@ -82,7 +82,7 @@ provider "helm" {
 }
 
 provider "kubernetes" {
-  host                   = module.cluster.api.endpoint
+  host                   = module.cluster.api.internal.endpoint
   cluster_ca_certificate = base64decode(module.cluster.api.ca)
   exec {
     api_version = "client.authentication.k8s.io/v1beta1"