feat: wip

2024-08-04 19:50:50 +02:00 · 2024-08-04 19:50:50 +02:00 · cc12c5d30d
commit cc12c5d30d
parent dd5b8f884b
17 changed files with 223 additions and 101 deletions
--- a/ingress.tjo.cloud/.terraform.lock.hcl
+++ b/ingress.tjo.cloud/.terraform.lock.hcl
@ -24,3 +24,21 @@ provider "registry.opentofu.org/digitalocean/digitalocean" {
    "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2",
  ]
 }
+
+provider "registry.opentofu.org/hashicorp/dns" {
+  version     = "3.4.1"
+  constraints = "~> 3.4.1"
+  hashes = [
+    "h1:6Tb2wZRxfKunvjLw47ihfkaoARWIQrJd+WCOXkx0hg4=",
+    "zh:53f46f16fc3b25d9bdce61d7cc9a67cea9c67ea5347fcde35833451d5011f1c4",
+    "zh:662669802c99c7e698ca52e7a32afc19acf66d97bbc05f5effacdf0fe471848f",
+    "zh:8289aee49df8a0cb2fcdfded575e68df1721fc3e6ce75555def499d31b7e910d",
+    "zh:90aa9b049d19137ed15ea3f88238824e1fde7d1a806378cb97520391e169a96d",
+    "zh:9e5795e10d65927ba689cfabc06b72085055543342d1db26ef5963ae5ab5d64a",
+    "zh:9f39b6e0ca35cf2405fe4e13af8a405cf9b0b0c34b6b06ad36f319631405a3aa",
+    "zh:c37a2c140c95b3eaa9d0aeb79496b220ba1a0456d01a77fcc68bd565bee7edd5",
+    "zh:cd40d356c5382f0b868df2d8539f95031e3cf0c210053674ec4c7cd20c73b82a",
+    "zh:fde59f09e4e62fc84012fff8493bc64eac287a0245e1c8abf29b0138d3018dfb",
+    "zh:ffee5b54936bd43247e9316de0ecc0dde36556a2cd50f090d7f6443a3f689b95",
+  ]
+}
--- a/ingress.tjo.cloud/nodes.tf
+++ b/ingress.tjo.cloud/nodes.tf
@ -1,14 +1,8 @@

 locals {
-  nodes = {
-    hetzner = {
-      ipv4 = "46.4.88.62"
-      ipv6 = "2a01:4f8:202:2395::"
-    }
-    odroid = {
-      ipv4 = "93.103.125.118"
-      ipv6 = "2a01:261:455:6c00:21e:6ff:fe45:c34"
-    }
+  locations = {
+    DE = ["46.4.88.62", "2a01:4f8:202:2395::"]
+    SI = ["93.103.125.118", "2a01:261:455:6c00:21e:6ff:fe45:c34"]
  }
 }

@ -16,22 +10,19 @@ data "digitalocean_domain" "ingress" {
  name = "ingress.tjo.cloud"
 }

-resource "digitalocean_record" "nodes-a" {
-  for_each = local.nodes
+resource "digitalocean_record" "locations" {
+  for_each = merge([
+    for location, ips in local.locations : {
+      for ip in ips : "${location} at ${ip}" => {
+        location = location,
+        ip       = ip,
+      }
+    }
+  ]...)

  domain = data.digitalocean_domain.ingress.id
-  type   = "A"
-  name   = each.key
-  value  = each.value.ipv4
-}
-
-resource "digitalocean_record" "nodes-aaaa" {
-  for_each = local.nodes
-
-  domain = data.digitalocean_domain.ingress.id
-  type   = "AAAA"
-  name   = each.key
-  value  = each.value.ipv6
-
-  ttl = 60
+  type   = strcontains(each.value.ip, ":") ? "AAAA" : "A"
+  name   = lower(each.value.location)
+  value  = each.value.ip
+  ttl    = 60
 }
--- a/ingress.tjo.cloud/records.tf
+++ b/ingress.tjo.cloud/records.tf
@ -1,39 +1,35 @@
 locals {
  listeners = [
    {
-      domain = "k8s.tjo.cloud"
-      name   = "api"
-      nodes  = ["hetzner", "odroid"]
+      domain    = "k8s.tjo.cloud"
+      name      = "api"
+      locations = ["SI", "DE"]
+    },
+    {
+      domain    = "k8s.tjo.cloud"
+      name      = "dashboard"
+      locations = ["SI", "DE"]
    }
  ]
 }

 resource "digitalocean_record" "listeners" {
-  for_each = merge([
-    for listener in local.listeners : merge(
-      {
-        for node in listener.nodes : "ipv4 ${listener.name}.${listener.domain} at ${node}" => {
-          ip     = local.nodes[node].ipv4
+  for_each = merge(flatten([
+    for listener in local.listeners :
+    [
+      for location in listener.locations : {
+        for ip in local.locations[location] : "${ip} for ${listener.name}.${listener.domain} at ${location}" => {
+          ip     = ip
          domain = listener.domain
          name   = listener.name
-          type   = "A"
-        }
-      },
-      {
-        for node in listener.nodes : "ipv6 ${listener.name}.${listener.domain} at ${node}" => {
-          ip     = local.nodes[node].ipv6
-          domain = listener.domain
-          name   = listener.name
-          type   = "AAAA"
        }
      }
-    )
-  ]...)
+    ]
+  ])...)

  domain = each.value.domain
-  type   = each.value.type
+  type   = strcontains(each.value.ip, ":") ? "AAAA" : "A"
  name   = each.value.name
  value  = each.value.ip
-
-  ttl = 60
+  ttl    = 60
 }
--- a/ingress.tjo.cloud/terraform.tf
+++ b/ingress.tjo.cloud/terraform.tf
@ -4,6 +4,10 @@ terraform {
      source  = "digitalocean/digitalocean"
      version = "~> 2.0"
    }
+    dns = {
+      source  = "hashicorp/dns"
+      version = "~> 3.4.1"
+    }
  }

  required_version = "~> 1.7.3"
--- a/k8s.tjo.cloud/README.md
+++ b/k8s.tjo.cloud/README.md
@ -0,0 +1,8 @@
+# `k8s.tjo.cloud`
+
+## Kubernetes JWT
+For federation to work, the OIDC JWKS needs to be manually added in to id.tjo.space.
+
+```
+kubectl get --raw /openid/v1/jwks
+````
--- a/k8s.tjo.cloud/kubeconfig
+++ b/k8s.tjo.cloud/kubeconfig
@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: k8s-tjo-cloud
+  cluster:
+    server: https://api.k8s.tjo.cloud:443
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRWXFNSG94UFhiTFMvZ21oTFhUOWd0akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGd3TXpFeU16TXpORm9YRFRNME1EZ3dNVEV5TXpNegpORm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCRzcvQVVjZ3VsVFhpZ1JpT2hkeWk0a1BycmJNenZqZXZYdXhKMDJkN1pEYVlkMmtzeStQVkZTcDM0a0YKSmE4ZEdWN0VnZHlWQlVZeGZVU0Z0UGpleFBHallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVUycmptRW1vbTJPYnVJYUhzTFNrQTFnamFYUHN3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU9CZWVyWlUKTXhHN1ErczV2OTR4QjUvVE9XaUd5MXRCOVk0aTUxZERFZkNYQWlBLzg1VEs3VDRlWGtDakFqaW1QU0Y3ZUZWago2c1NJbGd0WGhZWHpUUUVSZWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+contexts:
+- name: oidc@k8s-tjo-cloud
+  context:
+    cluster: k8s-tjo-cloud
+    namespace: default
+    user: oidc
+current-context: oidc@k8s-tjo-cloud
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
+      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
+      - --oidc-extra-scope=profile
--- a/k8s.tjo.cloud/kubeconfig.tftpl
+++ b/k8s.tjo.cloud/kubeconfig.tftpl
@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Config
+clusters:
+- name: ${cluster.name}
+  cluster:
+    server: ${cluster.endpoint}
+    certificate-authority-data: ${cluster.ca}
+contexts:
+- name: oidc@${cluster.name}
+  context:
+    cluster: ${cluster.name}
+    namespace: default
+    user: oidc
+current-context: oidc@${cluster.name}
+users:
+- name: oidc
+  user:
+    exec:
+      apiVersion: client.authentication.k8s.io/v1beta1
+      command: kubectl
+      args:
+      - oidc-login
+      - get-token
+      - --oidc-issuer-url=${oidc.issuer}
+      - --oidc-client-id=${oidc.id}
+      - --oidc-extra-scope=profile
--- a/k8s.tjo.cloud/main.tf
+++ b/k8s.tjo.cloud/main.tf
@ -71,18 +71,28 @@ data "tailscale_device" "controlpane" {
  for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" }
  hostname = each.value.name
 }
-resource "digitalocean_record" "internal-api" {
+resource "digitalocean_record" "api-internal" {
  for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses]))

  domain = local.cluster_domain
  type   = strcontains(each.value, ":") ? "AAAA" : "A"
-  name   = "internal.api"
+  name   = trimsuffix(module.cluster.api.internal.domain, ".${local.cluster_domain}")
  value  = each.value
  ttl    = 30
 }

 resource "local_file" "kubeconfig" {
-  content  = module.cluster.kubeconfig
+  content = templatefile("${path.module}/kubeconfig.tftpl", {
+    cluster : {
+      name : module.cluster.name,
+      endpoint : module.cluster.api.public.endpoint,
+      ca : module.cluster.api.ca,
+    }
+    oidc : {
+      issuer : var.oidc_issuer_url,
+      id : var.oidc_client_id,
+    }
+  })
  filename = "${path.module}/kubeconfig"
 }

--- a/k8s.tjo.cloud/modules/cluster-components/external-dns.tf
+++ b/k8s.tjo.cloud/modules/cluster-components/external-dns.tf
@ -0,0 +1,48 @@
+resource "helm_release" "external-dns-privileged" {
+  name       = "external-dns-privileged"
+  chart      = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  version    = "v1.14.5"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  values = [<<-EOF
+    provider: digitalocean
+    env:
+      - name: DO_TOKEN
+        valueFrom:
+          secretKeyRef:
+            name: ${kubernetes_secret.digitalocean-token.metadata[0].name}
+            key: token
+    sources:
+      - ingress
+      - service
+    domainFilters:
+      - k8s.tjo.cloud
+      - internal.k8s.tjo.cloud
+  EOF
+  ]
+}
+
+resource "helm_release" "external-dns-user-content" {
+  name       = "external-dns-user-content"
+  chart      = "external-dns"
+  repository = "https://kubernetes-sigs.github.io/external-dns/"
+  version    = "v1.14.5"
+  namespace  = kubernetes_namespace.tjo-cloud.metadata[0].name
+
+  values = [<<-EOF
+    provider: digitalocean
+    env:
+      - name: DO_TOKEN
+        valueFrom:
+          secretKeyRef:
+            name: ${kubernetes_secret.digitalocean-token.metadata[0].name}
+            key: token
+    sources:
+      - ingress
+      - service
+    domainFilters:
+      - user-content.tjo.cloud
+  EOF
+  ]
+}
--- a/k8s.tjo.cloud/modules/cluster-components/gateway.tf
+++ b/k8s.tjo.cloud/modules/cluster-components/gateway.tf
@ -1,13 +1,3 @@
-resource "kubernetes_secret" "digitalocean-token" {
-  metadata {
-    name      = "digitalocean-token"
-    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
-  }
-  data = {
-    token = var.digitalocean_token
-  }
-}
-
 resource "kubernetes_manifest" "tjo-cloud-issuer" {
  manifest = {
    apiVersion = "cert-manager.io/v1"
@ -33,6 +23,11 @@ resource "kubernetes_manifest" "tjo-cloud-issuer" {
                }
              }
            }
+            selector : {
+              dnsZones : [
+                "tjo.cloud"
+              ]
+            }
          }
        ]
      }
@ -57,24 +52,10 @@ resource "kubernetes_manifest" "gateway_class_config" {
            type                  = "ClusterIP"
            externalTrafficPolicy = "Local"
            annotations = {
-              "io.cilium.nodeipam/match-node-labels" = "k8s.tjo.cloud/public=true"
+              "external-dns.alpha.kubernetes.io/internal-hostname" = "envoy.internal.k8s.tjo.cloud"
            }
-            loadBalancerClass = "io.cilium/node"
          }
          envoyDaemonSet = {
-            patch = {
-              type = "StrategicMerge"
-              value = {
-                spec = {
-                  template = {
-                    spec = {
-                      hostNetwork = true
-                      dnsPolicy   = "ClusterFirstWithHostNet"
-                    }
-                  }
-                }
-              }
-            }
            pod = {
              nodeSelector = {
                "node-role.kubernetes.io/control-plane" = ""
--- a/k8s.tjo.cloud/modules/cluster-components/main.tf
+++ b/k8s.tjo.cloud/modules/cluster-components/main.tf
@ -3,3 +3,13 @@ resource "kubernetes_namespace" "tjo-cloud" {
    name = "tjo-cloud"
  }
 }
+
+resource "kubernetes_secret" "digitalocean-token" {
+  metadata {
+    name      = "digitalocean-token"
+    namespace = kubernetes_namespace.tjo-cloud.metadata[0].name
+  }
+  data = {
+    token = var.digitalocean_token
+  }
+}
--- a/k8s.tjo.cloud/modules/cluster-core/monitoring.tf
+++ b/k8s.tjo.cloud/modules/cluster-core/monitoring.tf
@ -203,6 +203,7 @@ resource "helm_release" "grafana-alloy" {
                client_id = "o6Tz2215HLvhvZ4RCZCR8oMmCapTu30iwkoMkz6m"
                client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
                endpoint_params = {
+                  grant_type = "client_credentials",
                  client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
                }
              }
@ -282,6 +283,7 @@ resource "helm_release" "grafana-alloy" {
                client_id = "56TYXtgg7QwLjh4lPl1PTu3C4iExOvO1d6b15WuC"
                client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
                endpoint_params = {
+                  grant_type = "client_credentials",
                  client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
                }
              }
--- a/k8s.tjo.cloud/modules/cluster/components.tf
+++ b/k8s.tjo.cloud/modules/cluster/components.tf
@ -50,8 +50,8 @@ data "helm_template" "cilium" {
      autoMount:
        enabled: false

-    k8sServiceHost: ${var.cluster.api.domain}
-    k8sServicePort: ${var.cluster.api.port}
+    k8sServiceHost: ${var.cluster.api.internal.domain}
+    k8sServicePort: ${var.cluster.api.internal.port}

    hubble:
      ui:
--- a/k8s.tjo.cloud/modules/cluster/main.tf
+++ b/k8s.tjo.cloud/modules/cluster/main.tf
@ -1,5 +1,6 @@
 locals {
-  cluster_endpoint = "https://${var.cluster.api.domain}:${var.cluster.api.port}"
+  cluster_internal_endpoint = "https://${var.cluster.api.internal.domain}:${var.cluster.api.internal.port}"
+  cluster_public_endpoint   = "https://${var.cluster.api.public.domain}:${var.cluster.api.public.port}"

  podSubnets = [
    "10.200.0.0/16",
@ -40,6 +41,10 @@ locals {
      }
      allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes,
      apiServer = {
+        certSANs = [
+          var.cluster.api.internal.domain,
+          var.cluster.api.public.domain,
+        ]
        extraArgs = {
          "oidc-issuer-url"      = "https://id.tjo.space/application/o/k8stjocloud/",
          "oidc-client-id"       = "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
@ -97,8 +102,8 @@ locals {
        enabled = true
      }
      controlPlane = {
-        endpoint           = local.cluster_endpoint
-        localAPIServerPort = var.cluster.api.port
+        endpoint           = local.cluster_internal_endpoint
+        localAPIServerPort = var.cluster.api.internal.port
      }
      network = {
        cni = {
@ -172,7 +177,7 @@ resource "talos_machine_secrets" "this" {
 data "talos_machine_configuration" "controlplane" {
  cluster_name     = var.cluster.name
  machine_type     = "controlplane"
-  cluster_endpoint = local.cluster_endpoint
+  cluster_endpoint = local.cluster_internal_endpoint
  machine_secrets  = talos_machine_secrets.this.machine_secrets

  talos_version      = var.talos.version
@ -182,7 +187,7 @@ data "talos_machine_configuration" "controlplane" {
 data "talos_machine_configuration" "worker" {
  cluster_name     = var.cluster.name
  machine_type     = "worker"
-  cluster_endpoint = local.cluster_endpoint
+  cluster_endpoint = local.cluster_internal_endpoint
  machine_secrets  = talos_machine_secrets.this.machine_secrets

  talos_version      = var.talos.version
--- a/k8s.tjo.cloud/modules/cluster/outputs.tf
+++ b/k8s.tjo.cloud/modules/cluster/outputs.tf
@ -1,24 +1,15 @@
-output "kubeconfig" {
-  value = templatefile("${path.module}/kubeconfig.tftpl", {
-    cluster : {
-      name : var.cluster.name,
-      endpoint : local.cluster_endpoint,
-      ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
-    }
-    oidc : {
-      issuer : var.cluster.oidc.issuer_url,
-      id : var.cluster.oidc.client_id,
-    }
-  })
-}
-
 output "name" {
  value = var.cluster.name
 }

 output "api" {
  value = merge(var.cluster.api, {
-    endpoint : local.cluster_endpoint,
+    internal : merge(var.cluster.api.internal, {
+      endpoint : local.cluster_internal_endpoint,
+    }),
+    public : merge(var.cluster.api.public, {
+      endpoint : local.cluster_public_endpoint,
+    }),
    ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
  })
 }
--- a/k8s.tjo.cloud/modules/cluster/variables.tf
+++ b/k8s.tjo.cloud/modules/cluster/variables.tf
@ -40,8 +40,14 @@ variable "cluster" {
  type = object({
    name = string
    api = optional(object({
-      domain = optional(string, "internal.api.k8s.tjo.cloud")
-      port   = optional(number, 6443)
+      internal = optional(object({
+        domain = optional(string, "api.internal.k8s.tjo.cloud")
+        port   = optional(number, 6443)
+      }), {})
+      public = optional(object({
+        domain = optional(string, "api.k8s.tjo.cloud")
+        port   = optional(number, 443)
+      }), {})
    }), {})
    oidc = object({
      client_id  = string
--- a/k8s.tjo.cloud/terraform.tf
+++ b/k8s.tjo.cloud/terraform.tf
@ -62,7 +62,7 @@ provider "helm" {

 provider "helm" {
  kubernetes {
-    host                   = module.cluster.api.endpoint
+    host                   = module.cluster.api.internal.endpoint
    cluster_ca_certificate = base64decode(module.cluster.api.ca)
    exec {
      api_version = "client.authentication.k8s.io/v1beta1"
@ -82,7 +82,7 @@ provider "helm" {
 }

 provider "kubernetes" {
-  host                   = module.cluster.api.endpoint
+  host                   = module.cluster.api.internal.endpoint
  cluster_ca_certificate = base64decode(module.cluster.api.ca)
  exec {
    api_version = "client.authentication.k8s.io/v1beta1"