feat: prototyping new ingress setup

ingress.tjo.cloud/.terraform.lock.hcl

@@ -0,0 +1,26 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/digitalocean/digitalocean" {
+  version     = "2.40.0"
+  constraints = "~> 2.0"
+  hashes = [
+    "h1:Y7VkuuqOBgv+1jgL/4Hi247K2BskXwXtR/Uk9ssK/e4=",
+    "zh:00235830abae70642ebefc4d9c00e5eb978e28b74abc6b34f16b078f242aa217",
+    "zh:09d77785f768bd568f85a121d3d79316083befe903ce4ccd5567689a23236fb0",
+    "zh:0c9c4e19b411702d316a6bd044903e2ec506a69d38495ed32cc31e3f3f26acae",
+    "zh:12b34c88faad5b6149e9a3ad1396680588e1bae263b20d6b19835460f111c190",
+    "zh:15f041fc57ea46673a828919efe2ef3f05f7c4b863b7d7881336b93e92bd1159",
+    "zh:45e01972de2fab1687a09ea8fb3e4519be11c93ef93a63f28665630850858a20",
+    "zh:4e18bf5c1d2ec1ec6b6a9f4b58045309006f510edf770168fc18e273e6a09289",
+    "zh:575528b7e36e3489d2309e0c6cb9bd9952595cac5459b914f2d2827de1a1e4fc",
+    "zh:67462192212f810875d556462c79f574a8f5713b7a869ba4fce25953bfcf2dd2",
+    "zh:7024637b31e8276b653265fdf3f479220182edde4b300b034562b4c287faefa5",
+    "zh:a7904721b2680be8330dde98dd826be15c67eb274da7876f042cbcd6592ac970",
+    "zh:b225d4b67037a19392b0ab00d1f5fc9e729db4dfc32d18d4b36225693270ef52",
+    "zh:bd1e8768819d6113b2ec16f939196a1f2ae6d2803824fde463a20d06e071b212",
+    "zh:c5da40dc0749548ee2e1943776fb41b952c994e50bbc404251df20a81f730242",
+    "zh:dabc3387392aaba297739e1e97fadf059258fc3efb4dff2f499dbc407b6e088d",
+    "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2",
+  ]
+}

ingress.tjo.cloud/nodes.tf

@@ -0,0 +1,37 @@
+
+locals {
+  nodes = {
+    hetzner = {
+      ipv4 = "46.4.88.62"
+      ipv6 = "2a01:4f8:202:2395::"
+    }
+    odroid = {
+      ipv4 = "93.103.125.118"
+      ipv6 = "2a01:261:455:6c00:21e:6ff:fe45:c34"
+    }
+  }
+}
+
+data "digitalocean_domain" "ingress" {
+  name = "ingress.tjo.cloud"
+}
+
+resource "digitalocean_record" "nodes-a" {
+  for_each = local.nodes
+
+  domain = data.digitalocean_domain.ingress.id
+  type   = "A"
+  name   = each.key
+  value  = each.value.ipv4
+}
+
+resource "digitalocean_record" "nodes-aaaa" {
+  for_each = local.nodes
+
+  domain = data.digitalocean_domain.ingress.id
+  type   = "AAAA"
+  name   = each.key
+  value  = each.value.ipv6
+
+  ttl = 60
+}

ingress.tjo.cloud/records.tf

@@ -0,0 +1,39 @@
+locals {
+  listeners = [
+    {
+      domain = "k8s.tjo.cloud"
+      name   = "api"
+      nodes  = ["hetzner", "odroid"]
+    }
+  ]
+}
+
+resource "digitalocean_record" "listeners" {
+  for_each = merge([
+    for listener in local.listeners : merge(
+      {
+        for node in listener.nodes : "ipv4 ${listener.name}.${listener.domain} at ${node}" => {
+          ip     = local.nodes[node].ipv4
+          domain = listener.domain
+          name   = listener.name
+          type   = "A"
+        }
+      },
+      {
+        for node in listener.nodes : "ipv6 ${listener.name}.${listener.domain} at ${node}" => {
+          ip     = local.nodes[node].ipv6
+          domain = listener.domain
+          name   = listener.name
+          type   = "AAAA"
+        }
+      }
+    )
+  ]...)
+
+  domain = each.value.domain
+  type   = each.value.type
+  name   = each.value.name
+  value  = each.value.ip
+
+  ttl = 60
+}

ingress.tjo.cloud/terraform.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    digitalocean = {
+      source  = "digitalocean/digitalocean"
+      version = "~> 2.0"
+    }
+  }
+
+  required_version = "~> 1.7.3"
+}
+
+provider "digitalocean" {
+  token = var.digitalocean_token
+}

ingress.tjo.cloud/variables.tf

@@ -0,0 +1,4 @@
+variable "digitalocean_token" {
+  type      = string
+  sensitive = true
+}

justfile

@@ -3,6 +3,9 @@ set shell := ["devbox", "run"]
 # Load dotenv
 set dotenv-load
 
+default:
+  @just --list
+
 lint:
   @tofu fmt -check -recursive .
   @tflint --recursive
@@ -11,17 +14,17 @@ GATEWAY_API_VERSION := "v1.1.0"
 PROMETHEUS_CRDS_VERSION := "main"
 
 modules-cluster-manifests:
-  @rm -rf modules/cluster/manifests
-  @mkdir -p modules/cluster/manifests
-  @curl -L -o modules/cluster/manifests/gateway-api.crds.yaml \
+  @rm -rf k8s.tjo.cloud/modules/cluster/manifests
+  @mkdir -p k8s.tjo.cloud/modules/cluster/manifests
+  @curl -L -o k8s.tjo.cloud/modules/cluster/manifests/gateway-api.crds.yaml \
     "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{GATEWAY_API_VERSION}}/experimental-install.yaml"
 
 module-cluster-core-manifests:
-  @rm -rf modules/cluster-core/manifests
-  @mkdir -p modules/cluster-core/manifests
-  @curl -L -o modules/cluster-core/manifests/crd-podmonitors.yaml \
+  @rm -rf k8s.tjo.cloud/modules/cluster-core/manifests
+  @mkdir -p k8s.tjo.cloud/modules/cluster-core/manifests
+  @curl -L -o k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml \
     "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml"
-  @curl -L -o modules/cluster-core/manifests/crd-servicemonitors.yaml \
+  @curl -L -o k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml \
     "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml"
 
 k8s-apply: modules-cluster-manifests module-cluster-core-manifests

k8s.tjo.cloud/.terraform.lock.hcl

@@ -137,3 +137,25 @@ provider "registry.opentofu.org/siderolabs/talos" {
     "zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581",
   ]
 }
+
+provider "registry.opentofu.org/tailscale/tailscale" {
+  version     = "0.16.1"
+  constraints = "0.16.1"
+  hashes = [
+    "h1:NDIIkEo0G/leQSvGoh2Mk74ZE2xWrWgHX/S8ZVyBDYU=",
+    "zh:0a9d28e5195e0e29ebf9b12b345cafcb686125008151fa01677c399d8f8f1321",
+    "zh:249bce2fcfd3414211ae9e49e179e31b5d3c23dd9da24dc45acdea34ad308cb0",
+    "zh:3129fb52a2aaa0c8c30aff21e7d4c0601d80898b3ecb9d7604b5933c14f54924",
+    "zh:4ec3e255f34bb4f6362ab41aa9e05a3ce040a791bc07445dec86188dee867f85",
+    "zh:68d3995e5a1722e24f89a385899f56a63542159b884cac989196e9538b53c6ce",
+    "zh:799840b3bfbd14537397f157f4e6a5e54080cd4fee51521bac675aa188e0b33e",
+    "zh:99f1da9fdaddd8a1255dce56edf8eb3e235293c72738cf70f1fb9ee9631b40e6",
+    "zh:9b18fd51e260b2f3100937c34feae5f6fe3515df9b5e27ae23d00af75249a6d4",
+    "zh:a7154cdce28aeb80e822a97c6bc8b8acb7a074304fd198e265ac9cbcbda0ca06",
+    "zh:b0ce2ca42f018e5235a2171cdd8ba9829c90c54a6b2d602bd38e0e90c43d5d5d",
+    "zh:c67609f7018fc6e48b17befd6eeb21197e8f524496185c5e29707efa6967a0a5",
+    "zh:d4c9dc9d2a5a535851fc10049506bad1e7ab88193d5dcd371f91ac1b84f43a0a",
+    "zh:da27f2a9b9d5a4c02ec3893a763874513825c7c4dc2bb870ba741cf7725bcf9f",
+    "zh:e5bc1797b97607ff3d841c6c0d40da89c3843156ad43e15ded7d41fc0ac27717",
+  ]
+}

k8s.tjo.cloud/kubeconfig

@@ -1,26 +0,0 @@
-apiVersion: v1
-kind: Config
-clusters:
-- name: tjo-cloud
-  cluster:
-    server: https://api.k8s.tjo.cloud:6443
-    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRUFlySjYzMngyaUJXQnRCeWZOTVB4akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN5TlRFMU1qWXlNMW9YRFRNME1EY3lNekUxTWpZeQpNMW93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCR2NCTm84UXQ0djUrdnloWVl2SU5UcEllVTgvNFVjUU9jOUJHQm53OU9ITWEwUGU1MnhoYS9mdGl0L1kKZTBkVFBybzNMdjNYREFVMmN6NC9JVjBOVWVXallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVwbC9NbWlvRUdVTDFScFd4TVZpakNXdmsxOHd3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU41b0lTeWMKNVZhc2IvMmxQbmlBSVBkVWF4OHdJMldJVjlyMHd1SEVnVWNmQWlBL2RsMFZocEhGSUlEcldEajFBeGtHNEZXbgpUNHJ4K0IvRFBiQnhsM2VmclE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
-contexts:
-- name: oidc@tjo-cloud
-  context:
-    cluster: tjo-cloud
-    namespace: default
-    user: oidc
-current-context: oidc@tjo-cloud
-users:
-- name: oidc
-  user:
-    exec:
-      apiVersion: client.authentication.k8s.io/v1beta1
-      command: kubectl
-      args:
-      - oidc-login
-      - get-token
-      - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/
-      - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg
-      - --oidc-extra-scope=profile

k8s.tjo.cloud/main.tf

@@ -1,3 +1,16 @@
+locals {
+  cluster_domain = "k8s.tjo.cloud"
+}
+
+resource "tailscale_tailnet_key" "nodes" {
+  reusable      = true
+  ephemeral     = true
+  preauthorized = true
+  tags          = ["tag:kubernetes-tjo-cloud"]
+
+  description = "tailscale key for k8s-tjo-cloud nodes"
+}
+
 module "cluster" {
   source = "./modules/cluster"
 
@@ -11,8 +24,7 @@ module "cluster" {
   }
 
   cluster = {
-    name   = "tjo-cloud"
-    domain = "k8s.tjo.cloud"
+    name = "k8s-tjo-cloud"
     oidc = {
       client_id  = var.oidc_client_id
       issuer_url = var.oidc_issuer_url
@@ -25,11 +37,11 @@ module "cluster" {
     common_storage = "proxmox-backup-tjo-cloud"
   }
 
-  tailscale_authkey = var.tailscale_authkey
+  tailscale_authkey = tailscale_tailnet_key.nodes.key
 
   nodes = {
     pink = {
-      public  = true
+      public  = false
       type    = "controlplane"
       host    = "hetzner"
       storage = "main"
@@ -55,6 +67,20 @@ module "cluster" {
   }
 }
 
+data "tailscale_device" "controlpane" {
+  for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" }
+  hostname = each.value.name
+}
+resource "digitalocean_record" "internal-api" {
+  for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses]))
+
+  domain = local.cluster_domain
+  type   = strcontains(each.value, ":") ? "AAAA" : "A"
+  name   = "internal.api"
+  value  = each.value
+  ttl    = 30
+}
+
 resource "local_file" "kubeconfig" {
   content  = module.cluster.kubeconfig
   filename = "${path.module}/kubeconfig"
@@ -75,12 +101,5 @@ module "cluster-components" {
   digitalocean_token = var.digitalocean_token
 
   cluster_name   = module.cluster.name
-  cluster_domain = module.cluster.domain
-
-  loadbalancer_ips = {
-    hetzner-public = {
-      ipv4 = [for k, node in module.cluster.nodes : node.ipv4 if node.public]
-      ipv6 = [for k, node in module.cluster.nodes : node.ipv6 if node.public]
-    }
-  }
+  cluster_domain = "k8s.tjo.cloud"
 }

k8s.tjo.cloud/modules/cluster-components/variables.tf

@@ -1,8 +1,3 @@
-variable "loadbalancer_ips" {
-  description = "Map of loadbalancer IPs."
-  type        = map(object({ ipv4 = set(string), ipv6 = set(string) }))
-}
-
 variable "cluster_name" {
   description = "Name of the cluster."
   type        = string

k8s.tjo.cloud/modules/cluster/components.tf

@@ -50,7 +50,7 @@ data "helm_template" "cilium" {
       autoMount:
         enabled: false
 
-    k8sServiceHost: ${local.cluster_api_domain}
+    k8sServiceHost: ${var.cluster.api.domain}
     k8sServicePort: ${var.cluster.api.port}
 
     hubble:

k8s.tjo.cloud/modules/cluster/main.tf

@@ -1,6 +1,5 @@
 locals {
-  cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}"
-  cluster_endpoint   = "https://${local.cluster_api_domain}:${var.cluster.api.port}"
+  cluster_endpoint = "https://${var.cluster.api.domain}:${var.cluster.api.port}"
 
   podSubnets = [
     "10.200.0.0/16",
@@ -159,33 +158,13 @@ locals {
             "TS_AUTHKEY=${var.tailscale_authkey}",
             "TS_HOSTNAME=${node.name}",
             "TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}",
-            #"TS_EXTRA_ARGS=--accept-routes --snat-subnet-routes",
+            #"TS_EXTRA_ARGS=--accept-routes",
           ]
       })
     ]
   }
 }
 
-resource "digitalocean_record" "controlplane-A" {
-  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }
-
-  domain = var.cluster.domain
-  type   = "A"
-  name   = var.cluster.api.subdomain
-  value  = each.value.ipv4
-  ttl    = 30
-}
-
-resource "digitalocean_record" "controlplane-AAAA" {
-  for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" }
-
-  domain = var.cluster.domain
-  type   = "AAAA"
-  name   = var.cluster.api.subdomain
-  value  = each.value.ipv6
-  ttl    = 30
-}
-
 resource "talos_machine_secrets" "this" {
   talos_version = var.talos.version
 }
@@ -198,11 +177,6 @@ data "talos_machine_configuration" "controlplane" {
 
   talos_version      = var.talos.version
   kubernetes_version = var.talos.kubernetes
-
-  depends_on = [
-    digitalocean_record.controlplane-A,
-    digitalocean_record.controlplane-AAAA,
-  ]
 }
 
 data "talos_machine_configuration" "worker" {
@@ -213,11 +187,6 @@ data "talos_machine_configuration" "worker" {
 
   talos_version      = var.talos.version
   kubernetes_version = var.talos.kubernetes
-
-  depends_on = [
-    digitalocean_record.controlplane-A,
-    digitalocean_record.controlplane-AAAA
-  ]
 }
 
 resource "talos_machine_configuration_apply" "controlplane" {

k8s.tjo.cloud/modules/cluster/outputs.tf

@@ -16,10 +16,6 @@ output "name" {
   value = var.cluster.name
 }
 
-output "domain" {
-  value = var.cluster.domain
-}
-
 output "api" {
   value = merge(var.cluster.api, {
     endpoint : local.cluster_endpoint,

k8s.tjo.cloud/modules/cluster/proxmox.tf

@@ -2,7 +2,7 @@ locals {
   nodes_with_names = {
     for k, v in var.nodes : k => merge(v, {
       id   = 1000 + index(keys(var.nodes), k)
-      name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-")
+      name = replace("${k}.${v.type}.${var.cluster.name}", ".", "-")
     })
   }
   hashes = {

k8s.tjo.cloud/modules/cluster/variables.tf

@@ -39,9 +39,8 @@ variable "allow_scheduling_on_control_planes" {
 variable "cluster" {
   type = object({
     name = string
-    domain = string
     api = optional(object({
-      subdomain = optional(string, "api")
+      domain = optional(string, "internal.api.k8s.tjo.cloud")
       port   = optional(number, 6443)
     }), {})
     oidc = object({

k8s.tjo.cloud/terraform.tf

@@ -28,6 +28,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.31.0"
     }
+    tailscale = {
+      source  = "tailscale/tailscale"
+      version = "0.16.1"
+    }
   }
 
   required_version = "~> 1.7.3"
@@ -48,6 +52,10 @@ provider "digitalocean" {
   token = var.digitalocean_token
 }
 
+provider "tailscale" {
+  api_key = var.tailscale_apikey
+}
+
 provider "helm" {
   alias = "template"
 }

k8s.tjo.cloud/variables.tf

@@ -1,4 +1,4 @@
-variable "tailscale_authkey" {
+variable "tailscale_apikey" {
   type      = string
   sensitive = true
 }