From dd5b8f884bdaeddfbf4d871cf54e2742d1f7ba50 Mon Sep 17 00:00:00 2001 From: Tine Date: Fri, 2 Aug 2024 22:10:21 +0200 Subject: [PATCH] feat: prototyping new ingress setup --- ingress.tjo.cloud/.terraform.lock.hcl | 26 +++++++++++ ingress.tjo.cloud/nodes.tf | 37 ++++++++++++++++ ingress.tjo.cloud/records.tf | 39 +++++++++++++++++ ingress.tjo.cloud/terraform.tf | 14 ++++++ ingress.tjo.cloud/variables.tf | 4 ++ justfile | 17 +++++--- k8s.tjo.cloud/.terraform.lock.hcl | 22 ++++++++++ k8s.tjo.cloud/kubeconfig | 26 ----------- k8s.tjo.cloud/main.tf | 43 +++++++++++++------ .../modules/cluster-components/variables.tf | 5 --- k8s.tjo.cloud/modules/cluster/components.tf | 2 +- k8s.tjo.cloud/modules/cluster/main.tf | 35 +-------------- k8s.tjo.cloud/modules/cluster/outputs.tf | 4 -- k8s.tjo.cloud/modules/cluster/proxmox.tf | 2 +- k8s.tjo.cloud/modules/cluster/variables.tf | 7 ++- k8s.tjo.cloud/terraform.tf | 8 ++++ k8s.tjo.cloud/variables.tf | 2 +- 17 files changed, 199 insertions(+), 94 deletions(-) create mode 100644 ingress.tjo.cloud/.terraform.lock.hcl create mode 100644 ingress.tjo.cloud/nodes.tf create mode 100644 ingress.tjo.cloud/records.tf create mode 100644 ingress.tjo.cloud/terraform.tf create mode 100644 ingress.tjo.cloud/variables.tf delete mode 100755 k8s.tjo.cloud/kubeconfig diff --git a/ingress.tjo.cloud/.terraform.lock.hcl b/ingress.tjo.cloud/.terraform.lock.hcl new file mode 100644 index 0000000..efed827 --- /dev/null +++ b/ingress.tjo.cloud/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/digitalocean/digitalocean" { + version = "2.40.0" + constraints = "~> 2.0" + hashes = [ + "h1:Y7VkuuqOBgv+1jgL/4Hi247K2BskXwXtR/Uk9ssK/e4=", + "zh:00235830abae70642ebefc4d9c00e5eb978e28b74abc6b34f16b078f242aa217", + "zh:09d77785f768bd568f85a121d3d79316083befe903ce4ccd5567689a23236fb0", + "zh:0c9c4e19b411702d316a6bd044903e2ec506a69d38495ed32cc31e3f3f26acae", + "zh:12b34c88faad5b6149e9a3ad1396680588e1bae263b20d6b19835460f111c190", + "zh:15f041fc57ea46673a828919efe2ef3f05f7c4b863b7d7881336b93e92bd1159", + "zh:45e01972de2fab1687a09ea8fb3e4519be11c93ef93a63f28665630850858a20", + "zh:4e18bf5c1d2ec1ec6b6a9f4b58045309006f510edf770168fc18e273e6a09289", + "zh:575528b7e36e3489d2309e0c6cb9bd9952595cac5459b914f2d2827de1a1e4fc", + "zh:67462192212f810875d556462c79f574a8f5713b7a869ba4fce25953bfcf2dd2", + "zh:7024637b31e8276b653265fdf3f479220182edde4b300b034562b4c287faefa5", + "zh:a7904721b2680be8330dde98dd826be15c67eb274da7876f042cbcd6592ac970", + "zh:b225d4b67037a19392b0ab00d1f5fc9e729db4dfc32d18d4b36225693270ef52", + "zh:bd1e8768819d6113b2ec16f939196a1f2ae6d2803824fde463a20d06e071b212", + "zh:c5da40dc0749548ee2e1943776fb41b952c994e50bbc404251df20a81f730242", + "zh:dabc3387392aaba297739e1e97fadf059258fc3efb4dff2f499dbc407b6e088d", + "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2", + ] +} diff --git a/ingress.tjo.cloud/nodes.tf b/ingress.tjo.cloud/nodes.tf new file mode 100644 index 0000000..4e121ac --- /dev/null +++ b/ingress.tjo.cloud/nodes.tf @@ -0,0 +1,37 @@ + +locals { + nodes = { + hetzner = { + ipv4 = "46.4.88.62" + ipv6 = "2a01:4f8:202:2395::" + } + odroid = { + ipv4 = "93.103.125.118" + ipv6 = "2a01:261:455:6c00:21e:6ff:fe45:c34" + } + } +} + +data "digitalocean_domain" "ingress" { + name = "ingress.tjo.cloud" +} + +resource "digitalocean_record" "nodes-a" { + for_each = local.nodes + + domain = data.digitalocean_domain.ingress.id + type = "A" + name = each.key + value = each.value.ipv4 +} + +resource "digitalocean_record" "nodes-aaaa" { + for_each = local.nodes + + domain = data.digitalocean_domain.ingress.id + type = "AAAA" + name = each.key + value = each.value.ipv6 + + ttl = 60 +} diff --git a/ingress.tjo.cloud/records.tf b/ingress.tjo.cloud/records.tf new file mode 100644 index 0000000..a479b0c --- /dev/null +++ b/ingress.tjo.cloud/records.tf @@ -0,0 +1,39 @@ +locals { + listeners = [ + { + domain = "k8s.tjo.cloud" + name = "api" + nodes = ["hetzner", "odroid"] + } + ] +} + +resource "digitalocean_record" "listeners" { + for_each = merge([ + for listener in local.listeners : merge( + { + for node in listener.nodes : "ipv4 ${listener.name}.${listener.domain} at ${node}" => { + ip = local.nodes[node].ipv4 + domain = listener.domain + name = listener.name + type = "A" + } + }, + { + for node in listener.nodes : "ipv6 ${listener.name}.${listener.domain} at ${node}" => { + ip = local.nodes[node].ipv6 + domain = listener.domain + name = listener.name + type = "AAAA" + } + } + ) + ]...) + + domain = each.value.domain + type = each.value.type + name = each.value.name + value = each.value.ip + + ttl = 60 +} diff --git a/ingress.tjo.cloud/terraform.tf b/ingress.tjo.cloud/terraform.tf new file mode 100644 index 0000000..f061052 --- /dev/null +++ b/ingress.tjo.cloud/terraform.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + } + + required_version = "~> 1.7.3" +} + +provider "digitalocean" { + token = var.digitalocean_token +} diff --git a/ingress.tjo.cloud/variables.tf b/ingress.tjo.cloud/variables.tf new file mode 100644 index 0000000..bfca604 --- /dev/null +++ b/ingress.tjo.cloud/variables.tf @@ -0,0 +1,4 @@ +variable "digitalocean_token" { + type = string + sensitive = true +} diff --git a/justfile b/justfile index 8daf5f9..949b555 100644 --- a/justfile +++ b/justfile @@ -3,6 +3,9 @@ set shell := ["devbox", "run"] # Load dotenv set dotenv-load +default: + @just --list + lint: @tofu fmt -check -recursive . @tflint --recursive @@ -11,17 +14,17 @@ GATEWAY_API_VERSION := "v1.1.0" PROMETHEUS_CRDS_VERSION := "main" modules-cluster-manifests: - @rm -rf modules/cluster/manifests - @mkdir -p modules/cluster/manifests - @curl -L -o modules/cluster/manifests/gateway-api.crds.yaml \ + @rm -rf k8s.tjo.cloud/modules/cluster/manifests + @mkdir -p k8s.tjo.cloud/modules/cluster/manifests + @curl -L -o k8s.tjo.cloud/modules/cluster/manifests/gateway-api.crds.yaml \ "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{GATEWAY_API_VERSION}}/experimental-install.yaml" module-cluster-core-manifests: - @rm -rf modules/cluster-core/manifests - @mkdir -p modules/cluster-core/manifests - @curl -L -o modules/cluster-core/manifests/crd-podmonitors.yaml \ + @rm -rf k8s.tjo.cloud/modules/cluster-core/manifests + @mkdir -p k8s.tjo.cloud/modules/cluster-core/manifests + @curl -L -o k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml \ "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml" - @curl -L -o modules/cluster-core/manifests/crd-servicemonitors.yaml \ + @curl -L -o k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml \ "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml" k8s-apply: modules-cluster-manifests module-cluster-core-manifests diff --git a/k8s.tjo.cloud/.terraform.lock.hcl b/k8s.tjo.cloud/.terraform.lock.hcl index 1c682db..12016df 100644 --- a/k8s.tjo.cloud/.terraform.lock.hcl +++ b/k8s.tjo.cloud/.terraform.lock.hcl @@ -137,3 +137,25 @@ provider "registry.opentofu.org/siderolabs/talos" { "zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581", ] } + +provider "registry.opentofu.org/tailscale/tailscale" { + version = "0.16.1" + constraints = "0.16.1" + hashes = [ + "h1:NDIIkEo0G/leQSvGoh2Mk74ZE2xWrWgHX/S8ZVyBDYU=", + "zh:0a9d28e5195e0e29ebf9b12b345cafcb686125008151fa01677c399d8f8f1321", + "zh:249bce2fcfd3414211ae9e49e179e31b5d3c23dd9da24dc45acdea34ad308cb0", + "zh:3129fb52a2aaa0c8c30aff21e7d4c0601d80898b3ecb9d7604b5933c14f54924", + "zh:4ec3e255f34bb4f6362ab41aa9e05a3ce040a791bc07445dec86188dee867f85", + "zh:68d3995e5a1722e24f89a385899f56a63542159b884cac989196e9538b53c6ce", + "zh:799840b3bfbd14537397f157f4e6a5e54080cd4fee51521bac675aa188e0b33e", + "zh:99f1da9fdaddd8a1255dce56edf8eb3e235293c72738cf70f1fb9ee9631b40e6", + "zh:9b18fd51e260b2f3100937c34feae5f6fe3515df9b5e27ae23d00af75249a6d4", + "zh:a7154cdce28aeb80e822a97c6bc8b8acb7a074304fd198e265ac9cbcbda0ca06", + "zh:b0ce2ca42f018e5235a2171cdd8ba9829c90c54a6b2d602bd38e0e90c43d5d5d", + "zh:c67609f7018fc6e48b17befd6eeb21197e8f524496185c5e29707efa6967a0a5", + "zh:d4c9dc9d2a5a535851fc10049506bad1e7ab88193d5dcd371f91ac1b84f43a0a", + "zh:da27f2a9b9d5a4c02ec3893a763874513825c7c4dc2bb870ba741cf7725bcf9f", + "zh:e5bc1797b97607ff3d841c6c0d40da89c3843156ad43e15ded7d41fc0ac27717", + ] +} diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig deleted file mode 100755 index 507f82e..0000000 --- a/k8s.tjo.cloud/kubeconfig +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- name: tjo-cloud - cluster: - server: https://api.k8s.tjo.cloud:6443 - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRUFlySjYzMngyaUJXQnRCeWZOTVB4akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN5TlRFMU1qWXlNMW9YRFRNME1EY3lNekUxTWpZeQpNMW93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCR2NCTm84UXQ0djUrdnloWVl2SU5UcEllVTgvNFVjUU9jOUJHQm53OU9ITWEwUGU1MnhoYS9mdGl0L1kKZTBkVFBybzNMdjNYREFVMmN6NC9JVjBOVWVXallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVwbC9NbWlvRUdVTDFScFd4TVZpakNXdmsxOHd3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU41b0lTeWMKNVZhc2IvMmxQbmlBSVBkVWF4OHdJMldJVjlyMHd1SEVnVWNmQWlBL2RsMFZocEhGSUlEcldEajFBeGtHNEZXbgpUNHJ4K0IvRFBiQnhsM2VmclE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== -contexts: -- name: oidc@tjo-cloud - context: - cluster: tjo-cloud - namespace: default - user: oidc -current-context: oidc@tjo-cloud -users: -- name: oidc - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - command: kubectl - args: - - oidc-login - - get-token - - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/ - - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg - - --oidc-extra-scope=profile diff --git a/k8s.tjo.cloud/main.tf b/k8s.tjo.cloud/main.tf index aac3f61..0f2e1cd 100644 --- a/k8s.tjo.cloud/main.tf +++ b/k8s.tjo.cloud/main.tf @@ -1,3 +1,16 @@ +locals { + cluster_domain = "k8s.tjo.cloud" +} + +resource "tailscale_tailnet_key" "nodes" { + reusable = true + ephemeral = true + preauthorized = true + tags = ["tag:kubernetes-tjo-cloud"] + + description = "tailscale key for k8s-tjo-cloud nodes" +} + module "cluster" { source = "./modules/cluster" @@ -11,8 +24,7 @@ module "cluster" { } cluster = { - name = "tjo-cloud" - domain = "k8s.tjo.cloud" + name = "k8s-tjo-cloud" oidc = { client_id = var.oidc_client_id issuer_url = var.oidc_issuer_url @@ -25,11 +37,11 @@ module "cluster" { common_storage = "proxmox-backup-tjo-cloud" } - tailscale_authkey = var.tailscale_authkey + tailscale_authkey = tailscale_tailnet_key.nodes.key nodes = { pink = { - public = true + public = false type = "controlplane" host = "hetzner" storage = "main" @@ -55,6 +67,20 @@ module "cluster" { } } +data "tailscale_device" "controlpane" { + for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" } + hostname = each.value.name +} +resource "digitalocean_record" "internal-api" { + for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses])) + + domain = local.cluster_domain + type = strcontains(each.value, ":") ? "AAAA" : "A" + name = "internal.api" + value = each.value + ttl = 30 +} + resource "local_file" "kubeconfig" { content = module.cluster.kubeconfig filename = "${path.module}/kubeconfig" @@ -75,12 +101,5 @@ module "cluster-components" { digitalocean_token = var.digitalocean_token cluster_name = module.cluster.name - cluster_domain = module.cluster.domain - - loadbalancer_ips = { - hetzner-public = { - ipv4 = [for k, node in module.cluster.nodes : node.ipv4 if node.public] - ipv6 = [for k, node in module.cluster.nodes : node.ipv6 if node.public] - } - } + cluster_domain = "k8s.tjo.cloud" } diff --git a/k8s.tjo.cloud/modules/cluster-components/variables.tf b/k8s.tjo.cloud/modules/cluster-components/variables.tf index cdb6c1d..56acfc8 100644 --- a/k8s.tjo.cloud/modules/cluster-components/variables.tf +++ b/k8s.tjo.cloud/modules/cluster-components/variables.tf @@ -1,8 +1,3 @@ -variable "loadbalancer_ips" { - description = "Map of loadbalancer IPs." - type = map(object({ ipv4 = set(string), ipv6 = set(string) })) -} - variable "cluster_name" { description = "Name of the cluster." type = string diff --git a/k8s.tjo.cloud/modules/cluster/components.tf b/k8s.tjo.cloud/modules/cluster/components.tf index a3f3755..c5d669b 100644 --- a/k8s.tjo.cloud/modules/cluster/components.tf +++ b/k8s.tjo.cloud/modules/cluster/components.tf @@ -50,7 +50,7 @@ data "helm_template" "cilium" { autoMount: enabled: false - k8sServiceHost: ${local.cluster_api_domain} + k8sServiceHost: ${var.cluster.api.domain} k8sServicePort: ${var.cluster.api.port} hubble: diff --git a/k8s.tjo.cloud/modules/cluster/main.tf b/k8s.tjo.cloud/modules/cluster/main.tf index 0c6509a..25226d3 100644 --- a/k8s.tjo.cloud/modules/cluster/main.tf +++ b/k8s.tjo.cloud/modules/cluster/main.tf @@ -1,6 +1,5 @@ locals { - cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}" - cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}" + cluster_endpoint = "https://${var.cluster.api.domain}:${var.cluster.api.port}" podSubnets = [ "10.200.0.0/16", @@ -159,33 +158,13 @@ locals { "TS_AUTHKEY=${var.tailscale_authkey}", "TS_HOSTNAME=${node.name}", "TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}", - #"TS_EXTRA_ARGS=--accept-routes --snat-subnet-routes", + #"TS_EXTRA_ARGS=--accept-routes", ] }) ] } } -resource "digitalocean_record" "controlplane-A" { - for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } - - domain = var.cluster.domain - type = "A" - name = var.cluster.api.subdomain - value = each.value.ipv4 - ttl = 30 -} - -resource "digitalocean_record" "controlplane-AAAA" { - for_each = { for k, node in local.nodes_with_address : k => node if node.type == "controlplane" } - - domain = var.cluster.domain - type = "AAAA" - name = var.cluster.api.subdomain - value = each.value.ipv6 - ttl = 30 -} - resource "talos_machine_secrets" "this" { talos_version = var.talos.version } @@ -198,11 +177,6 @@ data "talos_machine_configuration" "controlplane" { talos_version = var.talos.version kubernetes_version = var.talos.kubernetes - - depends_on = [ - digitalocean_record.controlplane-A, - digitalocean_record.controlplane-AAAA, - ] } data "talos_machine_configuration" "worker" { @@ -213,11 +187,6 @@ data "talos_machine_configuration" "worker" { talos_version = var.talos.version kubernetes_version = var.talos.kubernetes - - depends_on = [ - digitalocean_record.controlplane-A, - digitalocean_record.controlplane-AAAA - ] } resource "talos_machine_configuration_apply" "controlplane" { diff --git a/k8s.tjo.cloud/modules/cluster/outputs.tf b/k8s.tjo.cloud/modules/cluster/outputs.tf index 228ffbe..d6df9d8 100644 --- a/k8s.tjo.cloud/modules/cluster/outputs.tf +++ b/k8s.tjo.cloud/modules/cluster/outputs.tf @@ -16,10 +16,6 @@ output "name" { value = var.cluster.name } -output "domain" { - value = var.cluster.domain -} - output "api" { value = merge(var.cluster.api, { endpoint : local.cluster_endpoint, diff --git a/k8s.tjo.cloud/modules/cluster/proxmox.tf b/k8s.tjo.cloud/modules/cluster/proxmox.tf index dfdea35..5ecbdc7 100644 --- a/k8s.tjo.cloud/modules/cluster/proxmox.tf +++ b/k8s.tjo.cloud/modules/cluster/proxmox.tf @@ -2,7 +2,7 @@ locals { nodes_with_names = { for k, v in var.nodes : k => merge(v, { id = 1000 + index(keys(var.nodes), k) - name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-") + name = replace("${k}.${v.type}.${var.cluster.name}", ".", "-") }) } hashes = { diff --git a/k8s.tjo.cloud/modules/cluster/variables.tf b/k8s.tjo.cloud/modules/cluster/variables.tf index 74eec42..724a1bf 100644 --- a/k8s.tjo.cloud/modules/cluster/variables.tf +++ b/k8s.tjo.cloud/modules/cluster/variables.tf @@ -38,11 +38,10 @@ variable "allow_scheduling_on_control_planes" { variable "cluster" { type = object({ - name = string - domain = string + name = string api = optional(object({ - subdomain = optional(string, "api") - port = optional(number, 6443) + domain = optional(string, "internal.api.k8s.tjo.cloud") + port = optional(number, 6443) }), {}) oidc = object({ client_id = string diff --git a/k8s.tjo.cloud/terraform.tf b/k8s.tjo.cloud/terraform.tf index 09338ea..86ee423 100644 --- a/k8s.tjo.cloud/terraform.tf +++ b/k8s.tjo.cloud/terraform.tf @@ -28,6 +28,10 @@ terraform { source = "hashicorp/kubernetes" version = "2.31.0" } + tailscale = { + source = "tailscale/tailscale" + version = "0.16.1" + } } required_version = "~> 1.7.3" @@ -48,6 +52,10 @@ provider "digitalocean" { token = var.digitalocean_token } +provider "tailscale" { + api_key = var.tailscale_apikey +} + provider "helm" { alias = "template" } diff --git a/k8s.tjo.cloud/variables.tf b/k8s.tjo.cloud/variables.tf index 2604f30..5411b8b 100644 --- a/k8s.tjo.cloud/variables.tf +++ b/k8s.tjo.cloud/variables.tf @@ -1,4 +1,4 @@ -variable "tailscale_authkey" { +variable "tailscale_apikey" { type = string sensitive = true }