diff --git a/ingress.tjo.cloud/.terraform.lock.hcl b/ingress.tjo.cloud/.terraform.lock.hcl index efed827..3536fdb 100644 --- a/ingress.tjo.cloud/.terraform.lock.hcl +++ b/ingress.tjo.cloud/.terraform.lock.hcl @@ -24,3 +24,21 @@ provider "registry.opentofu.org/digitalocean/digitalocean" { "zh:f42137cf424c3e7c9c935b3f73618e51096bd0367a8d364073e2d70588d2cbf2", ] } + +provider "registry.opentofu.org/hashicorp/dns" { + version = "3.4.1" + constraints = "~> 3.4.1" + hashes = [ + "h1:6Tb2wZRxfKunvjLw47ihfkaoARWIQrJd+WCOXkx0hg4=", + "zh:53f46f16fc3b25d9bdce61d7cc9a67cea9c67ea5347fcde35833451d5011f1c4", + "zh:662669802c99c7e698ca52e7a32afc19acf66d97bbc05f5effacdf0fe471848f", + "zh:8289aee49df8a0cb2fcdfded575e68df1721fc3e6ce75555def499d31b7e910d", + "zh:90aa9b049d19137ed15ea3f88238824e1fde7d1a806378cb97520391e169a96d", + "zh:9e5795e10d65927ba689cfabc06b72085055543342d1db26ef5963ae5ab5d64a", + "zh:9f39b6e0ca35cf2405fe4e13af8a405cf9b0b0c34b6b06ad36f319631405a3aa", + "zh:c37a2c140c95b3eaa9d0aeb79496b220ba1a0456d01a77fcc68bd565bee7edd5", + "zh:cd40d356c5382f0b868df2d8539f95031e3cf0c210053674ec4c7cd20c73b82a", + "zh:fde59f09e4e62fc84012fff8493bc64eac287a0245e1c8abf29b0138d3018dfb", + "zh:ffee5b54936bd43247e9316de0ecc0dde36556a2cd50f090d7f6443a3f689b95", + ] +} diff --git a/ingress.tjo.cloud/nodes.tf b/ingress.tjo.cloud/nodes.tf index 4e121ac..40e7713 100644 --- a/ingress.tjo.cloud/nodes.tf +++ b/ingress.tjo.cloud/nodes.tf @@ -1,14 +1,8 @@ locals { - nodes = { - hetzner = { - ipv4 = "46.4.88.62" - ipv6 = "2a01:4f8:202:2395::" - } - odroid = { - ipv4 = "93.103.125.118" - ipv6 = "2a01:261:455:6c00:21e:6ff:fe45:c34" - } + locations = { + DE = ["46.4.88.62", "2a01:4f8:202:2395::"] + SI = ["93.103.125.118", "2a01:261:455:6c00:21e:6ff:fe45:c34"] } } @@ -16,22 +10,19 @@ data "digitalocean_domain" "ingress" { name = "ingress.tjo.cloud" } -resource "digitalocean_record" "nodes-a" { - for_each = local.nodes +resource "digitalocean_record" "locations" { + for_each = merge([ + for location, ips in local.locations : { + for ip in ips : "${location} at ${ip}" => { + location = location, + ip = ip, + } + } + ]...) domain = data.digitalocean_domain.ingress.id - type = "A" - name = each.key - value = each.value.ipv4 -} - -resource "digitalocean_record" "nodes-aaaa" { - for_each = local.nodes - - domain = data.digitalocean_domain.ingress.id - type = "AAAA" - name = each.key - value = each.value.ipv6 - - ttl = 60 + type = strcontains(each.value.ip, ":") ? "AAAA" : "A" + name = lower(each.value.location) + value = each.value.ip + ttl = 60 } diff --git a/ingress.tjo.cloud/records.tf b/ingress.tjo.cloud/records.tf index a479b0c..5221de4 100644 --- a/ingress.tjo.cloud/records.tf +++ b/ingress.tjo.cloud/records.tf @@ -1,39 +1,35 @@ locals { listeners = [ { - domain = "k8s.tjo.cloud" - name = "api" - nodes = ["hetzner", "odroid"] + domain = "k8s.tjo.cloud" + name = "api" + locations = ["SI", "DE"] + }, + { + domain = "k8s.tjo.cloud" + name = "dashboard" + locations = ["SI", "DE"] } ] } resource "digitalocean_record" "listeners" { - for_each = merge([ - for listener in local.listeners : merge( - { - for node in listener.nodes : "ipv4 ${listener.name}.${listener.domain} at ${node}" => { - ip = local.nodes[node].ipv4 + for_each = merge(flatten([ + for listener in local.listeners : + [ + for location in listener.locations : { + for ip in local.locations[location] : "${ip} for ${listener.name}.${listener.domain} at ${location}" => { + ip = ip domain = listener.domain name = listener.name - type = "A" - } - }, - { - for node in listener.nodes : "ipv6 ${listener.name}.${listener.domain} at ${node}" => { - ip = local.nodes[node].ipv6 - domain = listener.domain - name = listener.name - type = "AAAA" } } - ) - ]...) + ] + ])...) domain = each.value.domain - type = each.value.type + type = strcontains(each.value.ip, ":") ? "AAAA" : "A" name = each.value.name value = each.value.ip - - ttl = 60 + ttl = 60 } diff --git a/ingress.tjo.cloud/terraform.tf b/ingress.tjo.cloud/terraform.tf index f061052..56e079a 100644 --- a/ingress.tjo.cloud/terraform.tf +++ b/ingress.tjo.cloud/terraform.tf @@ -4,6 +4,10 @@ terraform { source = "digitalocean/digitalocean" version = "~> 2.0" } + dns = { + source = "hashicorp/dns" + version = "~> 3.4.1" + } } required_version = "~> 1.7.3" diff --git a/k8s.tjo.cloud/README.md b/k8s.tjo.cloud/README.md new file mode 100644 index 0000000..2107b0f --- /dev/null +++ b/k8s.tjo.cloud/README.md @@ -0,0 +1,8 @@ +# `k8s.tjo.cloud` + +## Kubernetes JWT +For federation to work, the OIDC JWKS needs to be manually added in to id.tjo.space. + +``` +kubectl get --raw /openid/v1/jwks +```` diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig new file mode 100755 index 0000000..be57459 --- /dev/null +++ b/k8s.tjo.cloud/kubeconfig @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Config +clusters: +- name: k8s-tjo-cloud + cluster: + server: https://api.k8s.tjo.cloud:443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRWXFNSG94UFhiTFMvZ21oTFhUOWd0akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGd3TXpFeU16TXpORm9YRFRNME1EZ3dNVEV5TXpNegpORm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCRzcvQVVjZ3VsVFhpZ1JpT2hkeWk0a1BycmJNenZqZXZYdXhKMDJkN1pEYVlkMmtzeStQVkZTcDM0a0YKSmE4ZEdWN0VnZHlWQlVZeGZVU0Z0UGpleFBHallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVUycmptRW1vbTJPYnVJYUhzTFNrQTFnamFYUHN3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU9CZWVyWlUKTXhHN1ErczV2OTR4QjUvVE9XaUd5MXRCOVk0aTUxZERFZkNYQWlBLzg1VEs3VDRlWGtDakFqaW1QU0Y3ZUZWago2c1NJbGd0WGhZWHpUUUVSZWc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== +contexts: +- name: oidc@k8s-tjo-cloud + context: + cluster: k8s-tjo-cloud + namespace: default + user: oidc +current-context: oidc@k8s-tjo-cloud +users: +- name: oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: kubectl + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/ + - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg + - --oidc-extra-scope=profile diff --git a/k8s.tjo.cloud/kubeconfig.tftpl b/k8s.tjo.cloud/kubeconfig.tftpl new file mode 100644 index 0000000..37c7428 --- /dev/null +++ b/k8s.tjo.cloud/kubeconfig.tftpl @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Config +clusters: +- name: ${cluster.name} + cluster: + server: ${cluster.endpoint} + certificate-authority-data: ${cluster.ca} +contexts: +- name: oidc@${cluster.name} + context: + cluster: ${cluster.name} + namespace: default + user: oidc +current-context: oidc@${cluster.name} +users: +- name: oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: kubectl + args: + - oidc-login + - get-token + - --oidc-issuer-url=${oidc.issuer} + - --oidc-client-id=${oidc.id} + - --oidc-extra-scope=profile diff --git a/k8s.tjo.cloud/main.tf b/k8s.tjo.cloud/main.tf index 0f2e1cd..f382f67 100644 --- a/k8s.tjo.cloud/main.tf +++ b/k8s.tjo.cloud/main.tf @@ -71,18 +71,28 @@ data "tailscale_device" "controlpane" { for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" } hostname = each.value.name } -resource "digitalocean_record" "internal-api" { +resource "digitalocean_record" "api-internal" { for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses])) domain = local.cluster_domain type = strcontains(each.value, ":") ? "AAAA" : "A" - name = "internal.api" + name = trimsuffix(module.cluster.api.internal.domain, ".${local.cluster_domain}") value = each.value ttl = 30 } resource "local_file" "kubeconfig" { - content = module.cluster.kubeconfig + content = templatefile("${path.module}/kubeconfig.tftpl", { + cluster : { + name : module.cluster.name, + endpoint : module.cluster.api.public.endpoint, + ca : module.cluster.api.ca, + } + oidc : { + issuer : var.oidc_issuer_url, + id : var.oidc_client_id, + } + }) filename = "${path.module}/kubeconfig" } diff --git a/k8s.tjo.cloud/modules/cluster-components/external-dns.tf b/k8s.tjo.cloud/modules/cluster-components/external-dns.tf new file mode 100644 index 0000000..9babe8c --- /dev/null +++ b/k8s.tjo.cloud/modules/cluster-components/external-dns.tf @@ -0,0 +1,48 @@ +resource "helm_release" "external-dns-privileged" { + name = "external-dns-privileged" + chart = "external-dns" + repository = "https://kubernetes-sigs.github.io/external-dns/" + version = "v1.14.5" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + + values = [<<-EOF + provider: digitalocean + env: + - name: DO_TOKEN + valueFrom: + secretKeyRef: + name: ${kubernetes_secret.digitalocean-token.metadata[0].name} + key: token + sources: + - ingress + - service + domainFilters: + - k8s.tjo.cloud + - internal.k8s.tjo.cloud + EOF + ] +} + +resource "helm_release" "external-dns-user-content" { + name = "external-dns-user-content" + chart = "external-dns" + repository = "https://kubernetes-sigs.github.io/external-dns/" + version = "v1.14.5" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + + values = [<<-EOF + provider: digitalocean + env: + - name: DO_TOKEN + valueFrom: + secretKeyRef: + name: ${kubernetes_secret.digitalocean-token.metadata[0].name} + key: token + sources: + - ingress + - service + domainFilters: + - user-content.tjo.cloud + EOF + ] +} diff --git a/k8s.tjo.cloud/modules/cluster-components/gateway.tf b/k8s.tjo.cloud/modules/cluster-components/gateway.tf index 1d85555..b97ea02 100644 --- a/k8s.tjo.cloud/modules/cluster-components/gateway.tf +++ b/k8s.tjo.cloud/modules/cluster-components/gateway.tf @@ -1,13 +1,3 @@ -resource "kubernetes_secret" "digitalocean-token" { - metadata { - name = "digitalocean-token" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - } - data = { - token = var.digitalocean_token - } -} - resource "kubernetes_manifest" "tjo-cloud-issuer" { manifest = { apiVersion = "cert-manager.io/v1" @@ -33,6 +23,11 @@ resource "kubernetes_manifest" "tjo-cloud-issuer" { } } } + selector : { + dnsZones : [ + "tjo.cloud" + ] + } } ] } @@ -57,24 +52,10 @@ resource "kubernetes_manifest" "gateway_class_config" { type = "ClusterIP" externalTrafficPolicy = "Local" annotations = { - "io.cilium.nodeipam/match-node-labels" = "k8s.tjo.cloud/public=true" + "external-dns.alpha.kubernetes.io/internal-hostname" = "envoy.internal.k8s.tjo.cloud" } - loadBalancerClass = "io.cilium/node" } envoyDaemonSet = { - patch = { - type = "StrategicMerge" - value = { - spec = { - template = { - spec = { - hostNetwork = true - dnsPolicy = "ClusterFirstWithHostNet" - } - } - } - } - } pod = { nodeSelector = { "node-role.kubernetes.io/control-plane" = "" diff --git a/k8s.tjo.cloud/modules/cluster-components/main.tf b/k8s.tjo.cloud/modules/cluster-components/main.tf index 61fbd67..79095aa 100644 --- a/k8s.tjo.cloud/modules/cluster-components/main.tf +++ b/k8s.tjo.cloud/modules/cluster-components/main.tf @@ -3,3 +3,13 @@ resource "kubernetes_namespace" "tjo-cloud" { name = "tjo-cloud" } } + +resource "kubernetes_secret" "digitalocean-token" { + metadata { + name = "digitalocean-token" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + data = { + token = var.digitalocean_token + } +} diff --git a/k8s.tjo.cloud/modules/cluster-core/monitoring.tf b/k8s.tjo.cloud/modules/cluster-core/monitoring.tf index 3ab5a9d..233308a 100644 --- a/k8s.tjo.cloud/modules/cluster-core/monitoring.tf +++ b/k8s.tjo.cloud/modules/cluster-core/monitoring.tf @@ -203,6 +203,7 @@ resource "helm_release" "grafana-alloy" { client_id = "o6Tz2215HLvhvZ4RCZCR8oMmCapTu30iwkoMkz6m" client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" endpoint_params = { + grant_type = "client_credentials", client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", } } @@ -282,6 +283,7 @@ resource "helm_release" "grafana-alloy" { client_id = "56TYXtgg7QwLjh4lPl1PTu3C4iExOvO1d6b15WuC" client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" endpoint_params = { + grant_type = "client_credentials", client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", } } diff --git a/k8s.tjo.cloud/modules/cluster/components.tf b/k8s.tjo.cloud/modules/cluster/components.tf index c5d669b..b2bc5f1 100644 --- a/k8s.tjo.cloud/modules/cluster/components.tf +++ b/k8s.tjo.cloud/modules/cluster/components.tf @@ -50,8 +50,8 @@ data "helm_template" "cilium" { autoMount: enabled: false - k8sServiceHost: ${var.cluster.api.domain} - k8sServicePort: ${var.cluster.api.port} + k8sServiceHost: ${var.cluster.api.internal.domain} + k8sServicePort: ${var.cluster.api.internal.port} hubble: ui: diff --git a/k8s.tjo.cloud/modules/cluster/main.tf b/k8s.tjo.cloud/modules/cluster/main.tf index 25226d3..30beabd 100644 --- a/k8s.tjo.cloud/modules/cluster/main.tf +++ b/k8s.tjo.cloud/modules/cluster/main.tf @@ -1,5 +1,6 @@ locals { - cluster_endpoint = "https://${var.cluster.api.domain}:${var.cluster.api.port}" + cluster_internal_endpoint = "https://${var.cluster.api.internal.domain}:${var.cluster.api.internal.port}" + cluster_public_endpoint = "https://${var.cluster.api.public.domain}:${var.cluster.api.public.port}" podSubnets = [ "10.200.0.0/16", @@ -40,6 +41,10 @@ locals { } allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes, apiServer = { + certSANs = [ + var.cluster.api.internal.domain, + var.cluster.api.public.domain, + ] extraArgs = { "oidc-issuer-url" = "https://id.tjo.space/application/o/k8stjocloud/", "oidc-client-id" = "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg", @@ -97,8 +102,8 @@ locals { enabled = true } controlPlane = { - endpoint = local.cluster_endpoint - localAPIServerPort = var.cluster.api.port + endpoint = local.cluster_internal_endpoint + localAPIServerPort = var.cluster.api.internal.port } network = { cni = { @@ -172,7 +177,7 @@ resource "talos_machine_secrets" "this" { data "talos_machine_configuration" "controlplane" { cluster_name = var.cluster.name machine_type = "controlplane" - cluster_endpoint = local.cluster_endpoint + cluster_endpoint = local.cluster_internal_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version @@ -182,7 +187,7 @@ data "talos_machine_configuration" "controlplane" { data "talos_machine_configuration" "worker" { cluster_name = var.cluster.name machine_type = "worker" - cluster_endpoint = local.cluster_endpoint + cluster_endpoint = local.cluster_internal_endpoint machine_secrets = talos_machine_secrets.this.machine_secrets talos_version = var.talos.version diff --git a/k8s.tjo.cloud/modules/cluster/outputs.tf b/k8s.tjo.cloud/modules/cluster/outputs.tf index d6df9d8..1e4fe50 100644 --- a/k8s.tjo.cloud/modules/cluster/outputs.tf +++ b/k8s.tjo.cloud/modules/cluster/outputs.tf @@ -1,24 +1,15 @@ -output "kubeconfig" { - value = templatefile("${path.module}/kubeconfig.tftpl", { - cluster : { - name : var.cluster.name, - endpoint : local.cluster_endpoint, - ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate, - } - oidc : { - issuer : var.cluster.oidc.issuer_url, - id : var.cluster.oidc.client_id, - } - }) -} - output "name" { value = var.cluster.name } output "api" { value = merge(var.cluster.api, { - endpoint : local.cluster_endpoint, + internal : merge(var.cluster.api.internal, { + endpoint : local.cluster_internal_endpoint, + }), + public : merge(var.cluster.api.public, { + endpoint : local.cluster_public_endpoint, + }), ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate, }) } diff --git a/k8s.tjo.cloud/modules/cluster/variables.tf b/k8s.tjo.cloud/modules/cluster/variables.tf index 724a1bf..064925a 100644 --- a/k8s.tjo.cloud/modules/cluster/variables.tf +++ b/k8s.tjo.cloud/modules/cluster/variables.tf @@ -40,8 +40,14 @@ variable "cluster" { type = object({ name = string api = optional(object({ - domain = optional(string, "internal.api.k8s.tjo.cloud") - port = optional(number, 6443) + internal = optional(object({ + domain = optional(string, "api.internal.k8s.tjo.cloud") + port = optional(number, 6443) + }), {}) + public = optional(object({ + domain = optional(string, "api.k8s.tjo.cloud") + port = optional(number, 443) + }), {}) }), {}) oidc = object({ client_id = string diff --git a/k8s.tjo.cloud/terraform.tf b/k8s.tjo.cloud/terraform.tf index 86ee423..0934e0d 100644 --- a/k8s.tjo.cloud/terraform.tf +++ b/k8s.tjo.cloud/terraform.tf @@ -62,7 +62,7 @@ provider "helm" { provider "helm" { kubernetes { - host = module.cluster.api.endpoint + host = module.cluster.api.internal.endpoint cluster_ca_certificate = base64decode(module.cluster.api.ca) exec { api_version = "client.authentication.k8s.io/v1beta1" @@ -82,7 +82,7 @@ provider "helm" { } provider "kubernetes" { - host = module.cluster.api.endpoint + host = module.cluster.api.internal.endpoint cluster_ca_certificate = base64decode(module.cluster.api.ca) exec { api_version = "client.authentication.k8s.io/v1beta1"