From 2fb90bf6211023a81c1309a8385bed1d89b4905c Mon Sep 17 00:00:00 2001 From: Tine Date: Tue, 16 Jul 2024 21:13:04 +0200 Subject: [PATCH] feat: work done, but using wrong talos image, should use nocloud version not metal --- clusters/pink/kubeconfig | 26 -- clusters/pink/kubernetes.tf | 271 ------------------ .../.terraform.lock.hcl | 0 k8s.tjo.cloud/dashboard.tf | 140 +++++++++ k8s.tjo.cloud/gateway.tf | 142 +++++++++ k8s.tjo.cloud/kubeconfig | 26 ++ k8s.tjo.cloud/main.tf | 117 ++++++++ {clusters/pink => k8s.tjo.cloud}/providers.tf | 1 + {clusters/pink => k8s.tjo.cloud}/variables.tf | 19 +- modules/cluster/main.tf | 133 +++++---- modules/cluster/variables.tf | 23 ++ 11 files changed, 540 insertions(+), 358 deletions(-) delete mode 100755 clusters/pink/kubeconfig delete mode 100644 clusters/pink/kubernetes.tf rename {clusters/pink => k8s.tjo.cloud}/.terraform.lock.hcl (100%) create mode 100644 k8s.tjo.cloud/dashboard.tf create mode 100644 k8s.tjo.cloud/gateway.tf create mode 100755 k8s.tjo.cloud/kubeconfig create mode 100644 k8s.tjo.cloud/main.tf rename {clusters/pink => k8s.tjo.cloud}/providers.tf (98%) rename {clusters/pink => k8s.tjo.cloud}/variables.tf (58%) diff --git a/clusters/pink/kubeconfig b/clusters/pink/kubeconfig deleted file mode 100755 index 3871014..0000000 --- a/clusters/pink/kubeconfig +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: v1 -kind: Config -clusters: -- name: pink - cluster: - server: https://api.pink.k8s.tjo.cloud:6443 - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVRDZ0F3SUJBZ0lSQU9vTSthdnlEZkN1THc4TVptRnFnNll3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOREEzTVRReE1UQTFNRGRhRncwek5EQTNNVEl4TVRBMQpNRGRhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVNEU3JVdUk1RFRxMHR4MkM4SmJHWmsvZXJlU0dlQU1tUzgzcE9RZklMbEFaMWE1NjJ4MnZobi9lMHYKWHQ0NmJFbzQ3QWttTlh3eEFNTExSRmkrYjF5V28yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGSGNLNFBEZHNSMmxnbHdRZmUwN2c3L0gralpGTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSVFERm1qeTkKcm4yVXpUN1hTL1pUZW5nbHpIZC9OOFZHU1JIQlQzV0xCbkpZendJZ0hKMFdEd1NoOGJQTHpvSzNxSkNjVDlmQgp1ZmZ0U0ZMVHdhdkNvdmF6Y09BPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== -contexts: -- name: oidc@pink - context: - cluster: pink - namespace: default - user: oidc -current-context: oidc@pink -users: -- name: oidc - user: - exec: - apiVersion: client.authentication.k8s.io/v1beta1 - command: kubectl - args: - - oidc-login - - get-token - - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/ - - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg - - --oidc-extra-scope=profile diff --git a/clusters/pink/kubernetes.tf b/clusters/pink/kubernetes.tf deleted file mode 100644 index 0205132..0000000 --- a/clusters/pink/kubernetes.tf +++ /dev/null @@ -1,271 +0,0 @@ -module "cluster" { - source = "../../modules/cluster" - - providers = { - helm.template = helm.template - } - - nodes = { - one = { - public = true - type = "controlplane" - host = "hetzner" - boot_pool = "hetzner-main-data" - }, - two = { - public = true - type = "controlplane" - host = "hetzner" - boot_pool = "hetzner-main-data" - }, - three = { - public = true - type = "controlplane" - host = "hetzner" - boot_pool = "hetzner-main-data" - }, - four = { - public = false - type = "worker" - host = "hetzner" - boot_pool = "hetzner-main-data" - } - five = { - public = false - type = "worker" - host = "hetzner" - boot_pool = "hetzner-main-data" - } - } - - versions = { - talos = "v1.7.5" - kubernetes = "v1.30.0" - } - - iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso" - - cluster = { - name = "pink" - domain = "k8s.tjo.cloud" - oidc = { - client_id = var.oidc_client_id - issuer_url = var.oidc_issuer_url - } - } - - tailscale_authkey = var.tailscale_authkey -} - -resource "local_file" "kubeconfig" { - content = module.cluster.kubeconfig - filename = "${path.module}/kubeconfig" -} - -resource "kubernetes_manifest" "hetzner-nodes-as-loadbalancers" { - manifest = { - apiVersion = "cilium.io/v2alpha1" - kind = "CiliumLoadBalancerIPPool" - metadata = { - name = "hetzner-nodes" - } - spec = { - blocks = concat( - [for k, node in module.cluster.nodes : { start : node.address_ipv4 } if node.public], - # [for k, node in module.cluster.nodes : { start : node.address_ipv6 } if node.public], - ) - } - } -} - -resource "kubernetes_namespace" "tjo-cloud" { - metadata { - name = "tjo-cloud" - } -} - -resource "kubernetes_secret" "digitalocean-token" { - metadata { - name = "digitalocean-token" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - } - data = { - token = var.digitalocean_token - } -} - -resource "helm_release" "external-dns" { - name = "external-dns" - chart = "external-dns" - repository = "https://kubernetes-sigs.github.io/external-dns/" - version = "v1.14.5" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - - set { - name = "namespaced" - value = "true" - } - - set { - name = "provider" - value = "digitalocean" - } - - set { - name = "env[0].name" - value = "DO_TOKEN" - } - set { - name = "env[0].valueFrom.secretKeyRef.name" - value = kubernetes_secret.digitalocean-token.metadata[0].name - } - set { - name = "env[0].valueFrom.secretKeyRef.key" - value = "token" - } - - set_list { - name = "sources" - value = ["gateway-httproute", "gateway-tlsroute", "gateway-tcproute", "gateway-udproute", "ingress", "service"] - } -} - -resource "helm_release" "cert-manager" { - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = "v1.15.1" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - - set { - name = "crds.enabled" - value = true - } - - set_list { - name = "extraArgs" - value = ["--enable-gateway-api"] - } -} - - -resource "kubernetes_manifest" "tjo-cloud-issuer" { - manifest = { - apiVersion = "cert-manager.io/v1" - kind = "Issuer" - metadata = { - name = "tjo-cloud" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - } - spec = { - acme = { - email = "tine@tjo.space" - server = "https://acme-staging-v02.api.letsencrypt.org/directory" - privateKeySecretRef = { - name = "tjo-cloud-acme-account" - } - solvers = [ - { - dns01 = { - digitalocean = { - tokenSecretRef = { - name = kubernetes_secret.digitalocean-token.metadata[0].name - key = "token" - } - } - } - } - ] - } - } - } -} - -resource "kubernetes_manifest" "gateway" { - manifest = { - apiVersion = "gateway.networking.k8s.io/v1" - kind = "Gateway" - metadata = { - name = "gateway" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - annotations = { - "cert-manager.io/issuer" : "tjo-cloud" - } - } - spec = { - gatewayClassName = "cilium" - listeners = [ - { - name : "http" - hostname : "*.${module.cluster.name}.${module.cluster.domain}" - protocol : "HTTPS" - port : 443 - allowedRoutes : { - namespaces : { - from : "Same" - } - } - tls : { - mode : "Terminate" - certificateRefs : [ - { - name : "tjo-cloud-tls" - } - ] - } - } - ] - } - } -} - -resource "helm_release" "dashboard" { - name = "kubernetes-dashboard" - repository = "https://kubernetes.github.io/dashboard" - chart = "kubernetes-dashboard" - version = "7.5.0" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name -} - -resource "kubernetes_manifest" "dashoard-http-route" { - depends_on = [ - kubernetes_manifest.gateway, - ] - - manifest = { - apiVersion = "gateway.networking.k8s.io/v1" - kind = "HTTPRoute" - metadata = { - name = "dashboard" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name - } - spec = { - parentRefs = [ - { - name : "gateway" - } - ] - hostnames = [ - "dashboard.${module.cluster.name}.${module.cluster.domain}" - ] - rules = [ - { - matches = [ - { - path : { - value : "/" - type : "PathPrefix" - } - } - ] - backendRefs = [ - { - name : "kubernetes-dashboard-web" - port : 8000 - } - ] - } - ] - } - } -} diff --git a/clusters/pink/.terraform.lock.hcl b/k8s.tjo.cloud/.terraform.lock.hcl similarity index 100% rename from clusters/pink/.terraform.lock.hcl rename to k8s.tjo.cloud/.terraform.lock.hcl diff --git a/k8s.tjo.cloud/dashboard.tf b/k8s.tjo.cloud/dashboard.tf new file mode 100644 index 0000000..62ccd09 --- /dev/null +++ b/k8s.tjo.cloud/dashboard.tf @@ -0,0 +1,140 @@ +resource "helm_release" "dashboard" { + name = "kubernetes-dashboard" + repository = "https://kubernetes.github.io/dashboard" + chart = "kubernetes-dashboard" + version = "7.5.0" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + + set { + name = "kong.enabled" + value = false + } +} + +resource "kubernetes_manifest" "dashoard-http-route" { + depends_on = [ + kubernetes_manifest.gateway, + ] + + manifest = { + apiVersion = "gateway.networking.k8s.io/v1" + kind = "HTTPRoute" + metadata = { + name = "dashboard" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + spec = { + parentRefs = [ + { + name : "gateway" + } + ] + hostnames = [ + "dashboard.${module.cluster.domain}" + ] + rules = [ + { + matches = [ + { + path : { + value : "/" + type : "PathPrefix" + } + } + ] + backendRefs = [ + { + name : "kubernetes-dashboard-web" + port : 8000 + } + ] + }, + { + matches = [ + { + path : { + value : "/api/v1/login" + type : "PathPrefix" + } + }, + { + path : { + value : "/api/v1/csrftoken/login" + type : "PathPrefix" + } + }, + { + path : { + value : "/api/v1/me" + type : "PathPrefix" + } + }, + ] + backendRefs = [ + { + name : "kubernetes-dashboard-auth" + port : 8000 + } + ] + }, + { + matches = [ + { + path : { + value : "/api" + type : "PathPrefix" + } + } + ] + backendRefs = [ + { + name : "kubernetes-dashboard-api" + port : 8000 + } + ] + }, + ] + } + } +} + +resource "kubernetes_secret" "dashboard-oidc" { + metadata { + name = "dashboard-oidc" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + data = { + client-secret = "null" + } +} + +resource "kubernetes_manifest" "dashboard-oidc" { + manifest = { + apiVersion = "gateway.envoyproxy.io/v1alpha1" + kind = "SecurityPolicy" + metadata = { + name = "dashboard-oidc" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + spec = { + targetRef = { + group : "gateway.networking.k8s.io" + kind : "HTTPRoute" + name : "dashboard" + } + oidc = { + provider = { + issuer : var.oidc_issuer_url + } + clientID : var.oidc_client_id + clientSecret : { + name : "dashboard-oidc" + } + scopes : ["openid", "email", "profile"] + forwardAccessToken : true + + redirectURL : "https://dashboard.${module.cluster.domain}/login" + } + } + } +} diff --git a/k8s.tjo.cloud/gateway.tf b/k8s.tjo.cloud/gateway.tf new file mode 100644 index 0000000..d0b9b76 --- /dev/null +++ b/k8s.tjo.cloud/gateway.tf @@ -0,0 +1,142 @@ +resource "kubernetes_secret" "digitalocean-token" { + metadata { + name = "digitalocean-token" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + data = { + token = var.digitalocean_token + } +} + +resource "helm_release" "cert-manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = "v1.15.1" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + + set { + name = "crds.enabled" + value = true + } + + set_list { + name = "extraArgs" + value = ["--enable-gateway-api"] + } +} + +resource "kubernetes_manifest" "tjo-cloud-issuer" { + manifest = { + apiVersion = "cert-manager.io/v1" + kind = "Issuer" + metadata = { + name = "tjo-cloud" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + spec = { + acme = { + email = "tine@tjo.space" + server = "https://acme-staging-v02.api.letsencrypt.org/directory" + privateKeySecretRef = { + name = "tjo-cloud-acme-account" + } + solvers = [ + { + dns01 = { + digitalocean = { + tokenSecretRef = { + name = kubernetes_secret.digitalocean-token.metadata[0].name + key = "token" + } + } + } + } + ] + } + } + } +} + +resource "helm_release" "envoy" { + name = "envoy" + chart = "gateway-helm" + repository = "oci://docker.io/envoyproxy" + version = "v1.1.0-rc.1" + namespace = "kube-system" + + values = [ + yamlencode({ + config = { + envoyGateway = { + provider = { + type = "Kubernetes" + kubernetes = { + envoyDaemonSet = {} + envoyDeployment = null + } + } + gateway = { + controllerName = "gateway.envoyproxy.io/gatewayclass-controller" + } + logging = { + level = { + default = "info" + } + } + } + } + }) + ] +} + +resource "kubernetes_manifest" "gateway-class" { + manifest = { + apiVersion = "gateway.networking.k8s.io/v1" + kind = "GatewayClass" + metadata = { + name = "envoy" + } + spec = { + controllerName : "gateway.envoyproxy.io/gatewayclass-controller" + } + } +} + +resource "kubernetes_manifest" "gateway" { + manifest = { + apiVersion = "gateway.networking.k8s.io/v1" + kind = "Gateway" + metadata = { + name = "gateway" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + annotations = { + "cert-manager.io/issuer" : "tjo-cloud" + } + } + spec = { + gatewayClassName = "envoy" + listeners = [ + { + name : "http" + hostname : "*.${module.cluster.name}.${module.cluster.domain}" + protocol : "HTTPS" + port : 443 + allowedRoutes : { + namespaces : { + from : "Same" + } + } + tls : { + mode : "Terminate" + certificateRefs : [ + { + name : "tjo-cloud-tls" + } + ] + } + } + ] + } + } +} diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig new file mode 100755 index 0000000..fd325d0 --- /dev/null +++ b/k8s.tjo.cloud/kubeconfig @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Config +clusters: +- name: tjo-cloud + cluster: + server: https://api.k8s.tjo.cloud:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRVHV4SURwTXNNUXNYWitLb2k1RkdsakFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN4TmpFNE1qWXhPVm9YRFRNME1EY3hOREU0TWpZeApPVm93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCQ01zTVdqWjU1alRzaWd6aVcrT1BUSm1uS1VLQ2VTbktnTmptOFNRVE0ydm1tQklaM1I1UDlKcTY4ZmEKUU9zZ0RXSE96aFF6MjRNRUw1aFhpUzRjU3B1allUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVEVG41Sy8raXROdEgyczM0MTR3TEFjV0V0Wm93Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQU56bVV6OVEKZVhVVVJzei9KWlRlb29ibzF0Y1BHN2p3cGFlSDQ0VFVwVEE4QWlFQTFZcTdhUWdkb2lVTFNac2IrT3JLeDJyWQpGRTdSVkpxR0lyM1U4c3daY3k4PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== +contexts: +- name: oidc@tjo-cloud + context: + cluster: tjo-cloud + namespace: default + user: oidc +current-context: oidc@tjo-cloud +users: +- name: oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: kubectl + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/ + - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg + - --oidc-extra-scope=profile diff --git a/k8s.tjo.cloud/main.tf b/k8s.tjo.cloud/main.tf new file mode 100644 index 0000000..9df20ce --- /dev/null +++ b/k8s.tjo.cloud/main.tf @@ -0,0 +1,117 @@ +module "cluster" { + source = "../modules/cluster" + + providers = { + helm.template = helm.template + } + + versions = { + talos = "v1.7.5" + kubernetes = "v1.30.0" + } + + iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso" + + cluster = { + name = "tjo-cloud" + domain = "k8s.tjo.cloud" + oidc = { + client_id = var.oidc_client_id + issuer_url = var.oidc_issuer_url + } + } + + proxmox = { + name = "tjo-cloud" + url = "https://proxmox.tjo.cloud/api2/json" + ccm = { + username = var.proxmox_ccm_username + token = var.proxmox_ccm_token + } + csi = { + username = var.proxmox_csi_username + token = var.proxmox_csi_token + } + } + + tailscale_authkey = var.tailscale_authkey + + allow_scheduling_on_control_planes = true + nodes = { + pink = { + public = true + type = "controlplane" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 4096 + } + purple = { + public = true + type = "controlplane" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 4096 + } + violet = { + public = true + type = "controlplane" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 4096 + } + blue = { + public = false + type = "worker" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 16384 + } + cyan = { + public = false + type = "worker" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 16384 + } + green = { + public = false + type = "worker" + host = "hetzner" + boot_pool = "hetzner-main-data" + cores = 4 + memory = 16384 + } + } +} + +resource "local_file" "kubeconfig" { + content = module.cluster.kubeconfig + filename = "${path.module}/kubeconfig" +} + +resource "kubernetes_manifest" "hetzner-nodes-as-loadbalancers" { + manifest = { + apiVersion = "cilium.io/v2alpha1" + kind = "CiliumLoadBalancerIPPool" + metadata = { + name = "hetzner-nodes" + } + spec = { + blocks = concat( + [for k, node in module.cluster.nodes : { start : node.ipv4 } if node.public], + # [for k, node in module.cluster.nodes : { start : node.ipv6 } if node.public], + ) + } + } +} + +resource "kubernetes_namespace" "tjo-cloud" { + metadata { + name = "tjo-cloud" + } +} diff --git a/clusters/pink/providers.tf b/k8s.tjo.cloud/providers.tf similarity index 98% rename from clusters/pink/providers.tf rename to k8s.tjo.cloud/providers.tf index d9d32b6..c260a92 100644 --- a/clusters/pink/providers.tf +++ b/k8s.tjo.cloud/providers.tf @@ -42,6 +42,7 @@ provider "proxmox" { } provider "digitalocean" { + token = var.digitalocean_token } provider "helm" { diff --git a/clusters/pink/variables.tf b/k8s.tjo.cloud/variables.tf similarity index 58% rename from clusters/pink/variables.tf rename to k8s.tjo.cloud/variables.tf index 65c8e3c..21c0f46 100644 --- a/clusters/pink/variables.tf +++ b/k8s.tjo.cloud/variables.tf @@ -6,16 +6,13 @@ variable "tailscale_authkey" { variable "oidc_username" { type = string } - variable "oidc_password" { type = string sensitive = true } - variable "oidc_client_id" { type = string } - variable "oidc_issuer_url" { type = string } @@ -24,3 +21,19 @@ variable "digitalocean_token" { type = string sensitive = true } + +variable "proxmox_csi_username" { + type = string +} +variable "proxmox_csi_token" { + type = string + sensitive = true +} + +variable "proxmox_ccm_username" { + type = string +} +variable "proxmox_ccm_token" { + type = string + sensitive = true +} diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index 6668f40..f7128a8 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -1,12 +1,10 @@ locals { - cluster_domain = "${var.cluster.name}.${var.cluster.domain}" - cluster_api_domain = "${var.cluster.api.subdomain}.${local.cluster_domain}" + cluster_api_domain = "${var.cluster.api.subdomain}.${var.cluster.domain}" + cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}" - cluster_endpoint = "https://${local.cluster_api_domain}:${var.cluster.api.port}" + nodes = { for k, v in var.nodes : k => merge(v, { name = replace("${k}.${v.type}.${var.cluster.domain}", ".", "-") }) } - nodes = { for k, v in var.nodes : k => merge(v, { name = "${k}.node.${local.cluster_domain}" }) } - - nodes_with_address = { for k, v in local.nodes : k => merge(v, { address_ipv4 = proxmox_vm_qemu.this[k].default_ipv4_address, address_ipv6 = proxmox_vm_qemu.this[k].default_ipv6_address }) } + nodes_with_address = { for k, v in local.nodes : k => merge(v, { ipv4 = try(proxmox_vm_qemu.this[k].default_ipv4_address, false), ipv6 = try(proxmox_vm_qemu.this[k].default_ipv6_address, false) }) } first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0] @@ -25,9 +23,22 @@ locals { "100.64.0.0/10", "fd7a:115c:a1e0::/96" ] + + proxmox-cloud-controller-manager = { + clusters : [{ + region : var.proxmox.name, + url : var.proxmox.url, + insecure : var.proxmox.insecure, + token_id : var.proxmox.ccm.username, + token_secret : var.proxmox.ccm.token, + }] + } } -resource "macaddress" "this" { +resource "macaddress" "private" { + for_each = local.nodes +} +resource "macaddress" "public" { for_each = local.nodes } @@ -39,6 +50,7 @@ resource "proxmox_vm_qemu" "this" { tags = join(";", concat( ["kubernetes", "terraform"], each.value.public ? ["public"] : ["private"], + [each.value.type] )) cores = each.value.cores @@ -52,7 +64,7 @@ resource "proxmox_vm_qemu" "this" { network { model = "virtio" bridge = each.value.public ? "vmpublic0" : "vmprivate0" - macaddr = macaddress.this[each.key].address + macaddr = macaddress.private[each.key].address } disks { @@ -75,21 +87,21 @@ resource "proxmox_vm_qemu" "this" { } resource "digitalocean_record" "controlplane-A" { - for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" } + for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) } domain = var.cluster.domain type = "A" - name = "${var.cluster.api.subdomain}.${var.cluster.name}" + name = var.cluster.api.subdomain value = each.value.default_ipv4_address ttl = 30 } resource "digitalocean_record" "controlplane-AAAA" { - for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" } + for_each = { for k, v in proxmox_vm_qemu.this : k => v if try(var.nodes[k].public && var.nodes[k].type == "controlplane", false) } domain = var.cluster.domain type = "AAAA" - name = "${var.cluster.api.subdomain}.${var.cluster.name}" + name = var.cluster.api.subdomain value = each.value.default_ipv6_address ttl = 30 } @@ -126,18 +138,6 @@ data "talos_machine_configuration" "worker" { ] } -data "talos_machine_disks" "boot" { - for_each = local.nodes_with_address - - client_configuration = talos_machine_secrets.this.client_configuration - node = each.value.name - endpoint = each.value.address_ipv4 - - filters = { - size = "< 60GB" - } -} - data "helm_template" "cilium" { provider = helm.template @@ -211,25 +211,11 @@ data "helm_template" "cilium" { enabled : true } }, - # Ingress gateway gatewayAPI : { - enabled : true - hostNetwork : { - enabled : true - } + enabled : false } envoy : { - enabled : true - securityContext : { - capabilities : { - keepCapNetBindService : true - envoy : [ - "NET_ADMIN", - "SYS_ADMIN", - "NET_BIND_SERVICE" - ] - } - } + enabled : false } })] } @@ -241,7 +227,7 @@ resource "talos_machine_configuration_apply" "controlplane" { machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration node = each.value.name - endpoint = each.value.address_ipv4 + endpoint = each.value.ipv4 apply_mode = "reboot" @@ -267,7 +253,7 @@ resource "talos_machine_configuration_apply" "controlplane" { proxy : { disabled : true } - allowSchedulingOnControlPlanes : true, + allowSchedulingOnControlPlanes : var.allow_scheduling_on_control_planes, apiServer : { extraArgs : { "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/", @@ -296,32 +282,56 @@ resource "talos_machine_configuration_apply" "controlplane" { apiGroup: rbac.authorization.k8s.io EOF }, - { - name : "gateway-api-crds" - contents : file("${path.module}/gateway-api-crds.yaml") - }, { name : "cilium" contents : data.helm_template.cilium.manifest + }, + { + name : "proxmox-cloud-controller-manager" + contents : <<-EOF + apiVersion: v1 + kind: Secret + metadata: + name: proxmox-cloud-controller-manager + namespace: kube-system + data: + config.yaml: ${base64encode(yamlencode(local.proxmox-cloud-controller-manager))} + EOF } - ], + ] + externalCloudProvider : { + enabled : true + manifests : [ + "https://raw.githubusercontent.com/sergelogvinov/proxmox-csi-plugin/v0.7.0/docs/deploy/proxmox-csi-plugin-talos.yml", + "https://raw.githubusercontent.com/sergelogvinov/proxmox-cloud-controller-manager/v0.4.2/docs/deploy/cloud-controller-manager-talos.yml" + ] + } + extraManifests : [ + #"https://raw.githubusercontent.com/alex1989hu/kubelet-serving-cert-approver/v0.8.5/deploy/standalone-install.yaml", + "https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.1/components.yaml", + ] } machine = { kubelet = { nodeIP : { validSubnets : local.tailscaleSubnets } + extraArgs : { + rotate-server-certificates : true + cloud-provider : "external" + } } network = { hostname = each.value.name } install = { image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}" - disk = data.talos_machine_disks.boot[each.key].disks[0].name + disk = "/dev/vda" } nodeLabels = { - "k8s.tjo.cloud/public" = each.value.public ? "true" : "false" - "k8s.tjo.cloud/host" = each.value.host + "k8s.tjo.cloud/public" = each.value.public ? "true" : "false" + "k8s.tjo.cloud/host" = each.value.host + "k8s.tjo.cloud/proxmox" = var.proxmox.name } } }), @@ -330,7 +340,8 @@ resource "talos_machine_configuration_apply" "controlplane" { kind : "ExtensionServiceConfig" name : "tailscale" environment : [ - "TS_AUTHKEY=${var.tailscale_authkey}" + "TS_AUTHKEY=${var.tailscale_authkey}", + "TS_HOSTNAME=${replace(each.value.name, ".", "-")}", ] }) ] @@ -343,7 +354,7 @@ resource "talos_machine_configuration_apply" "worker" { machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration node = each.value.name - endpoint = each.value.address_ipv4 + endpoint = each.value.ipv4 apply_mode = "reboot" @@ -370,17 +381,22 @@ resource "talos_machine_configuration_apply" "worker" { nodeIP : { validSubnets : local.tailscaleSubnets } + extraArgs : { + rotate-server-certificates : true + cloud-provider : "external" + } } network = { hostname = each.value.name } install = { image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:${var.versions.talos}" - disk = data.talos_machine_disks.boot[each.key].disks[0].name + disk = "/dev/vda" } nodeLabels = { - "k8s.tjo.cloud/public" = each.value.public ? "true" : "false" - "k8s.tjo.cloud/host" = each.value.host + "k8s.tjo.cloud/public" = each.value.public ? "true" : "false" + "k8s.tjo.cloud/host" = each.value.host + "k8s.tjo.cloud/proxmox" = var.proxmox.name } } }), @@ -389,7 +405,8 @@ resource "talos_machine_configuration_apply" "worker" { kind : "ExtensionServiceConfig" name : "tailscale" environment : [ - "TS_AUTHKEY=${var.tailscale_authkey}" + "TS_AUTHKEY=${var.tailscale_authkey}", + "TS_HOSTNAME=${replace(each.value.name, ".", "-")}", ] }) ] @@ -402,7 +419,7 @@ resource "talos_machine_bootstrap" "this" { ] node = local.first_controlplane_node.name - endpoint = local.first_controlplane_node.address_ipv4 + endpoint = local.first_controlplane_node.ipv4 client_configuration = talos_machine_secrets.this.client_configuration } @@ -412,5 +429,5 @@ data "talos_cluster_kubeconfig" "this" { ] client_configuration = talos_machine_secrets.this.client_configuration - node = local.first_controlplane_node.address_ipv4 + node = local.first_controlplane_node.ipv4 } diff --git a/modules/cluster/variables.tf b/modules/cluster/variables.tf index 80bd562..87bafac 100644 --- a/modules/cluster/variables.tf +++ b/modules/cluster/variables.tf @@ -24,6 +24,12 @@ variable "iso" { description = "Downloaded from factory.talos.dev, select quemu agent and tailscale extensions." } +variable "allow_scheduling_on_control_planes" { + default = false + type = bool + description = "Allow scheduling on control plane nodes" +} + variable "cluster" { type = object({ name = string @@ -43,3 +49,20 @@ variable "tailscale_authkey" { type = string sensitive = true } + +variable "proxmox" { + type = object({ + name = string + url = string + insecure = optional(bool, false) + csi = object({ + username = string + token = string + }) + ccm = object({ + username = string + token = string + }) + }) + sensitive = true +}