diff --git a/.gitignore b/.gitignore index a9e8a0c..3c1c1eb 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ override.tf.json .terraformrc terraform.rc + +# ENV +.env diff --git a/example.env b/example.env new file mode 100644 index 0000000..95844fd --- /dev/null +++ b/example.env @@ -0,0 +1,6 @@ +TF_VAR_tailscale_authkey="" + +DIGITALOCEAN_TOKEN="" + +PM_API_TOKEN_ID="terraform@pve!terraform-provisioner" +PM_API_TOKEN_SECRET="" diff --git a/terraform/.gitignore b/terraform/.gitignore new file mode 100644 index 0000000..80bb3a7 --- /dev/null +++ b/terraform/.gitignore @@ -0,0 +1,2 @@ +kubeconfig +talosconfig diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..8d24e9d --- /dev/null +++ b/terraform/.terraform.lock.hcl @@ -0,0 +1,142 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/digitalocean/digitalocean" { + version = "2.39.2" + constraints = "~> 2.0" + hashes = [ + "h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=", + "zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f", + "zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0", + "zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f", + "zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee", + "zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea", + "zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5", + "zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e", + "zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb", + "zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761", + "zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9", + "zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf", + "zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720", + "zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad", + "zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0", + "zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f", + "zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba", + ] +} + +provider "registry.opentofu.org/hashicorp/helm" { + version = "2.14.0" + constraints = "2.14.0" + hashes = [ + "h1:ibK3MM61pVjBwBcrro56OLTHwUhhNglvGG9CloLvliI=", + "zh:1c84ca8c274564c46497e89055139c7af64c9e1a8dd4f1cd4c68503ac1322fb8", + "zh:211a763173934d30c2e49c0cc828b1e34a528b0fdec8bf48d2bb3afadd4f9095", + "zh:3dca0b703a2f82d3e283a9e9ca6259a3b9897b217201f3cddf430009a1ca00c9", + "zh:40c5cfd48dcef54e87129e19d31c006c2e3309ee6c09d566139eaf315a59a369", + "zh:6f23c00ca1e2663e2a208a7491aa6dbe2604f00e0af7e23ef9323206e8f2fc81", + "zh:77f8cfc4888600e0d12da137bbdb836de160db168dde7af26c2e44cf00cbf057", + "zh:97b99c945eafa9bafc57c3f628d496356ea30312c3df8dfac499e0f3ff6bf0c9", + "zh:a01cfc53e50d5f722dc2aabd26097a8e4d966d343ffd471034968c2dc7a8819d", + "zh:b69c51e921fe8c91e38f4a82118d0b6b0f47f6c71a76f506fde3642ecbf39911", + "zh:fb8bfc7b8106bef58cc5628c024103f0dd5276d573fe67ac16f343a2b38ecee8", + ] +} + +provider "registry.opentofu.org/hashicorp/local" { + version = "1.4.0" + constraints = "1.4.0" + hashes = [ + "h1:/i49KTSz7SNa1DUbqvmsUN6ou/mYgWv5onXTReHmE/E=", + "zh:0637972352fa0a83f83eea54fe91c4a67a43669ad90fc13dcd4218eccfb474af", + "zh:590451e43ae4377449944e3fcf74fb6a2a05c29037d32e53d9de49579ea6fd55", + "zh:6942cc057dae128a9b867c9446ed519d9394d941f092a4eafc90dd9a7c26ed5d", + "zh:cc7c57baaa5af4d2f88c84910d8041685c821b88dd9bfcbe8b18190f31bbf629", + "zh:e958bafa3d7c792623b5965900de8532b5a5addc1a573aeec387729aebaf633d", + ] +} + +provider "registry.opentofu.org/hashicorp/random" { + version = "3.6.2" + constraints = "3.6.2" + hashes = [ + "h1:PXvoOj9gj+Or+9k0tQWCQJKxnsVO0GqnQwVahgwRrsU=", + "zh:1f27612f7099441526d8af59f5b4bdcc35f46915df5d243043d7337ea5a3e38a", + "zh:2a58e66502825db8b4b96116c04bd0323bca1cf1f5752bdd8f9c26feb84d3b1e", + "zh:4f0a4fa479e29de0c3c90146fd58799c097f7a55401cb00560dd4e9b1e6fad9d", + "zh:9c93c0fe6ef685513734527e0c8078636b2cc07591427502a7260f4744b1af1d", + "zh:a466ff5219beb77fb3b18a3d7e7fe30e7edd4d95c8e5c87f4f4e3fe3eeb8c2d7", + "zh:ab33e6176d0c757ddb31e40e01a941e6918ad10f7a786c8e8e4f35e5cff81c96", + "zh:b6eabf377a1c12cb3f9ddd97aacdd5b49c1646dc959074124f81d40fcd216d7e", + "zh:ccec5d03d0d1c0f354be299cdd6a417b2700f1a6781df36bcce77246b2f57e50", + "zh:d2a7945eeb691fdd2b1474da76ddc2d1655e2aedbb14b57f06d4f5123d47adf9", + "zh:ed62351f4ad9d1469c6798b77dee5f63b18b29c473620a0046ba3d4f111b621d", + ] +} + +provider "registry.opentofu.org/ivoronin/macaddress" { + version = "0.3.2" + constraints = "0.3.2" + hashes = [ + "h1:yk0ASl2cAoc/22tvpi9Kke+WvowgXGq0QwaP93IQ+S0=", + "zh:00cb168d9210ed88cfa7de8a33d5666b2cf6660a5d20a7a96348b8b902833eca", + "zh:1366458320df0b6f1132e59b5410931c0c5626bbf27b05b29dd311311a710e9b", + "zh:2e8102c7f6046665c95b806752d692843f2e846554f7eba85690cd2087c9048a", + "zh:3c1ae52f855d0e694ad28eb34ec41c553344aaa7bd51adaa48cf15e3ee842e17", + "zh:496d8db2055cead9d264fdad83534318e3ab77ce06e38d43674a4ec25c0e860d", + "zh:54c5eeae7cc61d706080256e06aaf509869b1d86297b9e99948a2fe2af6d455b", + "zh:5f26e851048be3c56f3706b7fde25fe76dd30003ef6356216dc9ecff400218bb", + "zh:5fc1debcd0fe043dfce00ab110e180b896a1a9958edea7d81d05aacc9b630e5e", + "zh:650045261b382b4559fd1bd190d6cabbeb022b53d7e240eb6b66f6824ca81bf4", + "zh:7203dea017883e8fdd7ba66c9b1a9aac0cab101133e4eeab365c4d0995194272", + "zh:726a9222d15f11316587c199ee367bae1d5495ff16ebdfc41635f7628834a8d6", + "zh:c9f3bcaa073a0921189bd74ef6b2b57cad34b3eb01788c010df8a15fd9d8045c", + "zh:d3fba491b0ff0d3d64162216159232398a75ad81c31e4304335d6b76b74a864a", + "zh:e80011c6e3af4eeafdeda9bd118a774f8b7cdf1f133953abf827f313653ec184", + ] +} + +provider "registry.opentofu.org/siderolabs/talos" { + version = "0.5.0" + constraints = "0.5.0" + hashes = [ + "h1:xogkLLCrJJmd278E+vNMnmQgaMD05Gd1QXN914xgVec=", + "zh:0f71f2624576224c9bc924b136b601b734243efa7a7ad8280dfd8bd583e4afa5", + "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d", + "zh:33c50dacc5029fa20caed702001fb1439899c94f203b1f37dccb970f504bca45", + "zh:3c97a6e2692b88d3f4631a3f8769146f602c210e881b46fa1b3b82c545e51cd1", + "zh:44077a137613bcfe29eef00315b5aa50d83390c3c727580a4ff0f4b87f22d228", + "zh:5bd02f278aec5567f94dd057d1c758363998ce581ff17b0869515bb682c02186", + "zh:80f40939bc3b55f0005c03b77122ceea86ec4deb82f5557950a97ad96fbb1557", + "zh:94c1b17f25bc30eacde926e46f196f1f135032674730d9f50c986ef6b7a854f0", + "zh:95ad665b2fdeed38180f5c471164833a34d07c1ef0470c1652565fe8cf4e9c4a", + "zh:a50ef6088afcb129c176dd4ba86c345e9be7b14358bb3b21c34f06930d8f39ef", + "zh:aa71da1da00ed66f1dddf1b69c10b829f24ac89e207de07d32c455dd04482096", + "zh:abb7eeb2b089081b4814ed80a295673e1a92f82ce092dde37b5bc92e75efec2c", + "zh:db9b9b54a0db5ae151376d5a73e0d28497c3e06181840e71ef8349213ac03e50", + "zh:e50ed8aa90b736508fce63680e8339240cecb74709ab9563d34d2c2ce7bc8445", + "zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581", + ] +} + +provider "registry.opentofu.org/telmate/proxmox" { + version = "3.0.1-rc3" + constraints = "3.0.1-rc3" + hashes = [ + "h1:x7TfUaW+RpBtGov4DBuSJ5YPYBozapWuLyyZs0qjsKY=", + "zh:3699c41289c6fbe0f33b6c54360d43dcfba429de5fbf49506df9276d03aea915", + "zh:486c9ddda427d3fecdc6dfa189fce85c4a2aa1f490b024d636c0ac6a4dd3c692", + "zh:6091e141a0b8dcb1632c31e0f9555117bb023176c5d083f0e03441bbcf673a4e", + "zh:63d312c2c2994ed39dcb47b4d43c89990bd5fff20dbda63cddfb11c9202270f4", + "zh:6e69c70a85cfa720f543090ee3ce7d2eb2902df19657121b8b7ae64d44875d9f", + "zh:897b9f6075262fc9533f87d470217b14ae82614c6818a26b578a6d41c403d4eb", + "zh:91c24bd374fb8ee0c9e4e1c213d157139c047be78b0cafac3c4c9724db8083b0", + "zh:a224b58759314dc045fdbfc88b63b036b8ca6f75ad32606e94b553f150077c13", + "zh:a56e940c71b45e222c69a2a45388b58ed319836b922f84f62bded5b063662f4a", + "zh:b2e0a83aa535cd3493fbc7485d05d1a823c48bf487e313703f01a17edc631908", + "zh:ba0ad4fea8ba3b01c67fb164ed92fa927ac70d2d898378d192a01e818fcf6bee", + "zh:c49ebe13e7011d35d72e8e6a720df83f21c106444ef4383c5d6c0015aee55db6", + "zh:c53e2775040e103aedcce06b9acb79ca5fccdb4c578a4b6e32489c89e9c652dc", + "zh:c9002cc470ccfd8cd298d5655cf76af84b1d8a200207973d9ad80235818e89e3", + ] +} diff --git a/terraform/kubernetes.tf b/terraform/kubernetes.tf new file mode 100644 index 0000000..fd17a23 --- /dev/null +++ b/terraform/kubernetes.tf @@ -0,0 +1,296 @@ +locals { + // Downloaded from factory.talos.dev + // https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0 + iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso" + + boot_pool = "hetzner-main-data" + + cluster_endpoint = "https://api.${var.cluster_name}.${var.domain}:6443" + + nodes = { for k, v in var.nodes : k => merge(v, { name = "${k}.node.${var.cluster_name}.${var.domain}" }) } + nodes_with_address = { for k, v in local.nodes : k => merge(v, { address_ipv4 = proxmox_vm_qemu.this[k].default_ipv4_address, address_ipv6 = proxmox_vm_qemu.this[k].default_ipv6_address }) } + first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0] + nodes_public_controlplane = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" } + +} + +resource "macaddress" "this" { + for_each = local.nodes +} + +resource "proxmox_vm_qemu" "this" { + for_each = local.nodes + + name = each.value.name + target_node = each.value.host + tags = join(";", concat( + ["kubernetes", "terraform"], + each.value.public ? ["public"] : ["private"], + )) + + cores = 4 + memory = 4096 + + scsihw = "virtio-scsi-pci" + qemu_os = "l26" + + agent = 1 + + network { + model = "virtio" + bridge = each.value.public ? "vmpublic0" : "vmprivate0" + macaddr = macaddress.this[each.key].address + } + + disks { + scsi { + scsi0 { + cdrom { + iso = local.iso + } + } + } + virtio { + virtio0 { + disk { + size = "32G" + storage = local.boot_pool + } + } + } + } +} + +resource "digitalocean_record" "controlplane-A" { + for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" } + + domain = var.domain + type = "A" + name = "api.${var.cluster_name}" + value = each.value.default_ipv4_address + ttl = 30 +} + +resource "digitalocean_record" "controlplane-AAAA" { + for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" } + + domain = var.domain + type = "AAAA" + name = "api.${var.cluster_name}" + value = each.value.default_ipv6_address + ttl = 30 +} + +resource "talos_machine_secrets" "this" {} + +data "talos_machine_configuration" "controlplane" { + cluster_name = var.cluster_name + machine_type = "controlplane" + cluster_endpoint = local.cluster_endpoint + machine_secrets = talos_machine_secrets.this.machine_secrets + + depends_on = [ + digitalocean_record.controlplane-A, + digitalocean_record.controlplane-AAAA, + ] +} + +data "talos_machine_configuration" "worker" { + cluster_name = var.cluster_name + machine_type = "worker" + cluster_endpoint = local.cluster_endpoint + machine_secrets = talos_machine_secrets.this.machine_secrets + + depends_on = [ + digitalocean_record.controlplane-A, + digitalocean_record.controlplane-AAAA + ] +} + +data "talos_machine_disks" "boot" { + for_each = local.nodes_with_address + + client_configuration = talos_machine_secrets.this.client_configuration + node = each.value.name + endpoint = each.value.address_ipv4 + + filters = { + size = "< 60GB" + } +} + + +resource "talos_machine_configuration_apply" "this" { + for_each = local.nodes_with_address + + client_configuration = talos_machine_secrets.this.client_configuration + machine_configuration_input = each.value.type == "controlplane" ? data.talos_machine_configuration.controlplane.machine_configuration : data.talos_machine_configuration.worker.machine_configuration + + node = each.value.name + endpoint = each.value.address_ipv4 + + config_patches = [ + yamlencode({ + cluster : { + network : { + cni : { + name : "none" + } + } + allowSchedulingOnControlPlanes : true, + apiServer : { + extraArgs : { + "oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/", + "oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg", + "oidc-username-claim" : "sub", + "oidc-username-prefix" : "oidc:", + "oidc-groups-claim" : "groups", + "oidc-groups-prefix" : "oidc:groups:", + } + } + inlineManifests : [{ + name : "oidc-groups" + contents : <<-EOF + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: id-tjo-space:admins + subjects: + - kind: Group + name: oidc:groups:k8s.tjo.cloud admin + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io + EOF + }] + } + machine = { + kubelet = { + extraArgs = { + rotate-server-certificates : "true" + } + } + network = { + hostname = each.value.name + } + install = { + image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5" + disk = data.talos_machine_disks.boot[each.key].disks[0].name + } + } + }), + yamlencode({ + apiVersion : "v1alpha1" + kind : "ExtensionServiceConfig" + name : "tailscale" + environment : [ + "TS_AUTHKEY=${var.tailscale_authkey}" + ] + }) + + ] +} + +resource "talos_machine_bootstrap" "this" { + depends_on = [ + talos_machine_configuration_apply.this + ] + + node = local.first_controlplane_node.name + endpoint = local.first_controlplane_node.address_ipv4 + client_configuration = talos_machine_secrets.this.client_configuration +} + +data "talos_cluster_kubeconfig" "this" { + count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0 + + depends_on = [ + talos_machine_bootstrap.this + ] + + client_configuration = talos_machine_secrets.this.client_configuration + node = values(local.nodes_public_controlplane)[0].default_ipv4_address +} + +resource "local_file" "kubeconfig" { + count = length(local.nodes_public_controlplane) > 0 ? 1 : 0 + + content = data.talos_cluster_kubeconfig.this[0].kubeconfig_raw + filename = "${path.module}/kubeconfig" +} + +data "talos_client_configuration" "this" { + count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0 + + cluster_name = var.cluster_name + client_configuration = talos_machine_secrets.this.client_configuration + endpoints = values(local.nodes_public_controlplane)[*].default_ipv4_address +} + +resource "local_file" "talosconfig" { + count = length(local.nodes_public_controlplane) > 0 ? 1 : 0 + + content = nonsensitive(data.talos_client_configuration.this[0].talos_config) + filename = "${path.module}/talosconfig" +} + + +resource "helm_release" "cilium" { + depends_on = [ + talos_machine_bootstrap.this + ] + + name = "cilium" + repository = "https://helm.cilium.io/" + chart = "cilium" + version = "1.15.6" + namespace = "kube-system" + + set { + name = "ipam.mode" + value = "kubernetes" + } + set { + name = "kubeProxyReplacement" + value = "disabled" + } + set { + name = "securityContext.capabilities.ciliumAgent" + value = "{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}" + } + set { + name = "securityContext.capabilities.cleanCiliumState" + value = "{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}" + } + set { + name = "cgroup.autoMount.enabled" + value = false + } + set { + name = "cgroup.hostRoot" + value = "/sys/fs/cgroup" + } +} + +resource "helm_release" "dashboard" { + depends_on = [ + talos_machine_bootstrap.this + ] + + name = "kubernetes-dashboard" + repository = "https://kubernetes.github.io/dashboard" + chart = "kubernetes-dashboard" + version = "7.5.0" + namespace = "kube-system" + + set { + name = "ingress.enabled" + value = true + } + set { + name = "useDefaultIngressClass" + value = true + } +} diff --git a/terraform/providers.tf b/terraform/providers.tf new file mode 100644 index 0000000..0183c68 --- /dev/null +++ b/terraform/providers.tf @@ -0,0 +1,47 @@ +terraform { + required_providers { + proxmox = { + source = "Telmate/proxmox" + version = "3.0.1-rc3" + } + talos = { + source = "siderolabs/talos" + version = "0.5.0" + } + local = { + source = "hashicorp/local" + version = "1.4.0" + } + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.0" + } + random = { + source = "hashicorp/random" + version = "3.6.2" + } + macaddress = { + source = "ivoronin/macaddress" + version = "0.3.2" + } + helm = { + source = "hashicorp/helm" + version = "2.14.0" + } + } +} + +provider "proxmox" { + # FIXME: Traefik/NGINX breaks this! 500 ERROR + pm_api_url = "https://178.63.49.225:8006/api2/json" + pm_tls_insecure = true +} + +provider "digitalocean" { +} + +provider "helm" { + kubernetes { + config_path = "./kubeconfig" + } +} diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..3f8e5f1 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,20 @@ +variable "nodes" { + type = map(object({ + public = bool + type = string + host = string + })) +} + +variable "cluster_name" { + type = string +} + +variable "domain" { + type = string +} + +variable "tailscale_authkey" { + type = string + sensitive = true +}