feat: initial terraform

This commit is contained in:
Tine 2024-07-08 22:27:08 +02:00
parent 669dc95794
commit bb59b75d02
Signed by: mentos1386
SSH key fingerprint: SHA256:MNtTsLbihYaWF8j1fkOHfkKNlnN1JQfxEU/rBU8nCGw
7 changed files with 516 additions and 0 deletions

3
.gitignore vendored
View file

@ -34,3 +34,6 @@ override.tf.json
.terraformrc
terraform.rc
# ENV
.env

6
example.env Normal file
View file

@ -0,0 +1,6 @@
TF_VAR_tailscale_authkey=""
DIGITALOCEAN_TOKEN=""
PM_API_TOKEN_ID="terraform@pve!terraform-provisioner"
PM_API_TOKEN_SECRET=""

2
terraform/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
kubeconfig
talosconfig

View file

@ -0,0 +1,142 @@
# This file is maintained automatically by "tofu init".
# Manual edits may be lost in future updates.
provider "registry.opentofu.org/digitalocean/digitalocean" {
version = "2.39.2"
constraints = "~> 2.0"
hashes = [
"h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=",
"zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f",
"zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0",
"zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f",
"zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee",
"zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea",
"zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5",
"zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e",
"zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb",
"zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761",
"zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9",
"zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf",
"zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720",
"zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad",
"zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0",
"zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f",
"zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba",
]
}
provider "registry.opentofu.org/hashicorp/helm" {
version = "2.14.0"
constraints = "2.14.0"
hashes = [
"h1:ibK3MM61pVjBwBcrro56OLTHwUhhNglvGG9CloLvliI=",
"zh:1c84ca8c274564c46497e89055139c7af64c9e1a8dd4f1cd4c68503ac1322fb8",
"zh:211a763173934d30c2e49c0cc828b1e34a528b0fdec8bf48d2bb3afadd4f9095",
"zh:3dca0b703a2f82d3e283a9e9ca6259a3b9897b217201f3cddf430009a1ca00c9",
"zh:40c5cfd48dcef54e87129e19d31c006c2e3309ee6c09d566139eaf315a59a369",
"zh:6f23c00ca1e2663e2a208a7491aa6dbe2604f00e0af7e23ef9323206e8f2fc81",
"zh:77f8cfc4888600e0d12da137bbdb836de160db168dde7af26c2e44cf00cbf057",
"zh:97b99c945eafa9bafc57c3f628d496356ea30312c3df8dfac499e0f3ff6bf0c9",
"zh:a01cfc53e50d5f722dc2aabd26097a8e4d966d343ffd471034968c2dc7a8819d",
"zh:b69c51e921fe8c91e38f4a82118d0b6b0f47f6c71a76f506fde3642ecbf39911",
"zh:fb8bfc7b8106bef58cc5628c024103f0dd5276d573fe67ac16f343a2b38ecee8",
]
}
provider "registry.opentofu.org/hashicorp/local" {
version = "1.4.0"
constraints = "1.4.0"
hashes = [
"h1:/i49KTSz7SNa1DUbqvmsUN6ou/mYgWv5onXTReHmE/E=",
"zh:0637972352fa0a83f83eea54fe91c4a67a43669ad90fc13dcd4218eccfb474af",
"zh:590451e43ae4377449944e3fcf74fb6a2a05c29037d32e53d9de49579ea6fd55",
"zh:6942cc057dae128a9b867c9446ed519d9394d941f092a4eafc90dd9a7c26ed5d",
"zh:cc7c57baaa5af4d2f88c84910d8041685c821b88dd9bfcbe8b18190f31bbf629",
"zh:e958bafa3d7c792623b5965900de8532b5a5addc1a573aeec387729aebaf633d",
]
}
provider "registry.opentofu.org/hashicorp/random" {
version = "3.6.2"
constraints = "3.6.2"
hashes = [
"h1:PXvoOj9gj+Or+9k0tQWCQJKxnsVO0GqnQwVahgwRrsU=",
"zh:1f27612f7099441526d8af59f5b4bdcc35f46915df5d243043d7337ea5a3e38a",
"zh:2a58e66502825db8b4b96116c04bd0323bca1cf1f5752bdd8f9c26feb84d3b1e",
"zh:4f0a4fa479e29de0c3c90146fd58799c097f7a55401cb00560dd4e9b1e6fad9d",
"zh:9c93c0fe6ef685513734527e0c8078636b2cc07591427502a7260f4744b1af1d",
"zh:a466ff5219beb77fb3b18a3d7e7fe30e7edd4d95c8e5c87f4f4e3fe3eeb8c2d7",
"zh:ab33e6176d0c757ddb31e40e01a941e6918ad10f7a786c8e8e4f35e5cff81c96",
"zh:b6eabf377a1c12cb3f9ddd97aacdd5b49c1646dc959074124f81d40fcd216d7e",
"zh:ccec5d03d0d1c0f354be299cdd6a417b2700f1a6781df36bcce77246b2f57e50",
"zh:d2a7945eeb691fdd2b1474da76ddc2d1655e2aedbb14b57f06d4f5123d47adf9",
"zh:ed62351f4ad9d1469c6798b77dee5f63b18b29c473620a0046ba3d4f111b621d",
]
}
provider "registry.opentofu.org/ivoronin/macaddress" {
version = "0.3.2"
constraints = "0.3.2"
hashes = [
"h1:yk0ASl2cAoc/22tvpi9Kke+WvowgXGq0QwaP93IQ+S0=",
"zh:00cb168d9210ed88cfa7de8a33d5666b2cf6660a5d20a7a96348b8b902833eca",
"zh:1366458320df0b6f1132e59b5410931c0c5626bbf27b05b29dd311311a710e9b",
"zh:2e8102c7f6046665c95b806752d692843f2e846554f7eba85690cd2087c9048a",
"zh:3c1ae52f855d0e694ad28eb34ec41c553344aaa7bd51adaa48cf15e3ee842e17",
"zh:496d8db2055cead9d264fdad83534318e3ab77ce06e38d43674a4ec25c0e860d",
"zh:54c5eeae7cc61d706080256e06aaf509869b1d86297b9e99948a2fe2af6d455b",
"zh:5f26e851048be3c56f3706b7fde25fe76dd30003ef6356216dc9ecff400218bb",
"zh:5fc1debcd0fe043dfce00ab110e180b896a1a9958edea7d81d05aacc9b630e5e",
"zh:650045261b382b4559fd1bd190d6cabbeb022b53d7e240eb6b66f6824ca81bf4",
"zh:7203dea017883e8fdd7ba66c9b1a9aac0cab101133e4eeab365c4d0995194272",
"zh:726a9222d15f11316587c199ee367bae1d5495ff16ebdfc41635f7628834a8d6",
"zh:c9f3bcaa073a0921189bd74ef6b2b57cad34b3eb01788c010df8a15fd9d8045c",
"zh:d3fba491b0ff0d3d64162216159232398a75ad81c31e4304335d6b76b74a864a",
"zh:e80011c6e3af4eeafdeda9bd118a774f8b7cdf1f133953abf827f313653ec184",
]
}
provider "registry.opentofu.org/siderolabs/talos" {
version = "0.5.0"
constraints = "0.5.0"
hashes = [
"h1:xogkLLCrJJmd278E+vNMnmQgaMD05Gd1QXN914xgVec=",
"zh:0f71f2624576224c9bc924b136b601b734243efa7a7ad8280dfd8bd583e4afa5",
"zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d",
"zh:33c50dacc5029fa20caed702001fb1439899c94f203b1f37dccb970f504bca45",
"zh:3c97a6e2692b88d3f4631a3f8769146f602c210e881b46fa1b3b82c545e51cd1",
"zh:44077a137613bcfe29eef00315b5aa50d83390c3c727580a4ff0f4b87f22d228",
"zh:5bd02f278aec5567f94dd057d1c758363998ce581ff17b0869515bb682c02186",
"zh:80f40939bc3b55f0005c03b77122ceea86ec4deb82f5557950a97ad96fbb1557",
"zh:94c1b17f25bc30eacde926e46f196f1f135032674730d9f50c986ef6b7a854f0",
"zh:95ad665b2fdeed38180f5c471164833a34d07c1ef0470c1652565fe8cf4e9c4a",
"zh:a50ef6088afcb129c176dd4ba86c345e9be7b14358bb3b21c34f06930d8f39ef",
"zh:aa71da1da00ed66f1dddf1b69c10b829f24ac89e207de07d32c455dd04482096",
"zh:abb7eeb2b089081b4814ed80a295673e1a92f82ce092dde37b5bc92e75efec2c",
"zh:db9b9b54a0db5ae151376d5a73e0d28497c3e06181840e71ef8349213ac03e50",
"zh:e50ed8aa90b736508fce63680e8339240cecb74709ab9563d34d2c2ce7bc8445",
"zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581",
]
}
provider "registry.opentofu.org/telmate/proxmox" {
version = "3.0.1-rc3"
constraints = "3.0.1-rc3"
hashes = [
"h1:x7TfUaW+RpBtGov4DBuSJ5YPYBozapWuLyyZs0qjsKY=",
"zh:3699c41289c6fbe0f33b6c54360d43dcfba429de5fbf49506df9276d03aea915",
"zh:486c9ddda427d3fecdc6dfa189fce85c4a2aa1f490b024d636c0ac6a4dd3c692",
"zh:6091e141a0b8dcb1632c31e0f9555117bb023176c5d083f0e03441bbcf673a4e",
"zh:63d312c2c2994ed39dcb47b4d43c89990bd5fff20dbda63cddfb11c9202270f4",
"zh:6e69c70a85cfa720f543090ee3ce7d2eb2902df19657121b8b7ae64d44875d9f",
"zh:897b9f6075262fc9533f87d470217b14ae82614c6818a26b578a6d41c403d4eb",
"zh:91c24bd374fb8ee0c9e4e1c213d157139c047be78b0cafac3c4c9724db8083b0",
"zh:a224b58759314dc045fdbfc88b63b036b8ca6f75ad32606e94b553f150077c13",
"zh:a56e940c71b45e222c69a2a45388b58ed319836b922f84f62bded5b063662f4a",
"zh:b2e0a83aa535cd3493fbc7485d05d1a823c48bf487e313703f01a17edc631908",
"zh:ba0ad4fea8ba3b01c67fb164ed92fa927ac70d2d898378d192a01e818fcf6bee",
"zh:c49ebe13e7011d35d72e8e6a720df83f21c106444ef4383c5d6c0015aee55db6",
"zh:c53e2775040e103aedcce06b9acb79ca5fccdb4c578a4b6e32489c89e9c652dc",
"zh:c9002cc470ccfd8cd298d5655cf76af84b1d8a200207973d9ad80235818e89e3",
]
}

296
terraform/kubernetes.tf Normal file
View file

@ -0,0 +1,296 @@
locals {
// Downloaded from factory.talos.dev
// https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
boot_pool = "hetzner-main-data"
cluster_endpoint = "https://api.${var.cluster_name}.${var.domain}:6443"
nodes = { for k, v in var.nodes : k => merge(v, { name = "${k}.node.${var.cluster_name}.${var.domain}" }) }
nodes_with_address = { for k, v in local.nodes : k => merge(v, { address_ipv4 = proxmox_vm_qemu.this[k].default_ipv4_address, address_ipv6 = proxmox_vm_qemu.this[k].default_ipv6_address }) }
first_controlplane_node = values({ for k, v in local.nodes_with_address : k => v if v.type == "controlplane" })[0]
nodes_public_controlplane = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
}
resource "macaddress" "this" {
for_each = local.nodes
}
resource "proxmox_vm_qemu" "this" {
for_each = local.nodes
name = each.value.name
target_node = each.value.host
tags = join(";", concat(
["kubernetes", "terraform"],
each.value.public ? ["public"] : ["private"],
))
cores = 4
memory = 4096
scsihw = "virtio-scsi-pci"
qemu_os = "l26"
agent = 1
network {
model = "virtio"
bridge = each.value.public ? "vmpublic0" : "vmprivate0"
macaddr = macaddress.this[each.key].address
}
disks {
scsi {
scsi0 {
cdrom {
iso = local.iso
}
}
}
virtio {
virtio0 {
disk {
size = "32G"
storage = local.boot_pool
}
}
}
}
}
resource "digitalocean_record" "controlplane-A" {
for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
domain = var.domain
type = "A"
name = "api.${var.cluster_name}"
value = each.value.default_ipv4_address
ttl = 30
}
resource "digitalocean_record" "controlplane-AAAA" {
for_each = { for k, v in proxmox_vm_qemu.this : k => v if var.nodes[k].public && var.nodes[k].type == "controlplane" }
domain = var.domain
type = "AAAA"
name = "api.${var.cluster_name}"
value = each.value.default_ipv6_address
ttl = 30
}
resource "talos_machine_secrets" "this" {}
data "talos_machine_configuration" "controlplane" {
cluster_name = var.cluster_name
machine_type = "controlplane"
cluster_endpoint = local.cluster_endpoint
machine_secrets = talos_machine_secrets.this.machine_secrets
depends_on = [
digitalocean_record.controlplane-A,
digitalocean_record.controlplane-AAAA,
]
}
data "talos_machine_configuration" "worker" {
cluster_name = var.cluster_name
machine_type = "worker"
cluster_endpoint = local.cluster_endpoint
machine_secrets = talos_machine_secrets.this.machine_secrets
depends_on = [
digitalocean_record.controlplane-A,
digitalocean_record.controlplane-AAAA
]
}
data "talos_machine_disks" "boot" {
for_each = local.nodes_with_address
client_configuration = talos_machine_secrets.this.client_configuration
node = each.value.name
endpoint = each.value.address_ipv4
filters = {
size = "< 60GB"
}
}
resource "talos_machine_configuration_apply" "this" {
for_each = local.nodes_with_address
client_configuration = talos_machine_secrets.this.client_configuration
machine_configuration_input = each.value.type == "controlplane" ? data.talos_machine_configuration.controlplane.machine_configuration : data.talos_machine_configuration.worker.machine_configuration
node = each.value.name
endpoint = each.value.address_ipv4
config_patches = [
yamlencode({
cluster : {
network : {
cni : {
name : "none"
}
}
allowSchedulingOnControlPlanes : true,
apiServer : {
extraArgs : {
"oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
"oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
"oidc-username-claim" : "sub",
"oidc-username-prefix" : "oidc:",
"oidc-groups-claim" : "groups",
"oidc-groups-prefix" : "oidc:groups:",
}
}
inlineManifests : [{
name : "oidc-groups"
contents : <<-EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: id-tjo-space:admins
subjects:
- kind: Group
name: oidc:groups:k8s.tjo.cloud admin
apiGroup: rbac.authorization.k8s.io
roleRef:
kind: ClusterRole
name: cluster-admin
apiGroup: rbac.authorization.k8s.io
EOF
}]
}
machine = {
kubelet = {
extraArgs = {
rotate-server-certificates : "true"
}
}
network = {
hostname = each.value.name
}
install = {
image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5"
disk = data.talos_machine_disks.boot[each.key].disks[0].name
}
}
}),
yamlencode({
apiVersion : "v1alpha1"
kind : "ExtensionServiceConfig"
name : "tailscale"
environment : [
"TS_AUTHKEY=${var.tailscale_authkey}"
]
})
]
}
resource "talos_machine_bootstrap" "this" {
depends_on = [
talos_machine_configuration_apply.this
]
node = local.first_controlplane_node.name
endpoint = local.first_controlplane_node.address_ipv4
client_configuration = talos_machine_secrets.this.client_configuration
}
data "talos_cluster_kubeconfig" "this" {
count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0
depends_on = [
talos_machine_bootstrap.this
]
client_configuration = talos_machine_secrets.this.client_configuration
node = values(local.nodes_public_controlplane)[0].default_ipv4_address
}
resource "local_file" "kubeconfig" {
count = length(local.nodes_public_controlplane) > 0 ? 1 : 0
content = data.talos_cluster_kubeconfig.this[0].kubeconfig_raw
filename = "${path.module}/kubeconfig"
}
data "talos_client_configuration" "this" {
count = length(values(local.nodes_public_controlplane)) > 0 ? 1 : 0
cluster_name = var.cluster_name
client_configuration = talos_machine_secrets.this.client_configuration
endpoints = values(local.nodes_public_controlplane)[*].default_ipv4_address
}
resource "local_file" "talosconfig" {
count = length(local.nodes_public_controlplane) > 0 ? 1 : 0
content = nonsensitive(data.talos_client_configuration.this[0].talos_config)
filename = "${path.module}/talosconfig"
}
resource "helm_release" "cilium" {
depends_on = [
talos_machine_bootstrap.this
]
name = "cilium"
repository = "https://helm.cilium.io/"
chart = "cilium"
version = "1.15.6"
namespace = "kube-system"
set {
name = "ipam.mode"
value = "kubernetes"
}
set {
name = "kubeProxyReplacement"
value = "disabled"
}
set {
name = "securityContext.capabilities.ciliumAgent"
value = "{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}"
}
set {
name = "securityContext.capabilities.cleanCiliumState"
value = "{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}"
}
set {
name = "cgroup.autoMount.enabled"
value = false
}
set {
name = "cgroup.hostRoot"
value = "/sys/fs/cgroup"
}
}
resource "helm_release" "dashboard" {
depends_on = [
talos_machine_bootstrap.this
]
name = "kubernetes-dashboard"
repository = "https://kubernetes.github.io/dashboard"
chart = "kubernetes-dashboard"
version = "7.5.0"
namespace = "kube-system"
set {
name = "ingress.enabled"
value = true
}
set {
name = "useDefaultIngressClass"
value = true
}
}

47
terraform/providers.tf Normal file
View file

@ -0,0 +1,47 @@
terraform {
required_providers {
proxmox = {
source = "Telmate/proxmox"
version = "3.0.1-rc3"
}
talos = {
source = "siderolabs/talos"
version = "0.5.0"
}
local = {
source = "hashicorp/local"
version = "1.4.0"
}
digitalocean = {
source = "digitalocean/digitalocean"
version = "~> 2.0"
}
random = {
source = "hashicorp/random"
version = "3.6.2"
}
macaddress = {
source = "ivoronin/macaddress"
version = "0.3.2"
}
helm = {
source = "hashicorp/helm"
version = "2.14.0"
}
}
}
provider "proxmox" {
# FIXME: Traefik/NGINX breaks this! 500 ERROR
pm_api_url = "https://178.63.49.225:8006/api2/json"
pm_tls_insecure = true
}
provider "digitalocean" {
}
provider "helm" {
kubernetes {
config_path = "./kubeconfig"
}
}

20
terraform/variables.tf Normal file
View file

@ -0,0 +1,20 @@
variable "nodes" {
type = map(object({
public = bool
type = string
host = string
}))
}
variable "cluster_name" {
type = string
}
variable "domain" {
type = string
}
variable "tailscale_authkey" {
type = string
sensitive = true
}