diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig index c56d7d6..6343db2 100755 --- a/k8s.tjo.cloud/kubeconfig +++ b/k8s.tjo.cloud/kubeconfig @@ -4,7 +4,7 @@ clusters: - name: tjo-cloud cluster: server: https://api.k8s.tjo.cloud:6443 - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVRDZ0F3SUJBZ0lSQU5MekZhUjV4SENpZHFZQjB5K0Z2ZVF3Q2dZSUtvWkl6ajBFQXdJd0ZURVQKTUJFR0ExVUVDaE1LYTNWaVpYSnVaWFJsY3pBZUZ3MHlOREEzTWpBeE56STNNRFZhRncwek5EQTNNVGd4TnpJMwpNRFZhTUJVeEV6QVJCZ05WQkFvVENtdDFZbVZ5Ym1WMFpYTXdXVEFUQmdjcWhrak9QUUlCQmdncWhrak9QUU1CCkJ3TkNBQVN3T2xEMnBYeXRyQjdFUkFNZFQ5VVkwaDZNN1BtU1k1SjVBa2F4bk8vYm1FOW84bjNMTU9JbVZFaFYKdEdScFhvNTl6MHQ5dnJJRkd3VGNLeGlpaG5STW8yRXdYekFPQmdOVkhROEJBZjhFQkFNQ0FvUXdIUVlEVlIwbApCQll3RkFZSUt3WUJCUVVIQXdFR0NDc0dBUVVGQndNQ01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPCkJCWUVGRHg1bDd1aDdtNUlvNWc5b285NUwvOGpVVXZJTUFvR0NDcUdTTTQ5QkFNQ0EwZ0FNRVVDSUhOblhLV2cKcUZPbkQvbXVyU0Y4S3BlNHJyN3FkSzFLWGQ2dm03aXN2ZThOQWlFQW5TTHM3dy9ySUFQM09GWTJmTnJsbFd2cgpJZlRpRG1IV3AzWVU5UDFVM2ljPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpakNDQVMrZ0F3SUJBZ0lRZXNQaHZSS20xbHlhdTU2RndIbDZMekFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN5TWpJd01UVXhOMW9YRFRNME1EY3lNREl3TVRVeApOMW93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCTnVNUnl0K1lQUncxN094TFNRUDlJdngzZVk1am1pS1FSL2tEeTFENFI2ZVI4WUpqTlVDOXZGNmxzZFcKaWV3M09wekZybFl4eHl3Ym9vZVdDN3R1dlkyallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVU2TUhBdEhTZEJuUTZBbTRjeFVMOVc3b1Y2UFl3Q2dZSUtvWkl6ajBFQXdJRFNRQXdSZ0loQUpobVdzRXgKVjVnRW5na25uMURndjBBaVNjZTBHVUtrZWdBNStDK1VyOXlWQWlFQTVzQituQmFGVUl3R2JsYkcrSWEvOXFsZApFZEh0dXNkbDRRaHVmT0R5K1d3PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== contexts: - name: oidc@tjo-cloud context: diff --git a/modules/cluster/components.tf b/modules/cluster/components.tf index 5622aec..d32d48a 100644 --- a/modules/cluster/components.tf +++ b/modules/cluster/components.tf @@ -9,72 +9,67 @@ data "helm_template" "cilium" { kube_version = var.talos.kubernetes - values = [yamlencode({ - ipam : { - mode : "kubernetes" - }, - nodeIPAM : { - enabled : true - }, - kubeProxyReplacement : "true" - securityContext : { - capabilities : { - ciliumAgent : [ - "CHOWN", - "KILL", - "NET_ADMIN", - "NET_RAW", - "IPC_LOCK", - "SYS_ADMIN", - "SYS_RESOURCE", - "DAC_OVERRIDE", - "FOWNER", - "SETGID", - "SETUID" - ], - cleanCiliumState : [ - "NET_ADMIN", - "SYS_ADMIN", - "SYS_RESOURCE" - ] - } - }, - cgroup : { - autoMount : { - enabled : false - }, - hostRoot : "/sys/fs/cgroup" - }, - k8sServiceHost : local.cluster_api_domain - k8sServicePort : var.cluster.api.port - ipv4 : { - enabled : true - }, - #ipv6 : { - # enabled : true - #}, - hubble : { - tls : { - auto : { - enabled : true - method : "cronJob" - schedule : "0 0 1 */4 *" - } - } - ui : { - enabled : true - } - relay : { - enabled : true - } - }, - gatewayAPI : { - enabled : false - } - envoy : { - enabled : false - } - })] + values = [<<-EOF + ipam: + mode: "kubernetes" + + #routingMode: native + #ipv4NativeRoutingCIDR: pod and service cidrs? + enableIPv4Masquerade: true + ipv4: + enabled: true + + #enableIPv6Masquerade: true + ipv6: + enabled: false + + nodeIPAM: + enabled: true + + + kubeProxyReplacement: "true" + securityContext: + capabilities: + ciliumAgent: + - "CHOWN" + - "KILL" + - "NET_ADMIN" + - "NET_RAW" + - "IPC_LOCK" + - "SYS_ADMIN" + - "SYS_RESOURCE" + - "DAC_OVERRIDE" + - "FOWNER" + - "SETGID" + - "SETUID" + cleanCiliumState: + - "NET_ADMIN" + - "SYS_ADMIN" + - "SYS_RESOURCE" + cgroup: + hostRoot: "/sys/fs/cgroup" + autoMount: + enabled: false + + k8sServiceHost: ${local.cluster_api_domain} + k8sServicePort: ${var.cluster.api.port} + + hubble: + ui: + enabled: true + relay: + enabled: true + tls: + auto: + enabled: true + method: "cronJob" + schedule: "0 0 1 */4 *" + gatewayAPI: + enabled: false + envoy: + enabled: false + EOF + ] } data "helm_template" "proxmox-csi" { @@ -173,15 +168,14 @@ data "helm_template" "cert-manager" { include_crds = true - set { - name = "crds.enabled" - value = true - } + values = [<<-EOF + crds: + enabled: true - set_list { - name = "extraArgs" - value = ["--enable-gateway-api"] - } + extraArgs: + - --enable-gateway-api + EOF + ] } data "helm_template" "envoy" { diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index cde4ac7..ba11396 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -4,11 +4,11 @@ locals { podSubnets = [ "10.200.0.0/16", - #"fd9b:5314:fc70::/48", + #"fd9b:5314:fc70::/64", ] serviceSubnets = [ "10.201.0.0/16", - #"fd9b:5314:fc71::/48", + #"fd9b:5314:fc71::/108", ] # Nodes will use IPs from this subnets @@ -75,14 +75,14 @@ locals { name : "cilium" contents : data.helm_template.cilium.manifest }, - { - name : "envoy" - contents : data.helm_template.envoy.manifest - }, - { - name : "cert-manager" - contents : data.helm_template.cert-manager.manifest - }, + #{ + # name : "envoy" + # contents : data.helm_template.envoy.manifest + #}, + #{ + # name : "cert-manager" + # contents : data.helm_template.cert-manager.manifest + #}, { name : "oidc-admins" contents : <<-EOF @@ -149,17 +149,10 @@ locals { hostname = node.name } nodeLabels = { - "k8s.tjo.cloud/public" = node.public ? "true" : "false" - #"k8s.tjo.cloud/ipv4" = node.ipv4 - #"k8s.tjo.cloud/ipv6" = node.ipv6 + "k8s.tjo.cloud/public" = node.public ? "true" : "false" "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } - kubelet = { - extraConfig = { - podCIDR = "" - } - } } }), yamlencode( @@ -170,6 +163,8 @@ locals { environment : [ "TS_AUTHKEY=${var.tailscale_authkey}", "TS_HOSTNAME=${node.name}", + # IPV6: https://github.com/siderolabs/extensions/issues/432 + "TS_ROUTES=${local.podSubnets[0]},${local.serviceSubnets[0]}" ] }) ] diff --git a/modules/cluster/proxmox.tf b/modules/cluster/proxmox.tf index 09a73bb..dfdea35 100644 --- a/modules/cluster/proxmox.tf +++ b/modules/cluster/proxmox.tf @@ -126,10 +126,10 @@ resource "proxmox_virtual_environment_vm" "nodes" { iothread = true } - #initialization { - # datastore_id = each.value.storage - # meta_data_file_id = proxmox_virtual_environment_file.metadata[each.key].id - #} + initialization { + datastore_id = each.value.storage + meta_data_file_id = proxmox_virtual_environment_file.metadata[each.key].id + } } resource "proxmox_virtual_environment_role" "csi" {