diff --git a/justfile b/justfile index 887defd..54ea676 100644 --- a/justfile +++ b/justfile @@ -17,4 +17,5 @@ modules-cluster-manifests: k8s-apply: modules-cluster-manifests tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud init tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud apply -target module.cluster + tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud apply -target module.cluster-core tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud apply diff --git a/k8s.tjo.cloud/main.tf b/k8s.tjo.cloud/main.tf index 637459d..acefacf 100644 --- a/k8s.tjo.cloud/main.tf +++ b/k8s.tjo.cloud/main.tf @@ -60,7 +60,11 @@ resource "local_file" "kubeconfig" { filename = "${path.module}/kubeconfig" } -module "cluster_components" { +module "cluster-core" { + source = "../modules/cluster-core" +} + +module "cluster-components" { source = "../modules/cluster-components" oidc_issuer_url = var.oidc_issuer_url diff --git a/modules/cluster-components/gateway.tf b/modules/cluster-components/gateway.tf index 51199d0..e3ab61e 100644 --- a/modules/cluster-components/gateway.tf +++ b/modules/cluster-components/gateway.tf @@ -40,6 +40,50 @@ resource "kubernetes_manifest" "tjo-cloud-issuer" { } } +resource "kubernetes_manifest" "gateway_class_config" { + manifest = { + apiVersion = "gateway.envoyproxy.io/v1alpha1" + kind = "EnvoyProxy" + metadata = { + name = "daemonset" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + } + spec = { + provider = { + type = "Kubernetes" + kubernetes = { + envoyDaemonSet = { + patch : { + type : "StrategicMerge" + value : { + spec : { + template : { + spec : { + hostNetwork : true + dnsPolicy : "ClusterFirstWithHostNet" + } + } + } + } + } + pod = { + nodeSelector = { + "node-role.kubernetes.io/control-plane" = "" + } + tolerations = [ + { + key = "node-role.kubernetes.io/control-plane" + effect = "NoSchedule" + } + ] + } + } + } + } + } + } +} + resource "kubernetes_manifest" "gateway_class" { manifest = { apiVersion = "gateway.networking.k8s.io/v1" @@ -49,6 +93,12 @@ resource "kubernetes_manifest" "gateway_class" { } spec = { controllerName : "gateway.envoyproxy.io/gatewayclass-controller" + parametersRef : { + group : "gateway.envoyproxy.io" + kind : "EnvoyProxy" + name : kubernetes_manifest.gateway_class_config.object.metadata.name + namespace : kubernetes_manifest.gateway_class_config.object.metadata.namespace + } } } } diff --git a/modules/cluster-core/main.tf b/modules/cluster-core/main.tf new file mode 100644 index 0000000..ca55321 --- /dev/null +++ b/modules/cluster-core/main.tf @@ -0,0 +1,28 @@ +resource "helm_release" "cert-manager" { + name = "cert-manager" + chart = "cert-manager" + repository = "https://charts.jetstack.io" + version = "v1.15.1" + namespace = "kube-system" + atomic = true + cleanup_on_fail = true + + values = [<<-EOF + crds: + enabled: true + + extraArgs: + - --enable-gateway-api + EOF + ] +} + +resource "helm_release" "envoy" { + name = "envoy" + chart = "gateway-helm" + repository = "oci://docker.io/envoyproxy" + version = "v1.1.0" + namespace = "kube-system" + atomic = true + cleanup_on_fail = true +} diff --git a/modules/cluster-core/variables.tf b/modules/cluster-core/variables.tf new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/modules/cluster-core/variables.tf @@ -0,0 +1 @@ + diff --git a/modules/cluster-core/versions.tf b/modules/cluster-core/versions.tf new file mode 100644 index 0000000..34ccf41 --- /dev/null +++ b/modules/cluster-core/versions.tf @@ -0,0 +1,8 @@ +terraform { + required_providers { + helm = { + source = "hashicorp/helm" + version = "2.14.0" + } + } +} diff --git a/modules/cluster/components.tf b/modules/cluster/components.tf index d32d48a..7af75ca 100644 --- a/modules/cluster/components.tf +++ b/modules/cluster/components.tf @@ -12,20 +12,19 @@ data "helm_template" "cilium" { values = [<<-EOF ipam: mode: "kubernetes" + nodeIPAM: + enabled: true + + bpf: + masquerade: true - #routingMode: native - #ipv4NativeRoutingCIDR: pod and service cidrs? enableIPv4Masquerade: true ipv4: enabled: true #enableIPv6Masquerade: true - ipv6: - enabled: false - - nodeIPAM: - enabled: true - + #ipv6: + # enabled: true kubeProxyReplacement: "true" securityContext: @@ -152,44 +151,3 @@ data "helm_template" "talos-ccm" { kube_version = var.talos.kubernetes } - -data "helm_template" "cert-manager" { - provider = helm.template - name = "cert-manager" - chart = "cert-manager" - repository = "https://charts.jetstack.io" - version = "v1.15.1" - namespace = "kube-system" - - kube_version = var.talos.kubernetes - api_versions = [ - "gateway.networking.k8s.io/v1/GatewayClass", - ] - - include_crds = true - - values = [<<-EOF - crds: - enabled: true - - extraArgs: - - --enable-gateway-api - EOF - ] -} - -data "helm_template" "envoy" { - provider = helm.template - name = "envoy" - chart = "gateway-helm" - repository = "oci://docker.io/envoyproxy" - version = "v1.1.0-rc.1" - namespace = "kube-system" - - kube_version = var.talos.kubernetes - api_versions = [ - "gateway.networking.k8s.io/v1/GatewayClass", - ] - - include_crds = true -} diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index ba11396..e84ff1b 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -75,14 +75,6 @@ locals { name : "cilium" contents : data.helm_template.cilium.manifest }, - #{ - # name : "envoy" - # contents : data.helm_template.envoy.manifest - #}, - #{ - # name : "cert-manager" - # contents : data.helm_template.cert-manager.manifest - #}, { name : "oidc-admins" contents : <<-EOF @@ -153,6 +145,10 @@ locals { "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } + sysctls = { + "net.ipv4.ip_forward" = "1" + "net.ipv6.conf.all.forwarding" = "1" + } } }), yamlencode( @@ -163,8 +159,8 @@ locals { environment : [ "TS_AUTHKEY=${var.tailscale_authkey}", "TS_HOSTNAME=${node.name}", - # IPV6: https://github.com/siderolabs/extensions/issues/432 - "TS_ROUTES=${local.podSubnets[0]},${local.serviceSubnets[0]}" + "TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}", + "TS_EXTRA_ARGS=--accept-routes --snat-subnet-routes", ] }) ]