feat: trying to make cilium work
This commit is contained in:
parent
bb59b75d02
commit
f3146c13c2
SSH key fingerprint:
SHA256:MNtTsLbihYaWF8j1fkOHfkKNlnN1JQfxEU/rBU8nCGw
4 changed files with 13679 additions and 49 deletions
|
|
@ -43,6 +43,24 @@ provider "registry.opentofu.org/hashicorp/helm" {
|
|||
]
|
||||
}
|
||||
|
||||
provider "registry.opentofu.org/hashicorp/kubernetes" {
|
||||
version = "2.31.0"
|
||||
constraints = "2.31.0"
|
||||
hashes = [
|
||||
"h1:z2qlqn6WbrjbezwQo4vvlwAgVUGz59klzDU4rlYhYi8=",
|
||||
"zh:0dd25babf78a88a61dd329b8c18538a295ea63630f1b69575e7898c89307da39",
|
||||
"zh:3138753e4b2ce6e9ffa5d65d73e9236169ff077c10089c7dc71031a0a139ff6d",
|
||||
"zh:644f94692dc33de0bb1183c307ae373efbf4ef4cb92654ccc646a5716edf9593",
|
||||
"zh:6cc630e43193220b1599e3227286cc4e3ca195910e8c56b6bacb50c5b5176dbf",
|
||||
"zh:764173875e77aa482da4dca9fec5f77c455d028848edfc394aa7dac5dfed6afd",
|
||||
"zh:7b1d380362d50ffbb3697483036ae351b0571e93b33754255cde6968e62b839f",
|
||||
"zh:a1d93ca3d8d1ecdd3b69242d16ff21c91b34e2e98f02a3b2d02c908aeb45189b",
|
||||
"zh:b471d0ab56dbf19c95fba68d2ef127bdb353be96a2be4c4a3dcd4d0db4b4180a",
|
||||
"zh:d610f725ded4acd3d31a240472bb283aa5e657ed020395bdefea18d094b8c2bf",
|
||||
"zh:d7f3ddd636ad5af6049922f212feb24830b7158410819c32073bf81c359cd2fa",
|
||||
]
|
||||
}
|
||||
|
||||
provider "registry.opentofu.org/hashicorp/local" {
|
||||
version = "1.4.0"
|
||||
constraints = "1.4.0"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
locals {
|
||||
// Downloaded from factory.talos.dev
|
||||
// https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
|
||||
# Downloaded from factory.talos.dev
|
||||
# https://factory.talos.dev/?arch=amd64&board=undefined&cmdline-set=true&extensions=-&extensions=siderolabs%2Fqemu-guest-agent&extensions=siderolabs%2Ftailscale&platform=metal&secureboot=undefined&target=metal&version=1.7.0
|
||||
iso = "proxmox-backup-tjo-cloud:iso/talos-v1.7.5-tailscale-metal-amd64.iso"
|
||||
|
||||
boot_pool = "hetzner-main-data"
|
||||
|
|
@ -119,16 +119,106 @@ data "talos_machine_disks" "boot" {
|
|||
}
|
||||
}
|
||||
|
||||
data "helm_template" "cilium" {
|
||||
name = "cilium"
|
||||
repository = "https://helm.cilium.io/"
|
||||
chart = "cilium"
|
||||
version = "1.15.6"
|
||||
namespace = "kube-system"
|
||||
include_crds = true
|
||||
|
||||
resource "talos_machine_configuration_apply" "this" {
|
||||
for_each = local.nodes_with_address
|
||||
values = [yamlencode({
|
||||
ipam : {
|
||||
mode : "kubernetes"
|
||||
},
|
||||
kubeProxyReplacement : true
|
||||
securityContext : {
|
||||
capabilities : {
|
||||
ciliumAgent : [
|
||||
"CHOWN",
|
||||
"KILL",
|
||||
"NET_ADMIN",
|
||||
"NET_RAW",
|
||||
"IPC_LOCK",
|
||||
"PERFMON",
|
||||
"BPF",
|
||||
"SYS_RESOURCE",
|
||||
"DAC_OVERRIDE",
|
||||
"FOWNER",
|
||||
"SETGID",
|
||||
"SETUID"
|
||||
],
|
||||
cleanCiliumState : [
|
||||
"NET_ADMIN",
|
||||
"PERFMON",
|
||||
"BPF",
|
||||
"SYS_RESOURCE"
|
||||
]
|
||||
}
|
||||
},
|
||||
cgroup : {
|
||||
autoMount : {
|
||||
enabled : false
|
||||
},
|
||||
hostRoot : "/sys/fs/cgroup"
|
||||
},
|
||||
k8sServiceHost : "localhost"
|
||||
k8sServicePort : "7445"
|
||||
ipv4 : {
|
||||
enabled : true
|
||||
},
|
||||
#ipv6: {
|
||||
# enabled: true
|
||||
#},
|
||||
hubble : {
|
||||
ui : {
|
||||
enabled : true
|
||||
}
|
||||
relay : {
|
||||
enabled : true
|
||||
}
|
||||
},
|
||||
# Ingress gateway
|
||||
gatewayAPI : {
|
||||
enabled : true
|
||||
default : true
|
||||
hostNetwork : {
|
||||
enabled : true
|
||||
nodes : {
|
||||
matchLabels : {
|
||||
"k8s.tjo.cloud/gateway" : "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
envoy : {
|
||||
enabled : true
|
||||
securityContext : {
|
||||
capabilities : {
|
||||
keepCapNetBindService : true
|
||||
envoy : [
|
||||
"NET_ADMIN",
|
||||
"BPF",
|
||||
"PERFMON",
|
||||
"NET_BIND_SERVICE"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
})]
|
||||
}
|
||||
|
||||
resource "talos_machine_configuration_apply" "controlplane" {
|
||||
for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
|
||||
|
||||
client_configuration = talos_machine_secrets.this.client_configuration
|
||||
machine_configuration_input = each.value.type == "controlplane" ? data.talos_machine_configuration.controlplane.machine_configuration : data.talos_machine_configuration.worker.machine_configuration
|
||||
machine_configuration_input = data.talos_machine_configuration.controlplane.machine_configuration
|
||||
|
||||
node = each.value.name
|
||||
endpoint = each.value.address_ipv4
|
||||
|
||||
apply_mode = "reboot"
|
||||
|
||||
config_patches = [
|
||||
yamlencode({
|
||||
cluster : {
|
||||
|
|
@ -137,6 +227,9 @@ resource "talos_machine_configuration_apply" "this" {
|
|||
name : "none"
|
||||
}
|
||||
}
|
||||
proxy : {
|
||||
disabled : true
|
||||
}
|
||||
allowSchedulingOnControlPlanes : true,
|
||||
apiServer : {
|
||||
extraArgs : {
|
||||
|
|
@ -148,7 +241,8 @@ resource "talos_machine_configuration_apply" "this" {
|
|||
"oidc-groups-prefix" : "oidc:groups:",
|
||||
}
|
||||
}
|
||||
inlineManifests : [{
|
||||
inlineManifests : [
|
||||
{
|
||||
name : "oidc-groups"
|
||||
contents : <<-EOF
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
|
@ -164,7 +258,80 @@ resource "talos_machine_configuration_apply" "this" {
|
|||
name: cluster-admin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
EOF
|
||||
}]
|
||||
},
|
||||
{
|
||||
name : "gateway-api-crds"
|
||||
contents : file("${path.module}/manifests/gateway-api-crds.yaml")
|
||||
},
|
||||
{
|
||||
name : "cilium"
|
||||
contents : data.helm_template.cilium.manifest
|
||||
}
|
||||
],
|
||||
}
|
||||
machine = {
|
||||
kubelet = {
|
||||
extraArgs = {
|
||||
rotate-server-certificates : "true"
|
||||
}
|
||||
}
|
||||
network = {
|
||||
hostname = each.value.name
|
||||
}
|
||||
install = {
|
||||
image = "factory.talos.dev/installer/7d4c31cbd96db9f90c874990697c523482b2bae27fb4631d5583dcd9c281b1ff:v1.7.5"
|
||||
disk = data.talos_machine_disks.boot[each.key].disks[0].name
|
||||
}
|
||||
nodeLabels = {
|
||||
"k8s.tjo.cloud/gateway" = "true"
|
||||
}
|
||||
}
|
||||
}),
|
||||
yamlencode({
|
||||
apiVersion : "v1alpha1"
|
||||
kind : "ExtensionServiceConfig"
|
||||
name : "tailscale"
|
||||
environment : [
|
||||
"TS_AUTHKEY=${var.tailscale_authkey}"
|
||||
]
|
||||
})
|
||||
|
||||
]
|
||||
}
|
||||
|
||||
resource "talos_machine_configuration_apply" "worker" {
|
||||
for_each = { for k, v in local.nodes_with_address : k => v if v.type == "worker" }
|
||||
|
||||
client_configuration = talos_machine_secrets.this.client_configuration
|
||||
machine_configuration_input = data.talos_machine_configuration.worker.machine_configuration
|
||||
|
||||
node = each.value.name
|
||||
endpoint = each.value.address_ipv4
|
||||
|
||||
apply_mode = "reboot"
|
||||
|
||||
config_patches = [
|
||||
yamlencode({
|
||||
cluster : {
|
||||
network : {
|
||||
cni : {
|
||||
name : "none"
|
||||
}
|
||||
}
|
||||
proxy : {
|
||||
disabled : true
|
||||
}
|
||||
allowSchedulingOnControlPlanes : true,
|
||||
apiServer : {
|
||||
extraArgs : {
|
||||
"oidc-issuer-url" : "https://id.tjo.space/application/o/k8stjocloud/",
|
||||
"oidc-client-id" : "HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg",
|
||||
"oidc-username-claim" : "sub",
|
||||
"oidc-username-prefix" : "oidc:",
|
||||
"oidc-groups-claim" : "groups",
|
||||
"oidc-groups-prefix" : "oidc:groups:",
|
||||
}
|
||||
}
|
||||
}
|
||||
machine = {
|
||||
kubelet = {
|
||||
|
|
@ -195,7 +362,8 @@ resource "talos_machine_configuration_apply" "this" {
|
|||
|
||||
resource "talos_machine_bootstrap" "this" {
|
||||
depends_on = [
|
||||
talos_machine_configuration_apply.this
|
||||
talos_machine_configuration_apply.controlplane,
|
||||
talos_machine_configuration_apply.worker
|
||||
]
|
||||
|
||||
node = local.first_controlplane_node.name
|
||||
|
|
@ -236,44 +404,6 @@ resource "local_file" "talosconfig" {
|
|||
filename = "${path.module}/talosconfig"
|
||||
}
|
||||
|
||||
|
||||
resource "helm_release" "cilium" {
|
||||
depends_on = [
|
||||
talos_machine_bootstrap.this
|
||||
]
|
||||
|
||||
name = "cilium"
|
||||
repository = "https://helm.cilium.io/"
|
||||
chart = "cilium"
|
||||
version = "1.15.6"
|
||||
namespace = "kube-system"
|
||||
|
||||
set {
|
||||
name = "ipam.mode"
|
||||
value = "kubernetes"
|
||||
}
|
||||
set {
|
||||
name = "kubeProxyReplacement"
|
||||
value = "disabled"
|
||||
}
|
||||
set {
|
||||
name = "securityContext.capabilities.ciliumAgent"
|
||||
value = "{CHOWN,KILL,NET_ADMIN,NET_RAW,IPC_LOCK,SYS_ADMIN,SYS_RESOURCE,DAC_OVERRIDE,FOWNER,SETGID,SETUID}"
|
||||
}
|
||||
set {
|
||||
name = "securityContext.capabilities.cleanCiliumState"
|
||||
value = "{NET_ADMIN,SYS_ADMIN,SYS_RESOURCE}"
|
||||
}
|
||||
set {
|
||||
name = "cgroup.autoMount.enabled"
|
||||
value = false
|
||||
}
|
||||
set {
|
||||
name = "cgroup.hostRoot"
|
||||
value = "/sys/fs/cgroup"
|
||||
}
|
||||
}
|
||||
|
||||
resource "helm_release" "dashboard" {
|
||||
depends_on = [
|
||||
talos_machine_bootstrap.this
|
||||
|
|
|
|||
13478
terraform/manifests/gateway-api-crds.yaml
Normal file
13478
terraform/manifests/gateway-api-crds.yaml
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -28,6 +28,10 @@ terraform {
|
|||
source = "hashicorp/helm"
|
||||
version = "2.14.0"
|
||||
}
|
||||
kubernetes = {
|
||||
source = "hashicorp/kubernetes"
|
||||
version = "2.31.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue