From b0b3a3b11611b71047b4f01d9bf4057f3b64c35f Mon Sep 17 00:00:00 2001 From: Tine Date: Mon, 2 Dec 2024 20:13:50 +0100 Subject: [PATCH] feat(kubernetes): use new networking concepts --- README.md | 2 +- k8s.tjo.cloud/.terraform.lock.hcl | 92 ++--- k8s.tjo.cloud/justfile | 3 + k8s.tjo.cloud/main.tf | 66 ++-- .../manifests/crd-podmonitors.yaml | 366 +++++++++++++---- .../manifests/crd-servicemonitors.yaml | 368 ++++++++++++++---- k8s.tjo.cloud/modules/cluster/main.tf | 75 ++-- k8s.tjo.cloud/modules/cluster/outputs.tf | 2 +- k8s.tjo.cloud/modules/cluster/proxmox.tf | 10 +- k8s.tjo.cloud/modules/cluster/variables.tf | 32 +- k8s.tjo.cloud/modules/cluster/versions.tf | 2 +- k8s.tjo.cloud/terraform.tf | 34 +- k8s.tjo.cloud/variables.tf | 5 - 13 files changed, 722 insertions(+), 335 deletions(-) diff --git a/README.md b/README.md index 8981e96..7d37e80 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ tailscale up --ssh --accept-routes --accept-dns=false --advertise-tags=tag:syste ``` -### 2. Install intel-micropodes updates. +### 2. Install intel-firmware updates. ``` # Add non-free-firmware to the end diff --git a/k8s.tjo.cloud/.terraform.lock.hcl b/k8s.tjo.cloud/.terraform.lock.hcl index 12016df..03cab4c 100644 --- a/k8s.tjo.cloud/.terraform.lock.hcl +++ b/k8s.tjo.cloud/.terraform.lock.hcl @@ -25,26 +25,26 @@ provider "registry.opentofu.org/bpg/proxmox" { } provider "registry.opentofu.org/digitalocean/digitalocean" { - version = "2.39.2" + version = "2.44.1" constraints = "~> 2.0" hashes = [ - "h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=", - "zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f", - "zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0", - "zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f", - "zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee", - "zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea", - "zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5", - "zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e", - "zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb", - "zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761", - "zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9", - "zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf", - "zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720", - "zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad", - "zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0", - "zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f", - "zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba", + "h1:wIccPAQ8HhEOg/Eo7ZLLiADITIfDRBv3ncRtnuwwkKc=", + "zh:02e0bd7320167fed3b9ceea492ab218c2568abd619e816c14542c0d185eb969a", + "zh:309452ac92ddfe6402613a5a7dcaf780e1b648e8737da3fef068e587eb932d88", + "zh:32433f540e9feb9a22a015e83dc299d46f08adec3880f72bd6af89ac1032b13c", + "zh:347664ab9c218f26eac168c10c52f6d72d1ff084fd6e24418d8e4982ec2f880e", + "zh:3a917158aa57372fa2254e4578905211338b0452135b47f00c9444202bb53311", + "zh:593b7ec19653558bbb75d202b8ecdf9580545b24ba20584c4abe2497b232fd60", + "zh:64506619588bc381471183dca0d5bf457df697699b08a42d1ae2a5cdb261c58c", + "zh:6b0c6dfdb5b685e25d1505445a0dd26d93a515c86ace1187767f7fadc6c69206", + "zh:9a4595e36ae6fb3341724dd08a476234cdb28c0b12615792a5cf73d5d2cccd26", + "zh:9e88880489f3162440f166cf083adbe876a022a7558c1cb7e35b759778c0439e", + "zh:a48c72a7e0b67a13c054c6dc1024124e8637cbecb45c684985a6037f3abd51a2", + "zh:d21f16e85cd02e4e1a147aa7dc65e149723bd2c6844236608278a4433ee56f62", + "zh:dee7a153f4201831607749c5f02b1433589c1e39db8b1d19da16836e0f3eb6cf", + "zh:df40d88ef94fd98c5c9eeabd82ed5178da4618735eaff06b83817b2ef5717e47", + "zh:f7bcc22d9ff38b98bf48c02834f4861f5b7a37c0144f2e7464d17751e01cea32", + "zh:fbf47dc012166d6545cc33a6c00b5dbdb789f7fef5b4f59935a3763f2d74e670", ] } @@ -116,46 +116,24 @@ provider "registry.opentofu.org/hashicorp/random" { } provider "registry.opentofu.org/siderolabs/talos" { - version = "0.5.0" - constraints = "0.5.0" + version = "0.6.1" + constraints = "0.6.1" hashes = [ - "h1:xogkLLCrJJmd278E+vNMnmQgaMD05Gd1QXN914xgVec=", - "zh:0f71f2624576224c9bc924b136b601b734243efa7a7ad8280dfd8bd583e4afa5", + "h1:eFw5nEpptkVQ+SNXFEaYa8o++5Q3WVznDgrxJ78ROLA=", "zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d", - "zh:33c50dacc5029fa20caed702001fb1439899c94f203b1f37dccb970f504bca45", - "zh:3c97a6e2692b88d3f4631a3f8769146f602c210e881b46fa1b3b82c545e51cd1", - "zh:44077a137613bcfe29eef00315b5aa50d83390c3c727580a4ff0f4b87f22d228", - "zh:5bd02f278aec5567f94dd057d1c758363998ce581ff17b0869515bb682c02186", - "zh:80f40939bc3b55f0005c03b77122ceea86ec4deb82f5557950a97ad96fbb1557", - "zh:94c1b17f25bc30eacde926e46f196f1f135032674730d9f50c986ef6b7a854f0", - "zh:95ad665b2fdeed38180f5c471164833a34d07c1ef0470c1652565fe8cf4e9c4a", - "zh:a50ef6088afcb129c176dd4ba86c345e9be7b14358bb3b21c34f06930d8f39ef", - "zh:aa71da1da00ed66f1dddf1b69c10b829f24ac89e207de07d32c455dd04482096", - "zh:abb7eeb2b089081b4814ed80a295673e1a92f82ce092dde37b5bc92e75efec2c", - "zh:db9b9b54a0db5ae151376d5a73e0d28497c3e06181840e71ef8349213ac03e50", - "zh:e50ed8aa90b736508fce63680e8339240cecb74709ab9563d34d2c2ce7bc8445", - "zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581", - ] -} - -provider "registry.opentofu.org/tailscale/tailscale" { - version = "0.16.1" - constraints = "0.16.1" - hashes = [ - "h1:NDIIkEo0G/leQSvGoh2Mk74ZE2xWrWgHX/S8ZVyBDYU=", - "zh:0a9d28e5195e0e29ebf9b12b345cafcb686125008151fa01677c399d8f8f1321", - "zh:249bce2fcfd3414211ae9e49e179e31b5d3c23dd9da24dc45acdea34ad308cb0", - "zh:3129fb52a2aaa0c8c30aff21e7d4c0601d80898b3ecb9d7604b5933c14f54924", - "zh:4ec3e255f34bb4f6362ab41aa9e05a3ce040a791bc07445dec86188dee867f85", - "zh:68d3995e5a1722e24f89a385899f56a63542159b884cac989196e9538b53c6ce", - "zh:799840b3bfbd14537397f157f4e6a5e54080cd4fee51521bac675aa188e0b33e", - "zh:99f1da9fdaddd8a1255dce56edf8eb3e235293c72738cf70f1fb9ee9631b40e6", - "zh:9b18fd51e260b2f3100937c34feae5f6fe3515df9b5e27ae23d00af75249a6d4", - "zh:a7154cdce28aeb80e822a97c6bc8b8acb7a074304fd198e265ac9cbcbda0ca06", - "zh:b0ce2ca42f018e5235a2171cdd8ba9829c90c54a6b2d602bd38e0e90c43d5d5d", - "zh:c67609f7018fc6e48b17befd6eeb21197e8f524496185c5e29707efa6967a0a5", - "zh:d4c9dc9d2a5a535851fc10049506bad1e7ab88193d5dcd371f91ac1b84f43a0a", - "zh:da27f2a9b9d5a4c02ec3893a763874513825c7c4dc2bb870ba741cf7725bcf9f", - "zh:e5bc1797b97607ff3d841c6c0d40da89c3843156ad43e15ded7d41fc0ac27717", + "zh:14f377dd6c3786583e1e8e10d74c762fd7767f84ab048d02cd418920f42686e7", + "zh:2bff386f61360f306e0c7cd8d4e67048b7e38bfcb974dd7f70b1f385477fa08d", + "zh:3601a3e133867abacc5836392db329dc6dfe52116263e2931837c8dfdf5d0bde", + "zh:54b47cfd80a939ccfdc4ebb693796e930be98e2ca1b3676c3fe61b114ca12621", + "zh:5b7cde484b9534bf5238c0f50da704edd53658bc376df5ef5b27406e4c80ee92", + "zh:5e844e071112293b4fced2ac9dd0fa2f744e78db18732dd989fd54783408b667", + "zh:a5442065fdc1de0bd38f70418b843d82570fb05a66e0a47c1358d0d9dab4418f", + "zh:b140dae2b6d0a09c2160841bf75fc7a654d7249b5b9f59db07df980ed950ffec", + "zh:b3cbf898cab3ae26be1dc3ed24b43f3a91510e6a190f5442c08957aaf1b6537e", + "zh:ba5eca495b37a2fd8647c138f1d50090fcaeb266508b87e7b8c931f0b6bdb735", + "zh:c0202c98f555fd7ecdc1b75255c3438351a557534c4ee0e9b55d678c007f785f", + "zh:d4bf2b894ecba7437906a450ecf136f2885b85108b3d49f8e1a046611535c841", + "zh:d89a71c1a3e2ea9cb109e2cbea7fd202a9ede5f5f0cc263ef50cb7f70c249c8e", + "zh:d98a6963b680db5a91ac51ede3be175fa9621070df2f3774197b34db0fc2e964", ] } diff --git a/k8s.tjo.cloud/justfile b/k8s.tjo.cloud/justfile index c0a22bd..2dfcda5 100644 --- a/k8s.tjo.cloud/justfile +++ b/k8s.tjo.cloud/justfile @@ -18,6 +18,9 @@ module-cluster-core-manifests: @curl -L -o modules/cluster-core/manifests/crd-servicemonitors.yaml \ "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml" +destroy: + tofu destroy -target module.cluster + apply: modules-cluster-manifests module-cluster-core-manifests tofu init tofu apply -target module.cluster diff --git a/k8s.tjo.cloud/main.tf b/k8s.tjo.cloud/main.tf index f382f67..5793497 100644 --- a/k8s.tjo.cloud/main.tf +++ b/k8s.tjo.cloud/main.tf @@ -2,15 +2,6 @@ locals { cluster_domain = "k8s.tjo.cloud" } -resource "tailscale_tailnet_key" "nodes" { - reusable = true - ephemeral = true - preauthorized = true - tags = ["tag:kubernetes-tjo-cloud"] - - description = "tailscale key for k8s-tjo-cloud nodes" -} - module "cluster" { source = "./modules/cluster" @@ -19,8 +10,8 @@ module "cluster" { } talos = { - version = "v1.7.5" - kubernetes = "v1.30.0" + version = "v1.8.3" + kubernetes = "v1.31.0" } cluster = { @@ -34,53 +25,46 @@ module "cluster" { proxmox = { name = "tjo-cloud" url = "https://proxmox.tjo.cloud/api2/json" - common_storage = "proxmox-backup-tjo-cloud" + common_storage = "synology.storage.tjo.cloud" } - tailscale_authkey = tailscale_tailnet_key.nodes.key - nodes = { pink = { - public = false type = "controlplane" - host = "hetzner" - storage = "main" + host = "nevaroo" + storage = "local-nvme-lvm" cores = 4 memory = 4096 + pod_cidr = { + ipv4 = "10.0.56.0/20" + ipv6 = "fd74:6a6f:0:3800::/52" + } } blue = { - public = false type = "worker" - host = "hetzner" - storage = "main" - cores = 6 - memory = 16384 + host = "nevaroo" + storage = "local-nvme-lvm" + cores = 8 + memory = 24576 + pod_cidr = { + ipv4 = "10.0.52.0/20" + ipv6 = "fd74:6a6f:0:3400::/52" + } } cyan = { - public = false type = "worker" - host = "hetzner" - storage = "main" - cores = 6 - memory = 16384 + host = "mustafar" + storage = "local" + cores = 2 + memory = 4096 + pod_cidr = { + ipv4 = "10.0.68.0/20" + ipv6 = "fd74:6a6f:0:4000::/52" + } } } } -data "tailscale_device" "controlpane" { - for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" } - hostname = each.value.name -} -resource "digitalocean_record" "api-internal" { - for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses])) - - domain = local.cluster_domain - type = strcontains(each.value, ":") ? "AAAA" : "A" - name = trimsuffix(module.cluster.api.internal.domain, ".${local.cluster_domain}") - value = each.value - ttl = 30 -} - resource "local_file" "kubeconfig" { content = templatefile("${path.module}/kubeconfig.tftpl", { cluster : { diff --git a/k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml b/k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml index ece0148..e51c40d 100644 --- a/k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml +++ b/k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.78.2/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 - operator.prometheus.io/version: 0.75.1 + controller-gen.kubebuilder.io/version: v0.16.4 + operator.prometheus.io/version: 0.78.2 name: podmonitors.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -23,7 +23,15 @@ spec: - name: v1 schema: openAPIV3Schema: - description: PodMonitor defines monitoring for a set of pods. + description: |- + The `PodMonitor` custom resource definition (CRD) defines how `Prometheus` and `PrometheusAgent` can scrape metrics from a group of pods. + Among other things, it allows to specify: + * The pods to scrape via label selectors. + * The container ports to scrape. + * Authentication credentials to use. + * Target and metric relabeling. + + `Prometheus` and `PrometheusAgent` objects select `PodMonitor` objects using label and namespace selectors. properties: apiVersion: description: |- @@ -51,13 +59,15 @@ spec: `attachMetadata` defines additional metadata which is added to the discovered targets. - - It requires Prometheus >= v2.37.0. + It requires Prometheus >= v2.35.0. properties: node: description: |- - When set to true, Prometheus must have the `get` permission on the - `Nodes` objects. + When set to true, Prometheus attaches node metadata to the discovered + targets. + + The Prometheus service account must have the `list` and `watch` + permissions on the `Nodes` objects. type: boolean type: object bodySizeLimit: @@ -65,7 +75,6 @@ spec: When defined, bodySizeLimit specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. - It requires Prometheus >= v2.28.0. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string @@ -75,12 +84,10 @@ spec: `jobLabel` selects the label from the associated Kubernetes `Pod` object which will be used as the `job` label for all metrics. - For example if `jobLabel` is set to `foo` and the Kubernetes `Pod` object is labeled with `foo: bar`, then Prometheus adds the `job="bar"` label to all ingested metrics. - If the value of this field is empty, the `job` label of the metrics defaults to the namespace and name of the PodMonitor object (e.g. `/`). type: string @@ -89,7 +96,6 @@ spec: Per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. - It requires Prometheus >= v2.47.0. format: int64 type: integer @@ -97,7 +103,6 @@ spec: description: |- Per-scrape limit on number of labels that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer @@ -105,7 +110,6 @@ spec: description: |- Per-scrape limit on length of labels name that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer @@ -113,14 +117,13 @@ spec: description: |- Per-scrape limit on length of labels value that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - Selector to select which namespaces the Kubernetes `Pods` objects - are discovered from. + `namespaceSelector` defines in which namespace(s) Prometheus should discover the pods. + By default, the pods are discovered in the same namespace as the `PodMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- @@ -133,8 +136,25 @@ spec: type: string type: array type: object + nativeHistogramBucketLimit: + description: |- + If there are more than this many buckets in a native histogram, + buckets will be merged to stay within the limit. + It requires Prometheus >= v2.45.0. + format: int64 + type: integer + nativeHistogramMinBucketFactor: + anyOf: + - type: integer + - type: string + description: |- + If the growth factor of one bucket to the next is smaller than this, + buckets will be merged to increase the factor sufficiently. + It requires Prometheus >= v2.50.0. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true podMetricsEndpoints: - description: List of endpoints part of this PodMonitor. + description: Defines how to scrape metrics from the selected pods. items: description: |- PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by @@ -145,7 +165,6 @@ spec: `authorization` configures the Authorization header credentials to use when scraping the target. - Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: @@ -163,9 +182,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -179,10 +196,8 @@ spec: description: |- Defines the authentication type. The value is case-insensitive. - "Basic" is not a supported value. - Default: "Bearer" type: string type: object @@ -191,7 +206,6 @@ spec: `basicAuth` configures the Basic Authentication credentials to use when scraping the target. - Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: @@ -210,9 +224,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -238,9 +250,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -257,7 +267,6 @@ spec: token for scraping targets. The secret needs to be in the same namespace as the PodMonitor object and readable by the Prometheus Operator. - Deprecated: use `authorization` instead. properties: key: @@ -271,9 +280,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -292,10 +299,8 @@ spec: When true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. - If unset, the filtering is enabled. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase type: boolean followRedirects: @@ -317,7 +322,6 @@ spec: description: |- Interval at which Prometheus scrapes the metrics from the target. - If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string @@ -330,7 +334,6 @@ spec: RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config properties: action: @@ -338,11 +341,9 @@ spec: description: |- Action to perform based on the regex matching. - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - Default: "Replace" enum: - replace @@ -372,7 +373,6 @@ spec: description: |- Modulus to take of the hash of the source label values. - Only applicable when the action is `HashMod`. format: int64 type: integer @@ -385,7 +385,6 @@ spec: Replacement value against which a Replace action is performed if the regular expression matches. - Regex capture groups are available. type: string separator: @@ -408,11 +407,9 @@ spec: description: |- Label to which the resulting string is written in a replacement. - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. - Regex capture groups are available. type: string type: object @@ -421,10 +418,8 @@ spec: description: |- `oauth2` configures the OAuth2 settings to use when scraping the target. - It requires Prometheus >= 2.27.0. - Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: @@ -446,9 +441,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -472,9 +465,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -501,9 +492,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -520,12 +509,232 @@ spec: `endpointParams` configures the HTTP parameters to append to the token URL. type: object + noProxy: + description: |- + `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + that should be excluded from proxying. IP and domain names can + contain port numbers. + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: string + proxyConnectHeader: + additionalProperties: + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + description: |- + ProxyConnectHeader optionally specifies headers to send to + proxies during CONNECT requests. + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: object + x-kubernetes-map-type: atomic + proxyFromEnvironment: + description: |- + Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: boolean + proxyUrl: + description: '`proxyURL` defines the HTTP proxy server to + use.' + pattern: ^http(s)?://.+$ + type: string scopes: description: '`scopes` defines the OAuth2 scopes used for the token request.' items: type: string type: array + tlsConfig: + description: |- + TLS configuration to use when connecting to the OAuth2 server. + It requires Prometheus >= v2.43.0. + properties: + ca: + description: Certificate authority used when verifying + server certificates. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + description: Client certificate to present when doing + client-authentication. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keySecret: + description: Secret containing the client key file for + the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + maxVersion: + description: |- + Maximum acceptable TLS version. + + It requires Prometheus >= v2.41.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + minVersion: + description: |- + Minimum acceptable TLS version. + + It requires Prometheus >= v2.35.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object tokenUrl: description: '`tokenURL` configures the URL to fetch the token from.' @@ -547,14 +756,12 @@ spec: description: |- HTTP path from which to scrape for metrics. - If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- Name of the Pod port which this endpoint refers to. - It takes precedence over `targetPort`. type: string proxyUrl: @@ -567,20 +774,16 @@ spec: `relabelings` configures the relabeling rules to apply the target's metadata labels. - The Operator automatically adds relabelings for a few standard Kubernetes fields. - The original scrape job's name is available via the `__tmp_prometheus_job_name` label. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config properties: action: @@ -588,11 +791,9 @@ spec: description: |- Action to perform based on the regex matching. - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - Default: "Replace" enum: - replace @@ -622,7 +823,6 @@ spec: description: |- Modulus to take of the hash of the source label values. - Only applicable when the action is `HashMod`. format: int64 type: integer @@ -635,7 +835,6 @@ spec: Replacement value against which a Replace action is performed if the regular expression matches. - Regex capture groups are available. type: string separator: @@ -658,11 +857,9 @@ spec: description: |- Label to which the resulting string is written in a replacement. - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. - Regex capture groups are available. type: string type: object @@ -671,11 +868,9 @@ spec: description: |- HTTP scheme to use for scraping. - `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. - If empty, Prometheus uses the default value `http`. enum: - http @@ -685,7 +880,6 @@ spec: description: |- Timeout after which Prometheus considers the scrape to be failed. - If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ @@ -698,7 +892,6 @@ spec: Name or number of the target port of the `Pod` object behind the Service, the port must be specified with container port property. - Deprecated: use 'port' instead. x-kubernetes-int-or-string: true tlsConfig: @@ -722,9 +915,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -748,9 +939,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -778,9 +967,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -804,9 +991,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -835,9 +1020,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -847,6 +1030,28 @@ spec: - key type: object x-kubernetes-map-type: atomic + maxVersion: + description: |- + Maximum acceptable TLS version. + + It requires Prometheus >= v2.41.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + minVersion: + description: |- + Minimum acceptable TLS version. + + It requires Prometheus >= v2.35.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string serverName: description: Used to verify the hostname for the targets. type: string @@ -857,7 +1062,6 @@ spec: the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. - It requires Prometheus >= v2.48.0. type: boolean type: object @@ -879,15 +1083,18 @@ spec: description: The scrape class to apply. minLength: 1 type: string + scrapeClassicHistograms: + description: |- + Whether to scrape a classic histogram that is also exposed as a native histogram. + It requires Prometheus >= v2.45.0. + type: boolean scrapeProtocols: description: |- `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). - If unset, Prometheus uses its default value. - It requires Prometheus >= v2.49.0. items: description: |- @@ -906,7 +1113,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Pod` objects. + description: Label selector to select the Kubernetes `Pod` objects + to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. diff --git a/k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml b/k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml index dbd6710..ebfe496 100644 --- a/k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml +++ b/k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml @@ -1,11 +1,11 @@ -# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.78.2/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.15.0 - operator.prometheus.io/version: 0.75.1 + controller-gen.kubebuilder.io/version: v0.16.4 + operator.prometheus.io/version: 0.78.2 name: servicemonitors.monitoring.coreos.com spec: group: monitoring.coreos.com @@ -23,7 +23,15 @@ spec: - name: v1 schema: openAPIV3Schema: - description: ServiceMonitor defines monitoring for a set of services. + description: |- + The `ServiceMonitor` custom resource definition (CRD) defines how `Prometheus` and `PrometheusAgent` can scrape metrics from a group of services. + Among other things, it allows to specify: + * The services to scrape via label selectors. + * The container ports to scrape. + * Authentication credentials to use. + * Target and metric relabeling. + + `Prometheus` and `PrometheusAgent` objects select `ServiceMonitor` objects using label and namespace selectors. properties: apiVersion: description: |- @@ -52,13 +60,15 @@ spec: `attachMetadata` defines additional metadata which is added to the discovered targets. - It requires Prometheus >= v2.37.0. properties: node: description: |- - When set to true, Prometheus must have the `get` permission on the - `Nodes` objects. + When set to true, Prometheus attaches node metadata to the discovered + targets. + + The Prometheus service account must have the `list` and `watch` + permissions on the `Nodes` objects. type: boolean type: object bodySizeLimit: @@ -66,12 +76,14 @@ spec: When defined, bodySizeLimit specifies a job level limit on the size of uncompressed response body that will be accepted by Prometheus. - It requires Prometheus >= v2.28.0. pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ type: string endpoints: - description: List of endpoints part of this ServiceMonitor. + description: |- + List of endpoints part of this ServiceMonitor. + Defines how to scrape metrics from Kubernetes [Endpoints](https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) objects. + In most cases, an Endpoints object is backed by a Kubernetes [Service](https://kubernetes.io/docs/concepts/services-networking/service/) object with the same name and labels. items: description: |- Endpoint defines an endpoint serving Prometheus metrics to be scraped by @@ -82,7 +94,6 @@ spec: `authorization` configures the Authorization header credentials to use when scraping the target. - Cannot be set at the same time as `basicAuth`, or `oauth2`. properties: credentials: @@ -100,9 +111,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -116,10 +125,8 @@ spec: description: |- Defines the authentication type. The value is case-insensitive. - "Basic" is not a supported value. - Default: "Bearer" type: string type: object @@ -128,7 +135,6 @@ spec: `basicAuth` configures the Basic Authentication credentials to use when scraping the target. - Cannot be set at the same time as `authorization`, or `oauth2`. properties: password: @@ -147,9 +153,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -175,9 +179,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -192,7 +194,6 @@ spec: description: |- File to read bearer token for scraping the target. - Deprecated: use `authorization` instead. type: string bearerTokenSecret: @@ -201,7 +202,6 @@ spec: token for scraping targets. The secret needs to be in the same namespace as the ServiceMonitor object and readable by the Prometheus Operator. - Deprecated: use `authorization` instead. properties: key: @@ -215,9 +215,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -236,10 +234,8 @@ spec: When true, the pods which are not running (e.g. either in Failed or Succeeded state) are dropped during the target discovery. - If unset, the filtering is enabled. - More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase type: boolean followRedirects: @@ -261,7 +257,6 @@ spec: description: |- Interval at which Prometheus scrapes the metrics from the target. - If empty, Prometheus uses the global scrape interval. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ type: string @@ -274,7 +269,6 @@ spec: RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config properties: action: @@ -282,11 +276,9 @@ spec: description: |- Action to perform based on the regex matching. - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - Default: "Replace" enum: - replace @@ -316,7 +308,6 @@ spec: description: |- Modulus to take of the hash of the source label values. - Only applicable when the action is `HashMod`. format: int64 type: integer @@ -329,7 +320,6 @@ spec: Replacement value against which a Replace action is performed if the regular expression matches. - Regex capture groups are available. type: string separator: @@ -352,11 +342,9 @@ spec: description: |- Label to which the resulting string is written in a replacement. - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. - Regex capture groups are available. type: string type: object @@ -365,10 +353,8 @@ spec: description: |- `oauth2` configures the OAuth2 settings to use when scraping the target. - It requires Prometheus >= 2.27.0. - Cannot be set at the same time as `authorization`, or `basicAuth`. properties: clientId: @@ -390,9 +376,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -416,9 +400,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -445,9 +427,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -464,12 +444,232 @@ spec: `endpointParams` configures the HTTP parameters to append to the token URL. type: object + noProxy: + description: |- + `noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names + that should be excluded from proxying. IP and domain names can + contain port numbers. + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: string + proxyConnectHeader: + additionalProperties: + items: + description: SecretKeySelector selects a key of a Secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: array + description: |- + ProxyConnectHeader optionally specifies headers to send to + proxies during CONNECT requests. + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: object + x-kubernetes-map-type: atomic + proxyFromEnvironment: + description: |- + Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY). + + It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0. + type: boolean + proxyUrl: + description: '`proxyURL` defines the HTTP proxy server to + use.' + pattern: ^http(s)?://.+$ + type: string scopes: description: '`scopes` defines the OAuth2 scopes used for the token request.' items: type: string type: array + tlsConfig: + description: |- + TLS configuration to use when connecting to the OAuth2 server. + It requires Prometheus >= v2.43.0. + properties: + ca: + description: Certificate authority used when verifying + server certificates. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + description: Client certificate to present when doing + client-authentication. + properties: + configMap: + description: ConfigMap containing data to use for + the targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or + its key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the + targets. + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keySecret: + description: Secret containing the client key file for + the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + maxVersion: + description: |- + Maximum acceptable TLS version. + + It requires Prometheus >= v2.41.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + minVersion: + description: |- + Minimum acceptable TLS version. + + It requires Prometheus >= v2.35.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object tokenUrl: description: '`tokenURL` configures the URL to fetch the token from.' @@ -491,14 +691,12 @@ spec: description: |- HTTP path from which to scrape for metrics. - If empty, Prometheus uses the default value (e.g. `/metrics`). type: string port: description: |- Name of the Service port which this endpoint refers to. - It takes precedence over `targetPort`. type: string proxyUrl: @@ -511,20 +709,16 @@ spec: `relabelings` configures the relabeling rules to apply the target's metadata labels. - The Operator automatically adds relabelings for a few standard Kubernetes fields. - The original scrape job's name is available via the `__tmp_prometheus_job_name` label. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config items: description: |- RelabelConfig allows dynamic rewriting of the label set for targets, alerts, scraped samples and remote write samples. - More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config properties: action: @@ -532,11 +726,9 @@ spec: description: |- Action to perform based on the regex matching. - `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. - Default: "Replace" enum: - replace @@ -566,7 +758,6 @@ spec: description: |- Modulus to take of the hash of the source label values. - Only applicable when the action is `HashMod`. format: int64 type: integer @@ -579,7 +770,6 @@ spec: Replacement value against which a Replace action is performed if the regular expression matches. - Regex capture groups are available. type: string separator: @@ -602,11 +792,9 @@ spec: description: |- Label to which the resulting string is written in a replacement. - It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, `KeepEqual` and `DropEqual` actions. - Regex capture groups are available. type: string type: object @@ -615,11 +803,9 @@ spec: description: |- HTTP scheme to use for scraping. - `http` and `https` are the expected values unless you rewrite the `__scheme__` label via relabeling. - If empty, Prometheus uses the default value `http`. enum: - http @@ -629,7 +815,6 @@ spec: description: |- Timeout after which Prometheus considers the scrape to be failed. - If empty, Prometheus uses the global scrape timeout unless it is less than the target's scrape interval value in which the latter is used. pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ @@ -663,9 +848,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -689,9 +872,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -723,9 +904,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the ConfigMap or its @@ -749,9 +928,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key @@ -788,9 +965,7 @@ spec: This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. - TODO: Add other useful fields. apiVersion, kind, uid? More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names - TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. type: string optional: description: Specify whether the Secret or its key must @@ -800,6 +975,28 @@ spec: - key type: object x-kubernetes-map-type: atomic + maxVersion: + description: |- + Maximum acceptable TLS version. + + It requires Prometheus >= v2.41.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string + minVersion: + description: |- + Minimum acceptable TLS version. + + It requires Prometheus >= v2.35.0. + enum: + - TLS10 + - TLS11 + - TLS12 + - TLS13 + type: string serverName: description: Used to verify the hostname for the targets. type: string @@ -810,7 +1007,6 @@ spec: the metrics that have an explicit timestamp present in scraped data. Has no effect if `honorTimestamps` is false. - It requires Prometheus >= v2.48.0. type: boolean type: object @@ -820,12 +1016,10 @@ spec: `jobLabel` selects the label from the associated Kubernetes `Service` object which will be used as the `job` label for all metrics. - For example if `jobLabel` is set to `foo` and the Kubernetes `Service` object is labeled with `foo: bar`, then Prometheus adds the `job="bar"` label to all ingested metrics. - If the value of this field is empty or if the label doesn't exist for the given Service, the `job` label of the metrics defaults to the name of the associated Kubernetes `Service`. @@ -835,7 +1029,6 @@ spec: Per-scrape limit on the number of targets dropped by relabeling that will be kept in memory. 0 means no limit. - It requires Prometheus >= v2.47.0. format: int64 type: integer @@ -843,7 +1036,6 @@ spec: description: |- Per-scrape limit on number of labels that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer @@ -851,7 +1043,6 @@ spec: description: |- Per-scrape limit on length of labels name that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer @@ -859,14 +1050,13 @@ spec: description: |- Per-scrape limit on length of labels value that will be accepted for a sample. - It requires Prometheus >= v2.27.0. format: int64 type: integer namespaceSelector: description: |- - Selector to select which namespaces the Kubernetes `Endpoints` objects - are discovered from. + `namespaceSelector` defines in which namespace(s) Prometheus should discover the services. + By default, the services are discovered in the same namespace as the `ServiceMonitor` object but it is possible to select pods across different/all namespaces. properties: any: description: |- @@ -879,6 +1069,23 @@ spec: type: string type: array type: object + nativeHistogramBucketLimit: + description: |- + If there are more than this many buckets in a native histogram, + buckets will be merged to stay within the limit. + It requires Prometheus >= v2.45.0. + format: int64 + type: integer + nativeHistogramMinBucketFactor: + anyOf: + - type: integer + - type: string + description: |- + If the growth factor of one bucket to the next is smaller than this, + buckets will be merged to increase the factor sufficiently. + It requires Prometheus >= v2.50.0. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true podTargetLabels: description: |- `podTargetLabels` defines the labels which are transferred from the @@ -896,15 +1103,18 @@ spec: description: The scrape class to apply. minLength: 1 type: string + scrapeClassicHistograms: + description: |- + Whether to scrape a classic histogram that is also exposed as a native histogram. + It requires Prometheus >= v2.45.0. + type: boolean scrapeProtocols: description: |- `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the protocols supported by Prometheus in order of preference (from most to least preferred). - If unset, Prometheus uses its default value. - It requires Prometheus >= v2.49.0. items: description: |- @@ -923,7 +1133,8 @@ spec: type: array x-kubernetes-list-type: set selector: - description: Label selector to select the Kubernetes `Endpoints` objects. + description: Label selector to select the Kubernetes `Endpoints` objects + to scrape metrics from. properties: matchExpressions: description: matchExpressions is a list of label selector requirements. @@ -982,6 +1193,7 @@ spec: format: int64 type: integer required: + - endpoints - selector type: object required: diff --git a/k8s.tjo.cloud/modules/cluster/main.tf b/k8s.tjo.cloud/modules/cluster/main.tf index 30beabd..cd377d0 100644 --- a/k8s.tjo.cloud/modules/cluster/main.tf +++ b/k8s.tjo.cloud/modules/cluster/main.tf @@ -1,21 +1,16 @@ locals { - cluster_internal_endpoint = "https://${var.cluster.api.internal.domain}:${var.cluster.api.internal.port}" - cluster_public_endpoint = "https://${var.cluster.api.public.domain}:${var.cluster.api.public.port}" + public_domain = "${var.cluster.api.public.subdomain}.${var.cluster.api.public.domain}" + internal_domain = "${var.cluster.api.internal.subdomain}.${var.cluster.api.internal.domain}" + cluster_internal_endpoint = "https://${local.internal_domain}:${var.cluster.api.internal.port}" + cluster_public_endpoint = "https://${local.public_domain}:${var.cluster.api.public.port}" podSubnets = [ - "10.200.0.0/16", - "fd9b:5314:fc70::/56", + "10.0.240.0/22", + "fd74:6a6f:0:f000::/54", ] serviceSubnets = [ - "10.201.0.0/16", - "fd9b:5314:fc71::/112", - ] - - # Nodes will use IPs from this subnets - # for communication between each other. - tailscaleSubnets = [ - "100.64.0.0/10", - "fd7a:115c:a1e0::/96" + "10.0.244.0/22", + "fd74:6a6f:0:f400::/54", ] talos_controlplane_config = { @@ -35,15 +30,11 @@ locals { } } cluster = { - etcd = { - advertisedSubnets = local.tailscaleSubnets - listenSubnets = local.tailscaleSubnets - } allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes, apiServer = { certSANs = [ - var.cluster.api.internal.domain, - var.cluster.api.public.domain, + local.public_domain, + local.internal_domain, ] extraArgs = { "oidc-issuer-url" = "https://id.tjo.space/application/o/k8stjocloud/", @@ -118,9 +109,6 @@ locals { } machine = { kubelet = { - nodeIP = { - validSubnets = local.tailscaleSubnets - } extraArgs = { rotate-server-certificates = true cloud-provider = "external" @@ -144,28 +132,15 @@ locals { } } nodeLabels = { - "k8s.tjo.cloud/public" = node.public ? "true" : "false" "k8s.tjo.cloud/host" = node.host "k8s.tjo.cloud/proxmox" = var.proxmox.name } - sysctls = { - "net.ipv4.ip_forward" = "1" - "net.ipv6.conf.all.forwarding" = "1" + nodeAnnotations = { + "network.cilium.io/ipv4-pod-cidr" : node.pod_cidr.ipv4 + "network.cilium.io/ipv6-pod-cidr" : node.pod_cidr.ipv6 } } }), - yamlencode( - { - apiVersion = "v1alpha1" - kind = "ExtensionServiceConfig" - name = "tailscale" - environment = [ - "TS_AUTHKEY=${var.tailscale_authkey}", - "TS_HOSTNAME=${node.name}", - "TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}", - #"TS_EXTRA_ARGS=--accept-routes", - ] - }) ] } } @@ -244,7 +219,7 @@ resource "talos_machine_bootstrap" "this" { client_configuration = talos_machine_secrets.this.client_configuration } -data "talos_cluster_kubeconfig" "this" { +resource "talos_cluster_kubeconfig" "this" { depends_on = [ talos_machine_bootstrap.this ] @@ -254,7 +229,7 @@ data "talos_cluster_kubeconfig" "this" { } resource "local_file" "kubeconfig" { - content = data.talos_cluster_kubeconfig.this.kubeconfig_raw + content = talos_cluster_kubeconfig.this.kubeconfig_raw filename = "${path.root}/admin.kubeconfig" lifecycle { @@ -276,3 +251,23 @@ resource "local_file" "talosconfig" { content = nonsensitive(data.talos_client_configuration.this[0].talos_config) filename = "${path.root}/admin.talosconfig" } + +resource "digitalocean_record" "api-internal-ipv4" { + for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } + + domain = var.cluster.api.internal.domain + type = "A" + name = var.cluster.api.internal.subdomain + value = each.value.ipv4 + ttl = 30 +} + +resource "digitalocean_record" "api-internal-ipv6" { + for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" } + + domain = var.cluster.api.internal.domain + type = "AAAA" + name = var.cluster.api.internal.subdomain + value = each.value.ipv6 + ttl = 30 +} diff --git a/k8s.tjo.cloud/modules/cluster/outputs.tf b/k8s.tjo.cloud/modules/cluster/outputs.tf index 1e4fe50..75f84a3 100644 --- a/k8s.tjo.cloud/modules/cluster/outputs.tf +++ b/k8s.tjo.cloud/modules/cluster/outputs.tf @@ -10,7 +10,7 @@ output "api" { public : merge(var.cluster.api.public, { endpoint : local.cluster_public_endpoint, }), - ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate, + ca : talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate, }) } diff --git a/k8s.tjo.cloud/modules/cluster/proxmox.tf b/k8s.tjo.cloud/modules/cluster/proxmox.tf index 5ecbdc7..184d837 100644 --- a/k8s.tjo.cloud/modules/cluster/proxmox.tf +++ b/k8s.tjo.cloud/modules/cluster/proxmox.tf @@ -1,7 +1,7 @@ locals { nodes_with_names = { for k, v in var.nodes : k => merge(v, { - id = 1000 + index(keys(var.nodes), k) + id = 6000 + index(keys(var.nodes), k) name = replace("${k}.${v.type}.${var.cluster.name}", ".", "-") }) } @@ -75,11 +75,7 @@ resource "proxmox_virtual_environment_vm" "nodes" { node_name = each.value.host description = "Node ${each.value.name} for cluster ${var.cluster.name}." - tags = concat( - ["kubernetes", "terraform"], - each.value.public ? ["public"] : ["private"], - [each.value.type] - ) + tags = ["kubernetes.tjo.cloud", each.value.type] stop_on_destroy = true timeout_start_vm = 60 @@ -106,7 +102,7 @@ resource "proxmox_virtual_environment_vm" "nodes" { } network_device { - bridge = each.value.public ? "vmpublic0" : "vmprivate0" + bridge = "vmbr1" mac_address = each.value.mac_address } diff --git a/k8s.tjo.cloud/modules/cluster/variables.tf b/k8s.tjo.cloud/modules/cluster/variables.tf index 064925a..15a9760 100644 --- a/k8s.tjo.cloud/modules/cluster/variables.tf +++ b/k8s.tjo.cloud/modules/cluster/variables.tf @@ -1,21 +1,25 @@ variable "nodes" { type = map(object({ - public = bool - type = string - host = string + type = string + host = string cores = optional(number, 4) memory = optional(number, 4096) storage = string boot_size = optional(number, 32) + + pod_cidr = object({ + ipv4 = string + ipv6 = string + }) })) } variable "talos" { type = object({ - version = optional(string, "v1.7.5") - kubernetes = optional(string, "v1.30.0") + version = optional(string, "v1.8.3") + kubernetes = optional(string, "v1.31.0") # Default is: # customization: @@ -23,9 +27,8 @@ variable "talos" { # officialExtensions: # - siderolabs/kata-containers # - siderolabs/qemu-guest-agent - # - siderolabs/tailscale # - siderolabs/wasmedge - schematic_id = optional(string, "a125b6d6becb63df5543edfae1231e351723dd6e4d551ba73e0f30229ad6ff59") + schematic_id = optional(string, "392092063ce5c8be7dfeba0bd466add2bc0b55a20939cc2c0060058fcc25d784") }) } @@ -41,12 +44,14 @@ variable "cluster" { name = string api = optional(object({ internal = optional(object({ - domain = optional(string, "api.internal.k8s.tjo.cloud") - port = optional(number, 6443) + domain = optional(string, "k8s.tjo.cloud") + subdomain = optional(string, "api.internal") + port = optional(number, 6443) }), {}) public = optional(object({ - domain = optional(string, "api.k8s.tjo.cloud") - port = optional(number, 443) + domain = optional(string, "k8s.tjo.cloud") + subdomain = optional(string, "api") + port = optional(number, 443) }), {}) }), {}) oidc = object({ @@ -56,11 +61,6 @@ variable "cluster" { }) } -variable "tailscale_authkey" { - type = string - sensitive = true -} - variable "proxmox" { type = object({ name = string diff --git a/k8s.tjo.cloud/modules/cluster/versions.tf b/k8s.tjo.cloud/modules/cluster/versions.tf index 689f191..593f519 100644 --- a/k8s.tjo.cloud/modules/cluster/versions.tf +++ b/k8s.tjo.cloud/modules/cluster/versions.tf @@ -8,7 +8,7 @@ terraform { } talos = { source = "siderolabs/talos" - version = "0.5.0" + version = "0.6.1" } local = { source = "hashicorp/local" diff --git a/k8s.tjo.cloud/terraform.tf b/k8s.tjo.cloud/terraform.tf index 0934e0d..c1497f4 100644 --- a/k8s.tjo.cloud/terraform.tf +++ b/k8s.tjo.cloud/terraform.tf @@ -6,7 +6,7 @@ terraform { } talos = { source = "siderolabs/talos" - version = "0.5.0" + version = "0.6.1" } local = { source = "hashicorp/local" @@ -28,10 +28,6 @@ terraform { source = "hashicorp/kubernetes" version = "2.31.0" } - tailscale = { - source = "tailscale/tailscale" - version = "0.16.1" - } } required_version = "~> 1.7.3" @@ -45,6 +41,30 @@ provider "proxmox" { ssh { agent = true username = "root" + + node { + name = "batuu" + address = "batuu.system.tjo.cloud" + port = 22 + } + + node { + name = "jakku" + address = "jakku.system.tjo.cloud" + port = 22 + } + + node { + name = "nevaroo" + address = "nevaroo.system.tjo.cloud" + port = 22 + } + + node { + name = "mustafar" + address = "mustafar.system.tjo.cloud" + port = 22 + } } } @@ -52,10 +72,6 @@ provider "digitalocean" { token = var.digitalocean_token } -provider "tailscale" { - api_key = var.tailscale_apikey -} - provider "helm" { alias = "template" } diff --git a/k8s.tjo.cloud/variables.tf b/k8s.tjo.cloud/variables.tf index 5411b8b..7248cae 100644 --- a/k8s.tjo.cloud/variables.tf +++ b/k8s.tjo.cloud/variables.tf @@ -1,8 +1,3 @@ -variable "tailscale_apikey" { - type = string - sensitive = true -} - variable "oidc_username" { type = string }