tjo-cloud/infrastructure
1
0
Fork 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions

feat(kubernetes): use new networking concepts
Some checks failed
/ lint (push) Has been cancelled
Details

Browse source
This commit is contained in:
Tine 2024-12-02 20:13:50 +01:00
parent 9d5b07ae50
commit b0b3a3b116
Signed by: mentos1386
SSH key fingerprint: SHA256:MNtTsLbihYaWF8j1fkOHfkKNlnN1JQfxEU/rBU8nCGw
13 changed files with 722 additions and 335 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README.md
View file

@ -12,7 +12,7 @@ tailscale up --ssh --accept-routes --accept-dns=false --advertise-tags=tag:syste
```
### 2. Install intel-micropodes updates.
### 2. Install intel-firmware updates.
```
# Add non-free-firmware to the end

92
k8s.tjo.cloud/.terraform.lock.hcl
View file

@ -25,26 +25,26 @@ provider "registry.opentofu.org/bpg/proxmox" {
}
provider "registry.opentofu.org/digitalocean/digitalocean" {
version = "2.39.2"
version = "2.44.1"
constraints = "~> 2.0"
hashes = [
"h1:ci1lDN5Jz3QTvNjuKxdGngXs1xxPba0eDv/2rTVqw60=",
"zh:00380bd275cdb15645d03880a5c219a6826a9edba43099f5c09475465f87eb5f",
"zh:1e40f4aa51ba898cf64b1f296450b2ae85e77af6e2706536242093550aa605b0",
"zh:3f5f0c9f8c0cad64a757e38c1098633904786db998ab772e44f5f981b1acc06f",
"zh:511d02b9cad7946cab21b5bab30c15edf92610b0316a5a035771c4681df848ee",
"zh:5e56c038b16c97ea33d94e105ad5db4ccec01e957dd6adf4572e9414b499d2ea",
"zh:763b49a44a911fcba6e4d6773951cb6a612f93faf504cebdcc548c09b65790e5",
"zh:848079d6e125c2491d980d96c2e1ff59e81b19cf05e7c0b338054f27ba90ee9e",
"zh:9f54e4bbf89e051ef8cad73e39f505ff054b155b87b5b1fd578e7709ad0d2eeb",
"zh:c14e8e0f989e68338ff2ec6230b9ec846ebc33a1d3a858a662d77f162cf45761",
"zh:d30792eff5441c26f47cb2181b6eb1f0340c2c330378bec726f40f88dba49ab9",
"zh:d660a22bb43427d9ceff604e28d5d8a3b4f21639c85614f6134b39e43ca58ecf",
"zh:de8b42065fe420127e430dbd0c5aa5bd2c51e76ceeabd436e7e1137627b2a720",
"zh:eec0295a9c24af2c00436fea5e40fef13f7104fcd15eab30025d81096eb59fad",
"zh:ef8602f1deb8bd522ceb17de950864f2432e2e3ef2fa467caffe79b10e60f2c0",
"zh:f28a340515ac9cd0eb21bf2a0d2dcbaa58ccb2996d1e30e18ceb9ae79caab87f",
"zh:f30ce538e6beb13c9fe7712c543ad6cfed5d079d7e2bd050fdbeac3cc356b1ba",
"h1:wIccPAQ8HhEOg/Eo7ZLLiADITIfDRBv3ncRtnuwwkKc=",
"zh:02e0bd7320167fed3b9ceea492ab218c2568abd619e816c14542c0d185eb969a",
"zh:309452ac92ddfe6402613a5a7dcaf780e1b648e8737da3fef068e587eb932d88",
"zh:32433f540e9feb9a22a015e83dc299d46f08adec3880f72bd6af89ac1032b13c",
"zh:347664ab9c218f26eac168c10c52f6d72d1ff084fd6e24418d8e4982ec2f880e",
"zh:3a917158aa57372fa2254e4578905211338b0452135b47f00c9444202bb53311",
"zh:593b7ec19653558bbb75d202b8ecdf9580545b24ba20584c4abe2497b232fd60",
"zh:64506619588bc381471183dca0d5bf457df697699b08a42d1ae2a5cdb261c58c",
"zh:6b0c6dfdb5b685e25d1505445a0dd26d93a515c86ace1187767f7fadc6c69206",
"zh:9a4595e36ae6fb3341724dd08a476234cdb28c0b12615792a5cf73d5d2cccd26",
"zh:9e88880489f3162440f166cf083adbe876a022a7558c1cb7e35b759778c0439e",
"zh:a48c72a7e0b67a13c054c6dc1024124e8637cbecb45c684985a6037f3abd51a2",
"zh:d21f16e85cd02e4e1a147aa7dc65e149723bd2c6844236608278a4433ee56f62",
"zh:dee7a153f4201831607749c5f02b1433589c1e39db8b1d19da16836e0f3eb6cf",
"zh:df40d88ef94fd98c5c9eeabd82ed5178da4618735eaff06b83817b2ef5717e47",
"zh:f7bcc22d9ff38b98bf48c02834f4861f5b7a37c0144f2e7464d17751e01cea32",
"zh:fbf47dc012166d6545cc33a6c00b5dbdb789f7fef5b4f59935a3763f2d74e670",
]
}
@ -116,46 +116,24 @@ provider "registry.opentofu.org/hashicorp/random" {
}
provider "registry.opentofu.org/siderolabs/talos" {
version = "0.5.0"
constraints = "0.5.0"
version = "0.6.1"
constraints = "0.6.1"
hashes = [
"h1:xogkLLCrJJmd278E+vNMnmQgaMD05Gd1QXN914xgVec=",
"zh:0f71f2624576224c9bc924b136b601b734243efa7a7ad8280dfd8bd583e4afa5",
"h1:eFw5nEpptkVQ+SNXFEaYa8o++5Q3WVznDgrxJ78ROLA=",
"zh:0fa82a384b25a58b65523e0ea4768fa1212b1f5cfc0c9379d31162454fedcc9d",
"zh:33c50dacc5029fa20caed702001fb1439899c94f203b1f37dccb970f504bca45",
"zh:3c97a6e2692b88d3f4631a3f8769146f602c210e881b46fa1b3b82c545e51cd1",
"zh:44077a137613bcfe29eef00315b5aa50d83390c3c727580a4ff0f4b87f22d228",
"zh:5bd02f278aec5567f94dd057d1c758363998ce581ff17b0869515bb682c02186",
"zh:80f40939bc3b55f0005c03b77122ceea86ec4deb82f5557950a97ad96fbb1557",
"zh:94c1b17f25bc30eacde926e46f196f1f135032674730d9f50c986ef6b7a854f0",
"zh:95ad665b2fdeed38180f5c471164833a34d07c1ef0470c1652565fe8cf4e9c4a",
"zh:a50ef6088afcb129c176dd4ba86c345e9be7b14358bb3b21c34f06930d8f39ef",
"zh:aa71da1da00ed66f1dddf1b69c10b829f24ac89e207de07d32c455dd04482096",
"zh:abb7eeb2b089081b4814ed80a295673e1a92f82ce092dde37b5bc92e75efec2c",
"zh:db9b9b54a0db5ae151376d5a73e0d28497c3e06181840e71ef8349213ac03e50",
"zh:e50ed8aa90b736508fce63680e8339240cecb74709ab9563d34d2c2ce7bc8445",
"zh:f3a279723ff31a095d7bfff21857abfcc9a2cfdeeea8521d179630ae6565d581",
]
}
provider "registry.opentofu.org/tailscale/tailscale" {
version = "0.16.1"
constraints = "0.16.1"
hashes = [
"h1:NDIIkEo0G/leQSvGoh2Mk74ZE2xWrWgHX/S8ZVyBDYU=",
"zh:0a9d28e5195e0e29ebf9b12b345cafcb686125008151fa01677c399d8f8f1321",
"zh:249bce2fcfd3414211ae9e49e179e31b5d3c23dd9da24dc45acdea34ad308cb0",
"zh:3129fb52a2aaa0c8c30aff21e7d4c0601d80898b3ecb9d7604b5933c14f54924",
"zh:4ec3e255f34bb4f6362ab41aa9e05a3ce040a791bc07445dec86188dee867f85",
"zh:68d3995e5a1722e24f89a385899f56a63542159b884cac989196e9538b53c6ce",
"zh:799840b3bfbd14537397f157f4e6a5e54080cd4fee51521bac675aa188e0b33e",
"zh:99f1da9fdaddd8a1255dce56edf8eb3e235293c72738cf70f1fb9ee9631b40e6",
"zh:9b18fd51e260b2f3100937c34feae5f6fe3515df9b5e27ae23d00af75249a6d4",
"zh:a7154cdce28aeb80e822a97c6bc8b8acb7a074304fd198e265ac9cbcbda0ca06",
"zh:b0ce2ca42f018e5235a2171cdd8ba9829c90c54a6b2d602bd38e0e90c43d5d5d",
"zh:c67609f7018fc6e48b17befd6eeb21197e8f524496185c5e29707efa6967a0a5",
"zh:d4c9dc9d2a5a535851fc10049506bad1e7ab88193d5dcd371f91ac1b84f43a0a",
"zh:da27f2a9b9d5a4c02ec3893a763874513825c7c4dc2bb870ba741cf7725bcf9f",
"zh:e5bc1797b97607ff3d841c6c0d40da89c3843156ad43e15ded7d41fc0ac27717",
"zh:14f377dd6c3786583e1e8e10d74c762fd7767f84ab048d02cd418920f42686e7",
"zh:2bff386f61360f306e0c7cd8d4e67048b7e38bfcb974dd7f70b1f385477fa08d",
"zh:3601a3e133867abacc5836392db329dc6dfe52116263e2931837c8dfdf5d0bde",
"zh:54b47cfd80a939ccfdc4ebb693796e930be98e2ca1b3676c3fe61b114ca12621",
"zh:5b7cde484b9534bf5238c0f50da704edd53658bc376df5ef5b27406e4c80ee92",
"zh:5e844e071112293b4fced2ac9dd0fa2f744e78db18732dd989fd54783408b667",
"zh:a5442065fdc1de0bd38f70418b843d82570fb05a66e0a47c1358d0d9dab4418f",
"zh:b140dae2b6d0a09c2160841bf75fc7a654d7249b5b9f59db07df980ed950ffec",
"zh:b3cbf898cab3ae26be1dc3ed24b43f3a91510e6a190f5442c08957aaf1b6537e",
"zh:ba5eca495b37a2fd8647c138f1d50090fcaeb266508b87e7b8c931f0b6bdb735",
"zh:c0202c98f555fd7ecdc1b75255c3438351a557534c4ee0e9b55d678c007f785f",
"zh:d4bf2b894ecba7437906a450ecf136f2885b85108b3d49f8e1a046611535c841",
"zh:d89a71c1a3e2ea9cb109e2cbea7fd202a9ede5f5f0cc263ef50cb7f70c249c8e",
"zh:d98a6963b680db5a91ac51ede3be175fa9621070df2f3774197b34db0fc2e964",
]
}

3
k8s.tjo.cloud/justfile
View file

@ -18,6 +18,9 @@ module-cluster-core-manifests:
@curl -L -o modules/cluster-core/manifests/crd-servicemonitors.yaml \
"https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml"
destroy:
tofu destroy -target module.cluster
apply: modules-cluster-manifests module-cluster-core-manifests
tofu init
tofu apply -target module.cluster

66
k8s.tjo.cloud/main.tf
View file

@ -2,15 +2,6 @@ locals {
cluster_domain = "k8s.tjo.cloud"
}
resource "tailscale_tailnet_key" "nodes" {
reusable = true
ephemeral = true
preauthorized = true
tags = ["tag:kubernetes-tjo-cloud"]
description = "tailscale key for k8s-tjo-cloud nodes"
}
module "cluster" {
source = "./modules/cluster"
@ -19,8 +10,8 @@ module "cluster" {
}
talos = {
version = "v1.7.5"
kubernetes = "v1.30.0"
version = "v1.8.3"
kubernetes = "v1.31.0"
}
cluster = {
@ -34,53 +25,46 @@ module "cluster" {
proxmox = {
name = "tjo-cloud"
url = "https://proxmox.tjo.cloud/api2/json"
common_storage = "proxmox-backup-tjo-cloud"
common_storage = "synology.storage.tjo.cloud"
}
tailscale_authkey = tailscale_tailnet_key.nodes.key
nodes = {
pink = {
public = false
type = "controlplane"
host = "hetzner"
storage = "main"
host = "nevaroo"
storage = "local-nvme-lvm"
cores = 4
memory = 4096
pod_cidr = {
ipv4 = "10.0.56.0/20"
ipv6 = "fd74:6a6f:0:3800::/52"
}
}
blue = {
public = false
type = "worker"
host = "hetzner"
storage = "main"
cores = 6
memory = 16384
host = "nevaroo"
storage = "local-nvme-lvm"
cores = 8
memory = 24576
pod_cidr = {
ipv4 = "10.0.52.0/20"
ipv6 = "fd74:6a6f:0:3400::/52"
}
}
cyan = {
public = false
type = "worker"
host = "hetzner"
storage = "main"
cores = 6
memory = 16384
host = "mustafar"
storage = "local"
cores = 2
memory = 4096
pod_cidr = {
ipv4 = "10.0.68.0/20"
ipv6 = "fd74:6a6f:0:4000::/52"
}
}
}
}
data "tailscale_device" "controlpane" {
for_each = { for k, v in module.cluster.nodes : k => v if v.type == "controlplane" }
hostname = each.value.name
}
resource "digitalocean_record" "api-internal" {
for_each = toset(flatten([for key, device in data.tailscale_device.controlpane : device.addresses]))
domain = local.cluster_domain
type = strcontains(each.value, ":") ? "AAAA" : "A"
name = trimsuffix(module.cluster.api.internal.domain, ".${local.cluster_domain}")
value = each.value
ttl = 30
}
resource "local_file" "kubeconfig" {
content = templatefile("${path.module}/kubeconfig.tftpl", {
cluster : {

366
k8s.tjo.cloud/modules/cluster-core/manifests/crd-podmonitors.yaml
View file

@ -1,11 +1,11 @@
# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.78.2/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.15.0
operator.prometheus.io/version: 0.75.1
controller-gen.kubebuilder.io/version: v0.16.4
operator.prometheus.io/version: 0.78.2
name: podmonitors.monitoring.coreos.com
spec:
group: monitoring.coreos.com
@ -23,7 +23,15 @@ spec:
- name: v1
schema:
openAPIV3Schema:
description: PodMonitor defines monitoring for a set of pods.
description: |-
The `PodMonitor` custom resource definition (CRD) defines how `Prometheus` and `PrometheusAgent` can scrape metrics from a group of pods.
Among other things, it allows to specify:
* The pods to scrape via label selectors.
* The container ports to scrape.
* Authentication credentials to use.
* Target and metric relabeling.
`Prometheus` and `PrometheusAgent` objects select `PodMonitor` objects using label and namespace selectors.
properties:
apiVersion:
description: |-
@ -51,13 +59,15 @@ spec:
`attachMetadata` defines additional metadata which is added to the
discovered targets.
It requires Prometheus >= v2.37.0.
It requires Prometheus >= v2.35.0.
properties:
node:
description: |-
When set to true, Prometheus must have the `get` permission on the
`Nodes` objects.
When set to true, Prometheus attaches node metadata to the discovered
targets.
The Prometheus service account must have the `list` and `watch`
permissions on the `Nodes` objects.
type: boolean
type: object
bodySizeLimit:
@ -65,7 +75,6 @@ spec:
When defined, bodySizeLimit specifies a job level limit on the size
of uncompressed response body that will be accepted by Prometheus.
It requires Prometheus >= v2.28.0.
pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$
type: string
@ -75,12 +84,10 @@ spec:
`jobLabel` selects the label from the associated Kubernetes `Pod`
object which will be used as the `job` label for all metrics.
For example if `jobLabel` is set to `foo` and the Kubernetes `Pod`
object is labeled with `foo: bar`, then Prometheus adds the `job="bar"`
label to all ingested metrics.
If the value of this field is empty, the `job` label of the metrics
defaults to the namespace and name of the PodMonitor object (e.g. `<namespace>/<name>`).
type: string
@ -89,7 +96,6 @@ spec:
Per-scrape limit on the number of targets dropped by relabeling
that will be kept in memory. 0 means no limit.
It requires Prometheus >= v2.47.0.
format: int64
type: integer
@ -97,7 +103,6 @@ spec:
description: |-
Per-scrape limit on number of labels that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
@ -105,7 +110,6 @@ spec:
description: |-
Per-scrape limit on length of labels name that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
@ -113,14 +117,13 @@ spec:
description: |-
Per-scrape limit on length of labels value that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
namespaceSelector:
description: |-
Selector to select which namespaces the Kubernetes `Pods` objects
are discovered from.
`namespaceSelector` defines in which namespace(s) Prometheus should discover the pods.
By default, the pods are discovered in the same namespace as the `PodMonitor` object but it is possible to select pods across different/all namespaces.
properties:
any:
description: |-
@ -133,8 +136,25 @@ spec:
type: string
type: array
type: object
nativeHistogramBucketLimit:
description: |-
If there are more than this many buckets in a native histogram,
buckets will be merged to stay within the limit.
It requires Prometheus >= v2.45.0.
format: int64
type: integer
nativeHistogramMinBucketFactor:
anyOf:
- type: integer
- type: string
description: |-
If the growth factor of one bucket to the next is smaller than this,
buckets will be merged to increase the factor sufficiently.
It requires Prometheus >= v2.50.0.
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
podMetricsEndpoints:
description: List of endpoints part of this PodMonitor.
description: Defines how to scrape metrics from the selected pods.
items:
description: |-
PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by
@ -145,7 +165,6 @@ spec:
`authorization` configures the Authorization header credentials to use when
scraping the target.
Cannot be set at the same time as `basicAuth`, or `oauth2`.
properties:
credentials:
@ -163,9 +182,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -179,10 +196,8 @@ spec:
description: |-
Defines the authentication type. The value is case-insensitive.
"Basic" is not a supported value.
Default: "Bearer"
type: string
type: object
@ -191,7 +206,6 @@ spec:
`basicAuth` configures the Basic Authentication credentials to use when
scraping the target.
Cannot be set at the same time as `authorization`, or `oauth2`.
properties:
password:
@ -210,9 +224,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -238,9 +250,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -257,7 +267,6 @@ spec:
token for scraping targets. The secret needs to be in the same namespace
as the PodMonitor object and readable by the Prometheus Operator.
Deprecated: use `authorization` instead.
properties:
key:
@ -271,9 +280,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -292,10 +299,8 @@ spec:
When true, the pods which are not running (e.g. either in Failed or
Succeeded state) are dropped during the target discovery.
If unset, the filtering is enabled.
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
type: boolean
followRedirects:
@ -317,7 +322,6 @@ spec:
description: |-
Interval at which Prometheus scrapes the metrics from the target.
If empty, Prometheus uses the global scrape interval.
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
type: string
@ -330,7 +334,6 @@ spec:
RelabelConfig allows dynamic rewriting of the label set for targets, alerts,
scraped samples and remote write samples.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
properties:
action:
@ -338,11 +341,9 @@ spec:
description: |-
Action to perform based on the regex matching.
`Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0.
`DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0.
Default: "Replace"
enum:
- replace
@ -372,7 +373,6 @@ spec:
description: |-
Modulus to take of the hash of the source label values.
Only applicable when the action is `HashMod`.
format: int64
type: integer
@ -385,7 +385,6 @@ spec:
Replacement value against which a Replace action is performed if the
regular expression matches.
Regex capture groups are available.
type: string
separator:
@ -408,11 +407,9 @@ spec:
description: |-
Label to which the resulting string is written in a replacement.
It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`,
`KeepEqual` and `DropEqual` actions.
Regex capture groups are available.
type: string
type: object
@ -421,10 +418,8 @@ spec:
description: |-
`oauth2` configures the OAuth2 settings to use when scraping the target.
It requires Prometheus >= 2.27.0.
Cannot be set at the same time as `authorization`, or `basicAuth`.
properties:
clientId:
@ -446,9 +441,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -472,9 +465,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -501,9 +492,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -520,12 +509,232 @@ spec:
`endpointParams` configures the HTTP parameters to append to the token
URL.
type: object
noProxy:
description: |-
`noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names
that should be excluded from proxying. IP and domain names can
contain port numbers.
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: string
proxyConnectHeader:
additionalProperties:
items:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: array
description: |-
ProxyConnectHeader optionally specifies headers to send to
proxies during CONNECT requests.
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: object
x-kubernetes-map-type: atomic
proxyFromEnvironment:
description: |-
Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY).
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: boolean
proxyUrl:
description: '`proxyURL` defines the HTTP proxy server to
use.'
pattern: ^http(s)?://.+$
type: string
scopes:
description: '`scopes` defines the OAuth2 scopes used for
the token request.'
items:
type: string
type: array
tlsConfig:
description: |-
TLS configuration to use when connecting to the OAuth2 server.
It requires Prometheus >= v2.43.0.
properties:
ca:
description: Certificate authority used when verifying
server certificates.
properties:
configMap:
description: ConfigMap containing data to use for
the targets.
properties:
key:
description: The key to select.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the ConfigMap or
its key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
secret:
description: Secret containing data to use for the
targets.
properties:
key:
description: The key of the secret to select
from. Must be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its
key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
cert:
description: Client certificate to present when doing
client-authentication.
properties:
configMap:
description: ConfigMap containing data to use for
the targets.
properties:
key:
description: The key to select.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the ConfigMap or
its key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
secret:
description: Secret containing data to use for the
targets.
properties:
key:
description: The key of the secret to select
from. Must be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its
key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
insecureSkipVerify:
description: Disable target certificate validation.
type: boolean
keySecret:
description: Secret containing the client key file for
the targets.
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
maxVersion:
description: |-
Maximum acceptable TLS version.
It requires Prometheus >= v2.41.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
minVersion:
description: |-
Minimum acceptable TLS version.
It requires Prometheus >= v2.35.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
serverName:
description: Used to verify the hostname for the targets.
type: string
type: object
tokenUrl:
description: '`tokenURL` configures the URL to fetch the
token from.'
@ -547,14 +756,12 @@ spec:
description: |-
HTTP path from which to scrape for metrics.
If empty, Prometheus uses the default value (e.g. `/metrics`).
type: string
port:
description: |-
Name of the Pod port which this endpoint refers to.
It takes precedence over `targetPort`.
type: string
proxyUrl:
@ -567,20 +774,16 @@ spec:
`relabelings` configures the relabeling rules to apply the target's
metadata labels.
The Operator automatically adds relabelings for a few standard Kubernetes fields.
The original scrape job's name is available via the `__tmp_prometheus_job_name` label.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
items:
description: |-
RelabelConfig allows dynamic rewriting of the label set for targets, alerts,
scraped samples and remote write samples.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
properties:
action:
@ -588,11 +791,9 @@ spec:
description: |-
Action to perform based on the regex matching.
`Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0.
`DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0.
Default: "Replace"
enum:
- replace
@ -622,7 +823,6 @@ spec:
description: |-
Modulus to take of the hash of the source label values.
Only applicable when the action is `HashMod`.
format: int64
type: integer
@ -635,7 +835,6 @@ spec:
Replacement value against which a Replace action is performed if the
regular expression matches.
Regex capture groups are available.
type: string
separator:
@ -658,11 +857,9 @@ spec:
description: |-
Label to which the resulting string is written in a replacement.
It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`,
`KeepEqual` and `DropEqual` actions.
Regex capture groups are available.
type: string
type: object
@ -671,11 +868,9 @@ spec:
description: |-
HTTP scheme to use for scraping.
`http` and `https` are the expected values unless you rewrite the
`__scheme__` label via relabeling.
If empty, Prometheus uses the default value `http`.
enum:
- http
@ -685,7 +880,6 @@ spec:
description: |-
Timeout after which Prometheus considers the scrape to be failed.
If empty, Prometheus uses the global scrape timeout unless it is less
than the target's scrape interval value in which the latter is used.
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
@ -698,7 +892,6 @@ spec:
Name or number of the target port of the `Pod` object behind the Service, the
port must be specified with container port property.
Deprecated: use 'port' instead.
x-kubernetes-int-or-string: true
tlsConfig:
@ -722,9 +915,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -748,9 +939,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -778,9 +967,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -804,9 +991,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -835,9 +1020,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -847,6 +1030,28 @@ spec:
- key
type: object
x-kubernetes-map-type: atomic
maxVersion:
description: |-
Maximum acceptable TLS version.
It requires Prometheus >= v2.41.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
minVersion:
description: |-
Minimum acceptable TLS version.
It requires Prometheus >= v2.35.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
serverName:
description: Used to verify the hostname for the targets.
type: string
@ -857,7 +1062,6 @@ spec:
the metrics that have an explicit timestamp present in scraped data.
Has no effect if `honorTimestamps` is false.
It requires Prometheus >= v2.48.0.
type: boolean
type: object
@ -879,15 +1083,18 @@ spec:
description: The scrape class to apply.
minLength: 1
type: string
scrapeClassicHistograms:
description: |-
Whether to scrape a classic histogram that is also exposed as a native histogram.
It requires Prometheus >= v2.45.0.
type: boolean
scrapeProtocols:
description: |-
`scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the
protocols supported by Prometheus in order of preference (from most to least preferred).
If unset, Prometheus uses its default value.
It requires Prometheus >= v2.49.0.
items:
description: |-
@ -906,7 +1113,8 @@ spec:
type: array
x-kubernetes-list-type: set
selector:
description: Label selector to select the Kubernetes `Pod` objects.
description: Label selector to select the Kubernetes `Pod` objects
to scrape metrics from.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.

368
k8s.tjo.cloud/modules/cluster-core/manifests/crd-servicemonitors.yaml
View file

@ -1,11 +1,11 @@
# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.78.2/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.15.0
operator.prometheus.io/version: 0.75.1
controller-gen.kubebuilder.io/version: v0.16.4
operator.prometheus.io/version: 0.78.2
name: servicemonitors.monitoring.coreos.com
spec:
group: monitoring.coreos.com
@ -23,7 +23,15 @@ spec:
- name: v1
schema:
openAPIV3Schema:
description: ServiceMonitor defines monitoring for a set of services.
description: |-
The `ServiceMonitor` custom resource definition (CRD) defines how `Prometheus` and `PrometheusAgent` can scrape metrics from a group of services.
Among other things, it allows to specify:
* The services to scrape via label selectors.
* The container ports to scrape.
* Authentication credentials to use.
* Target and metric relabeling.
`Prometheus` and `PrometheusAgent` objects select `ServiceMonitor` objects using label and namespace selectors.
properties:
apiVersion:
description: |-
@ -52,13 +60,15 @@ spec:
`attachMetadata` defines additional metadata which is added to the
discovered targets.
It requires Prometheus >= v2.37.0.
properties:
node:
description: |-
When set to true, Prometheus must have the `get` permission on the
`Nodes` objects.
When set to true, Prometheus attaches node metadata to the discovered
targets.
The Prometheus service account must have the `list` and `watch`
permissions on the `Nodes` objects.
type: boolean
type: object
bodySizeLimit:
@ -66,12 +76,14 @@ spec:
When defined, bodySizeLimit specifies a job level limit on the size
of uncompressed response body that will be accepted by Prometheus.
It requires Prometheus >= v2.28.0.
pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$
type: string
endpoints:
description: List of endpoints part of this ServiceMonitor.
description: |-
List of endpoints part of this ServiceMonitor.
Defines how to scrape metrics from Kubernetes [Endpoints](https://kubernetes.io/docs/concepts/services-networking/service/#endpoints) objects.
In most cases, an Endpoints object is backed by a Kubernetes [Service](https://kubernetes.io/docs/concepts/services-networking/service/) object with the same name and labels.
items:
description: |-
Endpoint defines an endpoint serving Prometheus metrics to be scraped by
@ -82,7 +94,6 @@ spec:
`authorization` configures the Authorization header credentials to use when
scraping the target.
Cannot be set at the same time as `basicAuth`, or `oauth2`.
properties:
credentials:
@ -100,9 +111,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -116,10 +125,8 @@ spec:
description: |-
Defines the authentication type. The value is case-insensitive.
"Basic" is not a supported value.
Default: "Bearer"
type: string
type: object
@ -128,7 +135,6 @@ spec:
`basicAuth` configures the Basic Authentication credentials to use when
scraping the target.
Cannot be set at the same time as `authorization`, or `oauth2`.
properties:
password:
@ -147,9 +153,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -175,9 +179,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -192,7 +194,6 @@ spec:
description: |-
File to read bearer token for scraping the target.
Deprecated: use `authorization` instead.
type: string
bearerTokenSecret:
@ -201,7 +202,6 @@ spec:
token for scraping targets. The secret needs to be in the same namespace
as the ServiceMonitor object and readable by the Prometheus Operator.
Deprecated: use `authorization` instead.
properties:
key:
@ -215,9 +215,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -236,10 +234,8 @@ spec:
When true, the pods which are not running (e.g. either in Failed or
Succeeded state) are dropped during the target discovery.
If unset, the filtering is enabled.
More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase
type: boolean
followRedirects:
@ -261,7 +257,6 @@ spec:
description: |-
Interval at which Prometheus scrapes the metrics from the target.
If empty, Prometheus uses the global scrape interval.
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
type: string
@ -274,7 +269,6 @@ spec:
RelabelConfig allows dynamic rewriting of the label set for targets, alerts,
scraped samples and remote write samples.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
properties:
action:
@ -282,11 +276,9 @@ spec:
description: |-
Action to perform based on the regex matching.
`Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0.
`DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0.
Default: "Replace"
enum:
- replace
@ -316,7 +308,6 @@ spec:
description: |-
Modulus to take of the hash of the source label values.
Only applicable when the action is `HashMod`.
format: int64
type: integer
@ -329,7 +320,6 @@ spec:
Replacement value against which a Replace action is performed if the
regular expression matches.
Regex capture groups are available.
type: string
separator:
@ -352,11 +342,9 @@ spec:
description: |-
Label to which the resulting string is written in a replacement.
It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`,
`KeepEqual` and `DropEqual` actions.
Regex capture groups are available.
type: string
type: object
@ -365,10 +353,8 @@ spec:
description: |-
`oauth2` configures the OAuth2 settings to use when scraping the target.
It requires Prometheus >= 2.27.0.
Cannot be set at the same time as `authorization`, or `basicAuth`.
properties:
clientId:
@ -390,9 +376,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -416,9 +400,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -445,9 +427,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -464,12 +444,232 @@ spec:
`endpointParams` configures the HTTP parameters to append to the token
URL.
type: object
noProxy:
description: |-
`noProxy` is a comma-separated string that can contain IPs, CIDR notation, domain names
that should be excluded from proxying. IP and domain names can
contain port numbers.
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: string
proxyConnectHeader:
additionalProperties:
items:
description: SecretKeySelector selects a key of a Secret.
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: array
description: |-
ProxyConnectHeader optionally specifies headers to send to
proxies during CONNECT requests.
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: object
x-kubernetes-map-type: atomic
proxyFromEnvironment:
description: |-
Whether to use the proxy configuration defined by environment variables (HTTP_PROXY, HTTPS_PROXY, and NO_PROXY).
It requires Prometheus >= v2.43.0 or Alertmanager >= 0.25.0.
type: boolean
proxyUrl:
description: '`proxyURL` defines the HTTP proxy server to
use.'
pattern: ^http(s)?://.+$
type: string
scopes:
description: '`scopes` defines the OAuth2 scopes used for
the token request.'
items:
type: string
type: array
tlsConfig:
description: |-
TLS configuration to use when connecting to the OAuth2 server.
It requires Prometheus >= v2.43.0.
properties:
ca:
description: Certificate authority used when verifying
server certificates.
properties:
configMap:
description: ConfigMap containing data to use for
the targets.
properties:
key:
description: The key to select.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the ConfigMap or
its key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
secret:
description: Secret containing data to use for the
targets.
properties:
key:
description: The key of the secret to select
from. Must be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its
key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
cert:
description: Client certificate to present when doing
client-authentication.
properties:
configMap:
description: ConfigMap containing data to use for
the targets.
properties:
key:
description: The key to select.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the ConfigMap or
its key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
secret:
description: Secret containing data to use for the
targets.
properties:
key:
description: The key of the secret to select
from. Must be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its
key must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
type: object
insecureSkipVerify:
description: Disable target certificate validation.
type: boolean
keySecret:
description: Secret containing the client key file for
the targets.
properties:
key:
description: The key of the secret to select from. Must
be a valid secret key.
type: string
name:
default: ""
description: |-
Name of the referent.
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
type: string
optional:
description: Specify whether the Secret or its key
must be defined
type: boolean
required:
- key
type: object
x-kubernetes-map-type: atomic
maxVersion:
description: |-
Maximum acceptable TLS version.
It requires Prometheus >= v2.41.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
minVersion:
description: |-
Minimum acceptable TLS version.
It requires Prometheus >= v2.35.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
serverName:
description: Used to verify the hostname for the targets.
type: string
type: object
tokenUrl:
description: '`tokenURL` configures the URL to fetch the
token from.'
@ -491,14 +691,12 @@ spec:
description: |-
HTTP path from which to scrape for metrics.
If empty, Prometheus uses the default value (e.g. `/metrics`).
type: string
port:
description: |-
Name of the Service port which this endpoint refers to.
It takes precedence over `targetPort`.
type: string
proxyUrl:
@ -511,20 +709,16 @@ spec:
`relabelings` configures the relabeling rules to apply the target's
metadata labels.
The Operator automatically adds relabelings for a few standard Kubernetes fields.
The original scrape job's name is available via the `__tmp_prometheus_job_name` label.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
items:
description: |-
RelabelConfig allows dynamic rewriting of the label set for targets, alerts,
scraped samples and remote write samples.
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config
properties:
action:
@ -532,11 +726,9 @@ spec:
description: |-
Action to perform based on the regex matching.
`Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0.
`DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0.
Default: "Replace"
enum:
- replace
@ -566,7 +758,6 @@ spec:
description: |-
Modulus to take of the hash of the source label values.
Only applicable when the action is `HashMod`.
format: int64
type: integer
@ -579,7 +770,6 @@ spec:
Replacement value against which a Replace action is performed if the
regular expression matches.
Regex capture groups are available.
type: string
separator:
@ -602,11 +792,9 @@ spec:
description: |-
Label to which the resulting string is written in a replacement.
It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`,
`KeepEqual` and `DropEqual` actions.
Regex capture groups are available.
type: string
type: object
@ -615,11 +803,9 @@ spec:
description: |-
HTTP scheme to use for scraping.
`http` and `https` are the expected values unless you rewrite the
`__scheme__` label via relabeling.
If empty, Prometheus uses the default value `http`.
enum:
- http
@ -629,7 +815,6 @@ spec:
description: |-
Timeout after which Prometheus considers the scrape to be failed.
If empty, Prometheus uses the global scrape timeout unless it is less
than the target's scrape interval value in which the latter is used.
pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$
@ -663,9 +848,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -689,9 +872,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -723,9 +904,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the ConfigMap or its
@ -749,9 +928,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key
@ -788,9 +965,7 @@ spec:
This field is effectively required, but due to backwards compatibility is
allowed to be empty. Instances of this type with an empty value here are
almost certainly wrong.
TODO: Add other useful fields. apiVersion, kind, uid?
More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896.
type: string
optional:
description: Specify whether the Secret or its key must
@ -800,6 +975,28 @@ spec:
- key
type: object
x-kubernetes-map-type: atomic
maxVersion:
description: |-
Maximum acceptable TLS version.
It requires Prometheus >= v2.41.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
minVersion:
description: |-
Minimum acceptable TLS version.
It requires Prometheus >= v2.35.0.
enum:
- TLS10
- TLS11
- TLS12
- TLS13
type: string
serverName:
description: Used to verify the hostname for the targets.
type: string
@ -810,7 +1007,6 @@ spec:
the metrics that have an explicit timestamp present in scraped data.
Has no effect if `honorTimestamps` is false.
It requires Prometheus >= v2.48.0.
type: boolean
type: object
@ -820,12 +1016,10 @@ spec:
`jobLabel` selects the label from the associated Kubernetes `Service`
object which will be used as the `job` label for all metrics.
For example if `jobLabel` is set to `foo` and the Kubernetes `Service`
object is labeled with `foo: bar`, then Prometheus adds the `job="bar"`
label to all ingested metrics.
If the value of this field is empty or if the label doesn't exist for
the given Service, the `job` label of the metrics defaults to the name
of the associated Kubernetes `Service`.
@ -835,7 +1029,6 @@ spec:
Per-scrape limit on the number of targets dropped by relabeling
that will be kept in memory. 0 means no limit.
It requires Prometheus >= v2.47.0.
format: int64
type: integer
@ -843,7 +1036,6 @@ spec:
description: |-
Per-scrape limit on number of labels that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
@ -851,7 +1043,6 @@ spec:
description: |-
Per-scrape limit on length of labels name that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
@ -859,14 +1050,13 @@ spec:
description: |-
Per-scrape limit on length of labels value that will be accepted for a sample.
It requires Prometheus >= v2.27.0.
format: int64
type: integer
namespaceSelector:
description: |-
Selector to select which namespaces the Kubernetes `Endpoints` objects
are discovered from.
`namespaceSelector` defines in which namespace(s) Prometheus should discover the services.
By default, the services are discovered in the same namespace as the `ServiceMonitor` object but it is possible to select pods across different/all namespaces.
properties:
any:
description: |-
@ -879,6 +1069,23 @@ spec:
type: string
type: array
type: object
nativeHistogramBucketLimit:
description: |-
If there are more than this many buckets in a native histogram,
buckets will be merged to stay within the limit.
It requires Prometheus >= v2.45.0.
format: int64
type: integer
nativeHistogramMinBucketFactor:
anyOf:
- type: integer
- type: string
description: |-
If the growth factor of one bucket to the next is smaller than this,
buckets will be merged to increase the factor sufficiently.
It requires Prometheus >= v2.50.0.
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
x-kubernetes-int-or-string: true
podTargetLabels:
description: |-
`podTargetLabels` defines the labels which are transferred from the
@ -896,15 +1103,18 @@ spec:
description: The scrape class to apply.
minLength: 1
type: string
scrapeClassicHistograms:
description: |-
Whether to scrape a classic histogram that is also exposed as a native histogram.
It requires Prometheus >= v2.45.0.
type: boolean
scrapeProtocols:
description: |-
`scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the
protocols supported by Prometheus in order of preference (from most to least preferred).
If unset, Prometheus uses its default value.
It requires Prometheus >= v2.49.0.
items:
description: |-
@ -923,7 +1133,8 @@ spec:
type: array
x-kubernetes-list-type: set
selector:
description: Label selector to select the Kubernetes `Endpoints` objects.
description: Label selector to select the Kubernetes `Endpoints` objects
to scrape metrics from.
properties:
matchExpressions:
description: matchExpressions is a list of label selector requirements.
@ -982,6 +1193,7 @@ spec:
format: int64
type: integer
required:
- endpoints
- selector
type: object
required:

75
k8s.tjo.cloud/modules/cluster/main.tf
View file

@ -1,21 +1,16 @@
locals {
cluster_internal_endpoint = "https://${var.cluster.api.internal.domain}:${var.cluster.api.internal.port}"
cluster_public_endpoint = "https://${var.cluster.api.public.domain}:${var.cluster.api.public.port}"
public_domain = "${var.cluster.api.public.subdomain}.${var.cluster.api.public.domain}"
internal_domain = "${var.cluster.api.internal.subdomain}.${var.cluster.api.internal.domain}"
cluster_internal_endpoint = "https://${local.internal_domain}:${var.cluster.api.internal.port}"
cluster_public_endpoint = "https://${local.public_domain}:${var.cluster.api.public.port}"
podSubnets = [
"10.200.0.0/16",
"fd9b:5314:fc70::/56",
"10.0.240.0/22",
"fd74:6a6f:0:f000::/54",
]
serviceSubnets = [
"10.201.0.0/16",
"fd9b:5314:fc71::/112",
]
# Nodes will use IPs from this subnets
# for communication between each other.
tailscaleSubnets = [
"100.64.0.0/10",
"fd7a:115c:a1e0::/96"
"10.0.244.0/22",
"fd74:6a6f:0:f400::/54",
]
talos_controlplane_config = {
@ -35,15 +30,11 @@ locals {
}
}
cluster = {
etcd = {
advertisedSubnets = local.tailscaleSubnets
listenSubnets = local.tailscaleSubnets
}
allowSchedulingOnControlPlanes = var.allow_scheduling_on_control_planes,
apiServer = {
certSANs = [
var.cluster.api.internal.domain,
var.cluster.api.public.domain,
local.public_domain,
local.internal_domain,
]
extraArgs = {
"oidc-issuer-url" = "https://id.tjo.space/application/o/k8stjocloud/",
@ -118,9 +109,6 @@ locals {
}
machine = {
kubelet = {
nodeIP = {
validSubnets = local.tailscaleSubnets
}
extraArgs = {
rotate-server-certificates = true
cloud-provider = "external"
@ -144,28 +132,15 @@ locals {
}
}
nodeLabels = {
"k8s.tjo.cloud/public" = node.public ? "true" : "false"
"k8s.tjo.cloud/host" = node.host
"k8s.tjo.cloud/proxmox" = var.proxmox.name
}
sysctls = {
"net.ipv4.ip_forward" = "1"
"net.ipv6.conf.all.forwarding" = "1"
nodeAnnotations = {
"network.cilium.io/ipv4-pod-cidr" : node.pod_cidr.ipv4
"network.cilium.io/ipv6-pod-cidr" : node.pod_cidr.ipv6
}
}
}),
yamlencode(
{
apiVersion = "v1alpha1"
kind = "ExtensionServiceConfig"
name = "tailscale"
environment = [
"TS_AUTHKEY=${var.tailscale_authkey}",
"TS_HOSTNAME=${node.name}",
"TS_ROUTES=${join(",", local.podSubnets)},${join(",", local.serviceSubnets)}",
#"TS_EXTRA_ARGS=--accept-routes",
]
})
]
}
}
@ -244,7 +219,7 @@ resource "talos_machine_bootstrap" "this" {
client_configuration = talos_machine_secrets.this.client_configuration
}
data "talos_cluster_kubeconfig" "this" {
resource "talos_cluster_kubeconfig" "this" {
depends_on = [
talos_machine_bootstrap.this
]
@ -254,7 +229,7 @@ data "talos_cluster_kubeconfig" "this" {
}
resource "local_file" "kubeconfig" {
content = data.talos_cluster_kubeconfig.this.kubeconfig_raw
content = talos_cluster_kubeconfig.this.kubeconfig_raw
filename = "${path.root}/admin.kubeconfig"
lifecycle {
@ -276,3 +251,23 @@ resource "local_file" "talosconfig" {
content = nonsensitive(data.talos_client_configuration.this[0].talos_config)
filename = "${path.root}/admin.talosconfig"
}
resource "digitalocean_record" "api-internal-ipv4" {
for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
domain = var.cluster.api.internal.domain
type = "A"
name = var.cluster.api.internal.subdomain
value = each.value.ipv4
ttl = 30
}
resource "digitalocean_record" "api-internal-ipv6" {
for_each = { for k, v in local.nodes_with_address : k => v if v.type == "controlplane" }
domain = var.cluster.api.internal.domain
type = "AAAA"
name = var.cluster.api.internal.subdomain
value = each.value.ipv6
ttl = 30
}

2
k8s.tjo.cloud/modules/cluster/outputs.tf
View file

@ -10,7 +10,7 @@ output "api" {
public : merge(var.cluster.api.public, {
endpoint : local.cluster_public_endpoint,
}),
ca : data.talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
ca : talos_cluster_kubeconfig.this.kubernetes_client_configuration.ca_certificate,
})
}

10
k8s.tjo.cloud/modules/cluster/proxmox.tf
View file

@ -1,7 +1,7 @@
locals {
nodes_with_names = {
for k, v in var.nodes : k => merge(v, {
id = 1000 + index(keys(var.nodes), k)
id = 6000 + index(keys(var.nodes), k)
name = replace("${k}.${v.type}.${var.cluster.name}", ".", "-")
})
}
@ -75,11 +75,7 @@ resource "proxmox_virtual_environment_vm" "nodes" {
node_name = each.value.host
description = "Node ${each.value.name} for cluster ${var.cluster.name}."
tags = concat(
["kubernetes", "terraform"],
each.value.public ? ["public"] : ["private"],
[each.value.type]
)
tags = ["kubernetes.tjo.cloud", each.value.type]
stop_on_destroy = true
timeout_start_vm = 60
@ -106,7 +102,7 @@ resource "proxmox_virtual_environment_vm" "nodes" {
}
network_device {
bridge = each.value.public ? "vmpublic0" : "vmprivate0"
bridge = "vmbr1"
mac_address = each.value.mac_address
}

32
k8s.tjo.cloud/modules/cluster/variables.tf
View file

@ -1,21 +1,25 @@
variable "nodes" {
type = map(object({
public = bool
type = string
host = string
type = string
host = string
cores = optional(number, 4)
memory = optional(number, 4096)
storage = string
boot_size = optional(number, 32)
pod_cidr = object({
ipv4 = string
ipv6 = string
})
}))
}
variable "talos" {
type = object({
version = optional(string, "v1.7.5")
kubernetes = optional(string, "v1.30.0")
version = optional(string, "v1.8.3")
kubernetes = optional(string, "v1.31.0")
# Default is:
# customization:
@ -23,9 +27,8 @@ variable "talos" {
# officialExtensions:
# - siderolabs/kata-containers
# - siderolabs/qemu-guest-agent
# - siderolabs/tailscale
# - siderolabs/wasmedge
schematic_id = optional(string, "a125b6d6becb63df5543edfae1231e351723dd6e4d551ba73e0f30229ad6ff59")
schematic_id = optional(string, "392092063ce5c8be7dfeba0bd466add2bc0b55a20939cc2c0060058fcc25d784")
})
}
@ -41,12 +44,14 @@ variable "cluster" {
name = string
api = optional(object({
internal = optional(object({
domain = optional(string, "api.internal.k8s.tjo.cloud")
port = optional(number, 6443)
domain = optional(string, "k8s.tjo.cloud")
subdomain = optional(string, "api.internal")
port = optional(number, 6443)
}), {})
public = optional(object({
domain = optional(string, "api.k8s.tjo.cloud")
port = optional(number, 443)
domain = optional(string, "k8s.tjo.cloud")
subdomain = optional(string, "api")
port = optional(number, 443)
}), {})
}), {})
oidc = object({
@ -56,11 +61,6 @@ variable "cluster" {
})
}
variable "tailscale_authkey" {
type = string
sensitive = true
}
variable "proxmox" {
type = object({
name = string

2
k8s.tjo.cloud/modules/cluster/versions.tf
View file

@ -8,7 +8,7 @@ terraform {
}
talos = {
source = "siderolabs/talos"
version = "0.5.0"
version = "0.6.1"
}
local = {
source = "hashicorp/local"

34
k8s.tjo.cloud/terraform.tf
View file

@ -6,7 +6,7 @@ terraform {
}
talos = {
source = "siderolabs/talos"
version = "0.5.0"
version = "0.6.1"
}
local = {
source = "hashicorp/local"
@ -28,10 +28,6 @@ terraform {
source = "hashicorp/kubernetes"
version = "2.31.0"
}
tailscale = {
source = "tailscale/tailscale"
version = "0.16.1"
}
}
required_version = "~> 1.7.3"
@ -45,6 +41,30 @@ provider "proxmox" {
ssh {
agent = true
username = "root"
node {
name = "batuu"
address = "batuu.system.tjo.cloud"
port = 22
}
node {
name = "jakku"
address = "jakku.system.tjo.cloud"
port = 22
}
node {
name = "nevaroo"
address = "nevaroo.system.tjo.cloud"
port = 22
}
node {
name = "mustafar"
address = "mustafar.system.tjo.cloud"
port = 22
}
}
}
@ -52,10 +72,6 @@ provider "digitalocean" {
token = var.digitalocean_token
}
provider "tailscale" {
api_key = var.tailscale_apikey
}
provider "helm" {
alias = "template"
}

5
k8s.tjo.cloud/variables.tf
View file

@ -1,8 +1,3 @@
variable "tailscale_apikey" {
type = string
sensitive = true
}
variable "oidc_username" {
type = string
}