From 0f179e4c0eb8950ed4debddbf55974ccf6c31288 Mon Sep 17 00:00:00 2001 From: Tine Date: Fri, 26 Jul 2024 23:40:38 +0200 Subject: [PATCH] feat: initial monitoring setup --- justfile | 14 +- k8s.tjo.cloud/kubeconfig | 26 + modules/cluster-components/dashboard.tf | 14 +- modules/cluster-components/gateway.tf | 9 + modules/cluster-core/main.tf | 16 + .../manifests/crd-podmonitors.yaml | 967 +++++++++++++++++ .../manifests/crd-servicemonitors.yaml | 991 ++++++++++++++++++ modules/cluster-core/monitoring.tf | 231 ++++ modules/cluster/main.tf | 4 - modules/cluster/manifests/metrics-server.yaml | 201 ---- 10 files changed, 2258 insertions(+), 215 deletions(-) create mode 100755 k8s.tjo.cloud/kubeconfig create mode 100644 modules/cluster-core/manifests/crd-podmonitors.yaml create mode 100644 modules/cluster-core/manifests/crd-servicemonitors.yaml create mode 100644 modules/cluster-core/monitoring.tf delete mode 100644 modules/cluster/manifests/metrics-server.yaml diff --git a/justfile b/justfile index fccc278..8daf5f9 100644 --- a/justfile +++ b/justfile @@ -8,17 +8,23 @@ lint: @tflint --recursive GATEWAY_API_VERSION := "v1.1.0" -METRICS_SERVER_VERSION := "v0.7.1" +PROMETHEUS_CRDS_VERSION := "main" modules-cluster-manifests: @rm -rf modules/cluster/manifests @mkdir -p modules/cluster/manifests @curl -L -o modules/cluster/manifests/gateway-api.crds.yaml \ "https://github.com/kubernetes-sigs/gateway-api/releases/download/{{GATEWAY_API_VERSION}}/experimental-install.yaml" - @curl -L -o modules/cluster/manifests/metrics-server.yaml \ - "https://github.com/kubernetes-sigs/metrics-server/releases/download/{{METRICS_SERVER_VERSION}}/components.yaml" -k8s-apply: modules-cluster-manifests +module-cluster-core-manifests: + @rm -rf modules/cluster-core/manifests + @mkdir -p modules/cluster-core/manifests + @curl -L -o modules/cluster-core/manifests/crd-podmonitors.yaml \ + "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-podmonitors.yaml" + @curl -L -o modules/cluster-core/manifests/crd-servicemonitors.yaml \ + "https://raw.githubusercontent.com/prometheus-community/helm-charts/{{PROMETHEUS_CRDS_VERSION}}/charts/kube-prometheus-stack/charts/crds/crds/crd-servicemonitors.yaml" + +k8s-apply: modules-cluster-manifests module-cluster-core-manifests tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud init tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud apply -target module.cluster tofu -chdir={{justfile_directory()}}/k8s.tjo.cloud apply -target module.cluster-core diff --git a/k8s.tjo.cloud/kubeconfig b/k8s.tjo.cloud/kubeconfig new file mode 100755 index 0000000..507f82e --- /dev/null +++ b/k8s.tjo.cloud/kubeconfig @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Config +clusters: +- name: tjo-cloud + cluster: + server: https://api.k8s.tjo.cloud:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJpVENDQVMrZ0F3SUJBZ0lRUFlySjYzMngyaUJXQnRCeWZOTVB4akFLQmdncWhrak9QUVFEQWpBVk1STXcKRVFZRFZRUUtFd3ByZFdKbGNtNWxkR1Z6TUI0WERUSTBNRGN5TlRFMU1qWXlNMW9YRFRNME1EY3lNekUxTWpZeQpNMW93RlRFVE1CRUdBMVVFQ2hNS2EzVmlaWEp1WlhSbGN6QlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VICkEwSUFCR2NCTm84UXQ0djUrdnloWVl2SU5UcEllVTgvNFVjUU9jOUJHQm53OU9ITWEwUGU1MnhoYS9mdGl0L1kKZTBkVFBybzNMdjNYREFVMmN6NC9JVjBOVWVXallUQmZNQTRHQTFVZER3RUIvd1FFQXdJQ2hEQWRCZ05WSFNVRQpGakFVQmdnckJnRUZCUWNEQVFZSUt3WUJCUVVIQXdJd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFCkZnUVVwbC9NbWlvRUdVTDFScFd4TVZpakNXdmsxOHd3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQU41b0lTeWMKNVZhc2IvMmxQbmlBSVBkVWF4OHdJMldJVjlyMHd1SEVnVWNmQWlBL2RsMFZocEhGSUlEcldEajFBeGtHNEZXbgpUNHJ4K0IvRFBiQnhsM2VmclE9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== +contexts: +- name: oidc@tjo-cloud + context: + cluster: tjo-cloud + namespace: default + user: oidc +current-context: oidc@tjo-cloud +users: +- name: oidc + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + command: kubectl + args: + - oidc-login + - get-token + - --oidc-issuer-url=https://id.tjo.space/application/o/k8stjocloud/ + - --oidc-client-id=HAI6rW0EWtgmSPGKAJ3XXzubQTUut2GMeTRS2spg + - --oidc-extra-scope=profile diff --git a/modules/cluster-components/dashboard.tf b/modules/cluster-components/dashboard.tf index 1617e9f..39ccae2 100644 --- a/modules/cluster-components/dashboard.tf +++ b/modules/cluster-components/dashboard.tf @@ -1,9 +1,11 @@ resource "helm_release" "dashboard" { - name = "kubernetes-dashboard" - repository = "https://kubernetes.github.io/dashboard" - chart = "kubernetes-dashboard" - version = "7.5.0" - namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + name = "kubernetes-dashboard" + repository = "https://kubernetes.github.io/dashboard" + chart = "kubernetes-dashboard" + version = "7.5.0" + namespace = kubernetes_namespace.tjo-cloud.metadata[0].name + atomic = true + cleanup_on_fail = true set { name = "kong.enabled" @@ -22,7 +24,7 @@ resource "kubernetes_manifest" "dashoard-http-route" { spec = { parentRefs = [ { - name : kubernetes_manifest.gateway.object.metadata.name + name = kubernetes_manifest.gateway.object.metadata.name } ] hostnames = [ diff --git a/modules/cluster-components/gateway.tf b/modules/cluster-components/gateway.tf index 41a783b..3bfdd23 100644 --- a/modules/cluster-components/gateway.tf +++ b/modules/cluster-components/gateway.tf @@ -49,9 +49,18 @@ resource "kubernetes_manifest" "gateway_class_config" { namespace = kubernetes_namespace.tjo-cloud.metadata[0].name } spec = { + mergeGateways = true provider = { type = "Kubernetes" kubernetes = { + envoyService = { + type = "LoadBalancer" + externalTrafficPolicy = "Local" + annotations = { + "io.cilium.nodeipam/match-node-labels" = "k8s.tjo.cloud/public=true" + } + loadBalancerClass = "io.cilium/node" + } envoyDaemonSet = { patch = { type = "StrategicMerge" diff --git a/modules/cluster-core/main.tf b/modules/cluster-core/main.tf index ca55321..dcdb796 100644 --- a/modules/cluster-core/main.tf +++ b/modules/cluster-core/main.tf @@ -26,3 +26,19 @@ resource "helm_release" "envoy" { atomic = true cleanup_on_fail = true } + +resource "helm_release" "metrics-server" { + name = "metrics-server" + chart = "metrics-server" + repository = "https://kubernetes-sigs.github.io/metrics-server/" + version = "3.11.0" + namespace = "kube-system" + atomic = true + cleanup_on_fail = true + + values = [<<-EOF + serviceMonitor: + enabled: true + EOF + ] +} diff --git a/modules/cluster-core/manifests/crd-podmonitors.yaml b/modules/cluster-core/manifests/crd-podmonitors.yaml new file mode 100644 index 0000000..ece0148 --- /dev/null +++ b/modules/cluster-core/manifests/crd-podmonitors.yaml @@ -0,0 +1,967 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + operator.prometheus.io/version: 0.75.1 + name: podmonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: PodMonitor + listKind: PodMonitorList + plural: podmonitors + shortNames: + - pmon + singular: podmonitor + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: PodMonitor defines monitoring for a set of pods. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of desired Pod selection for target discovery + by Prometheus. + properties: + attachMetadata: + description: |- + `attachMetadata` defines additional metadata which is added to the + discovered targets. + + + It requires Prometheus >= v2.37.0. + properties: + node: + description: |- + When set to true, Prometheus must have the `get` permission on the + `Nodes` objects. + type: boolean + type: object + bodySizeLimit: + description: |- + When defined, bodySizeLimit specifies a job level limit on the size + of uncompressed response body that will be accepted by Prometheus. + + + It requires Prometheus >= v2.28.0. + pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ + type: string + jobLabel: + description: |- + The label to use to retrieve the job name from. + `jobLabel` selects the label from the associated Kubernetes `Pod` + object which will be used as the `job` label for all metrics. + + + For example if `jobLabel` is set to `foo` and the Kubernetes `Pod` + object is labeled with `foo: bar`, then Prometheus adds the `job="bar"` + label to all ingested metrics. + + + If the value of this field is empty, the `job` label of the metrics + defaults to the namespace and name of the PodMonitor object (e.g. `/`). + type: string + keepDroppedTargets: + description: |- + Per-scrape limit on the number of targets dropped by relabeling + that will be kept in memory. 0 means no limit. + + + It requires Prometheus >= v2.47.0. + format: int64 + type: integer + labelLimit: + description: |- + Per-scrape limit on number of labels that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + labelNameLengthLimit: + description: |- + Per-scrape limit on length of labels name that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + labelValueLengthLimit: + description: |- + Per-scrape limit on length of labels value that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + namespaceSelector: + description: |- + Selector to select which namespaces the Kubernetes `Pods` objects + are discovered from. + properties: + any: + description: |- + Boolean describing whether all namespaces are selected in contrast to a + list restricting them. + type: boolean + matchNames: + description: List of namespace names to select from. + items: + type: string + type: array + type: object + podMetricsEndpoints: + description: List of endpoints part of this PodMonitor. + items: + description: |- + PodMetricsEndpoint defines an endpoint serving Prometheus metrics to be scraped by + Prometheus. + properties: + authorization: + description: |- + `authorization` configures the Authorization header credentials to use when + scraping the target. + + + Cannot be set at the same time as `basicAuth`, or `oauth2`. + properties: + credentials: + description: Selects a key of a Secret in the namespace + that contains the credentials for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + description: |- + Defines the authentication type. The value is case-insensitive. + + + "Basic" is not a supported value. + + + Default: "Bearer" + type: string + type: object + basicAuth: + description: |- + `basicAuth` configures the Basic Authentication credentials to use when + scraping the target. + + + Cannot be set at the same time as `authorization`, or `oauth2`. + properties: + password: + description: |- + `password` specifies a key of a Secret containing the password for + authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + description: |- + `username` specifies a key of a Secret containing the username for + authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenSecret: + description: |- + `bearerTokenSecret` specifies a key of a Secret containing the bearer + token for scraping targets. The secret needs to be in the same namespace + as the PodMonitor object and readable by the Prometheus Operator. + + + Deprecated: use `authorization` instead. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + enableHttp2: + description: '`enableHttp2` can be used to disable HTTP2 when + scraping the target.' + type: boolean + filterRunning: + description: |- + When true, the pods which are not running (e.g. either in Failed or + Succeeded state) are dropped during the target discovery. + + + If unset, the filtering is enabled. + + + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + type: boolean + followRedirects: + description: |- + `followRedirects` defines whether the scrape requests should follow HTTP + 3xx redirects. + type: boolean + honorLabels: + description: |- + When true, `honorLabels` preserves the metric's labels when they collide + with the target's labels. + type: boolean + honorTimestamps: + description: |- + `honorTimestamps` controls whether Prometheus preserves the timestamps + when exposed by the target. + type: boolean + interval: + description: |- + Interval at which Prometheus scrapes the metrics from the target. + + + If empty, Prometheus uses the global scrape interval. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + metricRelabelings: + description: |- + `metricRelabelings` configures the relabeling rules to apply to the + samples before ingestion. + items: + description: |- + RelabelConfig allows dynamic rewriting of the label set for targets, alerts, + scraped samples and remote write samples. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + properties: + action: + default: replace + description: |- + Action to perform based on the regex matching. + + + `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + + + Default: "Replace" + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + description: |- + Modulus to take of the hash of the source label values. + + + Only applicable when the action is `HashMod`. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. + type: string + replacement: + description: |- + Replacement value against which a Replace action is performed if the + regular expression matches. + + + Regex capture groups are available. + type: string + separator: + description: Separator is the string between concatenated + SourceLabels. + type: string + sourceLabels: + description: |- + The source labels select values from existing labels. Their content is + concatenated using the configured Separator and matched against the + configured regular expression. + items: + description: |- + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + description: |- + Label to which the resulting string is written in a replacement. + + + It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + `KeepEqual` and `DropEqual` actions. + + + Regex capture groups are available. + type: string + type: object + type: array + oauth2: + description: |- + `oauth2` configures the OAuth2 settings to use when scraping the target. + + + It requires Prometheus >= 2.27.0. + + + Cannot be set at the same time as `authorization`, or `basicAuth`. + properties: + clientId: + description: |- + `clientId` specifies a key of a Secret or ConfigMap containing the + OAuth2 client's ID. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + description: |- + `clientSecret` specifies a key of a Secret containing the OAuth2 + client's secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + description: |- + `endpointParams` configures the HTTP parameters to append to the token + URL. + type: object + scopes: + description: '`scopes` defines the OAuth2 scopes used for + the token request.' + items: + type: string + type: array + tokenUrl: + description: '`tokenURL` configures the URL to fetch the + token from.' + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + params: + additionalProperties: + items: + type: string + type: array + description: '`params` define optional HTTP URL parameters.' + type: object + path: + description: |- + HTTP path from which to scrape for metrics. + + + If empty, Prometheus uses the default value (e.g. `/metrics`). + type: string + port: + description: |- + Name of the Pod port which this endpoint refers to. + + + It takes precedence over `targetPort`. + type: string + proxyUrl: + description: |- + `proxyURL` configures the HTTP Proxy URL (e.g. + "http://proxyserver:2195") to go through when scraping the target. + type: string + relabelings: + description: |- + `relabelings` configures the relabeling rules to apply the target's + metadata labels. + + + The Operator automatically adds relabelings for a few standard Kubernetes fields. + + + The original scrape job's name is available via the `__tmp_prometheus_job_name` label. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + items: + description: |- + RelabelConfig allows dynamic rewriting of the label set for targets, alerts, + scraped samples and remote write samples. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + properties: + action: + default: replace + description: |- + Action to perform based on the regex matching. + + + `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + + + Default: "Replace" + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + description: |- + Modulus to take of the hash of the source label values. + + + Only applicable when the action is `HashMod`. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. + type: string + replacement: + description: |- + Replacement value against which a Replace action is performed if the + regular expression matches. + + + Regex capture groups are available. + type: string + separator: + description: Separator is the string between concatenated + SourceLabels. + type: string + sourceLabels: + description: |- + The source labels select values from existing labels. Their content is + concatenated using the configured Separator and matched against the + configured regular expression. + items: + description: |- + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + description: |- + Label to which the resulting string is written in a replacement. + + + It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + `KeepEqual` and `DropEqual` actions. + + + Regex capture groups are available. + type: string + type: object + type: array + scheme: + description: |- + HTTP scheme to use for scraping. + + + `http` and `https` are the expected values unless you rewrite the + `__scheme__` label via relabeling. + + + If empty, Prometheus uses the default value `http`. + enum: + - http + - https + type: string + scrapeTimeout: + description: |- + Timeout after which Prometheus considers the scrape to be failed. + + + If empty, Prometheus uses the global scrape timeout unless it is less + than the target's scrape interval value in which the latter is used. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the target port of the `Pod` object behind the Service, the + port must be specified with container port property. + + + Deprecated: use 'port' instead. + x-kubernetes-int-or-string: true + tlsConfig: + description: TLS configuration to use when scraping the target. + properties: + ca: + description: Certificate authority used when verifying server + certificates. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + cert: + description: Client certificate to present when doing client-authentication. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + trackTimestampsStaleness: + description: |- + `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + the metrics that have an explicit timestamp present in scraped data. + Has no effect if `honorTimestamps` is false. + + + It requires Prometheus >= v2.48.0. + type: boolean + type: object + type: array + podTargetLabels: + description: |- + `podTargetLabels` defines the labels which are transferred from the + associated Kubernetes `Pod` object onto the ingested metrics. + items: + type: string + type: array + sampleLimit: + description: |- + `sampleLimit` defines a per-scrape limit on the number of scraped samples + that will be accepted. + format: int64 + type: integer + scrapeClass: + description: The scrape class to apply. + minLength: 1 + type: string + scrapeProtocols: + description: |- + `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + protocols supported by Prometheus in order of preference (from most to least preferred). + + + If unset, Prometheus uses its default value. + + + It requires Prometheus >= v2.49.0. + items: + description: |- + ScrapeProtocol represents a protocol used by Prometheus for scraping metrics. + Supported values are: + * `OpenMetricsText0.0.1` + * `OpenMetricsText1.0.0` + * `PrometheusProto` + * `PrometheusText0.0.4` + enum: + - PrometheusProto + - OpenMetricsText0.0.1 + - OpenMetricsText1.0.0 + - PrometheusText0.0.4 + type: string + type: array + x-kubernetes-list-type: set + selector: + description: Label selector to select the Kubernetes `Pod` objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + targetLimit: + description: |- + `targetLimit` defines a limit on the number of scraped targets that will + be accepted. + format: int64 + type: integer + required: + - selector + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/modules/cluster-core/manifests/crd-servicemonitors.yaml b/modules/cluster-core/manifests/crd-servicemonitors.yaml new file mode 100644 index 0000000..dbd6710 --- /dev/null +++ b/modules/cluster-core/manifests/crd-servicemonitors.yaml @@ -0,0 +1,991 @@ +# https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.75.1/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.15.0 + operator.prometheus.io/version: 0.75.1 + name: servicemonitors.monitoring.coreos.com +spec: + group: monitoring.coreos.com + names: + categories: + - prometheus-operator + kind: ServiceMonitor + listKind: ServiceMonitorList + plural: servicemonitors + shortNames: + - smon + singular: servicemonitor + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ServiceMonitor defines monitoring for a set of services. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Specification of desired Service selection for target discovery by + Prometheus. + properties: + attachMetadata: + description: |- + `attachMetadata` defines additional metadata which is added to the + discovered targets. + + + It requires Prometheus >= v2.37.0. + properties: + node: + description: |- + When set to true, Prometheus must have the `get` permission on the + `Nodes` objects. + type: boolean + type: object + bodySizeLimit: + description: |- + When defined, bodySizeLimit specifies a job level limit on the size + of uncompressed response body that will be accepted by Prometheus. + + + It requires Prometheus >= v2.28.0. + pattern: (^0|([0-9]*[.])?[0-9]+((K|M|G|T|E|P)i?)?B)$ + type: string + endpoints: + description: List of endpoints part of this ServiceMonitor. + items: + description: |- + Endpoint defines an endpoint serving Prometheus metrics to be scraped by + Prometheus. + properties: + authorization: + description: |- + `authorization` configures the Authorization header credentials to use when + scraping the target. + + + Cannot be set at the same time as `basicAuth`, or `oauth2`. + properties: + credentials: + description: Selects a key of a Secret in the namespace + that contains the credentials for authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: + description: |- + Defines the authentication type. The value is case-insensitive. + + + "Basic" is not a supported value. + + + Default: "Bearer" + type: string + type: object + basicAuth: + description: |- + `basicAuth` configures the Basic Authentication credentials to use when + scraping the target. + + + Cannot be set at the same time as `authorization`, or `oauth2`. + properties: + password: + description: |- + `password` specifies a key of a Secret containing the password for + authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + username: + description: |- + `username` specifies a key of a Secret containing the username for + authentication. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + bearerTokenFile: + description: |- + File to read bearer token for scraping the target. + + + Deprecated: use `authorization` instead. + type: string + bearerTokenSecret: + description: |- + `bearerTokenSecret` specifies a key of a Secret containing the bearer + token for scraping targets. The secret needs to be in the same namespace + as the ServiceMonitor object and readable by the Prometheus Operator. + + + Deprecated: use `authorization` instead. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + enableHttp2: + description: '`enableHttp2` can be used to disable HTTP2 when + scraping the target.' + type: boolean + filterRunning: + description: |- + When true, the pods which are not running (e.g. either in Failed or + Succeeded state) are dropped during the target discovery. + + + If unset, the filtering is enabled. + + + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + type: boolean + followRedirects: + description: |- + `followRedirects` defines whether the scrape requests should follow HTTP + 3xx redirects. + type: boolean + honorLabels: + description: |- + When true, `honorLabels` preserves the metric's labels when they collide + with the target's labels. + type: boolean + honorTimestamps: + description: |- + `honorTimestamps` controls whether Prometheus preserves the timestamps + when exposed by the target. + type: boolean + interval: + description: |- + Interval at which Prometheus scrapes the metrics from the target. + + + If empty, Prometheus uses the global scrape interval. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + metricRelabelings: + description: |- + `metricRelabelings` configures the relabeling rules to apply to the + samples before ingestion. + items: + description: |- + RelabelConfig allows dynamic rewriting of the label set for targets, alerts, + scraped samples and remote write samples. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + properties: + action: + default: replace + description: |- + Action to perform based on the regex matching. + + + `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + + + Default: "Replace" + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + description: |- + Modulus to take of the hash of the source label values. + + + Only applicable when the action is `HashMod`. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. + type: string + replacement: + description: |- + Replacement value against which a Replace action is performed if the + regular expression matches. + + + Regex capture groups are available. + type: string + separator: + description: Separator is the string between concatenated + SourceLabels. + type: string + sourceLabels: + description: |- + The source labels select values from existing labels. Their content is + concatenated using the configured Separator and matched against the + configured regular expression. + items: + description: |- + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + description: |- + Label to which the resulting string is written in a replacement. + + + It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + `KeepEqual` and `DropEqual` actions. + + + Regex capture groups are available. + type: string + type: object + type: array + oauth2: + description: |- + `oauth2` configures the OAuth2 settings to use when scraping the target. + + + It requires Prometheus >= 2.27.0. + + + Cannot be set at the same time as `authorization`, or `basicAuth`. + properties: + clientId: + description: |- + `clientId` specifies a key of a Secret or ConfigMap containing the + OAuth2 client's ID. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + clientSecret: + description: |- + `clientSecret` specifies a key of a Secret containing the OAuth2 + client's secret. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + endpointParams: + additionalProperties: + type: string + description: |- + `endpointParams` configures the HTTP parameters to append to the token + URL. + type: object + scopes: + description: '`scopes` defines the OAuth2 scopes used for + the token request.' + items: + type: string + type: array + tokenUrl: + description: '`tokenURL` configures the URL to fetch the + token from.' + minLength: 1 + type: string + required: + - clientId + - clientSecret + - tokenUrl + type: object + params: + additionalProperties: + items: + type: string + type: array + description: params define optional HTTP URL parameters. + type: object + path: + description: |- + HTTP path from which to scrape for metrics. + + + If empty, Prometheus uses the default value (e.g. `/metrics`). + type: string + port: + description: |- + Name of the Service port which this endpoint refers to. + + + It takes precedence over `targetPort`. + type: string + proxyUrl: + description: |- + `proxyURL` configures the HTTP Proxy URL (e.g. + "http://proxyserver:2195") to go through when scraping the target. + type: string + relabelings: + description: |- + `relabelings` configures the relabeling rules to apply the target's + metadata labels. + + + The Operator automatically adds relabelings for a few standard Kubernetes fields. + + + The original scrape job's name is available via the `__tmp_prometheus_job_name` label. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + items: + description: |- + RelabelConfig allows dynamic rewriting of the label set for targets, alerts, + scraped samples and remote write samples. + + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + properties: + action: + default: replace + description: |- + Action to perform based on the regex matching. + + + `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + + + Default: "Replace" + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + description: |- + Modulus to take of the hash of the source label values. + + + Only applicable when the action is `HashMod`. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. + type: string + replacement: + description: |- + Replacement value against which a Replace action is performed if the + regular expression matches. + + + Regex capture groups are available. + type: string + separator: + description: Separator is the string between concatenated + SourceLabels. + type: string + sourceLabels: + description: |- + The source labels select values from existing labels. Their content is + concatenated using the configured Separator and matched against the + configured regular expression. + items: + description: |- + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + description: |- + Label to which the resulting string is written in a replacement. + + + It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + `KeepEqual` and `DropEqual` actions. + + + Regex capture groups are available. + type: string + type: object + type: array + scheme: + description: |- + HTTP scheme to use for scraping. + + + `http` and `https` are the expected values unless you rewrite the + `__scheme__` label via relabeling. + + + If empty, Prometheus uses the default value `http`. + enum: + - http + - https + type: string + scrapeTimeout: + description: |- + Timeout after which Prometheus considers the scrape to be failed. + + + If empty, Prometheus uses the global scrape timeout unless it is less + than the target's scrape interval value in which the latter is used. + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + targetPort: + anyOf: + - type: integer + - type: string + description: |- + Name or number of the target port of the `Pod` object behind the + Service. The port must be specified with the container's port property. + x-kubernetes-int-or-string: true + tlsConfig: + description: TLS configuration to use when scraping the target. + properties: + ca: + description: Certificate authority used when verifying server + certificates. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + caFile: + description: Path to the CA cert in the Prometheus container + to use for the targets. + type: string + cert: + description: Client certificate to present when doing client-authentication. + properties: + configMap: + description: ConfigMap containing data to use for the + targets. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + secret: + description: Secret containing data to use for the targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + certFile: + description: Path to the client cert file in the Prometheus + container for the targets. + type: string + insecureSkipVerify: + description: Disable target certificate validation. + type: boolean + keyFile: + description: Path to the client key file in the Prometheus + container for the targets. + type: string + keySecret: + description: Secret containing the client key file for the + targets. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + TODO: Add other useful fields. apiVersion, kind, uid? + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Drop `kubebuilder:default` when controller-gen doesn't need it https://github.com/kubernetes-sigs/kubebuilder/issues/3896. + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + serverName: + description: Used to verify the hostname for the targets. + type: string + type: object + trackTimestampsStaleness: + description: |- + `trackTimestampsStaleness` defines whether Prometheus tracks staleness of + the metrics that have an explicit timestamp present in scraped data. + Has no effect if `honorTimestamps` is false. + + + It requires Prometheus >= v2.48.0. + type: boolean + type: object + type: array + jobLabel: + description: |- + `jobLabel` selects the label from the associated Kubernetes `Service` + object which will be used as the `job` label for all metrics. + + + For example if `jobLabel` is set to `foo` and the Kubernetes `Service` + object is labeled with `foo: bar`, then Prometheus adds the `job="bar"` + label to all ingested metrics. + + + If the value of this field is empty or if the label doesn't exist for + the given Service, the `job` label of the metrics defaults to the name + of the associated Kubernetes `Service`. + type: string + keepDroppedTargets: + description: |- + Per-scrape limit on the number of targets dropped by relabeling + that will be kept in memory. 0 means no limit. + + + It requires Prometheus >= v2.47.0. + format: int64 + type: integer + labelLimit: + description: |- + Per-scrape limit on number of labels that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + labelNameLengthLimit: + description: |- + Per-scrape limit on length of labels name that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + labelValueLengthLimit: + description: |- + Per-scrape limit on length of labels value that will be accepted for a sample. + + + It requires Prometheus >= v2.27.0. + format: int64 + type: integer + namespaceSelector: + description: |- + Selector to select which namespaces the Kubernetes `Endpoints` objects + are discovered from. + properties: + any: + description: |- + Boolean describing whether all namespaces are selected in contrast to a + list restricting them. + type: boolean + matchNames: + description: List of namespace names to select from. + items: + type: string + type: array + type: object + podTargetLabels: + description: |- + `podTargetLabels` defines the labels which are transferred from the + associated Kubernetes `Pod` object onto the ingested metrics. + items: + type: string + type: array + sampleLimit: + description: |- + `sampleLimit` defines a per-scrape limit on the number of scraped samples + that will be accepted. + format: int64 + type: integer + scrapeClass: + description: The scrape class to apply. + minLength: 1 + type: string + scrapeProtocols: + description: |- + `scrapeProtocols` defines the protocols to negotiate during a scrape. It tells clients the + protocols supported by Prometheus in order of preference (from most to least preferred). + + + If unset, Prometheus uses its default value. + + + It requires Prometheus >= v2.49.0. + items: + description: |- + ScrapeProtocol represents a protocol used by Prometheus for scraping metrics. + Supported values are: + * `OpenMetricsText0.0.1` + * `OpenMetricsText1.0.0` + * `PrometheusProto` + * `PrometheusText0.0.4` + enum: + - PrometheusProto + - OpenMetricsText0.0.1 + - OpenMetricsText1.0.0 + - PrometheusText0.0.4 + type: string + type: array + x-kubernetes-list-type: set + selector: + description: Label selector to select the Kubernetes `Endpoints` objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + targetLabels: + description: |- + `targetLabels` defines the labels which are transferred from the + associated Kubernetes `Service` object onto the ingested metrics. + items: + type: string + type: array + targetLimit: + description: |- + `targetLimit` defines a limit on the number of scraped targets that will + be accepted. + format: int64 + type: integer + required: + - selector + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/modules/cluster-core/monitoring.tf b/modules/cluster-core/monitoring.tf new file mode 100644 index 0000000..5a7fd75 --- /dev/null +++ b/modules/cluster-core/monitoring.tf @@ -0,0 +1,231 @@ +resource "kubernetes_namespace" "monitoring-system" { + metadata { + name = "monitoring-system" + } +} + +resource "kubernetes_manifest" "prometheus-pod-monitors" { + manifest = yamldecode(file("${path.module}/manifests/crd-podmonitors.yaml")) +} + +resource "kubernetes_manifest" "prometheus-service-monitors" { + manifest = yamldecode(file("${path.module}/manifests/crd-servicemonitors.yaml")) +} + +resource "helm_release" "grafana-alloy" { + depends_on = [kubernetes_manifest.prometheus-pod-monitors, kubernetes_manifest.prometheus-service-monitors] + + name = "grafana-alloy-deamonset" + chart = "alloy" + repository = "https://grafana.github.io/helm-charts" + version = "0.5.1" + namespace = kubernetes_namespace.monitoring-system.metadata[0].name + atomic = true + cleanup_on_fail = true + + values = [<<-EOF + alloy: + extraEnv: + - name: "CLUSTER_NAME" + value: "tjo-cloud" + - name: "PROMETHEUS_CLIENT_ID" + value: "o6Tz2215HLvhvZ4RCZCR8oMmCapTu30iwkoMkz6m" + - name: "LOKI_CLIENT_ID" + value: "56TYXtgg7QwLjh4lPl1PTu3C4iExOvO1d6b15WuC" + configMap: + content: |- + logging { + level = "info" + format = "logfmt" + } + + discovery.kubernetes "pods" { + role = "pod" + selectors { + role = "pod" + field = "spec.nodeName=" + coalesce(env("HOSTNAME"), constants.hostname) + } + } + + // -- + // Metrics + // -- + prometheus.exporter.unix "self" {} + discovery.relabel "pod_metrics" { + targets = concat(discovery.kubernetes.pods.targets, prometheus.exporter.unix.self.targets) + + // allow override of http scheme with `promehteus.io/scheme` + rule { + action = "replace" + regex = "(https?)" + source_labels = [ + "__meta_kubernetes_service_annotation_prometheus_io_scheme", + "__meta_kubernetes_pod_annotation_prometheus_io_scheme", + ] + target_label = "__scheme__" + } + + // allow override of default /metrics path with `prometheus.io/path` + rule { + action = "replace" + regex = "(.+)" + source_labels = [ + "__meta_kubernetes_service_annotation_prometheus_io_path", + "__meta_kubernetes_pod_annotation_prometheus_io_path", + ] + target_label = "__metrics_path__" + } + + // allow override of default port with `prometheus.io/port` + rule { + action = "replace" + regex = "([^:]+)(?::\\d+)?;(\\d+)" + replacement = "$1:$2" + source_labels = [ + "__address__", + "__meta_kubernetes_service_annotation_prometheus_io_port", + "__meta_kubernetes_pod_annotation_prometheus_io_port", + ] + target_label = "__address__" + } + + // Add Namespace + rule { + action = "replace" + source_labels = ["__meta_kubernetes_namespace"] + target_label = "kubernetes_namespace" + } + // Add Pod Name + rule { + action = "replace" + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "kubernetes_pod" + } + // Add Service Name + rule { + action = "replace" + source_labels = ["__meta_kubernetes_service_name"] + target_label = "kubernetes_service" + } + + // Add all pod labels + rule { + action = "labelmap" + regex = "__meta_kubernetes_pod_label_(.+)" + } + // Add all service labels + rule { + action = "labelmap" + regex = "__meta_kubernetes_service_label_(.+)" + } + } + prometheus.scrape "containers" { + targets = discovery.relabel.pod_metrics.output + forward_to = [prometheus.remote_write.prometheus_monitor_tjo_space.receiver] + } + prometheus.remote_write "prometheus_monitor_tjo_space" { + external_labels = { + cluster = env("CLUSTER_NAME"), + } + + endpoint { + url = "https://prometheus.monitor.tjo.space/api/v1/write" + + oauth2 { + token_url = "https://id.tjo.space/application/o/token/" + client_id = env("PROMETHEUS_CLIENT_ID") + client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + endpoint_params = { + client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + } + } + } + } + + // -- + // Logs + // -- + local.file_match "node_logs" { + path_targets = [{ + // Monitor syslog to scrape node-logs + __path__ = "/var/log/syslog", + job = "node/syslog", + node_name = env("HOSTNAME"), + }] + } + loki.source.file "node_logs" { + targets = local.file_match.node_logs.targets + forward_to = [loki.write.loki_monitor_tjo_space.receiver] + } + + + discovery.relabel "pod_logs" { + targets = discovery.kubernetes.pod.targets + + rule { + source_labels = ["__meta_kubernetes_namespace"] + action = "replace" + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + action = "replace" + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "container" + } + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + action = "replace" + target_label = "app" + } + rule { + source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "job" + separator = "/" + replacement = "$1" + } + rule { + source_labels = ["__meta_kubernetes_pod_uid", "__meta_kubernetes_pod_container_name"] + action = "replace" + target_label = "__path__" + separator = "/" + replacement = "/var/log/pods/*$1/*.log" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_id"] + action = "replace" + target_label = "container_runtime" + regex = "^(\\S+):\\/\\/.+$" + replacement = "$1" + } + } + loki.source.kubernetes "pod_logs" { + targets = discovery.relabel.pod_logs.output + forward_to = [loki.write.loki_monitor_tjo_space.receiver] + } + loki.write "loki_monitor_tjo_space" { + external_labels = { + cluster = env("CLUSTER_NAME"), + } + + endpoint { + url = "https://loki.monitor.tjo.space/loki/api/v1/push" + + oauth2 { + token_url = "https://id.tjo.space/application/o/token/" + client_id = env("LOKI_CLIENT_ID") + client_secret_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + endpoint_params = { + client_assertion_type = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + } + } + } + } + EOF + ] +} diff --git a/modules/cluster/main.tf b/modules/cluster/main.tf index c32bb3c..0b2f7fa 100644 --- a/modules/cluster/main.tf +++ b/modules/cluster/main.tf @@ -67,10 +67,6 @@ locals { name = "gateway-api-crds" contents = file("${path.module}/manifests/gateway-api.crds.yaml") }, - { - name = "metrics-server" - contents = file("${path.module}/manifests/metrics-server.yaml") - }, { name = "cilium" contents = data.helm_template.cilium.manifest diff --git a/modules/cluster/manifests/metrics-server.yaml b/modules/cluster/manifests/metrics-server.yaml deleted file mode 100644 index 00c9812..0000000 --- a/modules/cluster/manifests/metrics-server.yaml +++ /dev/null @@ -1,201 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - rbac.authorization.k8s.io/aggregate-to-admin: "true" - rbac.authorization.k8s.io/aggregate-to-edit: "true" - rbac.authorization.k8s.io/aggregate-to-view: "true" - name: system:aggregated-metrics-reader -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -rules: -- apiGroups: - - "" - resources: - - nodes/metrics - verbs: - - get -- apiGroups: - - "" - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server-auth-reader - namespace: kube-system -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: metrics-server:system:auth-delegator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:auth-delegator -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - labels: - k8s-app: metrics-server - name: system:metrics-server -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:metrics-server -subjects: -- kind: ServiceAccount - name: metrics-server - namespace: kube-system ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: https - selector: - k8s-app: metrics-server ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: metrics-server - name: metrics-server - namespace: kube-system -spec: - selector: - matchLabels: - k8s-app: metrics-server - strategy: - rollingUpdate: - maxUnavailable: 0 - template: - metadata: - labels: - k8s-app: metrics-server - spec: - containers: - - args: - - --cert-dir=/tmp - - --secure-port=10250 - - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname - - --kubelet-use-node-status-port - - --metric-resolution=15s - image: registry.k8s.io/metrics-server/metrics-server:v0.7.1 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 3 - httpGet: - path: /livez - port: https - scheme: HTTPS - periodSeconds: 10 - name: metrics-server - ports: - - containerPort: 10250 - name: https - protocol: TCP - readinessProbe: - failureThreshold: 3 - httpGet: - path: /readyz - port: https - scheme: HTTPS - initialDelaySeconds: 20 - periodSeconds: 10 - resources: - requests: - cpu: 100m - memory: 200Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - readOnlyRootFilesystem: true - runAsNonRoot: true - runAsUser: 1000 - seccompProfile: - type: RuntimeDefault - volumeMounts: - - mountPath: /tmp - name: tmp-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: metrics-server - volumes: - - emptyDir: {} - name: tmp-dir ---- -apiVersion: apiregistration.k8s.io/v1 -kind: APIService -metadata: - labels: - k8s-app: metrics-server - name: v1beta1.metrics.k8s.io -spec: - group: metrics.k8s.io - groupPriorityMinimum: 100 - insecureSkipTLSVerify: true - service: - name: metrics-server - namespace: kube-system - version: v1beta1 - versionPriority: 100