diff --git a/ingress.tjo.cloud/terraform/node.tf b/ingress.tjo.cloud/terraform/node.tf index 6e5e83b..fde13b2 100644 --- a/ingress.tjo.cloud/terraform/node.tf +++ b/ingress.tjo.cloud/terraform/node.tf @@ -1,5 +1,5 @@ locals { - domain = "ingress.tjo.cloud" + domain = "postgresql.tjo.cloud" nodes = { for k, v in var.nodes : k => merge(v, { @@ -11,25 +11,11 @@ locals { username = authentik_user.service_account[k].username password = authentik_token.service_account[k].key } - tailscale = { - auth_key = tailscale_tailnet_key.key.key - } - dnsimple = { - token = var.dnsimple_token - } } }) } } -resource "tailscale_tailnet_key" "key" { - reusable = true - ephemeral = false - preauthorized = true - description = "ingress-tjo-cloud terraform key" - tags = ["tag:ingress-tjo-cloud"] -} - resource "proxmox_virtual_environment_download_file" "ubuntu" { for_each = local.nodes @@ -57,7 +43,10 @@ resource "proxmox_virtual_environment_file" "userdata" { - path: /etc/tjo.cloud/meta.json encoding: base64 content: ${base64encode(jsonencode(each.value.meta))} - ssh_authorized_keys: ${jsonencode(var.ssh_keys)} + - path: /tmp/provision.sh + encoding: base64 + content: ${base64encode(file("${path.module}/../provision.sh"))} + ssh_authorized_keys: ${jsonencode(values(var.ssh_keys))} packages: - qemu-guest-agent power_state: @@ -66,9 +55,9 @@ resource "proxmox_virtual_environment_file" "userdata" { filename: /swapfile size: 512M runcmd: - - git clone --depth 1 --no-checkout --filter=tree:0 https://github.com/tjo-space/tjo-cloud-infrastructure.git /srv - - cd /srv && git sparse-checkout set --no-cone /ingress.tjo.cloud && git checkout - - /srv/ingress.tjo.cloud/install.sh + - "chmod +x /tmp/provision.sh" + - "/tmp/provision.sh" + - "rm /tmp/provision.sh" EOF file_name = "${each.value.host}.${each.value.domain}.userconfig.yaml" } @@ -84,7 +73,7 @@ resource "proxmox_virtual_environment_vm" "nodes" { description = </etc/apt/keyrings/grafana.gpg +echo "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" >/etc/apt/sources.list.d/grafana.list +apt update -y +apt install -y alloy + +# Tailscale +curl -fsSL https://pkgs.tailscale.com/stable/ubuntu/noble.noarmor.gpg >/usr/share/keyrings/tailscale-archive-keyring.gpg +curl -fsSL https://pkgs.tailscale.com/stable/ubuntu/noble.tailscale-keyring.list >/etc/apt/sources.list.d/tailscale.list +apt update -y +apt install -y tailscale + +## +echo "== Configure Grafana Alloy" +cp -r root/etc/alloy/* /etc/alloy/ +cp -r root/etc/default/alloy /etc/default/alloy +# Set Attributes +ATTRIBUTES="" +ATTRIBUTES+="service.name=${SERVICE_NAME}," +ATTRIBUTES+="service.version=${SERVICE_VERSION}," +ATTRIBUTES+="cloud.region=${CLOUD_REGION}" +{ + echo "" + echo "OTEL_RESOURCE_ATTRIBUTES=${ATTRIBUTES}" + echo "ALLOY_USERNAME=${SERVICE_ACCOUNT_USERNAME}" + echo "ALLOY_PASSWORD=${SERVICE_ACCOUNT_PASSWORD}" +} >>/etc/default/alloy +systemctl enable --now alloy +systemctl restart alloy + +## +echo "== Configure Dyndns" +cp root/etc/systemd/system/dyndns.service /etc/systemd/system/dyndns.service +cp root/usr/local/bin/dyndns /usr/local/bin/dyndns +cp -r root/etc/default/dyndns /etc/default/dyndns +{ + echo "" + echo "DNSIMPLE_TOKEN=${DNSIMPLE_TOKEN}" + echo "CLOUD_REGION=${CLOUD_REGION}" +} >>/etc/default/dyndns +systemctl enable --now dyndns +systemctl restart dyndns + +## +echo "== Configure Tailscale" +systemctl enable --now tailscaled +if tailscale status --json | jq -e -r '.BackendState != "Running"' >/dev/null; then + tailscale up \ + --ssh=true \ + --accept-routes=true \ + --accept-dns=false \ + --advertise-tags="tag:ingress-tjo-cloud" \ + --hostname="$(hostname -f | sed 's/\./-/g')" \ + --authkey="${TAILSCALE_AUTH_KEY}" +else + echo "Tailscale is already running" +fi + +## +echo "== Configure SSH" +cat </etc/ssh/sshd_config.d/port-2222.conf +Port 2222 +EOF +systemctl restart ssh + +## +echo "== Configure UFW" +# Should basically match nginx.conf +ufw default deny incoming +ufw default allow outgoing + +ufw allow in on tailscale0 + +ufw allow 22 # GIT +ufw allow 25 # EMAIL +ufw allow 143 # EMAIL +ufw allow 443 # HTTPS +ufw allow 465 # EMAIL +ufw allow 587 # EMAIL +ufw allow 993 # EMAIL +ufw allow 1337 # HTTP (healthcheck) +ufw allow 4190 # EMAIL + +ufw allow 2222 # SSH ACCESS + +ufw --force enable +systemctl enable ufw + +## +echo "== Configure NGINX" +cp assets/dbip-city-lite-2023-07.mmdb /var/geoip.mmdb +cp -r root/etc/nginx/* /etc/nginx/ +unlink /etc/nginx/sites-enabled/default || true +systemctl enable --now nginx +systemctl reload nginx diff --git a/postgresql.tjo.cloud/justfile b/postgresql.tjo.cloud/justfile new file mode 100644 index 0000000..5b279c3 --- /dev/null +++ b/postgresql.tjo.cloud/justfile @@ -0,0 +1,56 @@ +default: + @just --list + +apply: + #!/usr/bin/env sh + cd {{source_directory()}}/terraform + tofu init + tofu apply + +apply-only node: + #!/usr/bin/env sh + cd {{source_directory()}}/terraform + tofu init + tofu apply --target 'proxmox_virtual_environment_vm.nodes["{{node}}"]' + +destroy: + #!/usr/bin/env sh + cd {{source_directory()}}/terraform + tofu destroy + +destroy-only node: + #!/usr/bin/env sh + cd {{source_directory()}}/terraform + tofu init + tofu destroy --target 'proxmox_virtual_environment_vm.nodes["{{node}}"]' + +configure: + #!/usr/bin/env sh + set -eou pipefail + + pushd {{source_directory()}}/terraform > /dev/null + NODES=$(tofu output -json | jq -r '.nodes.value[]') + popd > /dev/null + + for NODE in $NODES + do + echo "= Provisioning node ${NODE}" + cat install.sh | tailscale ssh ubuntu@${NODE} 'sudo bash -s' + done + +configure-only node: + #!/usr/bin/env sh + set -eou pipefail + + pushd {{source_directory()}}/terraform > /dev/null + NODES=$(tofu output -json | jq -r '.nodes.value[]') + popd > /dev/null + + for NODE in $NODES + do + if [ "$NODE" = "{{node}}-ingress-tjo-cloud" ] + then + echo "= Provisioning node ${NODE}" + cat install.sh | tailscale ssh ubuntu@${NODE} 'sudo bash -s' + fi + done diff --git a/postgresql.tjo.cloud/root/etc/alloy/config.alloy b/postgresql.tjo.cloud/root/etc/alloy/config.alloy new file mode 100644 index 0000000..2983097 --- /dev/null +++ b/postgresql.tjo.cloud/root/etc/alloy/config.alloy @@ -0,0 +1,109 @@ +logging { + level = "info" + format = "logfmt" +} + +//=== +// Metrics +//=== +prometheus.exporter.self "default" { +} +prometheus.exporter.unix "default" { +} +prometheus.scrape "exporters" { + targets = concat( + prometheus.exporter.self.default.targets, + prometheus.exporter.unix.default.targets, + ) + forward_to = [ + otelcol.receiver.prometheus.default.receiver, + ] +} + +//=== +// Logs +//=== +loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "journal_unit" + } +} + +loki.source.journal "default" { + forward_to = [loki.process.drop_old.receiver] + relabel_rules = loki.relabel.journal.rules +} +loki.process "drop_old" { + stage.drop { + older_than = "1h" + drop_counter_reason = "too old" + } + forward_to = [ + otelcol.receiver.loki.default.receiver, + ] +} + +//=== +// OTEL +//=== +otelcol.receiver.prometheus "default" { + output { + metrics = [otelcol.processor.attributes.default.input] + } +} +otelcol.receiver.loki "default" { + output { + logs = [otelcol.processor.attributes.default.input] + } +} +otelcol.processor.attributes "default" { + output { + metrics = [otelcol.processor.resourcedetection.default.input] + logs = [otelcol.processor.resourcedetection.default.input] + traces = [otelcol.processor.resourcedetection.default.input] + } +} +otelcol.processor.resourcedetection "default" { + detectors = ["env", "system"] + system { + hostname_sources = ["os"] + resource_attributes { + host.arch { enabled = true } + host.id { enabled = true } + host.name { enabled = true } + os.type { enabled = true } + } + } + output { + metrics = [otelcol.processor.batch.default.input] + logs = [otelcol.processor.batch.default.input] + traces = [otelcol.processor.batch.default.input] + } +} +otelcol.processor.batch "default" { + timeout = "10s" + output { + metrics = [otelcol.exporter.otlp.default.input] + logs = [otelcol.exporter.otlp.default.input] + traces = [otelcol.exporter.otlp.default.input] + } +} +otelcol.auth.oauth2 "default" { + token_url = "https://id.tjo.space/application/o/token/" + client_id = "Vlw69HXoTJn1xMQaDX71ymGuLVoD9d2WxscGhksh" + client_secret = "none" + endpoint_params = { + grant_type = ["client_credentials"], + username = [env("ALLOY_USERNAME")], + password = [env("ALLOY_PASSWORD")], + } +} +otelcol.exporter.otlp "default" { + client { + endpoint = "grpc.otel.monitor.tjo.cloud:443" + auth = otelcol.auth.oauth2.default.handler + } +} diff --git a/postgresql.tjo.cloud/root/etc/default/alloy b/postgresql.tjo.cloud/root/etc/default/alloy new file mode 100644 index 0000000..06144ec --- /dev/null +++ b/postgresql.tjo.cloud/root/etc/default/alloy @@ -0,0 +1,16 @@ +## Path: +## Description: Grafana Alloy settings +## Type: string +## Default: "" +## ServiceRestart: alloy +# +# Command line options for alloy +# +# The configuration file holding the Grafana Alloy configuration. +CONFIG_FILE="/etc/alloy/config.alloy" + +# User-defined arguments to pass to the run command. +CUSTOM_ARGS="" + +# Restart on system upgrade. Defaults to true. +RESTART_ON_UPGRADE=true diff --git a/postgresql.tjo.cloud/terraform/.terraform.lock.hcl b/postgresql.tjo.cloud/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..edbe760 --- /dev/null +++ b/postgresql.tjo.cloud/terraform/.terraform.lock.hcl @@ -0,0 +1,95 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/bpg/proxmox" { + version = "0.61.1" + constraints = "0.61.1" + hashes = [ + "h1:6kz2Rdjc8+TVq2aUxEQXLOwbb9OdhJJei0L1fC4K2R4=", + "h1:SQSHTHj2ThcF08cON2gHYcnkS/XLmoF8E4cRIgpagtE=", + "zh:27d8b589a2dc1e0a5b0f8ab299b9f3704a2f0b69799d1d4d8845c68056986d1f", + "zh:46dfa6b33ddd7007a2144f38090457604eb56a59a303b37bb0ad1be5c84ddaca", + "zh:47a1b14a759393c5ecc76f2feb950677c418c910b8c677fde0dd3e4675c41579", + "zh:582e49d109d1c2b1f3b1268a7cbc43548f3c6d96a87c92a5428767097a5e383e", + "zh:5e98ad6afae5969a4c3ffb14c0484936550c66c8313d7686551c29b633ff32f2", + "zh:7b9e24b76f947ab8f1e571cf61beefc983b7d2aa1b85df35c4f015728fe37a38", + "zh:8255ca210f279a0f7b8ca2762df26d2ea1a01704298c5e3d5cf601bd39a743f0", + "zh:85d7655fdc95dedced9cf8105a0beeb0d7bc8f668c55f62019a7215a76d60300", + "zh:8aeea5a1d001b06baaf923b754e1a14d06c75eb8c8b87a7f65a3c8205fc8b079", + "zh:a9cfab6c06f613658c5fdd83742cd22c0eb7563778924b1407965ef8c36c1ce0", + "zh:ceaab67801d49a92eb5858b1ddae6df2569462e5ffbe31f9dbd79dcb684ea142", + "zh:dc25b506d5c55d1d78a335d3ebd03213c99b4b2a5859812349a955c2f746ff7e", + "zh:e04b477fd77a0d37a0bdb76a7cf69184dad9e7fbba9b4f3a378a8901b82b75e5", + "zh:f1e6838d9141557f73340df9b21fce5a82b41cc16ae36f063a920ccc36bc0758", + "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597", + ] +} + +provider "registry.opentofu.org/dnsimple/dnsimple" { + version = "1.8.0" + constraints = "1.8.0" + hashes = [ + "h1:Nwu+3tVJnNmSJQoctRSWAamUX3AiTCZ5mOMtAUPtg7Q=", + "zh:0852fd9523268b30fb637a03a0cb6d6a5878cbbf7e0e4219615c9ba073fbdf17", + "zh:0ac43193082dd467abad4937b0abb97ea349205726fc450cb3a94dc0db6e9a49", + "zh:10e4aad54c2d6cbd9328a1661d72a978357743eda7099a3f120a497119be4ff1", + "zh:211d481935dec36903928c51f5f4f15d98313f6d50649ea064bc20a4d6541678", + "zh:2705b5ebac4219449f9126cc19fa982cf0644e5df60d3d5254131d2e2d676afd", + "zh:27f0df80af6652e96f85a0856daa571af495d2119ab126199d6d5ab53f6eb887", + "zh:27fbb2fb69291a660d8e99ba960f01051b7fc28658f7932772ce7e80a42bd6e9", + "zh:3ecf20ead1f044f08ae9e411c9341d47319eb6af5d6543b58f2f6932c6b288b0", + "zh:635055f0af3eb27d30801aeead51d8b960c386f369a378fad7146350ec6b4d68", + "zh:7ca26f64221a9c6634a02296e30a87e3fffed1144ac57e0ae9a86a448f42d4ca", + "zh:895e0732da00942b2eb13c78673a9c9268e87e92a225999cddf2d13b823f3295", + "zh:b3806e5b687faf97ad8cb2a23e105729059693ae07a229fecef52da5279d7bd1", + "zh:c3c284a54aab3ddea2dba140af4a707ce077c9c2d9d34556902afdb25fe6ca8e", + "zh:d2539f2cc5960a55a53eaaa90248abfb3167275e34af7e93735ec4571eb879eb", + "zh:f809ab383cca0a5f83072981c64208cbd7fa67e986a86ee02dd2c82333221e32", + ] +} + +provider "registry.opentofu.org/goauthentik/authentik" { + version = "2024.8.3" + constraints = "2024.8.3" + hashes = [ + "h1:8ZYjDZc+RMO9vFxOPXjc4PEZimV9gMKk1vxDPjc+TZQ=", + "h1:NiXi1gn1BH2tk1MIqgl6hQotwVe8FN8RJqvE7ix+EWs=", + "zh:1d2d165662d36dae0aacb478a6bae055546979dea58ee3762dd7d398b7f60e8c", + "zh:3a118d3c123eab3e26c33821607d2f70f9e317d3d33289f9d615e4b6d353b877", + "zh:3fa67bd9c64c1277a107205becdbd2d35649aeb97b591bc8a5bdd8444164f754", + "zh:40bbc8a31e7568ad68100620aa229fbb1837846b79ad8a468bf486b519d19c8c", + "zh:4ffb5344ae5ec44edf0f5c92f600455a731683b13b7a322760153eb53ff544af", + "zh:5b52f1268ca28b7c6869e69363ffff139d965fab0ae7d2e1158688cb076a7298", + "zh:7c598a517e358eb4a83d0805845e6e8b1aa9320143d225fc14d6987e8dd12506", + "zh:843627dd43a5df89f907ccd499b7264e00df0e1269dccec0738f1d5efb5db969", + "zh:8604f50738667066406c31775a32497eca69f52a085bcd14862736b1d0183de1", + "zh:9de948d1df56fe6a6eb4279c704554ea70f8791b6dbd301a3432ab7859718360", + "zh:9f95520468bf49ae11e9d2493cafdb99910faeac34bb25586105e5326461949b", + "zh:d25048f3cbe96981dc72894c7ceae839846c240e2c270909aaf93cdf8af75a14", + "zh:e2e72159b9a1d91c7bd4eb62e09eaf7440478a493d853cb3aa3076b9acd8793b", + "zh:f6af0fd2e89ea7b7e692ef893cf5fdcc6f53c37fc0c6e066a28d9c834226c539", + ] +} + +provider "registry.opentofu.org/tailscale/tailscale" { + version = "0.17.2" + constraints = "0.17.2" + hashes = [ + "h1:0bZpffptYi/bXOXEnFjUYD6UwaR4vqUdMULdeeBhz84=", + "h1:Hb7w+ibr6O6jvQSJbLAH0DI/r7sgnkxKLiAofAjEzpQ=", + "zh:13d21db507bfb17018005c5c4f19314591a5734c76bcd51ab6e80984164c2a71", + "zh:13dbb3d978aca16f66c49596e5a38d236264d10a66879dc0d06839aca9cdad3f", + "zh:1589a8b006da14d60e3fcd55fbc465ccdce7a99e833b6a7455fbf81be59f07f3", + "zh:1de3673533c0c20c4fc6070822f0c416a64734656f2e181e6bab5e9df5383ed9", + "zh:24eaaf37dacb48e26b53a2a0491ffa7bc5c1977d9c27753ada734ed0191f28aa", + "zh:2a0890a012829aa370bb930a8155af49accf53832324e8124e123d0679878c3c", + "zh:4f8a462d462b0942add33cf376655c0470b6826db34e57aecc9a62742e286283", + "zh:5cf38de52c7e2e8f3a5f8e05e1fbef4db4545c5b2dc2f89b0bfb4b8eea293a14", + "zh:8bbf0a4c9a6c37b31dda332a8a7436516fc62ce777e0e586772883f39de56e52", + "zh:9213bbdea053d1edbeccb51a7e86829e1539b5295fba08bf0eda9af729e8ba60", + "zh:9a645a49430297e27304e93ebc699fcb0d1a068ba8b431c4ec0f9ad4a4e134bf", + "zh:b3b70b083161cb97ef0618be579453d13b25ba95c785744cd0c4a84eecc7a0f9", + "zh:b3e1e5ac6087120ef548d2ceeafef1b0b469aad17a84eb873f0f4d5eaa2bf6f9", + "zh:e323626e070442308bcadfcc51a3ce5b0e6ae41a7632f82bb24318706920a9d3", + ] +} diff --git a/postgresql.tjo.cloud/terraform/dns.tf b/postgresql.tjo.cloud/terraform/dns.tf new file mode 100644 index 0000000..8551163 --- /dev/null +++ b/postgresql.tjo.cloud/terraform/dns.tf @@ -0,0 +1,20 @@ +resource "dnsimple_zone" "tjo_cloud" { + name = "tjo.cloud" +} + +resource "dnsimple_zone_record" "management" { + zone_name = dnsimple_zone.tjo_cloud.name + name = "postgresql" + value = "any.ingress.tjo.cloud" + type = "ALIAS" + ttl = 300 +} + +# TODO: For each node or some VIP + BGP thing? +resource "dnsimple_zone_record" "nodes" { + zone_name = dnsimple_zone.tjo_cloud.name + name = "postgresql" + value = "any.ingress.tjo.cloud" + type = "ALIAS" + ttl = 300 +} diff --git a/postgresql.tjo.cloud/terraform/node.tf b/postgresql.tjo.cloud/terraform/node.tf new file mode 100644 index 0000000..6e5e83b --- /dev/null +++ b/postgresql.tjo.cloud/terraform/node.tf @@ -0,0 +1,151 @@ +locals { + domain = "ingress.tjo.cloud" + + nodes = { + for k, v in var.nodes : k => merge(v, { + domain = local.domain + meta = { + name = v.host + domain = local.domain + service_account = { + username = authentik_user.service_account[k].username + password = authentik_token.service_account[k].key + } + tailscale = { + auth_key = tailscale_tailnet_key.key.key + } + dnsimple = { + token = var.dnsimple_token + } + } + }) + } +} + +resource "tailscale_tailnet_key" "key" { + reusable = true + ephemeral = false + preauthorized = true + description = "ingress-tjo-cloud terraform key" + tags = ["tag:ingress-tjo-cloud"] +} + +resource "proxmox_virtual_environment_download_file" "ubuntu" { + for_each = local.nodes + + content_type = "iso" + datastore_id = each.value.iso_storage + node_name = each.value.host + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + overwrite = true +} + +resource "proxmox_virtual_environment_file" "userdata" { + for_each = local.nodes + + node_name = each.value.host + content_type = "snippets" + datastore_id = each.value.iso_storage + + source_raw { + data = <<-EOF + #cloud-config + hostname: ${each.value.host} + fqdn: ${each.value.host}.${each.value.domain} + prefer_fqdn_over_hostname: true + write_files: + - path: /etc/tjo.cloud/meta.json + encoding: base64 + content: ${base64encode(jsonencode(each.value.meta))} + ssh_authorized_keys: ${jsonencode(var.ssh_keys)} + packages: + - qemu-guest-agent + power_state: + mode: reboot + swap: + filename: /swapfile + size: 512M + runcmd: + - git clone --depth 1 --no-checkout --filter=tree:0 https://github.com/tjo-space/tjo-cloud-infrastructure.git /srv + - cd /srv && git sparse-checkout set --no-cone /ingress.tjo.cloud && git checkout + - /srv/ingress.tjo.cloud/install.sh + EOF + file_name = "${each.value.host}.${each.value.domain}.userconfig.yaml" + } +} + +resource "proxmox_virtual_environment_vm" "nodes" { + for_each = local.nodes + + vm_id = each.value.id + name = "${each.value.host}.${each.value.domain}" + node_name = each.value.host + + description = <