diff --git a/configuration.nix b/configuration.nix deleted file mode 100644 index dd72fb4..0000000 --- a/configuration.nix +++ /dev/null @@ -1,142 +0,0 @@ -{ - lib, - config, - pkgs, - modulesPath, - ... -}: -let - ngx_http_geoip2_module = pkgs.stdenv.mkDerivation { - name = "ngx_http_geoip2_module-a28ceff"; - src = pkgs.fetchgit { - url = "https://github.com/leev/ngx_http_geoip2_module"; - rev = "445df24ef3781e488cee3dfe8a1e111997fc1dfe"; - sha256 = "1h2xkxpb2nk4r3pkbzgas5rbl95i59jpa59rh94x2hyzxmzrzvv8"; - }; - installPhase = '' - mkdir $out - cp *.c config $out/ - ''; - fixupPhase = ""; - }; - instance = builtins.fromJSON (builtins.readFile "/etc/tjo.cloud/meta.json"); -in -{ - system.stateVersion = "24.05"; - - ## FROM infrastructure/proxmox.tjo.cloud/configuration.nix - # Couldn't figure out the import to work. - imports = [ - "${toString modulesPath}/profiles/qemu-guest.nix" - ]; - fileSystems."/" = { - device = "/dev/disk/by-label/nixos"; - autoResize = true; - fsType = "ext4"; - }; - fileSystems."/boot" = { - device = "/dev/disk/by-label/ESP"; - fsType = "vfat"; - }; - boot.growPartition = true; - boot.kernelParams = [ "console=ttyS0" ]; - boot.loader.systemd-boot.enable = true; - boot.loader.timeout = 0; - services.qemuGuest.enable = true; - services.cloud-init = { - enable = true; - network.enable = true; - settings = lib.mkOptionDefault { - datasource = { - NoCloud = { }; - ConfigDrive = { }; - }; - }; - }; - networking.useNetworkd = true; - nix.settings.experimental-features = [ - "nix-command" - "flakes" - ]; - environment.systemPackages = [ pkgs.nginx ]; - ## END FROM - - nix.nixPath = [ "nixos-config=/etc/tjo.cloud/configuration.nix" ]; - system.autoUpgrade = { - enable = true; - dates = "06:00"; - randomizedDelaySec = "45min"; - }; - - # NETWORK - networking.hostName = instance.name; - networking.domain = instance.domain; - - # USER MANAGEMENT - security.sudo.wheelNeedsPassword = false; - nix.settings.trusted-users = [ "nixos" ]; - users.users.nixos = { - isNormalUser = true; - password = "hunter2"; - extraGroups = [ "wheel" ]; - openssh.authorizedKeys.keys = instance.ssh_keys; - }; - - # SSH - services.openssh = { - enable = true; - settings.PasswordAuthentication = false; - settings.KbdInteractiveAuthentication = false; - settings.PermitRootLogin = "no"; - }; - - # TAILSCALE - services.tailscale = { - enable = true; - authKeyFile = "/etc/tjo.cloud/secrets/tailscale.com/authkey"; - extraUpFlags = [ - "--ssh" - "--accept-routes" - ]; - }; - systemd.services.qemu-guest-agent.after = [ "tailscaled-autoconnect.service" ]; - systemd.services.qemu-guest-agent.requires = [ "tailscaled-autoconnect.service" ]; - - # FIREWALL - networking.firewall = { - enable = true; - - trustedInterfaces = [ "tailscale0" ]; - - allowedUDPPorts = [ config.services.tailscale.port ]; - allowedTCPPorts = [ - 22 - 80 - 443 - ]; - }; - - # NGINX - services.nginx = { - enable = true; - package = pkgs.nginx.overrideAttrs (oldAttrs: { - configureFlags = oldAttrs.configureFlags ++ [ "--add-module=${ngx_http_geoip2_module}" ]; - buildInputs = oldAttrs.buildInputs ++ [ pkgs.libmaxminddb ]; - }); - }; - - # WEBHOOK - # TODO: we will have multiple instances of these, - # should they somehow broadcast changes to eachother? - # Should this be a GO service instead? With some raft mechanism? - # At that point, we could also switch from nginx to envoy or something... - services.webhook = { - enable = true; - port = 9000; - hooks = { - test = { - execute-command = "echo 'test'"; - }; - }; - }; -} diff --git a/terraform/dns.tf b/terraform/dns.tf index 0a93a88..fcc9b43 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -1,3 +1,7 @@ +locals { + nodes_for_dns = { for k, v in var.nodes : k => v if v.public != null && v.internal != null } +} + data "digitalocean_domain" "ingress" { name = "ingress.tjo.cloud" } @@ -5,16 +9,16 @@ data "digitalocean_domain" "ingress" { resource "digitalocean_record" "public" { for_each = merge( { - for key, node in local.nodes_with_address : key => { + for key, node in local.nodes_for_dns : key => { node = node.name, - ip = node.public_ipv4, + ip = node.public.ipv4, type = "A", } }, { - for key, node in local.nodes_with_address : key => { + for key, node in local.nodes_for_dns : key => { node = node.name, - ip = node.public_ipv6, + ip = node.public.ipv6, type = "AAAA", } } @@ -30,16 +34,16 @@ resource "digitalocean_record" "public" { resource "digitalocean_record" "internal" { for_each = merge( { - for key, node in local.nodes_with_address : key => { + for key, node in local.nodes_for_dns : key => { node = node.name, - ip = node.internal_ipv4, + ip = node.internal.ipv4, type = "A", } }, { - for key, node in local.nodes_with_address : key => { + for key, node in local.nodes_for_dns : key => { node = node.name, - ip = node.internal_ipv6, + ip = node.internal.ipv6, type = "AAAA", } } @@ -53,7 +57,7 @@ resource "digitalocean_record" "internal" { } resource "digitalocean_record" "srv" { - for_each = local.nodes_with_address + for_each = local.nodes_for_dns domain = data.digitalocean_domain.ingress.id type = "SRV" diff --git a/terraform/node.tf b/terraform/node.tf index eee2d72..2a16340 100644 --- a/terraform/node.tf +++ b/terraform/node.tf @@ -9,39 +9,14 @@ locals { domain = local.domain }) } - - ipv4_addresses = { - for key, node in local.nodes : key => { - for k, v in proxmox_virtual_environment_vm.nodes[key].ipv4_addresses : - proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v - } - } - ipv6_addresses = { - for key, node in local.nodes : key => { - for k, v in proxmox_virtual_environment_vm.nodes[key].ipv6_addresses : - proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v - } - } - - nodes_with_address = { - for k, v in local.nodes : - k => merge(v, { - public_ipv4 = local.ipv4_addresses[k]["ens18"][0] - public_ipv6 = local.ipv6_addresses[k]["ens18"][0] - internal_ipv4 = "" # local.ipv4_addresses[k]["tailscale0"][0] - internal_ipv6 = "" # local.ipv6_addresses[k]["tailscale0"][0] - #internal_ipv4 = data.tailscale_device.ingress[k].addresses[0] - #internal_ipv6 = data.tailscale_device.ingress[k].addresses[1] - }) - } } -resource "tailscale_tailnet_key" "ingress" { - reusable = true - ephemeral = true - preauthorized = true - tags = ["tag:ingress-tjo-cloud"] - description = "tailscale key for ingress-tjo-cloud instances" +resource "proxmox_virtual_environment_download_file" "ubuntu" { + content_type = "iso" + datastore_id = var.common_storage + node_name = var.nodes[keys(var.nodes)[0]].host + url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" + overwrite = false } resource "proxmox_virtual_environment_file" "userdata" { @@ -54,20 +29,18 @@ resource "proxmox_virtual_environment_file" "userdata" { source_raw { data = <<-EOF #cloud-config + hostname: ${each.value.name}.${each.value.domain} write_files: - path: /etc/tjo.cloud/meta.json encoding: base64 - content: ${base64encode(jsonencode({ name : each.value.name, domain : each.value.domain, ssh_keys : var.ssh_keys }))} - - path: /etc/tjo.cloud/configuration.nix - encoding: base64 - content: ${base64encode(file("${path.module}/../configuration.nix"))} - - path: /etc/tjo.cloud/secrets/tailscale.com/authkey - permissions: '0600' - content: ${var.tailscale_apikey} - runcmd: - - source /etc/profile && nixos-rebuild switch -I nixos-config=/etc/tjo.cloud/configuration.nix - power_state: - mode: reboot + content: ${base64encode(jsonencode({ name : each.value.name, domain : each.value.domain }))} + ssh_authorized_keys: ${jsonencode(var.ssh_keys)} + packages: + - htop + - git + - qemu-guest-agent + run_cmd: + - systemctl enable --now qemu-guest-agent EOF file_name = "${each.value.name}.ingress.tjo.cloud.userconfig.yaml" } @@ -107,11 +80,6 @@ resource "proxmox_virtual_environment_vm" "nodes" { type = "l26" } - agent { - enabled = true - timeout = "5m" - } - network_device { bridge = each.value.bridge mac_address = each.value.mac_address @@ -119,7 +87,7 @@ resource "proxmox_virtual_environment_vm" "nodes" { scsi_hardware = "virtio-scsi-single" disk { - file_id = "proxmox-backup-tjo-cloud:iso/nixos-cloudinit.img" + file_id = proxmox_virtual_environment_download_file.ubuntu.id interface = "virtio0" datastore_id = each.value.storage size = each.value.boot_size @@ -129,8 +97,17 @@ resource "proxmox_virtual_environment_vm" "nodes" { } initialization { - interface = "sata0" + interface = "scsi0" datastore_id = each.value.storage user_data_file_id = proxmox_virtual_environment_file.userdata[each.key].id + + ip_config { + ipv4 { + address = "dhcp" + } + ipv6 { + address = "dhcp" + } + } } } diff --git a/terraform/variables.tf b/terraform/variables.tf index 3c30b93..120083d 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -2,11 +2,20 @@ variable "nodes" { type = map(object({ name = string + internal = optional(object({ + ipv4 = string + ipv6 = string + })) + public = optional(object({ + ipv4 = string + ipv6 = string + })) + host = string bridge = string cores = optional(number, 1) - memory = optional(number, 3072) + memory = optional(number, 512) storage = optional(string, "main") boot_size = optional(number, 8)