feat: replace nixos with ubuntu

configuration.nix

@@ -1,142 +0,0 @@
-{
-  lib,
-  config,
-  pkgs,
-  modulesPath,
-  ...
-}:
-let
-  ngx_http_geoip2_module = pkgs.stdenv.mkDerivation {
-    name = "ngx_http_geoip2_module-a28ceff";
-    src = pkgs.fetchgit {
-      url = "https://github.com/leev/ngx_http_geoip2_module";
-      rev = "445df24ef3781e488cee3dfe8a1e111997fc1dfe";
-      sha256 = "1h2xkxpb2nk4r3pkbzgas5rbl95i59jpa59rh94x2hyzxmzrzvv8";
-    };
-    installPhase = ''
-      mkdir $out
-      cp *.c config $out/
-    '';
-    fixupPhase = "";
-  };
-  instance = builtins.fromJSON (builtins.readFile "/etc/tjo.cloud/meta.json");
-in
-{
-  system.stateVersion = "24.05";
-
-  ## FROM infrastructure/proxmox.tjo.cloud/configuration.nix
-  # Couldn't figure out the import to work.
-  imports = [
-    "${toString modulesPath}/profiles/qemu-guest.nix"
-  ];
-  fileSystems."/" = {
-    device = "/dev/disk/by-label/nixos";
-    autoResize = true;
-    fsType = "ext4";
-  };
-  fileSystems."/boot" = {
-    device = "/dev/disk/by-label/ESP";
-    fsType = "vfat";
-  };
-  boot.growPartition = true;
-  boot.kernelParams = [ "console=ttyS0" ];
-  boot.loader.systemd-boot.enable = true;
-  boot.loader.timeout = 0;
-  services.qemuGuest.enable = true;
-  services.cloud-init = {
-    enable = true;
-    network.enable = true;
-    settings = lib.mkOptionDefault {
-      datasource = {
-        NoCloud = { };
-        ConfigDrive = { };
-      };
-    };
-  };
-  networking.useNetworkd = true;
-  nix.settings.experimental-features = [
-    "nix-command"
-    "flakes"
-  ];
-  environment.systemPackages = [ pkgs.nginx ];
-  ## END FROM
-
-  nix.nixPath = [ "nixos-config=/etc/tjo.cloud/configuration.nix" ];
-  system.autoUpgrade = {
-    enable = true;
-    dates = "06:00";
-    randomizedDelaySec = "45min";
-  };
-
-  # NETWORK
-  networking.hostName = instance.name;
-  networking.domain = instance.domain;
-
-  # USER MANAGEMENT
-  security.sudo.wheelNeedsPassword = false;
-  nix.settings.trusted-users = [ "nixos" ];
-  users.users.nixos = {
-    isNormalUser = true;
-    password = "hunter2";
-    extraGroups = [ "wheel" ];
-    openssh.authorizedKeys.keys = instance.ssh_keys;
-  };
-
-  # SSH
-  services.openssh = {
-    enable = true;
-    settings.PasswordAuthentication = false;
-    settings.KbdInteractiveAuthentication = false;
-    settings.PermitRootLogin = "no";
-  };
-
-  # TAILSCALE
-  services.tailscale = {
-    enable = true;
-    authKeyFile = "/etc/tjo.cloud/secrets/tailscale.com/authkey";
-    extraUpFlags = [
-      "--ssh"
-      "--accept-routes"
-    ];
-  };
-  systemd.services.qemu-guest-agent.after = [ "tailscaled-autoconnect.service" ];
-  systemd.services.qemu-guest-agent.requires = [ "tailscaled-autoconnect.service" ];
-
-  # FIREWALL
-  networking.firewall = {
-    enable = true;
-
-    trustedInterfaces = [ "tailscale0" ];
-
-    allowedUDPPorts = [ config.services.tailscale.port ];
-    allowedTCPPorts = [
-      22
-      80
-      443
-    ];
-  };
-
-  # NGINX
-  services.nginx = {
-    enable = true;
-    package = pkgs.nginx.overrideAttrs (oldAttrs: {
-      configureFlags = oldAttrs.configureFlags ++ [ "--add-module=${ngx_http_geoip2_module}" ];
-      buildInputs = oldAttrs.buildInputs ++ [ pkgs.libmaxminddb ];
-    });
-  };
-
-  # WEBHOOK
-  # TODO: we will have multiple instances of these,
-  #       should they somehow broadcast changes to eachother?
-  # Should this be a GO service instead? With some raft mechanism?
-  # At that point, we could also switch from nginx to envoy or something...
-  services.webhook = {
-    enable = true;
-    port = 9000;
-    hooks = {
-      test = {
-        execute-command = "echo 'test'";
-      };
-    };
-  };
-}

terraform/dns.tf

@@ -1,3 +1,7 @@
+locals {
+  nodes_for_dns = { for k, v in var.nodes : k => v if v.public != null && v.internal != null }
+}
+
 data "digitalocean_domain" "ingress" {
   name = "ingress.tjo.cloud"
 }
@@ -5,16 +9,16 @@ data "digitalocean_domain" "ingress" {
 resource "digitalocean_record" "public" {
   for_each = merge(
     {
-      for key, node in local.nodes_with_address : key => {
+      for key, node in local.nodes_for_dns : key => {
         node = node.name,
-        ip   = node.public_ipv4,
+        ip   = node.public.ipv4,
         type = "A",
       }
     },
     {
-      for key, node in local.nodes_with_address : key => {
+      for key, node in local.nodes_for_dns : key => {
         node = node.name,
-        ip   = node.public_ipv6,
+        ip   = node.public.ipv6,
         type = "AAAA",
       }
     }
@@ -30,16 +34,16 @@ resource "digitalocean_record" "public" {
 resource "digitalocean_record" "internal" {
   for_each = merge(
     {
-      for key, node in local.nodes_with_address : key => {
+      for key, node in local.nodes_for_dns : key => {
         node = node.name,
-        ip   = node.internal_ipv4,
+        ip   = node.internal.ipv4,
         type = "A",
       }
     },
     {
-      for key, node in local.nodes_with_address : key => {
+      for key, node in local.nodes_for_dns : key => {
         node = node.name,
-        ip   = node.internal_ipv6,
+        ip   = node.internal.ipv6,
         type = "AAAA",
       }
     }
@@ -53,7 +57,7 @@ resource "digitalocean_record" "internal" {
 }
 
 resource "digitalocean_record" "srv" {
-  for_each = local.nodes_with_address
+  for_each = local.nodes_for_dns
 
   domain   = data.digitalocean_domain.ingress.id
   type     = "SRV"

terraform/node.tf

@@ -9,39 +9,14 @@ locals {
       domain      = local.domain
     })
   }
-
-  ipv4_addresses = {
-    for key, node in local.nodes : key => {
-      for k, v in proxmox_virtual_environment_vm.nodes[key].ipv4_addresses :
-      proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v
-    }
-  }
-  ipv6_addresses = {
-    for key, node in local.nodes : key => {
-      for k, v in proxmox_virtual_environment_vm.nodes[key].ipv6_addresses :
-      proxmox_virtual_environment_vm.nodes[key].network_interface_names[k] => v
-    }
-  }
-
-  nodes_with_address = {
-    for k, v in local.nodes :
-    k => merge(v, {
-      public_ipv4   = local.ipv4_addresses[k]["ens18"][0]
-      public_ipv6   = local.ipv6_addresses[k]["ens18"][0]
-      internal_ipv4 = "" # local.ipv4_addresses[k]["tailscale0"][0]
-      internal_ipv6 = "" # local.ipv6_addresses[k]["tailscale0"][0]
-      #internal_ipv4 = data.tailscale_device.ingress[k].addresses[0]
-      #internal_ipv6 = data.tailscale_device.ingress[k].addresses[1]
-    })
-  }
 }
 
-resource "tailscale_tailnet_key" "ingress" {
-  reusable      = true
-  ephemeral     = true
-  preauthorized = true
-  tags          = ["tag:ingress-tjo-cloud"]
-  description   = "tailscale key for ingress-tjo-cloud instances"
+resource "proxmox_virtual_environment_download_file" "ubuntu" {
+  content_type = "iso"
+  datastore_id = var.common_storage
+  node_name    = var.nodes[keys(var.nodes)[0]].host
+  url          = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+  overwrite    = false
 }
 
 resource "proxmox_virtual_environment_file" "userdata" {
@@ -54,20 +29,18 @@ resource "proxmox_virtual_environment_file" "userdata" {
   source_raw {
     data      = <<-EOF
     #cloud-config
+    hostname: ${each.value.name}.${each.value.domain}
     write_files:
     - path: /etc/tjo.cloud/meta.json
       encoding: base64
-      content: ${base64encode(jsonencode({ name : each.value.name, domain : each.value.domain, ssh_keys : var.ssh_keys }))}
-    - path: /etc/tjo.cloud/configuration.nix
-      encoding: base64
-      content: ${base64encode(file("${path.module}/../configuration.nix"))}
-    - path: /etc/tjo.cloud/secrets/tailscale.com/authkey
-      permissions: '0600'
-      content: ${var.tailscale_apikey}
-    runcmd:
-    - source /etc/profile && nixos-rebuild switch -I nixos-config=/etc/tjo.cloud/configuration.nix
-    power_state:
-      mode: reboot
+      content: ${base64encode(jsonencode({ name : each.value.name, domain : each.value.domain }))}
+    ssh_authorized_keys: ${jsonencode(var.ssh_keys)}
+    packages:
+      - htop
+      - git
+      - qemu-guest-agent
+    run_cmd:
+      - systemctl enable --now qemu-guest-agent
     EOF
     file_name = "${each.value.name}.ingress.tjo.cloud.userconfig.yaml"
   }
@@ -107,11 +80,6 @@ resource "proxmox_virtual_environment_vm" "nodes" {
     type = "l26"
   }
 
-  agent {
-    enabled = true
-    timeout = "5m"
-  }
-
   network_device {
     bridge      = each.value.bridge
     mac_address = each.value.mac_address
@@ -119,7 +87,7 @@ resource "proxmox_virtual_environment_vm" "nodes" {
 
   scsi_hardware = "virtio-scsi-single"
   disk {
-    file_id      = "proxmox-backup-tjo-cloud:iso/nixos-cloudinit.img"
+    file_id      = proxmox_virtual_environment_download_file.ubuntu.id
     interface    = "virtio0"
     datastore_id = each.value.storage
     size         = each.value.boot_size
@@ -129,8 +97,17 @@ resource "proxmox_virtual_environment_vm" "nodes" {
   }
 
   initialization {
-    interface         = "sata0"
+    interface         = "scsi0"
     datastore_id      = each.value.storage
     user_data_file_id = proxmox_virtual_environment_file.userdata[each.key].id
+
+    ip_config {
+      ipv4 {
+        address = "dhcp"
+      }
+      ipv6 {
+        address = "dhcp"
+      }
+    }
   }
 }

terraform/variables.tf

@@ -2,11 +2,20 @@ variable "nodes" {
   type = map(object({
     name = string
 
+    internal = optional(object({
+      ipv4 = string
+      ipv6 = string
+    }))
+    public = optional(object({
+      ipv4 = string
+      ipv6 = string
+    }))
+
     host   = string
     bridge = string
 
     cores  = optional(number, 1)
-    memory = optional(number, 3072)
+    memory = optional(number, 512)
 
     storage   = optional(string, "main")
     boot_size = optional(number, 8)