nixfiles/nixos/boxes/colony/default.nix

{ lib, ... }:
let
  inherit (lib.my) net;
  inherit (lib.my.c.colony) domain prefixes firewallForwards;
in
{
  imports = [ ./vms ];

  nixos.systems.colony = {
    system = "x86_64-linux";
    nixpkgs = "mine-stable";
    home-manager = "mine-stable";

    assignments = {
      routing = {
        name = "colony-routing";
        inherit domain;
        ipv4.address = net.cidr.host 2 prefixes.base.v4;
      };
      internal = {
        altNames = [ "vm" ];
        inherit domain;
        ipv4 = {
          address = net.cidr.host 0 prefixes.vip1;
          mask = 32;
          gateway = null;
          genPTR = false;
        };
        ipv6 = {
          iid = "::2";
          address = net.cidr.host 2 prefixes.base.v6;
        };
      };
      vms = {
        name = "colony-vms";
        inherit domain;
        ipv4 = {
          address = net.cidr.host 1 prefixes.vms.v4;
          gateway = null;
        };
        ipv6.address = net.cidr.host 1 prefixes.vms.v6;
      };
    };

    configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
      let
        inherit (lib) mkIf mkMerge mkForce;
        inherit (lib.my) networkdAssignment;
      in
      {
        hardware = {
          enableRedistributableFirmware = true;
          cpu = {
            amd.updateMicrocode = true;
          };
          rasdaemon.enable = true;
        };

        boot = {
          kernelPackages = (lib.my.c.kernel.lts pkgs).extend (self: super: {
            kernel = super.kernel.override {
              structuredExtraConfig = with lib.kernel; {
                #SOME_OPT = yes;
                #A_MOD = module;
              };
            };
          });
          kernelModules = [ "kvm-amd" ];
          kernelParams = [ "amd_iommu=on" "console=ttyS0,115200n8" "console=ttyS1,115200n8" "console=tty0" ];
          initrd = {
            kernelModules = [ "dm-raid" ];
            availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
          };
        };

        fileSystems = {
          "/boot" = {
            device = "/dev/disk/by-uuid/C1C9-9CBC";
            fsType = "vfat";
          };
          "/nix" = {
            device = "/dev/main/colony-nix";
            fsType = "ext4";
          };
          "/persist" = {
            device = "/dev/main/colony-persist";
            fsType = "ext4";
            neededForBoot = true;
          };
          "/mnt/backup" = {
            device = "/dev/main/backup";
            fsType = "ext4";
          };
        };

        programs.ssh.knownHostsFiles = [
          lib.my.c.sshKeyFiles.rsyncNet
        ];

        services = {
          fstrim = lib.my.c.colony.fstrimConfig;
          lvm = {
            boot.thin.enable = true;
            dmeventd.enable = true;
          };

          netdata = {
            enable = true;
            config = {
              # Ignore the VCCM sensor (RAM voltage is 1.35V with XMP enabled)
              "plugin:freeipmi"."command options" = "ignore 5";
            };
          };

          smartd = {
            enable = true;
            autodetect = true;
            extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
          };
        };

        environment.systemPackages = with pkgs; [
          pciutils
          usbutils
          partclone
          lm_sensors
          linuxPackages.cpupower
          smartmontools
          xfsprogs
        ];

        systemd = {
          tmpfiles.rules = [
            "d /var/log/smartd 0755 root root"
          ];

          services = {
            "serial-getty@ttyS0".enable = true;
            "serial-getty@ttyS1".enable = true;

            rsync-lvm-meta = {
              description = "rsync lvm metadata backups / archives to rsync.net";
              serviceConfig = {
                Type = "oneshot";

                # Only run when no other process is using CPU or disk
                CPUSchedulingPolicy = "idle";
                IOSchedulingClass = "idle";
              };
              script = ''
                ${pkgs.rsync}/bin/rsync -av --delete --delete-after \
                  -e "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."colony/rsync.key".path}" \
                  /etc/lvm/{archive,backup} zh2855@zh2855.rsync.net:colony/lvm/
              '';
              wantedBy = [ "borgthin-job-main.service" ];
              after = [ "borgthin-job-main.service" ];
            };
            borgthin-rsync = {
              description = "rsync borgthin backups to rsync.net";
              serviceConfig = {
                Type = "oneshot";

                # Only run when no other process is using CPU or disk
                CPUSchedulingPolicy = "idle";
                IOSchedulingClass = "idle";
              };
              script = ''
                ${pkgs.rsync}/bin/rsync -av --delete --delete-after \
                  -e "${pkgs.openssh}/bin/ssh -i ${config.age.secrets."colony/rsync.key".path}" \
                  /mnt/backup/main/ zh2855@zh2855.rsync.net:borg/colony/
              '';
              wantedBy = [ "borgthin-job-main.service" ];
              after = [ "borgthin-job-main.service" ];
            };
          };

          network = {
            links = {
              "10-wan0" = {
                matchConfig.MACAddress = "d0:50:99:fa:a7:99";
                linkConfig.Name = "wan0";
              };
              "10-wan1" = {
                matchConfig.MACAddress = "d0:50:99:fa:a7:9a";
                linkConfig.Name = "wan1";
              };
            };
            netdevs = {
              "25-base".netdevConfig = {
                Name = "base";
                Kind = "bridge";
              };
              "30-base-dummy".netdevConfig = {
                Name = "base0";
                Kind = "dummy";
              };

              "25-vms".netdevConfig = {
                Name = "vms";
                Kind = "bridge";
              };
              "30-vms-dummy".netdevConfig = {
                Name = "vms0";
                Kind = "dummy";
              };
            };

            networks = {
              "80-base" = mkMerge [
                (networkdAssignment "base" assignments.routing)
                (networkdAssignment "base" assignments.internal)
              ];
              "80-base-dummy" = {
                matchConfig.Name = "base0";
                networkConfig.Bridge = "base";
              };
              "80-base-ext" = {
                matchConfig.Name = "base-ext";
                networkConfig.Bridge = "base";
              };

              "80-vms" = mkMerge [
                (networkdAssignment "vms" assignments.vms)
                {
                  networkConfig = {
                    IPv6AcceptRA = mkForce false;
                    IPv6SendRA = true;
                  };
                  ipv6SendRAConfig = {
                    DNS = [ allAssignments.estuary.base.ipv6.address ];
                    Domains = [ config.networking.domain ];
                  };
                  ipv6Prefixes = [
                    {
                      ipv6PrefixConfig.Prefix = prefixes.vms.v6;
                    }
                  ];
                  routes = map (r: { routeConfig = r; }) [
                    {
                      Destination = prefixes.ctrs.v4;
                      Gateway = allAssignments.shill.routing.ipv4.address;
                    }
                    {
                      Destination = prefixes.ctrs.v6;
                      Gateway = allAssignments.shill.internal.ipv6.address;
                    }
                    {
                      Destination = allAssignments.shill.internal.ipv4.address;
                      Gateway = allAssignments.shill.routing.ipv4.address;
                    }

                    {
                      Destination = prefixes.oci.v4;
                      Gateway = allAssignments.whale2.routing.ipv4.address;
                    }
                    {
                      Destination = prefixes.oci.v6;
                      Gateway = allAssignments.whale2.internal.ipv6.address;
                    }
                    {
                      Destination = allAssignments.whale2.internal.ipv4.address;
                      Gateway = allAssignments.whale2.routing.ipv4.address;
                    }

                    {
                      Destination = allAssignments.git.internal.ipv4.address;
                      Gateway = allAssignments.git.routing.ipv4.address;
                    }
                  ];
                }
              ];
              # Just so the vms interface will come up (in networkd's eyes), allowing dependant VMs to start.
              # Could tweak the `waitOnline` for a single VM, but this seems better overall?
              "80-vms-dummy" = {
                matchConfig.Name = "vms0";
                networkConfig.Bridge = "vms";
              };

              "90-vm-mail" = {
                matchConfig.Name = "vm-mail";
                address = [
                  (net.cidr.subnet 8 1 prefixes.cust.v4)
                  prefixes.mail.v6
                ];
                networkConfig = {
                  IPv6AcceptRA = false;
                  IPv6SendRA = true;
                };
                ipv6Prefixes = [
                  {
                    ipv6PrefixConfig.Prefix = prefixes.mail.v6;
                  }
                ];
                routes = map (r: { routeConfig = r; }) [
                  {
                    Destination = prefixes.mail.v4;
                    Scope = "link";
                  }
                ];
              };

              "90-vm-darts" = {
                matchConfig.Name = "vm-darts";
                address = [
                  (net.cidr.subnet 8 2 prefixes.cust.v4)
                  prefixes.darts.v6
                ];
                networkConfig = {
                  IPv6AcceptRA = false;
                  IPv6SendRA = true;
                };
                ipv6Prefixes = [
                  {
                    ipv6PrefixConfig.Prefix = prefixes.darts.v6;
                  }
                ];
                routes = map (r: { routeConfig = r; }) [
                  {
                    Destination = prefixes.darts.v4;
                    Scope = "link";
                  }
                ];
              };
            };
          };
        };

        #environment.etc."udev/udev.conf".text = "udev_log=debug";
        #systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
        virtualisation = {
          cores = 8;
          memorySize = 8192;
          qemu.options = [
            "-machine q35"
            "-accel kvm,kernel-irqchip=split"
            "-device intel-iommu,intremap=on,caching-mode=on"
          ];
        };

        my = {
          #deploy.generate.system.mode = "boot";
          secrets = {
            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPIijqzAWF6OxKr4aeCa1TAc5xGn4rdIjVTt0wAPU6uY";
            files = {
              "colony/borg-pass.txt" = {};
              "colony/rsync.key" = {};
            };
          };

          server.enable = true;

          firewall = {
            trustedInterfaces = [ "vms" ];
            nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
            extraRules = ''
              define cust = { vm-mail, vm-darts }
              table inet filter {
                chain forward {
                  # Trust that the outer firewall has done the filtering!
                  iifname base oifname { vms, $cust } accept
                  iifname $cust accept # trust for now...
                }
              }
            '';
          };

          borgthin = {
            enable = true;
            jobs = {
              main = {
                repo = "/mnt/backup/main";
                passFile = config.age.secrets."colony/borg-pass.txt".path;
                lvs = map (lv: "main/${lv}") [
                  "colony-persist"
                  "vm-shill-persist"
                  "minio"
                  "oci"
                  "vm-estuary-persist"
                  "vm-whale2-persist"
                  "vm-mail-data"
                  "vm-git-persist"
                  "git"
                ];
                compression = "zstd,5";
                extraCreateArgs = [ "--stats" ];
                prune.keep = {
                  last = 1;
                  within = "1d";
                  daily = 7;
                  weekly = 4;
                  monthly = 12;
                  yearly = -1;
                };
              };
            };
          };
        };
      };
  };
}