nixfiles/nixos/boxes/colony/vms/estuary/default.nix

{ lib, ... }: {
  nixos.systems.estuary = {
    system = "x86_64-linux";
    nixpkgs = "mine";
    home-manager = "mine";

    assignments = {
      internal = {
        name = "estuary-vm";
        altNames = [ "fw" ];
        domain = lib.my.colony.domain;
        ipv4 = {
          address = "94.142.240.44";
          mask = 24;
          gateway = "94.142.240.254";
          genPTR = false;
        };
        ipv6 = {
          address = "2a02:898:0:20::329:1";
          mask = 64;
          gateway = "2a02:898:0:20::1";
          genPTR = false;
        };
      };
      base = {
        name = "estuary-vm-base";
        domain = lib.my.colony.domain;
        ipv4 = {
          address = "${lib.my.colony.start.base.v4}1";
          gateway = null;
        };
        ipv6.address = "${lib.my.colony.start.base.v6}1";
      };
    };

    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
      let
        inherit (lib) flatten mkIf mkMerge mkForce;
        inherit (lib.my) networkdAssignment;
      in
      {
        imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./dns.nix ./bgp.nix ];

        config = mkMerge [
          {
            boot.kernelParams = [ "console=ttyS0,115200n8" ];
            fileSystems = {
              "/boot" = {
                device = "/dev/disk/by-label/ESP";
                fsType = "vfat";
              };
              "/nix" = {
                device = "/dev/disk/by-label/nix";
                fsType = "ext4";
              };
              "/persist" = {
                device = "/dev/disk/by-label/persist";
                fsType = "ext4";
                neededForBoot = true;
              };
            };

            services = {
              lvm = {
                dmeventd.enable = true;
              };
              netdata.enable = true;

              iperf3 = {
                enable = true;
                openFirewall = true;
              };
            };

            systemd = {
              services = {
                # Use this as a way to make sure the router always knows we're here (NDP seems kindy funky)
                ipv6-neigh-keepalive =
                let
                  waitOnline = "systemd-networkd-wait-online@wan.service";
                in
                {
                  description = "Frequent ICMP6 neighbour solicitations";
                  enable = false;
                  requires = [ waitOnline ];
                  after = [ waitOnline ];
                  script = ''
                    while true; do
                      ${pkgs.ndisc6}/bin/ndisc6 ${assignments.internal.ipv6.gateway} wan
                      sleep 10
                    done
                  '';
                  wantedBy = [ "multi-user.target" ];
                };

                bird2 =
                let
                  waitOnline = "systemd-networkd-wait-online@wan.service";
                in
                {
                  after = [ waitOnline ];
                  requires = [ waitOnline ];
                };
              };
            };

            systemd.network = {
              config = {
                networkConfig = {
                  ManageForeignRoutes = false;
                };
              };

              links = {
                "10-wan" = {
                  matchConfig.MACAddress = "d0:50:99:fa:a7:99";
                  linkConfig.Name = "wan";
                };
                # Mellanox ConnectX-2
                #"10-wan" = {
                #  matchConfig.MACAddress = "00:02:c9:56:24:6e";
                #  linkConfig.Name = "wan";
                #};

                "10-base" = {
                  matchConfig.MACAddress = "52:54:00:15:1a:53";
                  linkConfig.Name = "base";
                };
              };

              networks = {
                "80-wan" = {
                  matchConfig.Name = "wan";
                  DHCP = "no";
                  address = with assignments.internal; [
                    (with ipv4; "${address}/${toString mask}")
                    (with ipv6; "${address}/${toString mask}")
                  ];
                  gateway = with assignments.internal; [
                    ipv4.gateway
                    ipv6.gateway
                  ];
                  networkConfig = {
                    # We're using an explicit gateway and Linux uses link local address for neighbour discovery, so we
                    # get lost to the router... (this was true in 23M Frankfurt)
                    #LinkLocalAddressing = "no";
                    IPv6AcceptRA = false;
                  };
                };

                "80-base" = mkMerge [
                  (networkdAssignment "base" assignments.base)
                  {
                    dns = [ "127.0.0.1" "::1" ];
                    domains = [ config.networking.domain ];
                    networkConfig = {
                      IPv6AcceptRA = mkForce false;
                      IPv6SendRA = true;
                    };
                    ipv6SendRAConfig = {
                      DNS = [ assignments.base.ipv6.address ];
                      Domains = [ config.networking.domain ];
                    };
                    ipv6Prefixes = [
                      {
                        ipv6PrefixConfig.Prefix = lib.my.colony.prefixes.base.v6;
                      }
                    ];
                    routes = map (r: { routeConfig = r; }) (flatten
                      ([  ] ++
                      (map (pName: [
                        {
                          Gateway = allAssignments.colony.internal.ipv4.address;
                          Destination = lib.my.colony.prefixes."${pName}".v4;
                        }
                        {
                          Gateway = allAssignments.colony.internal.ipv6.address;
                          Destination = lib.my.colony.prefixes."${pName}".v6;
                        }
                      ]) [ "vms" "ctrs" "oci" ])));
                  }
                ];
              };
            };

            my = {
              #deploy.generate.system.mode = "boot";
              secrets.key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9up7pXu6M/OWCKufTOfSiGcxMUk4VqUe7fLuatNFFA";
              server.enable = true;

              firewall = {
                trustedInterfaces = [ "base" ];
                udp.allowed = [ 5353 ];
                tcp.allowed = [ 5353 ];
                nat = {
                  enable = true;
                  externalInterface = "wan";
                  forwardPorts = [
                    {
                      port = "http";
                      dst = allAssignments.middleman.internal.ipv4.address + ":http";
                    }
                    {
                      port = "https";
                      dst = allAssignments.middleman.internal.ipv4.address + ":https";
                    }
                    {
                      port = 8448;
                      dst = allAssignments.middleman.internal.ipv4.address + ":8448";
                    }

                    {
                      port = 2456;
                      dst = allAssignments.valheim-oci.internal.ipv4.address + ":2456";
                      proto = "udp";
                    }
                    {
                      port = 2457;
                      dst = allAssignments.valheim-oci.internal.ipv4.address + ":2457";
                      proto = "udp";
                    }
                  ];
                };
                extraRules =
                let
                  aa = allAssignments;
                  matchInet = rule: sys: ''
                    ip daddr ${aa."${sys}".internal.ipv4.address} ${rule}
                    ip6 daddr ${aa."${sys}".internal.ipv6.address} ${rule}
                  '';
                in
                ''
                  table inet filter {
                    chain routing-tcp {
                      # Safe enough to allow all SSH
                      tcp dport ssh accept

                      ${matchInet "tcp dport { http, https, 8448 } accept" "middleman"}
                      ${matchInet "udp dport { 2456-2457 } accept" "valheim-oci"}

                      return
                    }
                    chain routing-udp {
                      return
                    }
                    chain filter-routing {
                      tcp flags & (fin|syn|rst|ack) == syn ct state new jump routing-tcp
                      meta l4proto udp ct state new jump routing-udp
                      return
                    }
                    chain forward {
                      iifname wan oifname base jump filter-routing
                    }
                  }
                  table inet nat {
                    chain prerouting {
                      ${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" "estuary"}
                    }
                    chain postrouting {
                      ip saddr ${lib.my.colony.prefixes.all.v4} masquerade
                    }
                  }
                '';
              };
            };
          }
          (mkIf config.my.build.isDevVM {
            systemd.network = {
              netdevs."05-dummy-base".netdevConfig = {
                Name = "base";
                Kind = "dummy";
              };
            };
          })
        ];
      };
  };
}