docs: Document home-router HA pattern and boxes terminology

Capture what this session surfaced: the routing-common indexed-module
active/backup pattern, VRRP VIPs for gateway+DNS, the wan-online.target
convention, the networkd helper locations, and the "boxes not fleet"
terminology. Also note the fish login shell caveat for ssh scripting.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

AGENTS.md

@@ -4,9 +4,9 @@ This file provides guidance to coding agents when working with code in this repo
 
 ## Overview
 
-Personal Nix flake managing NixOS systems and home-manager configurations for a fleet of
+Personal Nix flake managing NixOS systems and home-manager configurations for a set of
-machines (servers, home boxes, routers). It is built around a **custom module system** layered
+machines — always called **"boxes"**, never "fleet". It is built around a **custom module
-on top of NixOS/home-manager, not the stock flake `nixosConfigurations` pattern.
+system** layered on top of NixOS/home-manager, not the stock flake `nixosConfigurations` pattern.
 
 ## Commands
 
@@ -31,7 +31,8 @@ Common ones:
   VMs and containers).
 - `ssh-machine <name> [cmd]` — SSH to a NixOS system or home-manager config by name. Resolves the
   target and ssh options (identity, port) from its deploy-rs node, so it needs `my.deploy.enable`
-  (same gate as `deploy`).
+  (same gate as `deploy`). Boxes default to the `fish` login shell, so pipe multi-statement remote
+  scripts through `bash` (e.g. `ssh-machine <name> bash -s < script.sh`) rather than `&&`/`for`.
 - `ragenix` — edit age secrets using `.keys/dev.key` as identity (see Secrets).
 - `repl` — `nix repl .#`.
 - `update-nixpkgs` / `update-home-manager` — bump pinned inputs.
@@ -91,6 +92,29 @@ Per-host configs live under `nixos/boxes/<host>` (some are single `.nix` files,
 with nested VMs/containers under e.g. `colony/vms`). Many "systems" are VMs or containers managed
 via the `vms` / `containers` modules and the `l2mesh` VXLAN module.
 
+### Home routers (`nixos/boxes/home/routing-common`)
+The two home routers, `river` and `stream`, share `routing-common`, which is a **function of an
+`index`** (`import ../../routing-common 0` for river, `1` for stream). The index derives per-box
+addresses, keepalived VRRP priorities/state, DNS `ns` numbering, etc., so the two boxes are an
+active/backup HA pair from one definition. They differ where hardware/uplink differ: `stream` has a
+DHCP WAN, `river` runs PPPoE (`services.pppd`, Digiweb) — box-specific bits live in the respective
+box file, not `routing-common`.
+
+- **HA is VRRP (`keepalived`).** Per-VLAN floating **VIPs** (`lib.my.c.home.vips`) are what clients
+  use as both gateway *and* DNS server. `kea` (DHCP) and `radvd` (RAs; started only on the master)
+  hand out the VIP, and `pdns-recursor` binds the VIPs (with `net.ipv*.ip_nonlocal_bind` so the
+  backup can pre-bind). Point client-facing services at the VIP, not a box's real address, so
+  failover follows the master instead of relying on client resolver timeouts.
+- **`wan-online.target`** is a shared abstract target meaning "the public WAN/IPv4 route is up".
+  `routing-common` only declares it; each box wires *how it is reached* (`stream`: a oneshot that
+  waits for the DHCP default route; `river`: the pppd `ip-up`/`ip-down` hooks). Services that need
+  the WAN attach **to** it via `wantedBy` + `partOf` + `after` (not `requires`/`wants`), so an empty
+  target is never pulled in and prematurely activated, and they re-load on WAN flap.
+- networkd helpers used heavily here: `lib.my.networkdAssignment` and `lib.my.mkVLAN` live under
+  **`lib.my`**, while networkd snippet constants like `networkd.noL3` live under **`lib.my.c`** —
+  easy to mix up. Set an interface MTU via the `.network`'s `linkConfig.MTUBytes` (`[Link]`), not
+  `netdevConfig` (`[NetDev]` rejects `MTUBytes`).
+
 ## Secrets
 
 age-encrypted secrets in `secrets/`, managed with **ragenix**. Each module declares
@@ -111,3 +135,6 @@ private keys) is required for editing secrets, deploying, and running dev VMs.
   as `overlays.default`.
 - In prose and commit messages, quote code-like identifiers (commands, options, paths, package and
   attribute names) in backticks.
+- Call the machines **"boxes"**, never "fleet".
+- Commit subjects follow `area/scope: Capitalized summary` (e.g. `nixos/home: ...`); keep logically
+  distinct changes in separate commits.

lib/constants.nix

@@ -312,6 +312,9 @@ rec {
       lo = 110;
       untrusted = 120;
       wan = 130;
+      wan-pon = 131;
+
+      pon-isp = 10;
     };
     hiMTU = 9000;
     routers = [

lib/default.nix

@@ -175,11 +175,11 @@ rec {
   };
 
   vm = rec {
-    lvmDisk' = name: lv: {
+    lvmDisk'' = name: vg: lv: {
       inherit name;
       backend = {
         driver = "host_device";
-        filename = "/dev/main/${lv}";
+        filename = "/dev/${vg}/${lv}";
         # It appears this needs to be set on the backend _and_ the format
         discard = "unmap";
       };
@@ -189,7 +189,8 @@ rec {
       };
       frontend = "virtio-blk";
     };
-    lvmDisk = lv: lvmDisk' lv lv;
+    lvmDisk' = vg: lv: lvmDisk'' lv vg lv;
+    lvmDisk = lvmDisk' "main";
     disk = vm: lv: lvmDisk' lv "vm-${vm}-${lv}";
   };
 

nixos/boxes/colony/vms/shill/containers/object.nix

@@ -95,6 +95,7 @@ in
                   shell = pkgs.bashInteractive;
                   openssh.authorizedKeys.keyFiles = [
                     lib.my.c.sshKeyFiles.harmonia
+                    lib.my.c.sshKeyFiles.me
                   ];
                 };
               };

nixos/boxes/home/palace/default.nix

@@ -53,7 +53,7 @@ in
               };
             };
           });
-          kernelModules = [ "kvm-amd" ];
+          kernelModules = [ "dm-raid" "kvm-amd" ];
           kernelParams = [ "amd_iommu=on" ];
           initrd = {
             availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
@@ -126,9 +126,9 @@ in
                 };
                 linkConfig.Name = "et1g0";
               };
-              "10-lan-core" = {
+              "10-lan-core-phy" = {
                 matchConfig.PermanentMACAddress = "e0:d5:5e:68:0c:70";
-                linkConfig.Name = "lan-core";
+                linkConfig.Name = "lan-core-phy";
               };
               "10-et100g" = {
                 matchConfig = {
@@ -145,6 +145,12 @@ in
             netdevs = mkMerge [
               (mkVLAN "lan-hi" vlans.hi)
               (mkVLAN "lan-lo-phy" vlans.lo)
+              {
+                "25-lan-core".netdevConfig = {
+                  Name = "lan-core";
+                  Kind = "bridge";
+                };
+              }
               {
                 "25-lan-lo".netdevConfig = {
                   Name = "lan-lo";
@@ -199,6 +205,12 @@ in
               };
               "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
 
+              "50-lan-core-phy" = {
+                matchConfig.Name = "lan-core-phy";
+                networkConfig = {
+                  Bridge = "lan-core";
+                } // networkd.noL3;
+              };
               "50-lan-lo-phy" = {
                 matchConfig.Name = "lan-lo-phy";
                 networkConfig = {

nixos/boxes/home/palace/vms/default.nix

@@ -172,12 +172,23 @@
             };
             memory = 32768;
             cleanShutdown.timeout = 120;
-            networks.netboot = {
+            networks = {
-              bridge = "lan-lo";
+              netboot = {
-              waitOnline = "carrier";
+                bridge = "lan-lo";
-              mac = "52:54:00:a5:7e:93";
+                waitOnline = "carrier";
-              extraOptions.bootindex = 1;
+                mac = "52:54:00:a5:7e:93";
+                extraOptions.bootindex = 1;
+              };
+              core = {
+                bridge = "lan-core";
+                ifname = "vm-sfh-core";
+                waitOnline = "carrier";
+                mac = "52:54:00:72:67:51";
+              };
             };
+            drives = [
+              (vm.lvmDisk' "hdds" "frigate")
+            ];
             hostDevices = {
               et100g0vf2 = {
                 index = 0;

nixos/boxes/home/palace/vms/river.nix

@@ -8,6 +8,7 @@
 
     configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
     let
+      inherit (lib) mkForce mkMerge;
       inherit (lib.my) networkdAssignment mkVLAN;
       inherit (lib.my.c) networkd;
       inherit (lib.my.c.home) vlans domain prefixes roceBootModules;
@@ -71,11 +72,72 @@
             dmeventd.enable = true;
           };
           fstrim.enable = true;
+
+          # TODO: re-enable once scheduling is tested
+          networkd-dispatcher.enable = mkForce false;
+
+          pppd = {
+            enable = true;
+            peers.digiweb = {
+              autostart = true;
+              enable = true;
+              # Password is shared across all Digiweb customers, so no need for a secret
+              config = ''
+                plugin pppoe.so wan-vlan-inner
+                name "digiweb@nga.digiweb.ie"
+                password "digiweb"
+                noipdefault
+                # no usepeerdns: we ignore Digiweb's resolvers and use the local recursive resolver
+                lcp-echo-interval 1
+                lcp-echo-failure 4
+                noauth
+                persist
+                maxfail 0
+                holdoff 5
+                mtu 1500
+                mru 1500
+                noaccomp
+                default-asyncmap
+                ifname wan
+              '';
+            };
+          };
+        };
+
+        # PPPoE WAN (Digiweb): pppd owns the `wan` interface on top of VLAN 10, and its
+        # ip-up/ip-down hooks toggle the shared wan-online.target. Nothing else Wants the
+        # target, so it stays inactive until the link is actually up.
+        systemd.targets.wan-online.unitConfig.DefaultDependencies = false;
+
+        environment.etc = {
+          ppp-up = {
+            target = "ppp/ip-up";
+            mode = "0755";
+            text = ''
+              #!${pkgs.runtimeShell}
+              ${pkgs.iproute2}/bin/ip route add default dev wan scope link metric 100
+              ${config.systemd.package}/bin/systemctl --no-block start wan-online.target
+            '';
+          };
+          ppp-down = {
+            target = "ppp/ip-down";
+            mode = "0755";
+            text = ''
+              #!${pkgs.runtimeShell}
+              ${config.systemd.package}/bin/systemctl --no-block stop wan-online.target
+              ${pkgs.iproute2}/bin/ip route del default dev wan scope link metric 100
+            '';
+          };
         };
 
         systemd.network = {
+          netdevs = mkMerge [
+            (mkVLAN "wan-vlan-outer" vlans.wan-pon)
+            (mkVLAN "wan-vlan-inner" vlans.pon-isp)
+          ];
+
           links = {
-            "10-wan" = {
+            "10-wan-old" = {
               matchConfig = {
                 # Matching against MAC address seems to break VLAN interfaces
                 # (since they share the same MAC address)
@@ -83,7 +145,7 @@
                 PermanentMACAddress = "e0:d5:5e:68:0c:6e";
               };
               linkConfig = {
-                Name = "wan";
+                Name = "wan-old";
                 RxBufferSize = 4096;
                 TxBufferSize = 4096;
               };
@@ -101,8 +163,31 @@
             };
           };
 
-          # So we don't drop the IP we use to connect to NVMe-oF!
+          networks = {
-          networks."60-lan-hi".networkConfig.KeepConfiguration = "static";
+            "55-lan" = {
+              vlan = [ "wan-vlan-outer" ];
+            };
+            # So we don't drop the IP we use to connect to NVMe-oF!
+            "60-lan-hi".networkConfig.KeepConfiguration = "static";
+
+            "70-wan-vlan-outer" = {
+              matchConfig.Name = "wan-vlan-outer";
+              vlan = [ "wan-vlan-inner" ];
+              networkConfig = networkd.noL3;
+              # baby jumbo: 1508 so the inner VLAN below can also carry it
+              linkConfig.MTUBytes = "1508";
+            };
+            # pppd attaches PPPoE to this; just needs to be up with no L3
+            "71-wan-vlan-inner" = {
+              matchConfig.Name = "wan-vlan-inner";
+              linkConfig = {
+                RequiredForOnline = "no";
+                # baby jumbo: PPPoE's 8B overhead leaves a clean 1500 on ppp
+                MTUBytes = "1508";
+              };
+              networkConfig = networkd.noL3;
+            };
+          };
         };
 
         my = {

nixos/boxes/home/palace/vms/sfh/containers/default.nix

@@ -1,6 +1,6 @@
 {
   imports = [
-    # ./unifi.nix
+    ./unifi.nix
     ./hass.nix
   ];
 }

nixos/boxes/home/palace/vms/sfh/containers/unifi.nix

@@ -24,6 +24,15 @@ in
           address = net.cidr.host (65536*5+1) prefixes.hi.v6;
         };
       };
+      core = {
+        inherit domain;
+        name = "unifi-ctr-core";
+        mtu = 1500;
+        ipv4 = {
+          address = net.cidr.host 21 prefixes.core.v4;
+          gateway = null;
+        };
+      };
     };
 
     configuration = { lib, config, pkgs, assignments, ... }:
@@ -48,7 +57,10 @@ in
         };
 
         systemd = {
-          network.networks."80-container-host0" = networkdAssignment "host0" assignments.hi;
+          network.networks = {
+            "80-container-host0" = networkdAssignment "host0" assignments.hi;
+            "80-lan-core" = networkdAssignment "lan-core" assignments.core;
+          };
         };
 
         services = {

nixos/boxes/home/palace/vms/sfh/default.nix

@@ -72,6 +72,10 @@ in
             fsType = "ext4";
             neededForBoot = true;
           };
+          "/mnt/frigate" = {
+            device = "/dev/disk/by-label/frigate";
+            fsType = "ext4";
+          };
         };
 
         networking = { inherit domain; };
@@ -111,6 +115,13 @@ in
                 MTUBytes = toString lib.my.c.home.hiMTU;
               };
             };
+            "10-lan-core-ctrs" = {
+              matchConfig = {
+                Driver = "virtio_net";
+                PermanentMACAddress = "52:54:00:72:67:51";
+              };
+              linkConfig.Name = "lan-core-ctrs";
+            };
             "10-lan-lo-ctrs" = {
               matchConfig = {
                 Driver = "virtio_net";
@@ -131,6 +142,11 @@ in
               linkConfig.RequiredForOnline = "no";
               networkConfig = networkd.noL3;
             };
+            "30-lan-core-ctrs" = {
+              matchConfig.Name = "lan-core-ctrs";
+              linkConfig.RequiredForOnline = "no";
+              networkConfig = networkd.noL3;
+            };
             "30-lan-lo-ctrs" = {
               matchConfig.Name = "lan-lo-ctrs";
               linkConfig.RequiredForOnline = "no";
@@ -145,6 +161,11 @@ in
               MACVLAN = mkForce "lan-hi-ctrs:host0 lan-lo-ctrs:lan-lo";
             };
           };
+          unifi = {
+            networkConfig = {
+              MACVLAN = mkForce "lan-hi-ctrs:host0 lan-core-ctrs:lan-core";
+            };
+          };
         };
 
         systemd.services = {
@@ -174,9 +195,13 @@ in
           containers.instances =
           let
             instances = {
-              # unifi = {};
+              unifi = {};
               hass = {
                 bindMounts = {
+                  "/mnt/frigate" = {
+                    mountPoint = "/var/lib/frigate";
+                    readOnly = false;
+                  };
                   "/dev/bus/usb/001/002".readOnly = false;
                   "/dev/video0".readOnly = false;
                   "/dev/serial/by-id/usb-Nabu_Casa_Home_Assistant_Connect_ZBT-1_ce549704fe38ef11a2c2e5d154516304-if00-port0" = {

nixos/boxes/home/routing-common/default.nix

@@ -152,26 +152,34 @@ in
 
           networking = { inherit domain; };
 
-          systemd.services =
+          # Uniform "WAN is up" gate. Consumers attach to this target (via wantedBy +
-          let
+          # partOf) rather than depending on it, so it is never pulled in / prematurely
-            waitOnline = "systemd-networkd-wait-online@wan.service";
+          # activated. Each box wires up how the target actually gets reached: stream
-          in
+          # gates it on networkd's wait-online, river drives it from the pppd hooks.
-          {
+          systemd.targets.wan-online.description = "WAN is online";
+
+          systemd.services = {
             ipsec = {
-              after = [ waitOnline ];
+              after = [ "wan-online.target" ];
-              requires = [ waitOnline ];
+              # strongswan/libreswan force wantedBy=multi-user.target; drop it so the
+              # target is a true gate rather than mere ordering. This matters most on
+              # river, where the target is hook-driven and not in the boot transaction,
+              # so plain ordering wouldn't hold ipsec back at all. partOf re-loads ipsec
+              # (re-orienting its connections) whenever the WAN drops and returns.
+              wantedBy = mkForce [ "wan-online.target" ];
+              partOf = [ "wan-online.target" ];
             };
 
             ipv6-clear-default-route = {
               description = "Clear IPv6 RA default route";
-              after = [ waitOnline ];
+              after = [ "wan-online.target" ];
-              requires = [ waitOnline ];
+              wantedBy = [ "wan-online.target" ];
+              partOf = [ "wan-online.target" ];
               script = ''
                 # Seems like we can sometimes pick up a default route somehow...
                 ${pkgs.iproute2}/bin/ip -6 route del default via fe80::1 || true
               '';
               serviceConfig.Type = "oneshot";
-              wantedBy = [ "multi-user.target" ];
             };
           };
 
@@ -220,41 +228,6 @@ in
             in
             mkMerge [
               {
-                "50-wan-ifb" = {
-                  matchConfig.Name = "wan-ifb";
-                  networkConfig = networkd.noL3;
-                  extraConfig = ''
-                    [CAKE]
-                    Bandwidth=490M
-                    RTTSec=50ms
-                    PriorityQueueingPreset=besteffort
-                    # DOCSIS preset
-                    OverheadBytes=18
-                    MPUBytes=64
-                    CompensationMode=none
-                  '';
-                };
-                "50-wan" = mkMerge [
-                  (networkdAssignment "wan" assignments.modem)
-                  {
-                    matchConfig.Name = "wan";
-                    DHCP = "ipv4";
-                    dns = [ "127.0.0.1" "::1" ];
-                    dhcpV4Config.UseDNS = false;
-
-                    qdiscConfig = {
-                      Parent = "ingress";
-                      Handle = "0xffff";
-                    };
-                    extraConfig = ''
-                      [CAKE]
-                      Parent=root
-                      Bandwidth=48M
-                      RTTSec=50ms
-                    '';
-                  }
-                ];
-
                 "55-lan" = {
                   matchConfig.Name = "lan";
                   vlan = [ "lan-hi" "lan-lo" "lan-untrusted" "wan-tunnel" ];

nixos/boxes/home/routing-common/dns.nix

@@ -13,6 +13,13 @@ let
 in
 {
   config = {
+    # Let pdns-recursor bind the VRRP VIPs even on the backup, where the addresses
+    # aren't present locally
+    boot.kernel.sysctl = {
+      "net.ipv4.ip_nonlocal_bind" = 1;
+      "net.ipv6.ip_nonlocal_bind" = 1;
+    };
+
     my = {
       secrets.files = {
         "home/pdns/auth.conf" = {
@@ -40,6 +47,10 @@ in
               "127.0.0.1" "::1"
               assignments.hi.ipv4.address assignments.hi.ipv6.address
               assignments.lo.ipv4.address assignments.lo.ipv6.address
+              # VRRP VIPs: DNS follows the master, so clients only ever have one
+              # (always-live) resolver address and never hang on a dead router
+              vips.hi.v4 vips.hi.v6
+              vips.lo.v4 vips.lo.v6
             ];
             allow_from = [
               "127.0.0.0/8" "::1/128"
@@ -223,6 +234,7 @@ in
             dave-lo IN AAAA ${net.cidr.host (65536+2) prefixes.lo.v6}
 
             shytzel IN A ${net.cidr.host 12 prefixes.core.v4}
+            brian IN A ${net.cidr.host 13 prefixes.core.v4}
 
             wave IN A ${net.cidr.host 12 prefixes.hi.v4}
             ; wave IN AAAA ${net.cidr.host (65536+3) prefixes.hi.v6}

nixos/boxes/home/routing-common/dns_update.py

@@ -33,7 +33,7 @@ def main():
 
     print(f'Updating {args.record} -> {address}')
     cf.dns.records.edit(
-        zone_id=zone.id, dns_record_id=record.id, name=args.record,
+        zone_id=zone.id, dns_record_id=record.id, name=args.record, ttl=60,
         type='A', content=address)
 
 if __name__ == '__main__':

nixos/boxes/home/routing-common/kea.nix

@@ -83,7 +83,8 @@ in
                 }
                 {
                   name = "domain-name-servers";
-                  data = "${net.cidr.host 1 prefixes.hi.v4}, ${net.cidr.host 2 prefixes.hi.v4}";
+                  # VRRP VIP so DNS follows the master and clients never hit a dead router
+                  data = vips.hi.v4;
                 }
                 {
                   name = "interface-mtu";
@@ -116,7 +117,8 @@ in
                 }
                 {
                   name = "domain-name-servers";
-                  data = "${net.cidr.host 1 prefixes.lo.v4}, ${net.cidr.host 2 prefixes.lo.v4}";
+                  # VRRP VIP so DNS follows the master and clients never hit a dead router
+                  data = vips.lo.v4;
                 }
               ];
               pools = [

nixos/boxes/home/routing-common/mstpd.nix

@@ -3,11 +3,14 @@ let
   # TODO: Move into nixpkgs
   mstpd = pkgs.mstpd.overrideAttrs {
     patches = [ ./mstpd.patch ];
+    # Delete postInstall since it nukes the bridge-stp script we need
+    postInstall = "";
   };
 in
 {
   environment = {
     systemPackages = [
+      # For kernel to call bridge-stp (see ./pkgs/os-specific/linux/kernel/bridge-stp-helper.patch)
       mstpd
     ];
     etc = {
@@ -39,8 +42,8 @@ in
         before = [ "network-pre.target" ];
         serviceConfig = {
           Type = "forking";
-          ExecStart = "${mstpd}/sbin/bridge-stp restart";
+          ExecStart = "${mstpd}/bin/bridge-stp restart";
-          ExecReload = "${mstpd}/sbin/bridge-stp restart_config";
+          ExecReload = "${mstpd}/bin/bridge-stp restart_config";
           PIDFile = "/run/mstpd.pid";
           Restart = "always";
           PrivateTmp = true;

nixos/boxes/home/routing-common/radvd.nix

@@ -2,7 +2,14 @@ index: { lib, pkgs, ... }:
 let
   inherit (lib) mkForce concatMapStringsSep;
   inherit (lib.my) net;
-  inherit (lib.my.c.home) domain prefixes;
+  inherit (lib.my.c.home) domain prefixes vips;
+
+  # untrusted uses external (Cloudflare) resolvers, matching the v4 kea config;
+  # trusted VLANs use the internal recursor via its floating VRRP VIP
+  rdnss = name:
+    if name == "untrusted"
+    then "2606:4700:4700::1111 2606:4700:4700::1001"
+    else vips."${name}".v6;
 
   mkInterface = name: ''
     interface lan-${name} {
@@ -10,7 +17,7 @@ let
       AdvRASrcAddress { fe80::1; };
       AdvLinkMTU ${toString prefixes."${name}".mtu};
       prefix ${prefixes."${name}".v6} {};
-      RDNSS ${net.cidr.host 1 prefixes."${name}".v6} ${net.cidr.host 2 prefixes."${name}".v6} {};
+      RDNSS ${rdnss name} {};
       DNSSL ${domain} dyn.${domain} ${lib.my.c.colony.domain} ${lib.my.c.britway.domain} {};
      };
   '';

nixos/boxes/home/stream.nix

@@ -6,9 +6,11 @@
     nixpkgs = "mine";
     home-manager = "mine";
 
-    configuration = { lib, pkgs, config, ... }:
+    configuration = { lib, pkgs, config, assignments, ... }:
     let
-      inherit (lib);
+      inherit (lib) mkMerge;
+      inherit (lib.my) networkdAssignment;
+      inherit (lib.my.c) networkd;
     in
     {
       imports = [ ./routing-common/mstpd.nix ];
@@ -75,6 +77,32 @@
           };
         };
 
+        # wan carries a permanent static modem-management address (assignments.modem)
+        # alongside the DHCP public IP, so wait-online@wan reports "online" as soon as
+        # the static address is up - before the DHCP lease arrives. ipsec's left= is the
+        # public IP, so gating on wait-online lets it start unoriented and never connect.
+        # Gate instead on the DHCP default route, which only exists once the public lease
+        # is up (the static modem address has no gateway).
+        systemd.services.wan-wait-online = {
+          description = "Wait for the wan default route (public DHCP lease)";
+          after = [ "systemd-networkd.service" ];
+          serviceConfig = {
+            Type = "oneshot";
+            RemainAfterExit = true;
+            TimeoutStartSec = "300";
+          };
+          script = ''
+            until [ -n "$(${pkgs.iproute2}/bin/ip -4 route show default dev wan)" ]; do
+              sleep 1
+            done
+          '';
+        };
+        systemd.targets.wan-online = {
+          requires = [ "wan-wait-online.service" ];
+          after = [ "wan-wait-online.service" ];
+          wantedBy = [ "multi-user.target" ];
+        };
+
         systemd.network = {
           netdevs = {
             "25-lan" = {
@@ -147,6 +175,41 @@
               matchConfig.Name = "lan-dave";
               networkConfig.Bridge = "lan";
             };
+
+            "50-wan-ifb" = {
+              matchConfig.Name = "wan-ifb";
+              networkConfig = networkd.noL3;
+              extraConfig = ''
+                [CAKE]
+                Bandwidth=490M
+                RTTSec=50ms
+                PriorityQueueingPreset=besteffort
+                # DOCSIS preset
+                OverheadBytes=18
+                MPUBytes=64
+                CompensationMode=none
+              '';
+            };
+            "50-wan" = mkMerge [
+              (networkdAssignment "wan" assignments.modem)
+              {
+                matchConfig.Name = "wan";
+                DHCP = "ipv4";
+                dns = [ "127.0.0.1" "::1" ];
+                dhcpV4Config.UseDNS = false;
+
+                qdiscConfig = {
+                  Parent = "ingress";
+                  Handle = "0xffff";
+                };
+                extraConfig = ''
+                  [CAKE]
+                  Parent=root
+                  Bandwidth=48M
+                  RTTSec=50ms
+                '';
+              }
+            ];
           };
         };
 

nixos/modules/netboot/default.nix

@@ -8,7 +8,7 @@ let
   tftpRoot = pkgs.linkFarm "tftp-root" [
     {
       name = "ipxe-x86_64.efi";
-      path = "${pkgs.ipxe}/ipxe.efi";
+      path = "${pkgs.ipxe}/snp.efi";
     }
   ];
   menuFile = pkgs.runCommand "menu.ipxe" {