lib: River IP update

The old build-n-parse seemed to output null sometimes.....

.envrc

@@ -1,2 +1,2 @@
-nix_direnv_watch_file devshell/{default,commands,install,vm-tasks}.nix
+watch_file devshell/{default,commands,install,vm-tasks}.nix
 use flake

.gitea/workflows/ci.yaml

@@ -6,11 +6,11 @@ on:
 
 jobs:
   check:
-    name: Check, build and cache Nix flake
+    name: Check, build and cache nixfiles
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
-      - uses: cachix/install-nix-action@v23
+      - uses: cachix/install-nix-action@v27
         with:
           # Gitea will supply a token in GITHUB_TOKEN, which this action will
           # try to pass to Nix when downloading from GitHub
@@ -18,22 +18,30 @@ jobs:
           extra_nix_config: |
             # Make sure we're using sandbox
             sandbox-fallback = false
+            # Big C++ projects fill up memory...
+            cores = 6
 
-            extra-substituters = https://nix-cache.nul.ie/main
+            extra-substituters = https://nix-cache.nul.ie
-            extra-trusted-public-keys = main:mMChkG8LwXrFirVfudqjSHasK1jV31OVElYD3eImYl8=
+            extra-trusted-public-keys = nix-cache.nul.ie-1:BzH5yMfF4HbzY1C977XzOxoPhEc9Zbu39ftPkUbH+m4=
-      - name: Set up attic
-        run: |
-          nix run .#nixpkgs.mine.x86_64-linux.attic-client -- \
-            login --set-default colony https://nix-cache.nul.ie "${{ secrets.NIX_CACHE_TOKEN }}"
 
       - name: Check flake
-        run: nix flake check
+        run: nix flake check --no-build
-      - name: Build the world
+
+      - name: Build (and cache) the world
         id: build
+        env:
+          HARMONIA_SSH_KEY: ${{ secrets.HARMONIA_SSH_KEY }}
         run: |
-          path=$(nix build --no-link .#ci.x86_64-linux --json | jq -r .[0].outputs.out)
+          nix eval --json --apply "builtins.attrNames" .#ci.x86_64-linux | jq -cr '.[]' | while read job; do
-          echo "path=$path" >> "$GITHUB_OUTPUT"
+            echo "::group::Build $job"
-      - name: Push to cache
+            nix build --no-link .#ci.x86_64-linux."$job"
-        run: |
+            echo "::endgroup::"
-          nix run .#nixpkgs.mine.x86_64-linux.attic-client -- \
+
-            push main ${{ steps.build.outputs.path }}
+            echo "::group::Cache $job"
+            ci/push-to-cache.sh "$(nix eval --raw .#ci.x86_64-linux."$job")"
+            echo "::endgroup::"
+          done
+
+          echo "Building and caching CI derivation"
+          nix build --no-link .#ciDrv.x86_64-linux
+          UPDATE_PROFILE=1 ci/push-to-cache.sh "$(nix eval --raw .#ciDrv.x86_64-linux)"

.gitea/workflows/installer.yaml

@@ -0,0 +1,49 @@
+name: Installer
+
+on:
+  push:
+    tags: [installer]
+
+jobs:
+  installer:
+    name: Build installer
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Go
+        uses: https://github.com/actions/setup-go@v4
+        with:
+          go-version: '>=1.20.1'
+      - uses: cachix/install-nix-action@v27
+        with:
+          github_access_token: ${{ secrets.GH_PULL_TOKEN }}
+          extra_nix_config: |
+            # Make sure we're using sandbox
+            sandbox-fallback = false
+
+            extra-substituters = https://nix-cache.nul.ie
+            extra-trusted-public-keys = nix-cache.nul.ie-1:BzH5yMfF4HbzY1C977XzOxoPhEc9Zbu39ftPkUbH+m4=
+
+      - name: Set up vars
+        id: setup
+        run: |
+          echo "short_rev=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+      - name: Build installer ISO
+        run: |
+          nix build .#nixfiles.config.nixos.systems.installer.configuration.config.my.buildAs.iso
+          ln -s "$(readlink result)"/iso/jackos-installer.iso \
+            jackos-installer-${{ steps.setup.outputs.short_rev }}.iso
+      - name: Build installer netboot archive
+        run: |
+          nix build .#nixfiles.config.nixos.systems.installer.configuration.config.my.buildAs.netbootArchive
+          ln -s "$(readlink result)" \
+            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst
+
+      - name: Create release
+        uses: https://gitea.com/actions/release-action@main
+        with:
+          title: Latest installer
+          api_key: '${{ secrets.RELEASE_TOKEN }}'
+          files: |
+            jackos-installer-${{ steps.setup.outputs.short_rev }}.iso
+            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst

.keys/harmonia.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKXRXkYnBf2opIjN+bXE7HmhUpa4hyXJUGmBT+MRccT4 harmonia

ci/known_hosts

@@ -0,0 +1 @@
+object-ctr.ams1.int.nul.ie ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdHbZErWLmTPO/aEWB1Fup/aGMf31Un5Wk66FJwTz/8

ci/push-to-cache.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+set -e
+
+REMOTE_STORE=/var/lib/harmonia
+SSH_HOST="harmonia@object-ctr.ams1.int.nul.ie"
+SSH_KEY=/tmp/harmonia.key
+STORE_URI="ssh-ng://$SSH_HOST?ssh-key=$SSH_KEY&remote-store=$REMOTE_STORE"
+
+remote_cmd() {
+  ssh -i "$SSH_KEY" "$SSH_HOST" env HOME=/run/harmonia NIX_REMOTE="$REMOTE_STORE" "$@"
+}
+
+umask_old=$(umask)
+umask 0066
+echo "$HARMONIA_SSH_KEY" | base64 -d > "$SSH_KEY"
+umask $umask_old
+
+mkdir -p ~/.ssh
+cp ci/known_hosts ~/.ssh/
+path="$1"
+
+echo "Pushing $path to cache..."
+nix copy --no-check-sigs --to "$STORE_URI" "$path"
+
+if [ -n "$UPDATE_PROFILE" ]; then
+  echo "Updating profile..."
+  remote_cmd nix-env -p "$REMOTE_STORE"/nix/var/nix/profiles/nixfiles --set "$path"
+
+  echo "Collecting garbage..."
+  remote_cmd nix-collect-garbage --delete-older-than 60d
+fi

deploy-rs.nix

@@ -47,8 +47,8 @@ in
           (nodesFor homes)
         );
 
-        autoRollback = true;
+        autoRollback = false;
-        magicRollback = true;
+        magicRollback = false;
       };
 
       # Filter out null values so deploy merges overriding options correctly

devshell/commands.nix

@@ -20,7 +20,7 @@ in
           [ -e "${homeFlake}" ] && echo "${homeFlake} already exists" && exit 1
 
           mkdir -p "$(dirname "${homeFlake}")"
-          ln -s "$(pwd)/flake.nix" "${homeFlake}"
+          ln -sf "$(pwd)/flake.nix" "${homeFlake}"
           echo "Installed link to $(pwd)/flake.nix at ${homeFlake}"
         '';
     }
@@ -48,6 +48,12 @@ in
       help = "Print the ed25519 pubkey for a host";
       command = "${pkgs.openssh}/bin/ssh-keyscan -t ed25519 \"$1\" 2> /dev/null | awk '{ print $2 \" \" $3 }'";
     }
+    {
+      name = "json2nix";
+      category = "utilities";
+      help = "Convert JSON to formatted Nix";
+      command = "nix eval --impure --expr 'builtins.fromJSON (builtins.readFile /dev/stdin)' | ${pkgs.nixfmt-rfc-style}/bin/nixfmt";
+    }
 
     {
       name = "fmt";
@@ -71,7 +77,12 @@ in
       name = "build-n-switch";
       category = "tasks";
       help = "Shortcut to nixos-rebuild for this flake";
-      command = ''doas nixos-rebuild --flake . "$@"'';
+      command = ''
+        # HACK: Upstream changes in Git + Nix makes this necessary
+        # https://github.com/NixOS/nix/issues/10202
+        doas git config --global --add safe.directory "$PWD"
+        doas nixos-rebuild --flake . "$@"
+      '';
     }
     {
       name = "run-vm";
@@ -91,35 +102,51 @@ in
       help = "Build NixOS configuration into an ISO";
       command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.iso"'';
     }
+    {
+      name = "build-kexec";
+      category = "tasks";
+      help = "Build NixOS configuration as kexec tree";
+      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.kexecTree"'';
+    }
+    {
+      name = "build-netboot";
+      category = "tasks";
+      help = "Build NixOS configuration as netboot tree";
+      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"'';
+    }
     {
       name = "build-home";
       category = "tasks";
       help = "Build home-manager configuration";
       command = ''nix build "''${@:2}" ".#homeConfigurations.\"$1\".activationPackage"'';
     }
-    {
-      name = "update-inputs";
-      category = "tasks";
-      help = "Update flake inputs";
-      command = ''
-        args=()
-        for f in "$@"; do
-          args+=(--update-input "$f")
-        done
-        nix flake lock "''${args[@]}"
-      '';
-    }
     {
       name = "update-nixpkgs";
       category = "tasks";
       help = "Update nixpkgs flake inputs";
-      command = ''update-inputs nixpkgs-{unstable,stable,mine,mine-stable}'';
+      command = ''nix flake update nixpkgs-{unstable,stable,mine,mine-stable}'';
     }
     {
       name = "update-home-manager";
       category = "tasks";
       help = "Update home-manager flake inputs";
-      command = ''update-inputs home-manager-{unstable,stable}'';
+      command = ''nix flake update home-manager-{unstable,stable}'';
+    }
+    {
+      name = "update-installer";
+      category = "tasks";
+      help = "Update installer tag (to trigger new release)";
+      command = ''git tag -f installer && git push -f origin installer'';
+    }
+    {
+      name = "deploy-multi";
+      category = "tasks";
+      help = "Deploy multiple flakes at once";
+      command = ''
+        for f in $@; do
+          deploy "$O" $f
+        done
+      '';
     }
   ];
 }

devshell/default.nix

@@ -11,7 +11,7 @@ in
 
     NIX_USER_CONF_FILES = toString (pkgs.writeText "nix.conf"
       ''
-        experimental-features = nix-command flakes ca-derivations repl-flake
+        experimental-features = nix-command flakes ca-derivations
         connect-timeout = 5
         fallback = true
         ${lib.my.c.nix.cache.conf}
@@ -24,8 +24,10 @@ in
     coreutils
     nixVersions.stable
     rage
-    deploy-rs.deploy-rs
+    wireguard-tools
+    (pkgs.writeShellScriptBin "deploy" ''
+      exec ${deploy-rs.deploy-rs}/bin/deploy --skip-checks "$@"
+    '')
     home-manager
-    attic-client
   ];
 }

flake.nix

@@ -7,26 +7,27 @@
     devshell.inputs.nixpkgs.follows = "nixpkgs-unstable";
 
     nixpkgs-unstable.url = "nixpkgs/nixos-unstable";
-    nixpkgs-stable.url = "nixpkgs/nixos-23.05";
+    nixpkgs-stable.url = "nixpkgs/nixos-24.11";
     nixpkgs-mine.url = "github:devplayer0/nixpkgs/devplayer0";
     nixpkgs-mine-stable.url = "github:devplayer0/nixpkgs/devplayer0-stable";
 
     home-manager-unstable.url = "home-manager";
     home-manager-unstable.inputs.nixpkgs.follows = "nixpkgs-unstable";
-    home-manager-stable.url = "home-manager/release-23.05";
+    home-manager-stable.url = "home-manager/release-24.11";
     home-manager-stable.inputs.nixpkgs.follows = "nixpkgs-stable";
 
     # Stuff used by the flake for build / deployment
-    ragenix.url = "github:yaxitech/ragenix";
+    # ragenix.url = "github:yaxitech/ragenix";
+    ragenix.url = "github:devplayer0/ragenix/add-rekey-one-flag";
     ragenix.inputs.nixpkgs.follows = "nixpkgs-unstable";
     deploy-rs.url = "github:serokell/deploy-rs";
     deploy-rs.inputs.nixpkgs.follows = "nixpkgs-unstable";
 
     # Stuff used by systems
     impermanence.url = "github:nix-community/impermanence";
-    boardie.url = "github:devplayer0/boardie";
+    boardie.url = "git+https://git.nul.ie/dev/boardie";
     boardie.inputs.nixpkgs.follows = "nixpkgs-unstable";
-    nixGL.url = "github:guibou/nixGL";
+    nixGL.url = "github:nix-community/nixGL";
     nixGL.inputs.nixpkgs.follows = "nixpkgs-unstable";
 
     # Packages not in nixpkgs
@@ -34,9 +35,6 @@
     sharry.inputs.nixpkgs.follows = "nixpkgs-unstable";
     borgthin.url = "github:devplayer0/borg";
     borgthin.inputs.nixpkgs.follows = "nixpkgs-mine";
-    attic.url = "github:zhaofengli/attic";
-    attic.inputs.nixpkgs.follows = "nixpkgs-unstable";
-    attic.inputs.nixpkgs-stable.follows = "nixpkgs-stable";
   };
 
   outputs =
@@ -51,7 +49,7 @@
       ...
     }:
     let
-      inherit (builtins) mapAttrs replaceStrings;
+      inherit (builtins) mapAttrs replaceStrings elem;
       inherit (lib) mapAttrs' filterAttrs nameValuePair recurseIntoAttrs evalModules;
       inherit (lib.flake) flattenTree eachDefaultSystem;
       inherit (lib.my) mkDefaultSystemsPkgs flakePackageOverlay;
@@ -63,7 +61,7 @@
         flake = flake-utils.lib;
       };
       pkgsLibOverlay = final: prev: { lib = prev.lib.extend libOverlay; };
-      myPkgsOverlay = final: prev: import ./pkgs { lib = prev.lib; pkgs = prev; };
+      myPkgsOverlay = final: prev: import ./pkgs { lib = final.lib; pkgs = prev; };
 
       # Override the flake-level lib since we're going to use it for non-config specific stuff
       pkgsFlakes = mapAttrs (_: pkgsFlake: pkgsFlake // { lib = pkgsFlake.lib.extend libOverlay; }) {
@@ -95,7 +93,6 @@
             inputs.ragenix.overlays.default
             inputs.deploy-rs.overlay
             (flakePackageOverlay inputs.home-manager-unstable system)
-            inputs.attic.overlays.default
           ];
         }))
         pkgsFlakes;
@@ -107,6 +104,16 @@
             pkgsLibOverlay
             myPkgsOverlay
           ];
+
+          config = {
+            # RMS forgive me...
+            # Normally this is set modularly, but sometimes we need to use other pkgs
+            allowUnfreePredicate = p: elem (lib.getName p) [
+              "widevine-cdm"
+              "chromium-unwrapped"
+              "chromium"
+            ];
+          };
         }))
         pkgsFlakes;
 
@@ -115,8 +122,10 @@
         nixos/installer.nix
         nixos/boxes/colony
         nixos/boxes/tower
-        nixos/boxes/castle
+        nixos/boxes/home/stream.nix
-        nixos/boxes/home/oxbow.nix
+        nixos/boxes/home/palace
+        nixos/boxes/home/castle
+        nixos/boxes/britway
         nixos/boxes/kelder
 
         # Homes
@@ -127,7 +136,7 @@
         modules = [
           {
             _module.args = {
-              inherit lib pkgsFlakes hmFlakes inputs;
+              inherit lib pkgsFlakes hmFlakes self inputs;
               pkgs' = configPkgs';
             };
 
@@ -195,8 +204,9 @@
         systems' = mapAttrs' (n: v: nameValuePair "system-${n}" v) systems;
         packages' = mapAttrs' (n: v: nameValuePair "package-${n}" v) packages;
       in
-        pkgs.linkFarm "ci" (homes' // systems' // packages' // {
+        homes' // systems' // packages' // {
           inherit shell;
-        });
+        };
+      ciDrv = pkgs.linkFarm "ci" ci;
     }));
 }

home-manager/modules/common.nix

@@ -47,9 +47,14 @@ in
 
       nix = {
         package = mkIf (!(versionAtLeast config.home.stateVersion "22.11")) pkgs.nix;
-        settings = {
+        settings = with lib.my.c.nix; {
           experimental-features = [ "nix-command" "flakes" "ca-derivations" ];
           max-jobs = mkDefault "auto";
+
+          extra-substituters = cache.substituters;
+          extra-trusted-public-keys = cache.keys;
+          connect-timeout = 5;
+          fallback = true;
         };
       };
 
@@ -75,6 +80,7 @@ in
 
         tmux = {
           enable = true;
+          keyMode = "vi";
         };
 
         bash = {
@@ -193,17 +199,20 @@ in
           file
           tree
           pwgen
+
+          minicom
           iperf3
           mosh
           wget
-          hyx
           whois
           ldns
-          minicom
           mtr
+
+          hyx
           ncdu
           jq
           yq-go
+          nix-tree
         ];
 
         sessionVariables = {
@@ -235,12 +244,6 @@ in
             exact = true;
           };
         };
-        settings = with lib.my.c.nix; {
-          extra-substituters = cache.substituters;
-          extra-trusted-public-keys = cache.keys;
-          connect-timeout = 5;
-          fallback = true;
-        };
       };
     })
     (mkIf config.my.isStandalone {

home-manager/modules/gui/alacritty-xterm.toml

@@ -0,0 +1,28 @@
+# XTerm's default colors
+
+# Default colors
+[colors.primary]
+background = '#000000'
+foreground = '#ffffff'
+
+# Normal colors
+[colors.normal]
+black   = '#000000'
+red     = '#cd0000'
+green   = '#00cd00'
+yellow  = '#cdcd00'
+blue    = '#0000ee'
+magenta = '#cd00cd'
+cyan    = '#00cdcd'
+white   = '#e5e5e5'
+
+# Bright colors
+[colors.bright]
+black   = '#7f7f7f'
+red     = '#ff0000'
+green   = '#00ff00'
+yellow  = '#ffff00'
+blue    = '#5c5cff'
+magenta = '#ff00ff'
+cyan    = '#00ffff'
+white   = '#ffffff'

home-manager/modules/gui/default.nix

@@ -1,6 +1,6 @@
-{ lib, pkgs, config, ... }:
+{ lib, pkgs', pkgs, config, ... }:
 let
-  inherit (lib) genAttrs mkIf mkMerge mkForce;
+  inherit (lib) genAttrs mkIf mkMerge mkForce mapAttrs mkOptionDefault;
   inherit (lib.my) mkBoolOpt';
 
   cfg = config.my.gui;
@@ -10,6 +10,23 @@ let
     name = "Monocraft";
     size = 10;
   };
+
+  doomWad = pkgs.fetchurl {
+    url = "https://distro.ibiblio.org/slitaz/sources/packages/d/doom1.wad";
+    hash = "sha256-HX1DvlAeZ9kn5BXguPPinDvzMHXoWXIYFvZSpSbKx3E=";
+  };
+
+  doomsaver = pkgs.runCommand "doomsaver" {
+    inherit (pkgs) windowtolayer;
+    chocoDoom = pkgs.chocolate-doom2xx;
+    python = pkgs.python3.withPackages (ps: [ ps.filelock ]);
+    inherit doomWad;
+    enojy = ./enojy.jpg;
+  } ''
+    mkdir -p "$out"/bin
+    substituteAll ${./screensaver.py} "$out"/bin/doomsaver
+    chmod +x "$out"/bin/doomsaver
+  '';
 in
 {
   options.my.gui = {
@@ -22,21 +39,32 @@ in
       {
         home = {
           packages = with pkgs; [
+            xdg-utils
+
             font.package
-            (nerdfonts.override {
+            nerd-fonts.sauce-code-pro
-              fonts = [ "DroidSansMono" "SourceCodePro" ];
+            nerd-fonts.droid-sans-mono
-            })
             noto-fonts-emoji
 
             grim
             slurp
             swappy
 
-            python310Packages.python-lsp-server
+            python3Packages.python-lsp-server
             nil # nix language server
             zls # zig language server
             rust-analyzer
+
+            cowsay
+            fortune
+            jp2a
+            terminaltexteffects
+            screenfetch
             neofetch
+            cmatrix
+            doomsaver
+
+            xournalpp
           ];
         };
 
@@ -51,7 +79,15 @@ in
           alacritty = {
             enable = true;
             settings = {
-              font.normal.family = font.name;
+              general.import = [ ./alacritty-xterm.toml ];
+
+              font = {
+                size = font.size;
+                normal = {
+                  family = font.name;
+                  style = "Regular";
+                };
+              };
             };
           };
 
@@ -59,8 +95,29 @@ in
             enable = true;
             inherit font;
             settings = {
-              background_opacity = "0.8";
+              background_opacity = "0.65";
               tab_bar_edge = "top";
+              shell_integration = "no-sudo";
+              font_features = "${font.name} -liga";
+            };
+          };
+
+          termite = {
+            enable = true;
+            font = "${font.name} ${toString font.size}";
+            backgroundColor = "rgba(0, 0, 0, 0.8)";
+          };
+
+          foot = {
+            enable = true;
+            settings = {
+              main = {
+                font = "${font.name}:size=${toString font.size}";
+              };
+              colors = {
+                alpha = 0.8;
+                background = "000000";
+              };
             };
           };
 
@@ -107,6 +164,19 @@ in
               };
               Install.RequiredBy = [ "sway-session.target" ];
             };
+
+            activate-linux = {
+              Unit = {
+                Description = "Linux activation watermark";
+                After = "graphical-session.target";
+                PartOf = "graphical-session.target";
+              };
+              Service = {
+                Type = "simple";
+                ExecStart = "${pkgs.activate-linux}/bin/activate-linux";
+              };
+              Install.RequiredBy = [ "graphical-session.target" ];
+            };
           };
         };
 
@@ -116,6 +186,7 @@ in
             wl-clipboard
             wev
             wdisplays
+            swaysome
 
             pavucontrol
             libsecret
@@ -125,10 +196,11 @@ in
           ];
 
           pointerCursor = {
-            package = pkgs.vanilla-dmz;
+            package = pkgs.posy-cursors;
-            name = "Vanilla-DMZ";
+            name = "Posy_Cursor";
-            size = 16;
+            size = 32;
             gtk.enable = true;
+            x11.enable = true;
           };
         };
 
@@ -137,9 +209,36 @@ in
         xsession.preferStatusNotifierItems = true;
         wayland = {
           windowManager = {
-            sway = {
+            sway =
+            let
+              cfg = config.wayland.windowManager.sway.config;
+              mod = cfg.modifier;
+
+              renameWs = pkgs.writeShellScript "sway-rename-ws" ''
+                focused_ws="$(swaymsg -t get_workspaces | jq ".[] | select(.focused)")"
+                focused_num="$(jq -r ".num" <<< "$focused_ws")"
+                focused_name="$(jq -r ".name" <<< "$focused_ws")"
+                placeholder="$(sed -E 's/[0-9]+: //' <<< "$focused_name")"
+
+                name="$(rofi -dmenu -p "rename ws $focused_num" -theme+entry+placeholder "\"$placeholder\"")"
+                if [ -n "$name" ]; then
+                  swaymsg rename workspace "$focused_name" to "$focused_num: $name"
+                fi
+              '';
+              clearWsName = pkgs.writeShellScript "sway-clear-ws-name" ''
+                focused_ws="$(swaymsg -t get_workspaces | jq ".[] | select(.focused)")"
+                focused_num="$(jq -r ".num" <<< "$focused_ws")"
+                focused_name="$(jq -r ".name" <<< "$focused_ws")"
+
+                swaymsg rename workspace "$focused_name" to "$focused_num"
+              '';
+            in
+            {
               enable = true;
               xwayland = true;
+              extraConfigEarly = ''
+                set $mod ${mod}
+              '';
               config = {
                 input = {
                   "type:touchpad" = {
@@ -154,31 +253,95 @@ in
 
                 modifier = "Mod4";
                 terminal = "kitty";
-                keybindings =
+                keybindings = mapAttrs (k: mkOptionDefault) {
-                  let
+                  "${mod}+Left" = "focus left";
-                    cfg = config.wayland.windowManager.sway.config;
+                  "${mod}+Down" = "focus down";
-                    mod = cfg.modifier;
+                  "${mod}+Up" = "focus up";
-                  in
+                  "${mod}+Right" = "focus right";
-                  lib.mkOptionDefault {
-                    "${mod}+d" = null;
-                    "${mod}+l" = "exec swaylock -i ${./lock.png} -s stretch";
-                    "${mod}+x" = "exec ${cfg.menu}";
-                    "${mod}+Shift+x" = "exec rofi -show drun";
-                    "${mod}+q" = "kill";
-                    "${mod}+Shift+q" = "exec swaynag -t warning -m 'bruh you really wanna kill sway?' -b 'ye' 'systemctl --user stop graphical-session.target && swaymsg exit'";
-                    "${mod}+Shift+d" = ''exec grim - | swappy -f -'';
-                    "${mod}+Shift+s" = ''exec grim -g "$(slurp)" - | swappy -f -'';
-                    "${mod}+Shift+e" = "exec rofi -show emoji";
-                    # Config for this doesn't seem to work :/
-                    "${mod}+c" = ''exec rofi -show calc -calc-command "echo -n '{result}' | ${pkgs.wl-clipboard}/bin/wl-copy"'';
 
-                    "XF86AudioRaiseVolume" = "exec ${pkgs.pamixer}/bin/pamixer -i 5";
+                  "${mod}+Shift+Left" = "move left";
-                    "XF86AudioLowerVolume" = "exec ${pkgs.pamixer}/bin/pamixer -d 5";
+                  "${mod}+Shift+Down" = "move down";
-                    "XF86AudioPlay" = "exec ${pkgs.playerctl}/bin/playerctl play";
+                  "${mod}+Shift+Up" = "move up";
-                    "XF86AudioPause" = "exec ${pkgs.playerctl}/bin/playerctl pause";
+                  "${mod}+Shift+Right" = "move right";
-                    "XF86AudioNext" = "exec ${pkgs.playerctl}/bin/playerctl next";
+
-                    "XF86AudioPrev" = "exec ${pkgs.playerctl}/bin/playerctl previous";
+                  "${mod}+b" = "splith";
-                  };
+                  "${mod}+v" = "splitv";
+                  "${mod}+f" = "fullscreen toggle";
+                  "${mod}+a" = "focus parent";
+
+                  "${mod}+s" = "layout stacking";
+                  "${mod}+w" = "layout tabbed";
+                  "${mod}+e" = "layout toggle split";
+
+                  "${mod}+Shift+space" = "floating toggle";
+                  "${mod}+space" = "focus mode_toggle";
+
+                  "${mod}+1" = "workspace number 1";
+                  "${mod}+2" = "workspace number 2";
+                  "${mod}+3" = "workspace number 3";
+                  "${mod}+4" = "workspace number 4";
+                  "${mod}+5" = "workspace number 5";
+                  "${mod}+6" = "workspace number 6";
+                  "${mod}+7" = "workspace number 7";
+                  "${mod}+8" = "workspace number 8";
+                  "${mod}+9" = "workspace number 9";
+                  "${mod}+0" = "workspace number 10";
+
+                  "${mod}+Shift+1" =
+                    "move container to workspace number 1";
+                  "${mod}+Shift+2" =
+                    "move container to workspace number 2";
+                  "${mod}+Shift+3" =
+                    "move container to workspace number 3";
+                  "${mod}+Shift+4" =
+                    "move container to workspace number 4";
+                  "${mod}+Shift+5" =
+                    "move container to workspace number 5";
+                  "${mod}+Shift+6" =
+                    "move container to workspace number 6";
+                  "${mod}+Shift+7" =
+                    "move container to workspace number 7";
+                  "${mod}+Shift+8" =
+                    "move container to workspace number 8";
+                  "${mod}+Shift+9" =
+                    "move container to workspace number 9";
+                  "${mod}+Shift+0" =
+                    "move container to workspace number 10";
+
+                  "${mod}+Shift+minus" = "move scratchpad";
+                  "${mod}+minus" = "scratchpad show";
+
+                  "${mod}+Return" = "exec ${cfg.terminal}";
+                  "${mod}+r" = "mode resize";
+                  "${mod}+d" = null;
+                  "${mod}+l" = "exec ${doomsaver}/bin/doomsaver";
+                  "${mod}+q" = "kill";
+                  "${mod}+Shift+c" = "reload";
+                  "${mod}+Shift+q" = "exec swaynag -t warning -m 'bruh you really wanna kill sway?' -b 'ye' 'systemctl --user stop graphical-session.target && swaymsg exit'";
+
+                  # rofi
+                  "${mod}+x" = "exec ${cfg.menu}";
+                  "${mod}+Shift+x" = "exec rofi -show drun";
+                  "${mod}+Shift+e" = "exec rofi -show emoji";
+                  # Config for this doesn't seem to work :/
+                  "${mod}+c" = ''exec rofi -show calc -calc-command "echo -n '{result}' | ${pkgs.wl-clipboard}/bin/wl-copy"'';
+                  "${mod}+Shift+r" = "exec ${renameWs}";
+                  "${mod}+Shift+n" = "exec ${clearWsName}";
+
+                  # Screenshots
+                  "${mod}+Shift+d" = ''exec grim - | swappy -f -'';
+                  "${mod}+Shift+s" = ''exec grim -g "$(slurp)" - | swappy -f -'';
+
+                  "XF86MonBrightnessDown" = "exec ${pkgs.brightnessctl}/bin/brightnessctl set 5%-";
+                  "XF86MonBrightnessUp" = "exec ${pkgs.brightnessctl}/bin/brightnessctl set +5%";
+
+                  "XF86AudioRaiseVolume" = "exec ${pkgs.pamixer}/bin/pamixer -i 5";
+                  "XF86AudioLowerVolume" = "exec ${pkgs.pamixer}/bin/pamixer -d 5";
+                  "XF86AudioPlay" = "exec ${pkgs.playerctl}/bin/playerctl play";
+                  "XF86AudioPause" = "exec ${pkgs.playerctl}/bin/playerctl pause";
+                  "XF86AudioNext" = "exec ${pkgs.playerctl}/bin/playerctl next";
+                  "XF86AudioPrev" = "exec ${pkgs.playerctl}/bin/playerctl previous";
+                };
                 keycodebindings = {
                   # keycode for XF86AudioPlayPause (no sym for some reason)
                   "172" = "exec ${pkgs.playerctl}/bin/playerctl play-pause";
@@ -187,6 +350,9 @@ in
                 menu = "rofi -show run";
                 bars = mkForce [ ];
               };
+              extraConfig = ''
+                include ${./swaysome.conf}
+              '';
 
               swaynag = {
                 enable = true;
@@ -209,17 +375,10 @@ in
         };
         qt = {
           enable = true;
-          platformTheme = "gtk";
+          platformTheme.name = "gtk";
         };
 
         services = {
-          swaync = {
-            enable = true;
-            settings = {
-              widgets = [ "title" "dnd" "mpris" "notifications" ];
-            };
-          };
-
           playerctld.enable = true;
           spotifyd = {
             enable = false;
@@ -244,6 +403,7 @@ in
             diff-so-fancy.enable = true;
             userEmail = "jackos1998@gmail.com";
             userName = "Jack O'Sullivan";
+            lfs.enable = true;
             extraConfig = {
               pull.rebase = true;
             };
@@ -251,11 +411,13 @@ in
 
           waybar = import ./waybar.nix { inherit lib pkgs config font; };
           rofi = {
+            package = pkgs.rofi-wayland;
             enable = true;
             font = "${font.name} ${toString font.size}";
-            plugins = with pkgs; [
+            plugins = with pkgs; (map (p: p.override { rofi-unwrapped = rofi-wayland-unwrapped; }) [
               rofi-calc
-              rofi-emoji
+            ]) ++ [
+              rofi-emoji-wayland
             ];
             extraConfig = {
               modes = "window,run,ssh,filebrowser,calc,emoji";
@@ -270,7 +432,7 @@ in
 
           chromium = {
             enable = true;
-            package = (pkgs.chromium.override { enableWideVine = true; }).overrideAttrs (old: {
+            package = (pkgs'.unstable.chromium.override { enableWideVine = true; }).overrideAttrs (old: {
               buildCommand = ''
                 ${old.buildCommand}
 
@@ -296,6 +458,15 @@ in
             ] (_: "chromium-browser.desktop");
           };
         };
+
+        my = {
+          swaync = {
+            enable = true;
+            settings = {
+              widgets = [ "title" "dnd" "mpris" "notifications" ];
+            };
+          };
+        };
       })
 
       (mkIf (cfg.standalone && !pkgs.stdenv.isDarwin) {

home-manager/modules/gui/screensaver.py

@@ -0,0 +1,209 @@
+#!@python@/bin/python
+import argparse
+import json
+import os
+import random
+import signal
+import subprocess
+import sys
+
+import filelock
+
+class Screensaver:
+    def __init__(self, cmd, env=None, weight=1):
+        self.cmd = cmd
+        self.weight = weight
+
+        if env is not None:
+            self.env = os.environ.copy()
+            for k, v in env.items():
+                self.env[k] = v
+        else:
+            self.env = None
+        self.proc = None
+
+    def start(self):
+        assert self.proc is None
+        self.proc = subprocess.Popen(self.cmd, env=self.env)
+
+    def wait(self):
+        assert self.proc is not None
+        self.proc.wait()
+
+    def stop(self, kill=False):
+        assert self.proc is not None
+        if kill:
+            self.proc.kill()
+        else:
+            self.proc.terminate()
+
+class DoomSaver(Screensaver):
+    wad = '@doomWad@'
+
+    def __init__(self, demo_index, weight=1.5):
+        super().__init__(
+            ['@chocoDoom@/bin/chocolate-doom',
+             '-iwad', self.wad,
+             '-demoloopi', str(demo_index)],
+            env={
+                'SDL_AUDIODRIVER': 'null',
+                'SDL_VIDEODRIVER': 'caca',
+                'CACA_DRIVER': 'ncurses',
+            },
+            weight=weight,
+        )
+
+    def stop(self):
+        super().stop(kill=True)
+
+class TTESaver(Screensaver):
+    effects = (
+        'beams,binarypath,blackhole,bouncyballs,bubbles,burn,colorshift,crumble,'
+        'decrypt,errorcorrect,expand,fireworks,middleout,orbittingvolley,overflow,'
+        'pour,print,rain,randomsequence,rings,scattered,slice,slide,spotlights,'
+        'spray,swarm,synthgrid,unstable,vhstape,waves,wipe'
+    ).split(',')
+
+    def __init__(self, cmd, env=None, weight=1):
+        super().__init__(cmd, env=env, weight=weight)
+        self.running = False
+
+    def start(self):
+        self.running = True
+
+    def wait(self):
+        while self.running:
+            effect_cmd = ['tte', random.choice(self.effects)]
+            print(f"$ {self.cmd} | {' '.join(effect_cmd)}")
+            content = subprocess.check_output(self.cmd, shell=True, env=self.env, stderr=subprocess.DEVNULL)
+
+            self.proc = subprocess.Popen(effect_cmd, stdin=subprocess.PIPE)
+            self.proc.stdin.write(content)
+            self.proc.stdin.close()
+            self.proc.wait()
+
+    def stop(self):
+        self.running = False
+        self.proc.terminate()
+
+class MultiSaver:
+    savers = [
+        DoomSaver(0),
+        DoomSaver(1),
+        DoomSaver(2),
+
+        Screensaver(['cmatrix']),
+
+        TTESaver('screenfetch -N'),
+        TTESaver('fortune | cowsay'),
+        TTESaver('top -bn1 | head -n50'),
+        TTESaver('ss -nltu'),
+        TTESaver('ss -ntu'),
+        TTESaver('jp2a --width=100 @enojy@'),
+    ]
+    state_filename = 'screensaver.json'
+
+    def __init__(self, select=None):
+        self.state_path = os.path.join(f'/run/user/{os.geteuid()}', self.state_filename)
+        self.lock = filelock.FileLock(f'{self.state_path}.lock')
+
+        if select is not None:
+            assert select >= 0 and select < len(self.savers), 'Invalid screensaver index'
+            self.selected = self.savers[select]
+        else:
+            self.selected = None
+        self.cleaned_up = False
+
+    def select(self):
+        with self.lock:
+            if not os.path.exists(self.state_path):
+                state = {'instances': []}
+            else:
+                with open(self.state_path) as f:
+                    state = json.load(f)
+
+            if self.selected is None:
+                available = set(range(len(self.savers)))
+                new_instances = []
+                for instance in state['instances']:
+                    if not os.path.exists(f"/proc/{instance['pid']}"):
+                        continue
+
+                    new_instances.append(instance)
+                    i = instance['saver']
+                    assert i in available
+                    available.remove(i)
+                assert available, 'No screensavers left'
+                available = list(available)
+
+                weights = []
+                for i in available:
+                    weights.append(self.savers[i].weight)
+                selected_i = random.choices(available, weights=weights)[0]
+
+                new_instances.append({'pid': os.getpid(), 'saver': selected_i})
+                state['instances'] = new_instances
+
+                # print(f'Selected saver {selected_i}')
+                self.selected = self.savers[selected_i]
+
+            with open(self.state_path, 'w') as f:
+                json.dump(state, f)
+
+    def cleanup(self):
+        if self.cleaned_up:
+            return
+        self.cleaned_up = True
+
+        with self.lock:
+            with open(self.state_path) as f:
+                state = json.load(f)
+
+            for i, instance in enumerate(state['instances']):
+                if instance['pid'] == os.getpid():
+                    del state['instances'][i]
+
+            with open(self.state_path, 'w') as f:
+                json.dump(state, f)
+
+    def run(self):
+        assert self.selected is not None
+        self.selected.start()
+
+        signal.signal(signal.SIGINT, self._sighandler)
+        signal.signal(signal.SIGTERM, self._sighandler)
+        signal.signal(signal.SIGHUP, self._sighandler)
+        self.selected.wait()
+        self.cleanup()
+
+    def stop(self):
+        assert self.selected is not None
+        print('Shutting down')
+        self.selected.stop()
+        self.cleanup()
+    def _sighandler(self, signum, frame):
+        self.stop()
+
+def main():
+    parser = argparse.ArgumentParser(description='Wayland terminal-based lock screen')
+    parser.add_argument('-l', '--locker-cmd', default='swaylock-plugin', help='swaylock-plugin command to use')
+    parser.add_argument('-t', '--terminal', default='alacritty', help='Terminal emulator to use')
+    parser.add_argument('-i', '--instance', action='store_true', help='Run as instance')
+    parser.add_argument('-s', '--screensaver', type=int, help='Force use of specific screensaver')
+
+    args = parser.parse_args()
+    if not args.instance:
+        cmd = [
+            args.locker_cmd, '--command-each',
+            f'@windowtolayer@/bin/windowtolayer -- {args.terminal} -e {sys.argv[0]} --instance']
+        if args.screensaver is not None:
+            cmd[-1] += f' --screensaver {args.screensaver}'
+        subprocess.check_call(cmd)
+        return
+
+    ms = MultiSaver(select=args.screensaver)
+    ms.select()
+    ms.run()
+
+if __name__ == '__main__':
+    main()

home-manager/modules/gui/swaysome.conf

@@ -0,0 +1,66 @@
+# Use (un)bindcode or (un)bindsym, depending on what you used in your main sway config file.
+# The `--no-warn` setting is only added to shortcuts that exist in the default config. You may want to add or remove
+# that flag on some bindings depending on your config.
+
+
+# Change focus between workspaces
+bindsym $mod+Alt+1 exec "swaysome focus 1"
+bindsym $mod+Alt+2 exec "swaysome focus 2"
+bindsym $mod+Alt+3 exec "swaysome focus 3"
+bindsym $mod+Alt+4 exec "swaysome focus 4"
+bindsym $mod+Alt+5 exec "swaysome focus 5"
+bindsym $mod+Alt+6 exec "swaysome focus 6"
+bindsym $mod+Alt+7 exec "swaysome focus 7"
+bindsym $mod+Alt+8 exec "swaysome focus 8"
+bindsym $mod+Alt+9 exec "swaysome focus 9"
+bindsym $mod+Alt+0 exec "swaysome focus 0"
+
+# Focus workspace groups
+bindsym --no-warn $mod+1 exec "swaysome focus-group 1"
+bindsym --no-warn $mod+2 exec "swaysome focus-group 2"
+bindsym --no-warn $mod+3 exec "swaysome focus-group 3"
+bindsym --no-warn $mod+4 exec "swaysome focus-group 4"
+bindsym --no-warn $mod+5 exec "swaysome focus-group 5"
+bindsym --no-warn $mod+6 exec "swaysome focus-group 6"
+bindsym --no-warn $mod+7 exec "swaysome focus-group 7"
+bindsym --no-warn $mod+8 exec "swaysome focus-group 8"
+bindsym --no-warn $mod+9 exec "swaysome focus-group 9"
+bindsym --no-warn $mod+0 exec "swaysome focus-group 0"
+
+# Move containers between workspaces
+bindsym $mod+Alt+Shift+1 exec "swaysome move 1"
+bindsym $mod+Alt+Shift+2 exec "swaysome move 2"
+bindsym $mod+Alt+Shift+3 exec "swaysome move 3"
+bindsym $mod+Alt+Shift+4 exec "swaysome move 4"
+bindsym $mod+Alt+Shift+5 exec "swaysome move 5"
+bindsym $mod+Alt+Shift+6 exec "swaysome move 6"
+bindsym $mod+Alt+Shift+7 exec "swaysome move 7"
+bindsym $mod+Alt+Shift+8 exec "swaysome move 8"
+bindsym $mod+Alt+Shift+9 exec "swaysome move 9"
+bindsym $mod+Alt+Shift+0 exec "swaysome move 0"
+
+# Move containers to other workspace groups
+bindsym --no-warn $mod+Shift+1 exec "swaysome move-to-group 1"
+bindsym --no-warn $mod+Shift+2 exec "swaysome move-to-group 2"
+bindsym --no-warn $mod+Shift+3 exec "swaysome move-to-group 3"
+bindsym --no-warn $mod+Shift+4 exec "swaysome move-to-group 4"
+bindsym --no-warn $mod+Shift+5 exec "swaysome move-to-group 5"
+bindsym --no-warn $mod+Shift+6 exec "swaysome move-to-group 6"
+bindsym --no-warn $mod+Shift+7 exec "swaysome move-to-group 7"
+bindsym --no-warn $mod+Shift+8 exec "swaysome move-to-group 8"
+bindsym --no-warn $mod+Shift+9 exec "swaysome move-to-group 9"
+bindsym --no-warn $mod+Shift+0 exec "swaysome move-to-group 0"
+
+# Move focused container to next output
+bindsym $mod+Alt+Right exec "swaysome next-output"
+# Move focused container to previous output
+bindsym $mod+Alt+Left exec "swaysome prev-output"
+
+# Move focused workspace group to next output
+bindsym $mod+Shift+Alt+Right exec "swaysome workspace-group-next-output"
+# Move focused workspace group to previous output
+bindsym $mod+Shift+Alt+Left exec "swaysome workspace-group-prev-output"
+
+# Init workspaces for every screen
+exec "swaysome init 1"
+

home-manager/modules/gui/waybar.nix

@@ -146,9 +146,9 @@ in
           dnd-none = "";
         };
         return-type = "json";
-        exec = "${config.services.swaync.package}/bin/swaync-client -swb";
+        exec = "${config.my.swaync.package}/bin/swaync-client -swb";
-        on-click = "${config.services.swaync.package}/bin/swaync-client -t -sw";
+        on-click = "${config.my.swaync.package}/bin/swaync-client -t -sw";
-        on-click-right = "${config.services.swaync.package}/bin/swaync-client -d -sw";
+        on-click-right = "${config.my.swaync.package}/bin/swaync-client -d -sw";
         escape = true;
       };
     };

home-manager/modules/swaync.nix

@@ -19,10 +19,10 @@ let
     };
   };
 
-  cfg = config.services.swaync;
+  cfg = config.my.swaync;
 in
 {
-  options.services.swaync = with lib.types; {
+  options.my.swaync = with lib.types; {
     enable = mkEnableOption "Sway Notification Center";
     package = mkOption {
       type = package;
@@ -70,13 +70,17 @@ in
         "swaync/config.json" = mkIf (cfg.settings != { }) {
           source = configSource;
           onChange = ''
-            ${cfg.package}/bin/swaync-client --reload-config
+            if ${pkgs.systemd}/bin/systemctl --user is-active --quiet swaync; then
+              ${cfg.package}/bin/swaync-client --reload-config
+            fi
           '';
         };
         "swaync/style.css" = mkIf (cfg.style != null) {
           source = styleSource;
           onChange = ''
-            ${cfg.package}/bin/swaync-client --reload-css
+            if ${pkgs.systemd}/bin/systemctl --user is-active --quiet swaync; then
+              ${cfg.package}/bin/swaync-client --reload-css
+            fi
           '';
         };
       };

lib/constants.nix

@@ -10,16 +10,27 @@ rec {
       gitea-runner = 401;
       jellyseerr = 402;
       atticd = 403;
+      kea = 404;
+      keepalived_script = 405;
+      photoprism = 406;
     };
     gids = {
       matrix-syncv3 = 400;
       gitea-runner = 401;
       jellyseerr = 402;
       atticd = 403;
+      kea = 404;
+      keepalived_script = 405;
+      photoprism = 406;
     };
   };
 
-  nginx = {
+  kernel = {
+    lts = pkgs: pkgs.linuxKernel.packages.linux_6_6;
+    latest = pkgs: pkgs.linuxKernel.packages.linux_6_12;
+  };
+
+  nginx = rec {
     proxyHeaders = ''
       # Setting any proxy_header in a child (e.g. location) will nuke the parents...
       proxy_set_header X-Origin-URI $request_uri;
@@ -33,15 +44,64 @@ rec {
       proxy_set_header X-Forwarded-Protocol $scheme;
       proxy_set_header X-Scheme $scheme;
     '';
+    baseHttpConfig = ''
+      # NixOS provides a logrotate config that auto-compresses :)
+      log_format main
+        '$remote_addr - $remote_user [$time_local] $scheme "$host" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"';
+      access_log /var/log/nginx/access.log main;
+
+      # optimisation
+      sendfile on;
+      tcp_nopush on;
+      tcp_nodelay on;
+      keepalive_timeout 65;
+
+      # gzip
+      gzip on;
+      gzip_proxied any;
+      gzip_comp_level 5;
+      gzip_types
+        application/atom+xml
+        application/javascript
+        application/json
+        application/xml
+        application/xml+rss
+        image/svg+xml
+        text/css
+        text/javascript
+        text/plain
+        text/xml;
+      gzip_vary on;
+
+      # proxying
+      proxy_buffering off;
+      proxy_redirect off;
+      proxy_connect_timeout 60s;
+      proxy_read_timeout 60s;
+      proxy_send_timeout 60s;
+      proxy_http_version 1.1;
+
+      ${proxyHeaders}
+    '';
+  };
+
+  networkd = {
+    noL3 = {
+      LinkLocalAddressing = "no";
+      DHCP = "no";
+      LLDP = false;
+      EmitLLDP = false;
+      IPv6AcceptRA = false;
+    };
   };
 
   nix = {
     cache = rec {
       substituters = [
-        "https://nix-cache.${pubDomain}/main"
+        "https://nix-cache.${pubDomain}"
       ];
       keys = [
-        "main:mMChkG8LwXrFirVfudqjSHasK1jV31OVElYD3eImYl8="
+        "nix-cache.nul.ie-1:BzH5yMfF4HbzY1C977XzOxoPhEc9Zbu39ftPkUbH+m4="
       ];
       conf = ''
         extra-substituters = ${concatStringsSep " " substituters}
@@ -51,8 +111,9 @@ rec {
   };
 
   pubDomain = "nul.ie";
-  colony = {
+  colony = rec {
     domain = "ams1.int.${pubDomain}";
+    pubV4 = "94.142.240.44";
     prefixes = with lib.my.net.cidr; rec {
       all = {
         v4 = "10.100.0.0/16";
@@ -74,6 +135,9 @@ rec {
         v4 = subnet 8 3 all.v4;
         v6 = subnet 4 3 all.v6;
       };
+      qclk = {
+        v4 = subnet 8 4 all.v4;
+      };
 
       cust = {
         v4 = subnet 8 100 all.v4; # single ip for routing only
@@ -87,16 +151,108 @@ rec {
         v4 = "94.142.242.255/32";
         v6 = subnet 8 1 cust.v6;
       };
+      jam = {
+        v4 = subnet 8 4 cust.v4;
+        v6 = subnet 8 2 cust.v6;
+      };
 
       vip1 = "94.142.241.224/30";
       vip2 = "94.142.242.254/31";
+      vip3 = "94.142.241.117/32";
+
+      as211024 = {
+        v4 = subnet 8 50 all.v4;
+        v6 = "2a0e:97c0:4df::/64";
+      };
+      home.v6 = "2a0e:97c0:4d0::/48";
     };
+
+    custRouting = with lib.my.net.cidr; {
+      mail-vm = host 1 prefixes.cust.v4;
+      darts-vm = host 2 prefixes.cust.v4;
+      jam-ctr = host 3 prefixes.cust.v4;
+    };
+
+    qclk = {
+      wgPort = 51821;
+    };
+
+    firewallForwards = aa: [
+      {
+        port = "http";
+        dst = aa.middleman.internal.ipv4.address;
+      }
+      {
+        port = "https";
+        dst = aa.middleman.internal.ipv4.address;
+      }
+      {
+        port = 8448;
+        dst = aa.middleman.internal.ipv4.address;
+      }
+
+      {
+        port = 25565;
+        dst = aa.simpcraft-oci.internal.ipv4.address;
+      }
+      {
+        port = 25566;
+        dst = aa.simpcraft-staging-oci.internal.ipv4.address;
+      }
+
+      {
+        port = 25575;
+        dst = aa.simpcraft-oci.internal.ipv4.address;
+      }
+
+      {
+        port = 2456;
+        dst = aa.valheim-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+      {
+        port = 2457;
+        dst = aa.valheim-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+
+      {
+        port = 41641;
+        dst = aa.waffletail.internal.ipv4.address;
+        proto = "udp";
+      }
+
+      {
+        port = 25565;
+        dst = aa.simpcraft-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+
+      {
+        port = 15636;
+        dst = aa.enshrouded-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+      {
+        port = 15637;
+        dst = aa.enshrouded-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+
+      {
+        port = qclk.wgPort;
+        dst = aa.qclk.internal.ipv4.address;
+        proto = "udp";
+      }
+    ];
+
     fstrimConfig = {
       enable = true;
       # backup happens at 05:00
       interval = "04:45";
     };
   };
+
   home = rec {
     domain = "h.${pubDomain}";
     vlans = {
@@ -106,6 +262,15 @@ rec {
       wan = 130;
     };
     hiMTU = 9000;
+    routers = [
+      "river"
+      "stream"
+    ];
+    routersPubV4 = [
+      "109.255.31.155"
+      "109.255.252.63"
+    ];
+
     prefixes = with lib.my.net.cidr; rec {
       modem = {
         v4 = "192.168.0.0/24";
@@ -120,15 +285,19 @@ rec {
       hi = {
         v4 = subnet 4 1 all.v4;
         v6 = subnet 4 1 all.v6;
+        mtu = hiMTU;
       };
       lo = {
         v4 = subnet 3 1 all.v4;
         v6 = subnet 4 2 all.v6;
+        mtu = 1500;
       };
       untrusted = {
         v4 = subnet 6 16 all.v4;
         v6 = subnet 4 3 all.v6;
+        mtu = 1500;
       };
+      inherit (colony.prefixes) as211024;
     };
     vips = with lib.my.net.cidr; {
       hi = {
@@ -143,8 +312,56 @@ rec {
         v4 = host 254 prefixes.untrusted.v4;
         v6 = host 65535 prefixes.untrusted.v6;
       };
+      as211024 = {
+        v4 = host 4 prefixes.as211024.v4;
+        v6 = host ((1*65536*65536*65536) + 65535) prefixes.as211024.v6;
+      };
+    };
+
+    roceBootModules = [ "ib_core" "ib_uverbs" "mlx5_core" "mlx5_ib" ];
+  };
+
+  britway = {
+    domain = "lon1.int.${pubDomain}";
+    pubV4 = "45.76.141.188";
+    prefixes = {
+      vultr = {
+        v6 = "2001:19f0:7402:128b::/64";
+      };
+      inherit (colony.prefixes) as211024;
+    };
+    # Need to use this IP as the source address for BGP
+    assignedV6 = "2001:19f0:7402:128b:5400:04ff:feac:6e06";
+  };
+
+  tailscale = {
+    prefix = {
+      v4 = "100.64.0.0/10";
+      v6 = "fd7a:115c:a1e0::/48";
     };
   };
+
+  as211024 = rec {
+    trusted = {
+      v4 = [
+        colony.prefixes.as211024.v4
+        colony.prefixes.all.v4
+        home.prefixes.all.v4
+        tailscale.prefix.v4
+      ];
+      v6 = [
+        colony.prefixes.as211024.v6
+        colony.prefixes.all.v6
+        home.prefixes.all.v6
+        tailscale.prefix.v6
+      ];
+    };
+    nftTrust = ''
+      iifname as211024 ip saddr { ${concatStringsSep ", " trusted.v4} } accept
+      iifname as211024 ip6 saddr { ${concatStringsSep ", " trusted.v6} } accept
+    '';
+  };
+
   kelder = {
     groups = {
       storage = 2000;
@@ -152,6 +369,7 @@ rec {
     };
 
     domain = "hentai.engineer";
+    ipv4MTU = 1460;
     vpn = {
       port = 51820;
     };
@@ -165,6 +383,7 @@ rec {
     deploy = ../.keys/deploy.pub;
     rsyncNet = ../.keys/zh2855.rsync.net.pub;
     mailcowAcme = ../.keys/mailcow-acme.pub;
+    harmonia = ../.keys/harmonia.pub;
   };
   sshHostKeys = {
     mail-vm = ../.keys/mail-vm-host.pub;

lib/default.nix

@@ -1,13 +1,26 @@
 { lib }:
 let
-  inherit (builtins) length match elemAt filter;
+  inherit (builtins) length match elemAt filter replaceStrings substring;
   inherit (lib)
     genAttrs mapAttrsToList filterAttrsRecursive nameValuePair types
     mkOption mkOverride mkForce mkIf mergeEqualOption optional
-    showWarnings concatStringsSep flatten unique;
+    showWarnings concatStringsSep flatten unique optionalAttrs
+    mkBefore toLower;
   inherit (lib.flake) defaultSystems;
 in
 rec {
+  pow =
+    let
+      pow' = base: exponent: value:
+        # FIXME: It will silently overflow on values > 2**62 :(
+        # The value will become negative or zero in this case
+        if exponent == 0
+        then 1
+        else if exponent <= 1
+        then value
+        else (pow' base (exponent - 1) (value * base));
+    in base: exponent: pow' base exponent base;
+
   attrsToNVList = mapAttrsToList nameValuePair;
 
   inherit (import ./net.nix { inherit lib; }) net;
@@ -29,6 +42,8 @@ rec {
       ports = checked (elemAt m 1);
     };
 
+  netBroadcast = net': net.cidr.host ((pow 2 (net.cidr.size net')) - 1) net';
+
   mkDefaultSystemsPkgs = path: args': genAttrs defaultSystems (system: import path ((args' system) // { inherit system; }));
   mkApp = program: { type = "app"; inherit program; };
   mkShellApp = pkgs: name: text: mkApp (pkgs.writeShellScript name text).outPath;
@@ -109,6 +124,21 @@ rec {
     home-manager = mkOpt' (enum [ "unstable" "stable" "mine" "mine-stable" ]) "unstable" "Branch of home-manager to use.";
   };
 
+  nft = rec {
+    ipEscape = replaceStrings ["." ":"] ["-" "-"];
+    natFilterChain = ip: "filter-fwd-${ipEscape ip}";
+    dnatChain = ip: "fwd-${ipEscape ip}";
+  };
+
+  mkVLAN = name: vid: {
+    "25-${name}" = {
+      netdevConfig = {
+        Name = name;
+        Kind = "vlan";
+      };
+      vlanConfig.Id = vid;
+    };
+  };
   networkdAssignment = iface: a: {
     matchConfig.Name = iface;
     address =
@@ -123,6 +153,9 @@ rec {
       LLDP = true;
       EmitLLDP = "customer-bridge";
     };
+    linkConfig = optionalAttrs (a.mtu != null) {
+      MTUBytes = toString a.mtu;
+    };
     ipv6AcceptRAConfig = {
       Token = mkIf (a.ipv6.iid != null) "static:${a.ipv6.iid}";
       UseDNS = true;
@@ -134,13 +167,32 @@ rec {
 
   systemdAwaitPostgres = pkg: host: {
     after = [ "systemd-networkd-wait-online.service" ];
-    preStart = ''
+    preStart = mkBefore ''
       until ${pkg}/bin/pg_isready -h ${host}; do
         sleep 0.5
       done
     '';
   };
 
+  vm = rec {
+    lvmDisk' = name: lv: {
+      inherit name;
+      backend = {
+        driver = "host_device";
+        filename = "/dev/main/${lv}";
+        # It appears this needs to be set on the backend _and_ the format
+        discard = "unmap";
+      };
+      format = {
+        driver = "raw";
+        discard = "unmap";
+      };
+      frontend = "virtio-blk";
+    };
+    lvmDisk = lv: lvmDisk' lv lv;
+    disk = vm: lv: lvmDisk' lv "vm-${vm}-${lv}";
+  };
+
   deploy-rs =
   with types;
   let
@@ -188,4 +240,45 @@ rec {
 
     filterOpts = filterAttrsRecursive (_: v: v != null);
   };
+
+  versionOverlay = { self, pkgsFlake }: final: prev:
+  let
+    date = substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101");
+    revCode = flake: flake.shortRev or "dirty";
+  in
+  {
+    trivial = prev.trivial // {
+      release = "24.12:u-${prev.trivial.release}";
+      codeName = "Epic";
+      revisionWithDefault = default: self.rev or default;
+      versionSuffix = ".${date}.${revCode self}:u-${revCode pkgsFlake}";
+    };
+  };
+
+  netbootKeaClientClasses = { tftpIP, hostname, systems }:
+  let
+    testIPXE = "substring(option[user-class].hex, 0, 4) == 'iPXE'";
+  in
+    (mapAttrsToList (name: mac: {
+      name = "nixos-${name}";
+      test = "(${testIPXE}) and (hexstring(pkt4.mac, ':') == '${toLower mac}')";
+      next-server = tftpIP;
+      server-hostname = hostname;
+      boot-file-name = "http://${hostname}/systems/${name}/menu.ipxe";
+    }) systems) ++ [
+    {
+      name = "ipxe";
+      test = testIPXE;
+      next-server = tftpIP;
+      server-hostname = hostname;
+      boot-file-name = "http://${hostname}/boot.ipxe";
+    }
+    {
+      name = "efi-x86_64";
+      test = "option[client-system].hex == 0x0007";
+      next-server = tftpIP;
+      server-hostname = hostname;
+      boot-file-name = "ipxe-x86_64.efi";
+    }
+  ];
 }

lib/dns.nix

@@ -67,4 +67,35 @@ rec {
           (a.ipv6.address != null && a.ipv6.genPTR)
           ''@@PTR:${a.ipv6.address}:${toString ndots}@@ IN PTR ${a.name}.${domain}.'';
     };
+
+  ifaceA = { pkgs, iface, skipBroadcasts ? [] }:
+  let
+    extraFilters = concatMapStringsSep " " (b: ''and .broadcast != \"${b}\"'') skipBroadcasts;
+    script = pkgs.writeText "if-${iface}-a.lua" ''
+      local proc = io.popen("${pkgs.iproute2}/bin/ip -j addr show dev ${iface} | ${pkgs.jq}/bin/jq -r '.[0].addr_info[] | select(.family == \"inet\" and .scope == \"global\" ${extraFilters}).local'", "r")
+      assert(proc, "failed to popen")
+
+      local addr_line = proc:read("*l")
+      assert(proc:close(), "command failed")
+      assert(addr_line, "no output from command")
+
+      return addr_line
+    '';
+  in
+    ''A "dofile('${script}')"'';
+
+  lookupIP = { pkgs, hostname, server, type ? "A" }:
+  let
+    script = pkgs.writeScript "drill-${hostname}-${server}.lua" ''
+      local proc = io.popen("${pkgs.ldns}/bin/drill -Q @${server} ${hostname} ${type}", "r")
+      assert(proc, "failed to popen")
+
+      local addr_line = proc:read("*l")
+      assert(proc:close(), "command failed")
+      assert(addr_line, "no output from command")
+
+      return addr_line
+    '';
+  in
+    ''${type} "dofile('${script}')"'';
 }

nixos/boxes/britway/bgp.nix

@@ -0,0 +1,197 @@
+{ lib, pkgs, config, assignments, ... }:
+let
+  inherit (lib.my.c.britway) assignedV6;
+
+  securebitSpace = "2a0e:97c0:4d0::/44";
+  intnet6 = "2a0e:97c0:4df::/48";
+  amsnet6 = "2a0e:97c0:4d2::/48";
+  homenet6 = "2a0e:97c0:4d0::/48";
+in
+{
+  config = {
+    my = {
+      secrets.files."britway/bgp-password-vultr.conf" = {
+        owner = "bird2";
+        group = "bird2";
+      };
+    };
+
+    environment.etc."bird/vultr-password.conf".source = config.age.secrets."britway/bgp-password-vultr.conf".path;
+
+    systemd = {
+      services.bird2.after = [ "systemd-networkd-wait-online@veth0.service" ];
+      network = {
+        config.networkConfig.ManageForeignRoutes = false;
+      };
+    };
+
+    services = {
+      bird2 = {
+        enable = true;
+        preCheckConfig = ''
+          echo '"dummy"' > vultr-password.conf
+        '';
+        # TODO: Clean up and modularise
+        config = ''
+          define OWNAS = 211024;
+
+          define OWNIP4 = ${assignments.vultr.ipv4.address};
+          define OWNNETSET4 = [ ${assignments.vultr.ipv4.address}/32 ];
+
+          define INTNET6 = ${intnet6};
+          define AMSNET6 = ${amsnet6};
+          define HOMENET6 = ${homenet6};
+
+          define OWNIP6 = ${assignments.vultr.ipv6.address};
+          define OWNNETSET6 = [ ${intnet6}, ${amsnet6}, ${homenet6} ];
+          #define TRANSSET6 = [ ::1/128 ];
+
+          define DUB1IP6 = ${lib.my.c.home.vips.as211024.v6};
+
+          define PREFIXP = 110;
+          define PREFPEER = 120;
+
+          filter bgp_import {
+            if net !~ OWNNETSET4 && net !~ OWNNETSET6 then accept; else reject;
+          }
+          filter bgp_export {
+            if net ~ OWNNETSET4 || net ~ OWNNETSET6 then accept; else reject;
+          }
+
+          router id from "veth0";
+
+          protocol device {}
+          protocol direct {
+            interface "veth0";
+            ipv4;
+            ipv6;
+          }
+          protocol static static4 {
+            ipv4 {
+              import all;
+              export none;
+            };
+          }
+          protocol static static6 {
+            # Special case: We have to do the routing on behalf of this _internal_ next-hop
+            route INTNET6 via "as211024";
+            route HOMENET6 via DUB1IP6;
+
+            ipv6 {
+              import all;
+              export none;
+            };
+          }
+
+          protocol kernel kernel4 {
+            ipv4 {
+              import none;
+              export none;
+            };
+          }
+          protocol kernel kernel6 {
+            ipv6 {
+              import none;
+              export filter {
+                if net = HOMENET6 then accept;
+                reject;
+              };
+            };
+          }
+
+          protocol bgp bgptools {
+            local as OWNAS;
+            multihop;
+            description "bgp.tools monitoring";
+            neighbor 2a0c:2f07:9459::b11 as 212232;
+            source address OWNIP6;
+            ipv4 {
+              import none;
+              export all;
+              add paths tx;
+            };
+            ipv6 {
+              import none;
+              export all;
+              add paths tx;
+            };
+          }
+
+          template bgp base_bgp4 {
+            local as OWNAS;
+            direct;
+            allow local as;
+            ipv4 {
+              import keep filtered;
+              export none;
+            };
+          }
+
+          template bgp upstream_bgp4 from base_bgp4 {
+            ipv4 {
+              #import none;
+              import filter bgp_import;
+            };
+          }
+          template bgp peer_bgp4 from base_bgp4 {
+            ipv4 {
+              import filter bgp_import;
+              preference PREFPEER;
+            };
+          }
+          template bgp ixp_bgp4 from base_bgp4 {
+            ipv4 {
+              import filter bgp_import;
+              preference PREFIXP;
+            };
+          }
+
+          template bgp base_bgp6 {
+            local ${assignedV6} as OWNAS;
+            direct;
+            # So we can see routes we announce from other routers
+            allow local as;
+            ipv6 {
+              import keep filtered;
+              export filter bgp_export;
+            };
+          }
+
+          template bgp upstream_bgp6 from base_bgp6 {
+            ipv6 {
+              #import none;
+              import filter bgp_import;
+            };
+          }
+          template bgp peer_bgp6 from base_bgp6 {
+            ipv6 {
+              import filter bgp_import;
+              preference PREFPEER;
+            };
+          }
+          template bgp ixp_bgp6 from base_bgp6 {
+            ipv6 {
+              import filter bgp_import;
+              preference PREFIXP;
+            };
+          }
+
+          protocol bgp upstream4_vultr from upstream_bgp4 {
+            description "Vultr transit (IPv4)";
+            neighbor 169.254.169.254 as 64515;
+            multihop 2;
+            password
+            include "vultr-password.conf";;
+          }
+          protocol bgp upstream6_vultr from upstream_bgp6 {
+            description "Vultr transit (IPv6)";
+            neighbor 2001:19f0:ffff::1 as 64515;
+            multihop 2;
+            password
+            include "vultr-password.conf";;
+          }
+        '';
+      };
+    };
+  };
+}

nixos/boxes/britway/default.nix

@@ -0,0 +1,174 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.britway) prefixes domain pubV4 assignedV6;
+in
+{
+  nixos.systems.britway = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+
+    assignments = {
+      vultr = {
+        inherit domain;
+        ipv4 = {
+          address = pubV4;
+          mask = 23;
+          gateway = "45.76.140.1";
+        };
+        ipv6 = {
+          iid = "::1";
+          address = "2001:19f0:7402:128b::1";
+        };
+      };
+      as211024 = {
+        ipv4 = {
+          address = net.cidr.host 5 prefixes.as211024.v4;
+          gateway = null;
+        };
+        ipv6.address = net.cidr.host ((2*65536*65536*65536) + 1) prefixes.as211024.v6;
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
+      let
+        inherit (lib) mkMerge mkForce;
+        inherit (lib.my) networkdAssignment;
+      in
+      {
+        imports = [
+          "${modulesPath}/profiles/qemu-guest.nix"
+          ./bgp.nix
+          ./nginx.nix
+          ./tailscale.nix
+        ];
+
+        config = mkMerge [
+          {
+            boot = {
+              initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "sr_mod" ];
+              loader = {
+                systemd-boot.enable = false;
+                grub = {
+                  enable = true;
+                  device = "/dev/vda";
+                };
+              };
+            };
+
+            fileSystems = {
+              "/boot" = {
+                device = "/dev/disk/by-partuuid/c557ef12-da44-41d1-84f5-d32a711feefd";
+                fsType = "ext4";
+              };
+              "/nix" = {
+                device = "/dev/disk/by-partuuid/d42d0853-b054-4104-8afd-6d36287c7ca3";
+                fsType = "ext4";
+              };
+              "/persist" = {
+                device = "/dev/disk/by-partuuid/f14fbcf4-5242-456b-a4db-ef15d053d62e";
+                fsType = "ext4";
+                neededForBoot = true;
+              };
+            };
+
+            services = {
+              iperf3 = {
+                enable = true;
+                openFirewall = true;
+              };
+            };
+
+            networking = { inherit domain; };
+
+            systemd.network = {
+              config = {
+                routeTables.ts-extra = 1337;
+              };
+
+              links = {
+                "10-veth0" = {
+                  matchConfig.PermanentMACAddress = "56:00:04:ac:6e:06";
+                  linkConfig.Name = "veth0";
+                };
+              };
+
+              networks = {
+                "20-veth0" = mkMerge [
+                  (networkdAssignment "veth0" assignments.vultr)
+                  {
+                    address = [ "${assignedV6}/64" ];
+                  }
+                ];
+                "90-l2mesh-as211024" = mkMerge [
+                  (networkdAssignment "as211024" assignments.as211024)
+                  {
+                    matchConfig.Name = "as211024";
+                    networkConfig.IPv6AcceptRA = mkForce false;
+                    routes = [
+                      {
+                        Destination = lib.my.c.colony.prefixes.all.v4;
+                        Gateway = allAssignments.estuary.as211024.ipv4.address;
+                      }
+                      {
+                        Destination = lib.my.c.home.prefixes.all.v4;
+                        Gateway = lib.my.c.home.vips.as211024.v4;
+                      }
+
+                      {
+                        # Just when routing traffic from Tailscale nodes, otherwise use WAN
+                        Destination = lib.my.c.colony.prefixes.all.v6;
+                        Gateway = allAssignments.estuary.as211024.ipv6.address;
+                        Table = "ts-extra";
+                      }
+                    ];
+                    routingPolicyRules = [
+                      {
+                        IncomingInterface = "tailscale0";
+                        To = lib.my.c.colony.prefixes.all.v6;
+                        Table = "ts-extra";
+                      }
+                    ];
+                  }
+                ];
+              };
+            };
+
+            my = {
+              server.enable = true;
+              secrets = {
+                key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAmXC9egI46Qtaiifhq2I+rv2s1yPyzTlO4BHzUb+3Su";
+                files = {
+                  "l2mesh/as211024.key" = {};
+                };
+              };
+              vpns = {
+                l2.pskFiles = {
+                  as211024 = config.age.secrets."l2mesh/as211024.key".path;
+                };
+              };
+
+              firewall = {
+                trustedInterfaces = [ "tailscale0" ];
+                extraRules = ''
+                  table inet filter {
+                    chain forward {
+                      ${lib.my.c.as211024.nftTrust}
+                      oifname as211024 accept
+                    }
+                  }
+                  table inet nat {
+                    chain postrouting {
+                      iifname tailscale0 oifname veth0 snat ip to ${assignments.vultr.ipv4.address}
+                      iifname tailscale0 oifname veth0 snat ip6 to ${assignments.as211024.ipv6.address}
+                    }
+                  }
+                '';
+              };
+            };
+          }
+        ];
+      };
+  };
+}

nixos/boxes/britway/nginx.nix

@@ -0,0 +1,109 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (builtins) mapAttrs;
+  inherit (lib) mkMerge mkDefault;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.nginx) baseHttpConfig proxyHeaders;
+in
+{
+  config = {
+    my = {
+      secrets.files = {
+        "dhparams.pem" = {
+          owner = "acme";
+          group = "acme";
+          mode = "440";
+        };
+        "britway/cloudflare-credentials.conf" = {
+          owner = "acme";
+          group = "acme";
+        };
+      };
+
+      firewall = {
+        tcp.allowed = [ "http" "https" ];
+      };
+    };
+
+    users = {
+      users = {
+        nginx.extraGroups = [ "acme" ];
+      };
+    };
+
+    security.acme = {
+      acceptTerms = true;
+      defaults = {
+        email = "dev@nul.ie";
+        server = "https://acme-v02.api.letsencrypt.org/directory";
+        reloadServices = [ "nginx" ];
+        dnsResolver = "8.8.8.8";
+      };
+      certs = {
+        "${pubDomain}" = {
+          extraDomainNames = [
+            "*.${pubDomain}"
+          ];
+          dnsProvider = "cloudflare";
+          credentialsFile = config.age.secrets."britway/cloudflare-credentials.conf".path;
+        };
+      };
+    };
+
+    services = {
+      nginx = {
+        enable = true;
+        enableReload = true;
+
+        logError = "stderr info";
+        recommendedTlsSettings = true;
+        serverTokens = true;
+        sslDhparam = config.age.secrets."dhparams.pem".path;
+
+        # Based on recommended*Settings, but probably better to be explicit about these
+        appendHttpConfig = ''
+          ${baseHttpConfig}
+
+          # caching
+          proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=512m;
+        '';
+
+        virtualHosts =
+        let
+          hosts = {
+            "_" = {
+              default = true;
+              forceSSL = true;
+              onlySSL = false;
+              locations = {
+                "/".root = "${pkgs.nginx}/html";
+              };
+            };
+
+            "hs.${pubDomain}" = {
+              locations."/" = {
+                proxyPass = "http://localhost:${toString config.services.headscale.port}";
+                proxyWebsockets = true;
+                extraConfig = ''
+                  proxy_buffering off;
+                  add_header Strict-Transport-Security "max-age=15552000; includeSubDomains" always;
+                '';
+              };
+            };
+          };
+
+          defaultsFor = mapAttrs (n: _: {
+            onlySSL = mkDefault true;
+            useACMEHost = mkDefault pubDomain;
+            kTLS = mkDefault true;
+            http2 = mkDefault true;
+          });
+        in
+        mkMerge [
+          hosts
+          (defaultsFor hosts)
+        ];
+      };
+    };
+  };
+}

nixos/boxes/britway/tailscale.nix

@@ -0,0 +1,112 @@
+{ lib, pkgs, config, assignments, allAssignments, ... }:
+let
+  inherit (lib) concatStringsSep;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.britway) prefixes domain;
+
+  # Can't use overrideAttrs because we need to override `vendorHash` within `buildGoModule`
+  headscale' = (pkgs.headscale.override {
+    buildGoModule = args: pkgs.buildGoModule (args // rec {
+      version = "0.23.0-alpha12";
+      src = pkgs.fetchFromGitHub {
+        owner = "juanfont";
+        repo = "headscale";
+        rev = "v${version}";
+        hash = "sha256-kZZK0cXnFARxblSMz01TDcBbTorkHGAwGpR+a4/mYfU=";
+      };
+      patches = [];
+      vendorHash = "sha256-EorT2AVwA3usly/LcNor6r5UIhLCdj3L4O4ilgTIC2o=";
+      doCheck = false;
+    });
+  });
+
+  advRoutes = concatStringsSep "," [
+    lib.my.c.home.prefixes.all.v4
+    lib.my.c.home.prefixes.all.v6
+  ];
+  pubNameservers = [
+    "1.1.1.1"
+    "1.0.0.1"
+    "2606:4700:4700::1111"
+    "2606:4700:4700::1001"
+  ];
+in
+{
+  config = {
+    environment.systemPackages = [
+      # For CLI
+      config.services.headscale.package
+    ];
+
+    services = {
+      headscale = {
+        enable = true;
+        settings = {
+          disable_check_updates = true;
+          unix_socket_permission = "0770";
+          server_url = "https://hs.${pubDomain}";
+          database = {
+            type = "sqlite3";
+            sqlite.path = "/var/lib/headscale/db.sqlite3";
+          };
+          noise.private_key_path = "/var/lib/headscale/noise_private.key";
+          prefixes = with lib.my.c.tailscale.prefix; { inherit v4 v6; };
+          dns = {
+            # Use IPs that will route inside the VPN to prevent interception
+            # (e.g. DNS rebinding filtering)
+            nameservers.split = {
+              "${domain}" = pubNameservers;
+              "${lib.my.c.colony.domain}" = with allAssignments.estuary.base; [
+                ipv4.address ipv6.address
+              ];
+              "${lib.my.c.home.domain}" = with allAssignments; [
+                river.hi.ipv4.address
+                river.hi.ipv6.address
+                stream.hi.ipv4.address
+                stream.hi.ipv6.address
+              ];
+            };
+            magic_dns = true;
+            base_domain = "ts.${pubDomain}";
+          };
+          oidc = {
+            only_start_if_oidc_is_available = true;
+            issuer = "https://accounts.google.com";
+            client_id = "545475967061-l45cln081mp8t4li2c34v7t7b8la6f4f.apps.googleusercontent.com";
+            client_secret_path = config.age.secrets."britway/oidc-secret.txt".path;
+            scope = [ "openid" "profile" "email" ];
+            allowed_users = [ "jackos1998@gmail.com" ];
+          };
+        };
+      };
+
+      tailscale = {
+        enable = true;
+        authKeyFile = config.age.secrets."tailscale-auth.key".path;
+        openFirewall = true;
+        interfaceName = "tailscale0";
+        extraUpFlags = [
+          "--operator=${config.my.user.config.name}"
+          "--login-server=https://hs.nul.ie"
+          "--netfilter-mode=off"
+          "--advertise-exit-node"
+          "--advertise-routes=${advRoutes}"
+          "--accept-routes=false"
+        ];
+      };
+    };
+
+    my = {
+      secrets = {
+        files = {
+          "britway/oidc-secret.txt" = {
+            owner = "headscale";
+            group = "headscale";
+            mode = "440";
+          };
+          "tailscale-auth.key" = {};
+        };
+      };
+    };
+  };
+}

nixos/boxes/colony/default.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 let
   inherit (lib.my) net;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my.c.colony) domain prefixes custRouting firewallForwards;
 in
 {
   imports = [ ./vms ];
@@ -57,19 +57,30 @@ in
         };
 
         boot = {
-          kernelPackages = pkgs.linuxKernel.packages.linux_6_1.extend (self: super: {
+          kernelPackages = (lib.my.c.kernel.lts pkgs).extend (self: super: {
             kernel = super.kernel.override {
               structuredExtraConfig = with lib.kernel; {
-                #SOME_OPT = yes;
+                ACPI_APEI_PCIEAER = yes;
-                #A_MOD = module;
+                PCIEAER = yes;
               };
             };
           });
           kernelModules = [ "kvm-amd" ];
-          kernelParams = [ "amd_iommu=on" "console=ttyS0,115200n8" "console=ttyS1,115200n8" "console=tty0" ];
+          kernelParams = [
+            "amd_iommu=on"
+            "console=ttyS0,115200n8" "console=ttyS1,115200n8" "console=tty0"
+            "systemd.setenv=SYSTEMD_SULOGIN_FORCE=1"
+          ];
           initrd = {
             kernelModules = [ "dm-raid" ];
             availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
+            systemd = {
+              enable = true;
+              # Onlu activate volumes needed for boot to prevent thin check from getting killed while switching root
+              contents."/etc/lvm/lvm.conf".text = ''
+                activation/auto_activation_volume_list = [ "main/colony-nix" "main/colony-persist" ]
+              '';
+            };
           };
         };
 
@@ -137,6 +148,15 @@ in
           services = {
             "serial-getty@ttyS0".enable = true;
             "serial-getty@ttyS1".enable = true;
+            lvm-activate-main = {
+              description = "Activate remaining LVs";
+              unitConfig.DefaultDependencies = false;
+              serviceConfig = {
+                Type = "oneshot";
+                ExecStart = "${pkgs.lvm2.bin}/bin/vgchange -aay main";
+              };
+              wantedBy = [ "local-fs-pre.target" ];
+            };
 
             rsync-lvm-meta = {
               description = "rsync lvm metadata backups / archives to rsync.net";
@@ -232,10 +252,10 @@ in
                   };
                   ipv6Prefixes = [
                     {
-                      ipv6PrefixConfig.Prefix = prefixes.vms.v6;
+                      Prefix = prefixes.vms.v6;
                     }
                   ];
-                  routes = map (r: { routeConfig = r; }) [
+                  routes = [
                     {
                       Destination = prefixes.ctrs.v4;
                       Gateway = allAssignments.shill.routing.ipv4.address;
@@ -244,11 +264,30 @@ in
                       Destination = prefixes.ctrs.v6;
                       Gateway = allAssignments.shill.internal.ipv6.address;
                     }
+
                     {
                       Destination = allAssignments.shill.internal.ipv4.address;
                       Gateway = allAssignments.shill.routing.ipv4.address;
                     }
 
+                    {
+                      Destination = lib.my.c.tailscale.prefix.v4;
+                      Gateway = allAssignments.shill.routing.ipv4.address;
+                    }
+                    {
+                      Destination = lib.my.c.tailscale.prefix.v6;
+                      Gateway = allAssignments.shill.internal.ipv6.address;
+                    }
+                    {
+                      Destination = prefixes.qclk.v4;
+                      Gateway = allAssignments.shill.routing.ipv4.address;
+                    }
+
+                    {
+                      Destination = prefixes.jam.v6;
+                      Gateway = allAssignments.shill.internal.ipv6.address;
+                    }
+
                     {
                       Destination = prefixes.oci.v4;
                       Gateway = allAssignments.whale2.routing.ipv4.address;
@@ -261,6 +300,11 @@ in
                       Destination = allAssignments.whale2.internal.ipv4.address;
                       Gateway = allAssignments.whale2.routing.ipv4.address;
                     }
+
+                    {
+                      Destination = allAssignments.git.internal.ipv4.address;
+                      Gateway = allAssignments.git.routing.ipv4.address;
+                    }
                   ];
                 }
               ];
@@ -274,7 +318,7 @@ in
               "90-vm-mail" = {
                 matchConfig.Name = "vm-mail";
                 address = [
-                  (net.cidr.subnet 8 1 prefixes.cust.v4)
+                  "${custRouting.mail-vm}/32"
                   prefixes.mail.v6
                 ];
                 networkConfig = {
@@ -283,10 +327,10 @@ in
                 };
                 ipv6Prefixes = [
                   {
-                    ipv6PrefixConfig.Prefix = prefixes.mail.v6;
+                    Prefix = prefixes.mail.v6;
                   }
                 ];
-                routes = map (r: { routeConfig = r; }) [
+                routes = [
                   {
                     Destination = prefixes.mail.v4;
                     Scope = "link";
@@ -297,7 +341,7 @@ in
               "90-vm-darts" = {
                 matchConfig.Name = "vm-darts";
                 address = [
-                  (net.cidr.subnet 8 2 prefixes.cust.v4)
+                  "${custRouting.darts-vm}/32"
                   prefixes.darts.v6
                 ];
                 networkConfig = {
@@ -306,10 +350,10 @@ in
                 };
                 ipv6Prefixes = [
                   {
-                    ipv6PrefixConfig.Prefix = prefixes.darts.v6;
+                    Prefix = prefixes.darts.v6;
                   }
                 ];
-                routes = map (r: { routeConfig = r; }) [
+                routes = [
                   {
                     Destination = prefixes.darts.v4;
                     Scope = "link";
@@ -346,6 +390,7 @@ in
 
           firewall = {
             trustedInterfaces = [ "vms" ];
+            nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
             extraRules = ''
               define cust = { vm-mail, vm-darts }
               table inet filter {
@@ -372,6 +417,7 @@ in
                   "vm-estuary-persist"
                   "vm-whale2-persist"
                   "vm-mail-data"
+                  "vm-git-persist"
                   "git"
                 ];
                 compression = "zstd,5";

nixos/boxes/colony/vms/default.nix

@@ -3,37 +3,22 @@
     ./estuary
     ./shill
     ./whale2
+    ./git
   ];
 
   nixos.systems.colony.configuration = { lib, pkgs, config, systems, ... }:
   let
     inherit (lib) mkIf mkMerge optionals;
-
+    inherit (lib.my) vm;
-    lvmDisk' = name: lv: {
-      inherit name;
-      backend = {
-        driver = "host_device";
-        filename = "/dev/main/${lv}";
-        # It appears this needs to be set on the backend _and_ the format
-        discard = "unmap";
-      };
-      format = {
-        driver = "raw";
-        discard = "unmap";
-      };
-      frontend = "virtio-blk";
-    };
-    lvmDisk = lv: lvmDisk' lv lv;
-    vmLVM = vm: lv: lvmDisk' lv "vm-${vm}-${lv}";
 
     installerDisk = {
       name = "installer";
       backend = {
         driver = "file";
         #filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos-installer-devplayer0.iso";
-        #filename = "/persist/home/dev/nixos-installer-devplayer0.iso";
+        filename = "/persist/home/dev/nixos-installer-devplayer0-b4d0d9a.iso";
         #filename = "/persist/home/dev/debian-12.1.0-amd64-netinst.iso";
-        filename = "/persist/home/dev/ubuntu-22.04.3-live-server-amd64.iso";
+        # filename = "/persist/home/dev/ubuntu-22.04.3-live-server-amd64.iso";
         read-only = "on";
       };
       format.driver = "raw";
@@ -44,6 +29,9 @@
     };
   in
   {
+    # Kernel Same-Page Merging to attempt memory usage reduction
+    hardware.ksm.enable = false;
+
     systemd = {
       network = {
         links = {
@@ -116,9 +104,9 @@
               };
             };
             drives = [ ] ++ (optionals (!config.my.build.isDevVM) [
-              (mkMerge [ (vmLVM "estuary" "esp") { frontendOpts.bootindex = 0; } ])
+              (mkMerge [ (vm.disk "estuary" "esp") { frontendOpts.bootindex = 0; } ])
-              (vmLVM "estuary" "nix")
+              (vm.disk "estuary" "nix")
-              (vmLVM "estuary" "persist")
+              (vm.disk "estuary" "persist")
             ]);
             hostDevices = {
               net-wan0 = {
@@ -135,18 +123,18 @@
               cpus = 12;
               threads = 2;
             };
-            memory = 65536;
+            memory = 40960;
             networks.vms.mac = "52:54:00:27:3d:5c";
             cleanShutdown.timeout = 120;
             drives = [ ] ++ (optionals (!config.my.build.isDevVM) [
-              (mkMerge [ (vmLVM "shill" "esp") { frontendOpts.bootindex = 0; } ])
+              (mkMerge [ (vm.disk "shill" "esp") { frontendOpts.bootindex = 0; } ])
-              (vmLVM "shill" "nix")
+              (vm.disk "shill" "nix")
-              (vmLVM "shill" "persist")
+              (vm.disk "shill" "persist")
 
-              (lvmDisk "media")
+              (vm.lvmDisk "media")
-              (lvmDisk "minio")
+              (vm.lvmDisk "minio")
-              (lvmDisk "nix-atticd")
+              (vm.lvmDisk "nix-cache")
-              (lvmDisk "git")
+              (vm.lvmDisk "jam")
             ]);
           };
 
@@ -157,19 +145,39 @@
               cpus = 8;
               threads = 2;
             };
-            memory = 32768;
+            memory = 16384;
             networks.vms.mac = "52:54:00:d5:d9:c6";
             cleanShutdown.timeout = 120;
             drives = [ ] ++ (optionals (!config.my.build.isDevVM) [
-              (mkMerge [ (vmLVM "whale2" "esp") { frontendOpts.bootindex = 0; } ])
+              (mkMerge [ (vm.disk "whale2" "esp") { frontendOpts.bootindex = 0; } ])
-              (vmLVM "whale2" "nix")
+              (vm.disk "whale2" "nix")
-              (vmLVM "whale2" "persist")
+              (vm.disk "whale2" "persist")
 
-              (lvmDisk "oci")
+              (vm.lvmDisk "oci")
-              (lvmDisk "gitea-actions-cache")
             ]);
           };
 
+          git = {
+            uuid = "c0659fdc-3356-4717-a6a1-5f289ef03c4a";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 12;
+              threads = 2;
+            };
+            memory = 40960;
+            networks.vms.mac = "52:54:00:75:78:a8";
+            cleanShutdown.timeout = 120;
+            drives = [
+              (mkMerge [ (vm.disk "git" "esp") { frontendOpts.bootindex = 0; } ])
+              (vm.disk "git" "nix")
+              (vm.disk "git" "persist")
+              (vm.disk "git" "oci")
+
+              (vm.lvmDisk "git")
+              (vm.lvmDisk "gitea-actions-cache")
+            ];
+          };
+
           mail = {
             uuid = "fd95fe0f-c204-4dd5-b16f-2b808e14a43a";
             cpu = "host,topoext";
@@ -177,15 +185,15 @@
               cpus = 3;
               threads = 2;
             };
-            memory = 8192;
+            memory = 6144;
             networks.public = {
               bridge = null;
               mac = "52:54:00:a8:d1:03";
             };
             cleanShutdown.timeout = 120;
             drives = [
-              (mkMerge [ (vmLVM "mail" "root") { frontendOpts.bootindex = 0; } ])
+              (mkMerge [ (vm.disk "mail" "root") { frontendOpts.bootindex = 0; } ])
-              (vmLVM "mail" "data")
+              (vm.disk "mail" "data")
             ];
           };
 
@@ -203,8 +211,9 @@
             };
             cleanShutdown.timeout = 120;
             drives = [
-              (mkMerge [ (vmLVM "darts" "root") { frontendOpts.bootindex = 0; } ])
+              (mkMerge [ (vm.disk "darts" "root") { frontendOpts.bootindex = 0; } ])
-              (lvmDisk' "media" "darts-media")
+              (vm.lvmDisk' "media" "darts-media")
+              (vm.lvmDisk' "ext" "darts-ext")
             ];
           };
         };

nixos/boxes/colony/vms/estuary/bgp.nix

@@ -16,20 +16,23 @@ in
 
           define CCVIP1 = ${lib.my.c.colony.prefixes.vip1};
           define CCVIP2 = ${lib.my.c.colony.prefixes.vip2};
+          define CCVIP3 = ${lib.my.c.colony.prefixes.vip3};
 
           define OWNIP4 = ${assignments.internal.ipv4.address};
           define OWNNETSET4 = [ ${assignments.internal.ipv4.address}/32 ];
-          define CCNETSET4 = [ ${lib.my.c.colony.prefixes.vip1}, ${lib.my.c.colony.prefixes.vip2} ];
+          define CCNETSET4 = [ ${lib.my.c.colony.prefixes.vip1}, ${lib.my.c.colony.prefixes.vip2}, ${lib.my.c.colony.prefixes.vip3} ];
 
           define INTNET6 = ${intnet6};
           define AMSNET6 = ${amsnet6};
           define HOMENET6 = ${homenet6};
 
           define OWNIP6 = ${assignments.base.ipv6.address};
-          define OWNNETSET6 = [ ${intnet6}, ${amsnet6}, ${homenet6} ];
+          # we have issues with sending ICMPv6 too big back on the wrong interface right now...
+          define OWNNETSET6 = [ ${intnet6}, ${amsnet6} ];
+          define CCNETSET6 = [ ];
           #define TRANSSET6 = [ ::1/128 ];
 
-          define DUB1IP6 = 2a0e:97c0:4df:0:2::1;
+          define DUB1IP6 = ${lib.my.c.home.vips.as211024.v6};
 
           define PREFIXP = 110;
           define PREFPEER = 120;
@@ -41,7 +44,7 @@ in
             if net ~ OWNNETSET4 || net ~ OWNNETSET6 then accept; else reject;
           }
           filter bgp_export_cc {
-            if net ~ OWNNETSET4 || net ~ OWNNETSET6 || net ~ CCNETSET4 then accept; else reject;
+            if net ~ OWNNETSET4 || net ~ OWNNETSET6 || net ~ CCNETSET4 || net ~ CCNETSET6 then accept; else reject;
           }
 
           router id from "wan";
@@ -55,6 +58,7 @@ in
           protocol static static4 {
             route CCVIP1 via "base";
             route CCVIP2 via "base";
+            route CCVIP3 via "base";
 
             ipv4 {
               import all;
@@ -186,10 +190,12 @@ in
           protocol bgp upstream6_coloclue_eun2 from upstream_bgp6 {
             description "ColoClue euNetworks 2 (IPv6)";
             neighbor 2a02:898:0:20::e2 as 8283;
+            ipv6 { export filter bgp_export_cc; };
           }
           protocol bgp upstream6_coloclue_eun3 from upstream_bgp6 {
             description "ColoClue euNetworks 3 (IPv6)";
             neighbor 2a02:898:0:20::e1 as 8283;
+            ipv6 { export filter bgp_export_cc; };
           }
 
           protocol bgp upstream6_ifog from upstream_bgp6 {
@@ -202,14 +208,15 @@ in
             neighbor 2001:7f8:10f::1b1b:154 as 6939;
           }
 
-          protocol bgp upstream4_fogixp_efero from upstream_bgp4 {
+          # Not working so well lately...
-            description "efero transit (on FogIXP, IPv4)";
+          # protocol bgp upstream4_fogixp_efero from upstream_bgp4 {
-            neighbor 185.1.147.107 as 208431;
+          #   description "efero transit (on FogIXP, IPv4)";
-          }
+          #   neighbor 185.1.147.107 as 208431;
-          protocol bgp upstream6_fogixp_efero from upstream_bgp6 {
+          # }
-            description "efero transit (on FogIXP, IPv6)";
+          # protocol bgp upstream6_fogixp_efero from upstream_bgp6 {
-            neighbor 2001:7f8:ca:1::107 as 208431;
+          #   description "efero transit (on FogIXP, IPv6)";
-          }
+          #   neighbor 2001:7f8:ca:1::107 as 208431;
+          # }
 
           protocol bgp peer4_cc_luje from peer_bgp4 {
             description "LUJE.net (on ColoClue, IPv4)";

nixos/boxes/colony/vms/estuary/default.nix

@@ -1,19 +1,22 @@
 { lib, ... }:
 let
-  inherit (lib.my) net;
+  inherit (builtins) elemAt;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my) net mkVLAN;
-
+  inherit (lib.my.c.colony) pubV4 domain prefixes firewallForwards;
-  pubV4 = "94.142.240.44";
 in
 {
   nixos = {
     vpns = {
       l2 = {
         as211024 = {
+          udpEncapsulation = true;
           vni = 211024;
+          security.enable = true;
           peers = {
             estuary.addr = pubV4;
-            home.addr = "188.141.75.2";
+            river.addr = elemAt lib.my.c.home.routersPubV4 0;
+            stream.addr = elemAt lib.my.c.home.routersPubV4 1;
+            britway.addr = lib.my.c.britway.pubV4;
           };
         };
       };
@@ -53,10 +56,10 @@ in
       };
       as211024 = {
         ipv4 = {
-          address = "10.255.3.1";
+          address = net.cidr.host 1 prefixes.as211024.v4;
           gateway = null;
         };
-        ipv6.address = "2a0e:97c0:4df:0:3::1";
+        ipv6.address = net.cidr.host 1 prefixes.as211024.v6;
       };
     };
 
@@ -64,6 +67,7 @@ in
       let
         inherit (lib) flatten mkIf mkMerge mkForce;
         inherit (lib.my) networkdAssignment;
+        inherit (lib.my.c) networkd;
       in
       {
         imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./dns.nix ./bgp.nix ];
@@ -90,6 +94,7 @@ in
             environment = {
               systemPackages = with pkgs; [
                 ethtool
+                conntrack-tools
                 wireguard-tools
               ];
             };
@@ -114,34 +119,19 @@ in
             };
 
             systemd = {
-              services = {
+              services =
-                # Use this as a way to make sure the router always knows we're here (NDP seems kindy funky)
+              let
-                ipv6-neigh-keepalive =
+                waitOnline = "systemd-networkd-wait-online@wan.service";
-                let
+              in
-                  waitOnline = "systemd-networkd-wait-online@wan.service";
+              {
-                in
+                bird2 = {
-                {
-                  description = "Frequent ICMP6 neighbour solicitations";
-                  enable = false;
-                  requires = [ waitOnline ];
-                  after = [ waitOnline ];
-                  script = ''
-                    while true; do
-                      ${pkgs.ndisc6}/bin/ndisc6 ${assignments.internal.ipv6.gateway} wan
-                      sleep 10
-                    done
-                  '';
-                  wantedBy = [ "multi-user.target" ];
-                };
-
-                bird2 =
-                let
-                  waitOnline = "systemd-networkd-wait-online@wan.service";
-                in
-                {
                   after = [ waitOnline ];
                   # requires = [ waitOnline ];
                 };
+                ipsec = {
+                  after = [ waitOnline ];
+                  requires = [ waitOnline ];
+                };
               };
             };
 
@@ -154,19 +144,7 @@ in
                 };
               };
 
-              netdevs =
+              netdevs = mkMerge [
-              let
-                mkVLAN = name: vid: {
-                  "25-${name}" = {
-                    netdevConfig = {
-                      Name = name;
-                      Kind = "vlan";
-                    };
-                    vlanConfig.Id = vid;
-                  };
-                };
-              in
-              mkMerge [
                 (mkVLAN "ifog" 409)
 
                 (mkVLAN "frys-ix" 701)
@@ -186,11 +164,9 @@ in
                     };
                     wireguardPeers = [
                       {
-                        wireguardPeerConfig = {
+                        PublicKey = "7N9YdQaCMWWIwAnW37vrthm9ZpbnG4Lx3gheHeRYz2E=";
-                          PublicKey = "7N9YdQaCMWWIwAnW37vrthm9ZpbnG4Lx3gheHeRYz2E=";
+                        AllowedIPs = [ allAssignments.kelder.estuary.ipv4.address ];
-                          AllowedIPs = [ allAssignments.kelder.estuary.ipv4.address ];
+                        PersistentKeepalive = 25;
-                          PersistentKeepalive = 25;
-                        };
                       }
                     ];
                   };
@@ -271,13 +247,7 @@ in
                     Kind = "vlan";
                   };
                   vlan = [ "frys-ix" "nl-ix" "fogixp" "ifog-transit" ];
-                  networkConfig = {
+                  networkConfig = networkd.noL3;
-                    LinkLocalAddressing = "no";
-                    DHCP = "no";
-                    LLDP = false;
-                    EmitLLDP = false;
-                    IPv6AcceptRA = false;
-                  };
                 };
                 "85-ifog-transit" = {
                   matchConfig.Name = "ifog-transit";
@@ -306,53 +276,73 @@ in
                     };
                     ipv6Prefixes = [
                       {
-                        ipv6PrefixConfig.Prefix = prefixes.base.v6;
+                        Prefix = prefixes.base.v6;
                       }
                     ];
-                    routes = map (r: { routeConfig = r; }) (flatten
+                    routes = flatten ([
-                      ([
+                      {
-                        {
+                        Destination = prefixes.vip1;
-                          Destination = prefixes.vip1;
+                        Gateway = allAssignments.colony.routing.ipv4.address;
-                          Gateway = allAssignments.colony.routing.ipv4.address;
+                      }
-                        }
+                      {
-                        {
+                        Destination = prefixes.vip3;
-                          Destination = prefixes.darts.v4;
+                        Gateway = allAssignments.colony.routing.ipv4.address;
-                          Gateway = allAssignments.colony.routing.ipv4.address;
+                      }
-                        }
+                      {
-                        {
+                        Destination = prefixes.darts.v4;
-                          Destination = prefixes.cust.v6;
+                        Gateway = allAssignments.colony.routing.ipv4.address;
-                          Gateway = allAssignments.colony.internal.ipv6.address;
+                      }
-                        }
+                      {
-                      ] ++
+                        Destination = prefixes.cust.v6;
-                      (map (pName: [
+                        Gateway = allAssignments.colony.internal.ipv6.address;
-                        {
+                      }
-                          Gateway = allAssignments.colony.routing.ipv4.address;
+
-                          Destination = prefixes."${pName}".v4;
+                      {
-                        }
+                        Destination = lib.my.c.tailscale.prefix.v4;
-                        {
+                        Gateway = allAssignments.colony.routing.ipv4.address;
-                          Destination = prefixes."${pName}".v6;
+                      }
-                          Gateway = allAssignments.colony.internal.ipv6.address;
+                      {
-                        }
+                        Destination = lib.my.c.tailscale.prefix.v6;
-                      ]) [ "vms" "ctrs" "oci" ])));
+                        Gateway = allAssignments.colony.internal.ipv6.address;
+                      }
+
+                      {
+                        Destination = prefixes.qclk.v4;
+                        Gateway = allAssignments.colony.routing.ipv4.address;
+                      }
+                    ] ++
+                    (map (pName: [
+                      {
+                        Gateway = allAssignments.colony.routing.ipv4.address;
+                        Destination = prefixes."${pName}".v4;
+                      }
+                      {
+                        Destination = prefixes."${pName}".v6;
+                        Gateway = allAssignments.colony.internal.ipv6.address;
+                      }
+                    ]) [ "vms" "ctrs" "oci" ]));
                   }
                 ];
 
-                "90-l2mesh-as211024" = {
+                "90-l2mesh-as211024" = mkMerge [
-                  matchConfig.Name = "as211024";
+                  (networkdAssignment "as211024" assignments.as211024)
-                  address = with assignments.as211024; [
+                  {
-                    (with ipv4; "${address}/${toString mask}")
+                    matchConfig.Name = "as211024";
-                    (with ipv6; "${address}/${toString mask}")
+                    networkConfig.IPv6AcceptRA = mkForce false;
-                  ];
+                    routes = [
-                  networkConfig.IPv6AcceptRA = false;
+                      {
-                };
+                        Destination = lib.my.c.home.prefixes.all.v4;
+                        Gateway = lib.my.c.home.vips.as211024.v4;
+                      }
+                    ];
+                  }
+                ];
                 "95-kelder" = {
                   matchConfig.Name = "kelder";
                   routes = [
                     {
-                      routeConfig = {
+                      Destination = allAssignments.kelder.estuary.ipv4.address;
-                        Destination = allAssignments.kelder.estuary.ipv4.address;
+                      Scope = "link";
-                        Scope = "link";
-                      };
                     }
                   ];
                 };
@@ -366,43 +356,23 @@ in
                   "estuary/kelder-wg.key" = {
                     owner = "systemd-network";
                   };
+                  "l2mesh/as211024.key" = {};
                 };
               };
               server.enable = true;
 
+              vpns = {
+                l2.pskFiles = {
+                  as211024 = config.age.secrets."l2mesh/as211024.key".path;
+                };
+              };
               firewall = {
-                trustedInterfaces = [ "as211024" ];
                 udp.allowed = [ 5353 lib.my.c.kelder.vpn.port ];
                 tcp.allowed = [ 5353 "bgp" ];
                 nat = {
                   enable = true;
                   externalInterface = "wan";
-                  externalIP = assignments.internal.ipv4.address;
+                  forwardPorts."${assignments.internal.ipv4.address}" = firewallForwards allAssignments;
-                  forwardPorts = [
-                    {
-                      port = "http";
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-                    {
-                      port = "https";
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-                    {
-                      port = 8448;
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-
-                    {
-                      port = 2456;
-                      dst = allAssignments.valheim-oci.internal.ipv4.address;
-                      proto = "udp";
-                    }
-                    {
-                      port = 2457;
-                      dst = allAssignments.valheim-oci.internal.ipv4.address;
-                      proto = "udp";
-                    }
-                  ];
                 };
                 extraRules =
                 let
@@ -424,12 +394,20 @@ in
                       # Safe enough to allow all SSH
                       tcp dport ssh accept
 
-                      ${matchInet "tcp dport { http, https, 8448 } accept" "middleman"}
+                      # jam-ctr forwards
-                      ${matchInet "udp dport { 2456-2457 } accept" "valheim-oci"}
+                      ip daddr ${aa.shill.internal.ipv4.address} tcp dport 60022 accept
 
+                      ip6 daddr ${aa.middleman.internal.ipv6.address} tcp dport { http, https, 8448 } accept
+                      ${matchInet "tcp dport { http, https } accept" "git"}
+                      ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} tcp dport { 25565, 25575 } accept
+                      ip6 daddr ${aa.simpcraft-staging-oci.internal.ipv6.address} tcp dport 25565 accept
                       return
                     }
                     chain routing-udp {
+                      ip6 daddr ${aa.valheim-oci.internal.ipv6.address} udp dport { 2456-2457 } accept
+                      ip6 daddr ${aa.waffletail.internal.ipv6.address} udp dport 41641 accept
+                      ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} udp dport 25565 accept
+                      ip6 daddr ${aa.enshrouded-oci.internal.ipv6.address} udp dport { 15636-15637 } accept
                       return
                     }
                     chain filter-routing {
@@ -446,7 +424,8 @@ in
                     }
 
                     chain forward {
-                      iifname { wan, $ixps } oifname base jump filter-routing
+                      ${lib.my.c.as211024.nftTrust}
+                      iifname { wan, as211024, $ixps } oifname base jump filter-routing
                       oifname $ixps jump ixp
                       iifname base oifname { base, wan, $ixps } accept
                       oifname { as211024, kelder } accept
@@ -459,11 +438,9 @@ in
                   table inet nat {
                     chain prerouting {
                       ${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" "estuary"}
-                      ip daddr ${allAssignments.shill.internal.ipv4.address} tcp dport { http, https } dnat to ${allAssignments.middleman.internal.ipv4.address}
-                      ip6 daddr ${allAssignments.shill.internal.ipv6.address} tcp dport { http, https } dnat to ${allAssignments.middleman.internal.ipv6.address}
                     }
                     chain postrouting {
-                      ip saddr ${prefixes.all.v4} snat to ${assignments.internal.ipv4.address}
+                      ip saddr ${prefixes.all.v4} oifname != as211024 snat to ${assignments.internal.ipv4.address}
                     }
                   }
                 '';

nixos/boxes/colony/vms/estuary/dns.nix

@@ -2,7 +2,7 @@
 let
   inherit (builtins) attrNames;
   inherit (lib.my) net;
-  inherit (lib.my.c.colony) prefixes;
+  inherit (lib.my.c.colony) prefixes custRouting;
 
   authZones = attrNames config.my.pdns.auth.bind.zones;
 in
@@ -52,7 +52,7 @@ in
           allowFrom = [
             "127.0.0.0/8" "::1/128"
             prefixes.all.v4 prefixes.all.v6
-          ];
+          ] ++ (with lib.my.c.tailscale.prefix; [ v4 v6 ]);
         };
 
         settings = {
@@ -145,9 +145,15 @@ in
 
             http IN A ${assignments.internal.ipv4.address}
             http IN AAAA ${allAssignments.middleman.internal.ipv6.address}
+            librespeed IN CNAME http.${config.networking.domain}.
 
             valheim IN A ${assignments.internal.ipv4.address}
             valheim IN AAAA ${allAssignments.valheim-oci.internal.ipv6.address}
+            simpcraft IN A ${assignments.internal.ipv4.address}
+            simpcraft IN AAAA ${allAssignments.simpcraft-oci.internal.ipv6.address}
+            simpcraft-staging IN A ${assignments.internal.ipv4.address}
+            simpcraft-staging IN AAAA ${allAssignments.simpcraft-staging-oci.internal.ipv6.address}
+            enshrouded IN A ${assignments.internal.ipv4.address}
 
             mail-vm IN A ${net.cidr.host 0 prefixes.mail.v4}
             mail-vm IN AAAA ${net.cidr.host 1 prefixes.mail.v6}
@@ -157,6 +163,10 @@ in
 
             andrey-cust IN A ${allAssignments.kelder.estuary.ipv4.address}
 
+            jam-cust IN A ${net.cidr.host 0 prefixes.jam.v4}
+            jam-fwd IN A ${allAssignments.shill.internal.ipv4.address}
+            jam-cust IN AAAA ${net.cidr.host 1 prefixes.jam.v6}
+
             $TTL 3
             _acme-challenge IN LUA TXT @@FILE@@
 

nixos/boxes/colony/vms/git/default.nix

@@ -0,0 +1,213 @@
+{ lib, ... }:
+let
+  inherit (builtins) mapAttrs;
+  inherit (lib) mkMerge mkDefault;
+  inherit (lib.my) net;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my.c.nginx) baseHttpConfig proxyHeaders;
+in
+{
+  nixos.systems.git = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+
+    assignments = {
+      routing = {
+        name = "git-vm-routing";
+        inherit domain;
+        ipv4.address = net.cidr.host 4 prefixes.vms.v4;
+      };
+      internal = {
+        name = "git-vm";
+        inherit domain;
+        ipv4 = {
+          address = net.cidr.host 0 prefixes.vip3;
+          mask = 32;
+          gateway = null;
+          genPTR = false;
+        };
+        ipv6 = {
+          iid = "::4";
+          address = net.cidr.host 4 prefixes.vms.v6;
+        };
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
+      let
+        inherit (lib) mkMerge;
+        inherit (lib.my) networkdAssignment;
+      in
+      {
+        imports = [
+          "${modulesPath}/profiles/qemu-guest.nix"
+
+          ./gitea.nix
+          ./gitea-actions.nix
+        ];
+
+        config = mkMerge [
+          {
+            boot = {
+              kernelParams = [ "console=ttyS0,115200n8" ];
+            };
+
+            fileSystems = {
+              "/boot" = {
+                device = "/dev/disk/by-label/ESP";
+                fsType = "vfat";
+              };
+              "/nix" = {
+                device = "/dev/disk/by-label/nix";
+                fsType = "ext4";
+              };
+              "/persist" = {
+                device = "/dev/disk/by-label/persist";
+                fsType = "ext4";
+                neededForBoot = true;
+              };
+
+              "/var/lib/containers" = {
+                device = "/dev/disk/by-label/oci";
+                fsType = "xfs";
+                options = [ "pquota" ];
+              };
+            };
+
+            users = {
+              users = {
+                nginx.extraGroups = [ "acme" ];
+              };
+            };
+
+            security.acme = {
+              acceptTerms = true;
+              defaults = {
+                email = "dev@nul.ie";
+                server = "https://acme-v02.api.letsencrypt.org/directory";
+                reloadServices = [ "nginx" ];
+                dnsResolver = "8.8.8.8";
+              };
+              certs = {
+                "${pubDomain}" = {
+                  extraDomainNames = [
+                    "*.${pubDomain}"
+                  ];
+                  dnsProvider = "cloudflare";
+                  credentialsFile = config.age.secrets."middleman/cloudflare-credentials.conf".path;
+                };
+              };
+            };
+
+            services = {
+              fstrim = lib.my.c.colony.fstrimConfig;
+              netdata.enable = true;
+              nginx = {
+                enable = true;
+                enableReload = true;
+
+                logError = "stderr info";
+                recommendedTlsSettings = true;
+                clientMaxBodySize = "0";
+                serverTokens = true;
+                sslDhparam = config.age.secrets."dhparams.pem".path;
+
+                # Based on recommended*Settings, but probably better to be explicit about these
+                appendHttpConfig = ''
+                  ${baseHttpConfig}
+
+                  # caching
+                  proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=512m;
+                '';
+
+                virtualHosts =
+                let
+                  hosts = {
+                    "_" = {
+                      default = true;
+                      forceSSL = true;
+                      onlySSL = false;
+                      locations = {
+                        "/".root = "${pkgs.nginx}/html";
+                      };
+                    };
+
+                    "git.${pubDomain}" = {
+                      locations."/".proxyPass = "http://localhost:3000";
+                    };
+                  };
+
+                  defaultsFor = mapAttrs (n: _: {
+                    onlySSL = mkDefault true;
+                    useACMEHost = mkDefault pubDomain;
+                    kTLS = mkDefault true;
+                    http2 = mkDefault true;
+                  });
+                in
+                mkMerge [
+                  hosts
+                  (defaultsFor hosts)
+                ];
+              };
+            };
+
+            virtualisation = {
+              podman = {
+                enable = true;
+              };
+              oci-containers = {
+                backend = "podman";
+              };
+              containers.containersConf.settings.network.default_subnet = "10.88.0.0/16";
+            };
+
+            systemd.network = {
+              links = {
+                "10-vms" = {
+                  matchConfig.MACAddress = "52:54:00:75:78:a8";
+                  linkConfig.Name = "vms";
+                };
+              };
+
+              networks = {
+                "80-vms" = mkMerge [
+                  (networkdAssignment "vms" assignments.routing)
+                  (networkdAssignment "vms" assignments.internal)
+                ];
+              };
+            };
+
+            my = {
+              secrets = {
+                key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP+KINpHLMduBuW96JzfSRDLUzkI+XaCBghu5/wHiW5R";
+                files = {
+                  "dhparams.pem" = {
+                    owner = "acme";
+                    group = "acme";
+                    mode = "440";
+                  };
+                  "middleman/cloudflare-credentials.conf" = {
+                    owner = "acme";
+                    group = "acme";
+                  };
+                };
+              };
+              server.enable = true;
+
+              firewall = {
+                tcp.allowed = [ 19999 "http" "https" ];
+                extraRules = ''
+                  table inet filter {
+                    chain forward {
+                      ip saddr 10.88.0.0/16 accept
+                    }
+                  }
+                '';
+              };
+            };
+          }
+        ];
+      };
+  };
+}

nixos/boxes/colony/vms/git/gitea-actions.nix

@@ -6,7 +6,7 @@ let
 
   cfgFile = pkgs.writeText "gitea-actions-runner.yaml" (toJSON {
     container = {
-      network = "colony";
+      network = "podman";
       privileged = true;
     };
     cache = {
@@ -35,6 +35,11 @@ in
           ];
           url = "https://git.${pubDomain}";
           tokenFile = config.age.secrets."gitea/actions-runner.env".path;
+          settings = {
+            runner = {
+              timeout = "8h";
+            };
+          };
         };
       };
     };

nixos/boxes/colony/vms/git/gitea.nix

@@ -1,5 +1,6 @@
 { lib, pkgs, config, assignments, allAssignments, ... }:
 let
+  inherit (lib) mkMerge;
   inherit (lib.my.c) pubDomain;
   inherit (lib.my.c.colony) prefixes;
 in
@@ -25,20 +26,25 @@ in
 
     systemd = {
       services = {
-        gitea.preStart =
+        gitea = mkMerge [
-        let
+          (lib.my.systemdAwaitPostgres pkgs.postgresql "colony-psql")
-          repSec = "${pkgs.replace-secret}/bin/replace-secret";
+          {
-          confPath = "${config.services.gitea.customDir}/conf/app.ini";
+            preStart =
-        in
+            let
-        ''
+              repSec = "${pkgs.replace-secret}/bin/replace-secret";
-          gitea_extra_setup() {
+              confPath = "${config.services.gitea.customDir}/conf/app.ini";
-            chmod u+w '${confPath}'
+            in
-            ${repSec} '#miniosecret#' '${config.age.secrets."gitea/minio.txt".path}' '${confPath}'
+            ''
-            chmod u-w '${confPath}'
+              gitea_extra_setup() {
-          }
+                chmod u+w '${confPath}'
+                ${repSec} '#miniosecret#' '${config.age.secrets."gitea/minio.txt".path}' '${confPath}'
+                chmod u-w '${confPath}'
+              }
 
-          (umask 027; gitea_extra_setup)
+              (umask 027; gitea_extra_setup)
-        '';
+            '';
+          }
+        ];
       };
     };
 
@@ -123,21 +129,6 @@ in
           "gitea/minio.txt" = ownedByGit;
         };
       };
-
-      firewall.extraRules = ''
-        table inet filter {
-          chain input {
-            ip saddr ${prefixes.all.v4} tcp dport 3000 accept
-            ip6 saddr ${prefixes.all.v6} tcp dport 3000 accept
-          }
-        }
-        table inet nat {
-          chain prerouting {
-            ip daddr ${assignments.internal.ipv4.address} tcp dport { http, https } dnat to ${allAssignments.middleman.internal.ipv4.address}
-            ip6 daddr ${assignments.internal.ipv6.address} tcp dport { http, https } dnat to ${allAssignments.middleman.internal.ipv6.address}
-          }
-        }
-      '';
     };
   };
 }

nixos/boxes/colony/vms/shill/containers-ext.nix

@@ -0,0 +1,105 @@
+{ lib, pkgs, assignments, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.colony) prefixes custRouting;
+in
+{
+  fileSystems = {
+    "/mnt/jam" = {
+      device = "/dev/disk/by-label/jam";
+      fsType = "ext4";
+    };
+
+    "/var/lib/machines/jam" = {
+      device = "/mnt/jam";
+      options = [ "bind" ];
+    };
+  };
+
+  systemd = {
+    nspawn = {
+      jam = {
+        enable = true;
+        execConfig = {
+          Boot = true;
+          PrivateUsers = "pick";
+          LinkJournal = false;
+        };
+        networkConfig = {
+          Private = true;
+          VirtualEthernet = true;
+        };
+      };
+    };
+    network.networks = {
+      "50-ve-jam" = {
+        matchConfig = {
+          Kind = "veth";
+          Name = "ve-jam";
+        };
+        address = [
+          custRouting.jam-ctr
+          prefixes.jam.v6
+        ];
+        networkConfig = {
+          IPv6AcceptRA = false;
+          IPv6SendRA = true;
+        };
+        ipv6Prefixes = [
+          {
+            Prefix = prefixes.jam.v6;
+          }
+        ];
+        routes = [
+          {
+            Destination = prefixes.jam.v4;
+            Scope = "link";
+          }
+        ];
+      };
+    };
+    services = {
+      "systemd-nspawn@jam" = {
+        overrideStrategy = "asDropin";
+
+        serviceConfig = {
+          CPUQuota = "400%";
+          MemoryHigh = "infinity";
+          MemoryMax = "4G";
+        };
+
+        wantedBy = [ "machines.target" ];
+      };
+    };
+  };
+
+  my = {
+    firewall =
+    let
+      jamIP = net.cidr.host 0 prefixes.jam.v4;
+    in
+    {
+      nat.forwardPorts."${assignments.internal.ipv4.address}" = [
+        {
+          port = 60022;
+          dst = jamIP;
+          dstPort = "ssh";
+        }
+      ];
+      extraRules = ''
+        table inet filter {
+          chain forward {
+            iifname { ve-jam } oifname vms accept
+            iifname vms oifname { ve-jam } accept
+          }
+        }
+
+        table inet nat {
+          chain postrouting {
+            ip saddr ${jamIP} snat to ${assignments.internal.ipv4.address}
+          }
+        }
+      '';
+    };
+  };
+}

nixos/boxes/colony/vms/shill/containers/chatterbox.nix

@@ -24,7 +24,7 @@ in
 
     configuration = { lib, pkgs, config, assignments, allAssignments, ... }:
     let
-      inherit (lib) mkMerge mkIf mkForce;
+      inherit (lib) genAttrs mkMerge mkIf mkForce;
       inherit (lib.my) networkdAssignment;
     in
     {
@@ -45,9 +45,22 @@ in
                   owner = "matrix-synapse";
                   group = "matrix-synapse";
                 };
-                "chatterbox/syncv3.env" = {
+                "chatterbox/doublepuppet.yaml" = {
-                  owner = "matrix-syncv3";
+                  owner = "matrix-synapse";
-                  group = "matrix-syncv3";
+                  group = "matrix-synapse";
+                };
+
+                "chatterbox/mautrix-whatsapp.env" = {
+                  owner = "mautrix-whatsapp";
+                  group = "mautrix-whatsapp";
+                };
+                "chatterbox/mautrix-messenger.env" = {
+                  owner = "mautrix-meta-messenger";
+                  group = "mautrix-meta";
+                };
+                "chatterbox/mautrix-instagram.env" = {
+                  owner = "mautrix-meta-instagram";
+                  group = "mautrix-meta";
                 };
               };
             };
@@ -59,29 +72,24 @@ in
 
           users = with lib.my.c.ids; {
             users = {
-              matrix-syncv3 = {
+              matrix-synapse.extraGroups = [
-                isSystemUser = true;
+                "mautrix-whatsapp"
-                uid = uids.matrix-syncv3;
+              ];
-                group = "matrix-syncv3";
-              };
-            };
-            groups = {
-              matrix-syncv3.gid = gids.matrix-syncv3;
             };
+            groups = { };
           };
 
           systemd = {
             network.networks."80-container-host0" = networkdAssignment "host0" assignments.internal;
-            services = {
+            services = { } // (genAttrs [ "mautrix-whatsapp" "mautrix-meta-messenger" "mautrix-meta-instagram" ] (_: {
-              matrix-sliding-sync.serviceConfig = {
+              # ffmpeg needed to convert GIFs to video
-                # Needs to be able to read its secrets
+              path = with pkgs; [ ffmpeg ];
-                DynamicUser = mkForce false;
+            }));
-                User = "matrix-syncv3";
-                Group = "matrix-syncv3";
-              };
-            };
           };
 
+          # TODO/FIXME: https://github.com/NixOS/nixpkgs/issues/336052
+          nixpkgs.config.permittedInsecurePackages = [ "olm-3.2.16" ];
+
           services = {
             netdata.enable = true;
             matrix-synapse = {
@@ -168,18 +176,10 @@ in
 
                 app_service_config_files = [
                   "/var/lib/heisenbridge/registration.yml"
+                  config.age.secrets."chatterbox/doublepuppet.yaml".path
                 ];
               };
 
-              sliding-sync = {
-                enable = true;
-                createDatabase = false;
-                environmentFile = config.age.secrets."chatterbox/syncv3.env".path;
-                settings = {
-                  SYNCV3_BINDADDR = "[::]:8009";
-                  SYNCV3_SERVER = "http://localhost:8008";
-                };
-              };
             };
 
             heisenbridge = {
@@ -195,6 +195,144 @@ in
                 ];
               };
             };
+
+            mautrix-whatsapp = {
+              enable = true;
+              environmentFile = config.age.secrets."chatterbox/mautrix-whatsapp.env".path;
+              settings = {
+                homeserver = {
+                  address = "http://localhost:8008";
+                  domain = "nul.ie";
+                };
+                appservice = {
+                  database = {
+                    type = "postgres";
+                    uri = "$MAU_WAPP_PSQL_URI";
+                  };
+                  id = "whatsapp2";
+                  bot = {
+                    username = "whatsapp2";
+                    displayname = "WhatsApp Bridge Bot";
+                  };
+                };
+                bridge = {
+                  username_template = "wapp2_{{.}}";
+                  displayname_template = "{{or .BusinessName .PushName .JID}} (WA)";
+                  personal_filtering_spaces = true;
+                  delivery_receipts = true;
+                  allow_user_invite = true;
+                  url_previews = true;
+                  command_prefix = "!wa";
+                  login_shared_secret_map."nul.ie" = "$MAU_WAPP_DOUBLE_PUPPET_TOKEN";
+                  encryption = {
+                    allow = true;
+                    default = true;
+                    require = true;
+                  };
+                  permissions = {
+                    "@dev:nul.ie" = "admin";
+                  };
+                };
+              };
+            };
+
+            mautrix-meta.instances = {
+              messenger = {
+                enable = true;
+                registerToSynapse = true;
+                dataDir = "mautrix-messenger";
+                environmentFile = config.age.secrets."chatterbox/mautrix-messenger.env".path;
+                settings = {
+                  homeserver = {
+                    address = "http://localhost:8008";
+                    domain = "nul.ie";
+                  };
+                  appservice = {
+                    database = {
+                      type = "postgres";
+                      uri = "$MAU_FBM_PSQL_URI";
+                    };
+                    id = "fbm2";
+                    bot = {
+                      username = "messenger2";
+                      displayname = "Messenger Bridge Bot";
+                      avatar = "mxc://maunium.net/ygtkteZsXnGJLJHRchUwYWak";
+                    };
+                  };
+                  network = {
+                    mode = "messenger";
+                    displayname_template = ''{{or .DisplayName .Username "Unknown user"}} (FBM)'';
+                  };
+                  bridge = {
+                    username_template = "fbm2_{{.}}";
+                    personal_filtering_spaces = true;
+                    delivery_receipts = true;
+                    management_room_text.welcome = "Hello, I'm a Messenger bridge bot.";
+                    command_prefix = "!fbm";
+                    login_shared_secret_map."nul.ie" = "$MAU_FBM_DOUBLE_PUPPET_TOKEN";
+                    backfill = {
+                      history_fetch_pages = 5;
+                    };
+                    encryption = {
+                      allow = true;
+                      default = true;
+                      require = true;
+                    };
+                    permissions = {
+                      "@dev:nul.ie" = "admin";
+                    };
+                  };
+                };
+              };
+
+              instagram = {
+                enable = true;
+                registerToSynapse = true;
+                dataDir = "mautrix-instagram";
+                environmentFile = config.age.secrets."chatterbox/mautrix-instagram.env".path;
+                settings = {
+                  homeserver = {
+                    address = "http://localhost:8008";
+                    domain = "nul.ie";
+                  };
+                  appservice = {
+                    database = {
+                      type = "postgres";
+                      uri = "$MAU_IG_PSQL_URI";
+                    };
+                    id = "instagram";
+                    bot = {
+                      username = "instagram";
+                      displayname = "Instagram Bridge Bot";
+                      avatar = "mxc://maunium.net/JxjlbZUlCPULEeHZSwleUXQv";
+                    };
+                  };
+                  network = {
+                    mode = "instagram";
+                    displayname_template = ''{{or .DisplayName .Username "Unknown user"}} (IG)'';
+                  };
+                  bridge = {
+                    username_template = "ig_{{.}}";
+                    personal_filtering_spaces = true;
+                    delivery_receipts = true;
+                    management_room_text.welcome = "Hello, I'm an Instagram bridge bot.";
+                    command_prefix = "!ig";
+                    login_shared_secret_map."nul.ie" = "$MAU_IG_DOUBLE_PUPPET_TOKEN";
+                    backfill = {
+                      history_fetch_pages = 5;
+                    };
+                    encryption = {
+                      allow = true;
+                      default = true;
+                      require = true;
+                    };
+                    permissions = {
+                      "@dev:nul.ie" = "admin";
+                    };
+                  };
+                };
+              };
+            };
           };
         }
         (mkIf config.my.build.isDevVM {

nixos/boxes/colony/vms/shill/containers/default.nix

@@ -7,5 +7,7 @@
     ./jackflix
     ./object.nix
     ./toot.nix
+    ./waffletail.nix
+    ./qclk
   ];
 }

nixos/boxes/colony/vms/shill/containers/jackflix/default.nix

@@ -1,6 +1,8 @@
 { lib, ... }:
 let
+  inherit (lib) concatStringsSep;
   inherit (lib.my) net;
+  inherit (lib.my.c) pubDomain;
   inherit (lib.my.c.colony) domain prefixes;
 in
 {
@@ -35,6 +37,9 @@ in
 
           secrets = {
             key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPUv1ntVrZv5ripsKpcOAnyDQX2PHjowzyhqWK10Ml53";
+            files = {
+              "jackflix/photoprism-pass.txt" = {};
+            };
           };
         };
 
@@ -50,10 +55,16 @@ in
               uid = uids.jellyseerr;
               group = "jellyseerr";
             };
+            photoprism = {
+              isSystemUser = true;
+              uid = uids.photoprism;
+              group = "photoprism";
+            };
           };
           groups = {
             media.gid = 2000;
             jellyseerr.gid = gids.jellyseerr;
+            photoprism.gid = gids.photoprism;
           };
         };
 
@@ -76,9 +87,21 @@ in
               RootDirectoryStartOnly = lib.mkForce false;
               RootDirectory = lib.mkForce "";
             };
+            photoprism.serviceConfig = {
+              # Needs to be able to access its data
+              DynamicUser = mkForce false;
+            };
           };
         };
 
+        nixpkgs.config.permittedInsecurePackages = [
+          # FIXME: This is needed for Sonarr
+          "aspnetcore-runtime-wrapped-6.0.36"
+          "aspnetcore-runtime-6.0.36"
+          "dotnet-sdk-wrapped-6.0.428"
+          "dotnet-sdk-6.0.428"
+        ];
+
         services = {
           netdata.enable = true;
 
@@ -117,6 +140,24 @@ in
           };
 
           jellyfin.enable = true;
+
+          photoprism = {
+            enable = true;
+            address = "[::]";
+            port = 2342;
+            originalsPath = "/mnt/media/photoprism/originals";
+            importPath = "/mnt/media/photoprism/import";
+            passwordFile = config.age.secrets."jackflix/photoprism-pass.txt".path;
+            settings = {
+              PHOTOPRISM_AUTH_MODE = "password";
+              PHOTOPRISM_ADMIN_USER = "dev";
+              PHOTOPRISM_APP_NAME = "/dev/player0 Photos";
+              PHOTOPRISM_SITE_URL = "https://photos.${pubDomain}/";
+              PHOTOPRISM_SITE_TITLE = "/dev/player0 Photos";
+              PHOTOPRISM_TRUSTED_PROXY = concatStringsSep "," (with prefixes.ctrs; [ v4 v6 ]);
+              PHOTOPRISM_DATABASE_DRIVER = "sqlite";
+            };
+          };
         };
       };
     };

nixos/boxes/colony/vms/shill/containers/jackflix/networking.nix

@@ -37,7 +37,7 @@ in
                 tcp dport ${toString transmissionPeerPort} accept
                 iifname vpn return
 
-                tcp dport { 19999, 9091, 9117, 7878, 8989, 8096 } accept
+                tcp dport { 19999, 9091, 9117, 7878, 8989, 8096, 2342 } accept
                 return
               }
               chain input {
@@ -71,14 +71,12 @@ in
               RouteTable = routeTable;
             };
             wireguardPeers = [
+              # AirVPN NL
               {
-                # AirVPN NL
+                Endpoint = "2a00:1678:1337:2329:e5f:35d4:4404:ef9f:1637";
-                wireguardPeerConfig = {
+                PublicKey = "PyLCXAQT8KkM4T+dUsOQfn+Ub3pGxfGlxkIApuig+hk=";
-                  Endpoint = "2a00:1678:1337:2329:e5f:35d4:4404:ef9f:1637";
+                PresharedKeyFile = config.age.secrets."${pskFile}".path;
-                  PublicKey = "PyLCXAQT8KkM4T+dUsOQfn+Ub3pGxfGlxkIApuig+hk=";
+                AllowedIPs = [ "0.0.0.0/0" "::/0" ];
-                  PresharedKeyFile = config.age.secrets."${pskFile}".path;
-                  AllowedIPs = [ "0.0.0.0/0" "::/0" ];
-                };
               }
             ];
           };
@@ -94,7 +92,7 @@ in
               matchConfig.Name = "vpn";
               address = [ "10.182.97.37/32" "fd7d:76ee:e68f:a993:735d:ef5e:6907:b122/128" ];
               dns = [ "10.128.0.1" "fd7d:76ee:e68f:a993::1" ];
-              routingPolicyRules = map (r: { routingPolicyRuleConfig = r; }) [
+              routingPolicyRules = [
                 {
                   Family = "both";
                   SuppressPrefixLength = 0;

nixos/boxes/colony/vms/shill/containers/middleman/default.nix

@@ -2,6 +2,7 @@
 let
   inherit (lib.my) net;
   inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.nginx) baseHttpConfig;
   inherit (lib.my.c.colony) domain prefixes;
 in
 {
@@ -65,6 +66,7 @@ in
                   owner = "nginx";
                   group = "nginx";
                 };
+                "librespeed.toml" = { };
               };
             };
 
@@ -121,6 +123,19 @@ in
                 baseURL = "https://sso.${pubDomain}";
               };
             };
+
+            librespeed = {
+              frontend.servers = [
+                {
+                  name = "Amsterdam, Netherlands";
+                  server = "//librespeed.${domain}";
+                }
+              ];
+              backend = {
+                enable = true;
+                extraSettingsFile = config.age.secrets."librespeed.toml".path;
+              };
+            };
           };
 
           users = {
@@ -131,6 +146,10 @@ in
 
           systemd = {
             network.networks."80-container-host0" = networkdAssignment "host0" assignments.internal;
+            services = {
+              # HACK: nginx seems to get stuck not being able to DNS early...
+              nginx = lib.my.systemdAwaitPostgres pkgs.postgresql "colony-psql";
+            };
           };
 
           security = {
@@ -220,6 +239,9 @@ in
               ];
 
               recommendedTlsSettings = true;
+              recommendedBrotliSettings = true;
+              # Uh so nginx is hanging with zstd enabled... maybe let's not for now
+              # recommendedZstdSettings = true;
               clientMaxBodySize = "0";
               serverTokens = true;
               resolver = {
@@ -231,43 +253,9 @@ in
 
               # Based on recommended*Settings, but probably better to be explicit about these
               appendHttpConfig = ''
-                # NixOS provides a logrotate config that auto-compresses :)
+                ${baseHttpConfig}
-                log_format main
-                  '$remote_addr - $remote_user [$time_local] $scheme "$host" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"';
-                access_log /var/log/nginx/access.log main;
 
-                # optimisation
+                resolver_timeout 5s;
-                sendfile on;
-                tcp_nopush on;
-                tcp_nodelay on;
-                keepalive_timeout 65;
-
-                # gzip
-                gzip on;
-                gzip_proxied any;
-                gzip_comp_level 5;
-                gzip_types
-                  application/atom+xml
-                  application/javascript
-                  application/json
-                  application/xml
-                  application/xml+rss
-                  image/svg+xml
-                  text/css
-                  text/javascript
-                  text/plain
-                  text/xml;
-                gzip_vary on;
-
-                # proxying
-                proxy_buffering off;
-                proxy_redirect off;
-                proxy_connect_timeout 60s;
-                proxy_read_timeout 60s;
-                proxy_send_timeout 60s;
-                proxy_http_version 1.1;
-
-                ${lib.my.c.nginx.proxyHeaders}
 
                 # caching
                 proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=4g;

nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix

@@ -2,7 +2,7 @@
 let
   inherit (builtins) mapAttrs toJSON;
   inherit (lib) mkMerge mkDefault genAttrs flatten concatStringsSep;
-  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c) pubDomain home;
   inherit (lib.my.c.nginx) proxyHeaders;
   inherit (config.networking) domain;
 
@@ -35,7 +35,6 @@ let
       # For clients
       (mkWellKnown "matrix/client" (toJSON {
         "m.homeserver".base_url = "https://matrix.nul.ie";
-        "org.matrix.msc3575.proxy".url = "https://matrix-syncv3.nul.ie";
       }))
     ];
   };
@@ -145,7 +144,7 @@ in
 
       "pass.${pubDomain}" =
       let
-        upstream = "http://vaultwarden-ctr.${domain}";
+        upstream = "http://vaultwarden-ctr.${domain}:8080";
       in
       {
         locations = {
@@ -182,10 +181,6 @@ in
         ];
         useACMEHost = pubDomain;
       };
-      "matrix-syncv3.${pubDomain}" = {
-        locations."/".proxyPass = "http://chatterbox-ctr.${domain}:8009";
-        useACMEHost = pubDomain;
-      };
 
       "element.${pubDomain}" =
       let
@@ -206,7 +201,8 @@ in
           # Currently it seems like single quotes aren't escaped like they should be...
           conf = {
             brand = "/dev/player0 Matrix";
-            showLabsSettings = true;
+            show_labs_settings = true;
+            default_country_code = "IE";
             disable_guests = true;
             default_server_config = {
               "m.homeserver" = {
@@ -214,9 +210,8 @@ in
                 server_name = "nul.ie";
               };
             };
-            roomDirectory.servers = [
+            room_directory.servers = [
               "nul.ie"
-              "netsoc.ie"
               "matrix.org"
             ];
           };
@@ -318,59 +313,12 @@ in
         useACMEHost = pubDomain;
       };
 
-      "toot.nul.ie" =
+      "toot.nul.ie" = {
-      let
+        locations."/" = {
-        mkAssetLoc = name: {
+          proxyPass = "http://toot-ctr.${domain}:80";
-          tryFiles = "$uri =404";
+          proxyWebsockets = true;
-          extraConfig = ''
+          extraConfig = proxyHeaders;
-            add_header Cache-Control "public, max-age=2419200, must-revalidate";
-            add_header Strict-Transport-Security "max-age=63072000; includeSubpubDomains";
-          '';
         };
-      in
-      {
-        root = "${pkgs.mastodon}/public";
-        locations = mkMerge [
-          (genAttrs [
-            "= /sw.js"
-            "~ ^/assets/"
-            "~ ^/avatars/"
-            "~ ^/emoji/"
-            "~ ^/headers/"
-            "~ ^/packs/"
-            "~ ^/shortcuts/"
-            "~ ^/sounds/"
-          ] mkAssetLoc)
-          {
-            "/".tryFiles = "$uri @proxy";
-
-            "^~ /api/v1/streaming" = {
-              proxyPass = "http://toot-ctr.${domain}:55000";
-              proxyWebsockets = true;
-              extraConfig = ''
-                ${proxyHeaders}
-                proxy_set_header Proxy "";
-
-                add_header Strict-Transport-Security "max-age=63072000; includeSubdomains";
-              '';
-            };
-            "@proxy" = {
-              proxyPass = "http://toot-ctr.${domain}:55001";
-              proxyWebsockets = true;
-              extraConfig = ''
-                ${proxyHeaders}
-                proxy_set_header Proxy "";
-                proxy_pass_header Server;
-
-                proxy_cache CACHE;
-                proxy_cache_valid 200 7d;
-                proxy_cache_valid 410 24h;
-                proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
-                add_header X-Cached $upstream_cache_status;
-              '';
-            };
-          }
-        ];
         useACMEHost = pubDomain;
       };
 
@@ -394,11 +342,77 @@ in
         };
         useACMEHost = pubDomain;
       };
-
+      "public.${pubDomain}" = {
-      "git.${pubDomain}" = {
+        serverAliases = [ "p.${pubDomain}" ];
-        locations."/".proxyPass = "http://shill-vm.${domain}:3000";
+        locations."/" = {
+          root = "/mnt/media/public";
+          extraConfig = ''
+            fancyindex on;
+            fancyindex_show_dotfiles on;
+          '';
+        };
         useACMEHost = pubDomain;
       };
+
+      "mc-map.${pubDomain}" = {
+        locations."/".proxyPass = "http://simpcraft-oci.${domain}:8100";
+        useACMEHost = pubDomain;
+      };
+      "mc-rail.${pubDomain}" = {
+        locations."/".proxyPass = "http://simpcraft-oci.${domain}:3876";
+        useACMEHost = pubDomain;
+      };
+
+      "librespeed.${domain}" = {
+        locations."/".proxyPass = "http://localhost:8989";
+      };
+      "speed.${pubDomain}" = {
+        locations."/".proxyPass = "http://localhost:8989";
+        useACMEHost = pubDomain;
+      };
+
+      "md.${pubDomain}" = {
+        locations."/" = {
+          proxyPass = "http://object-ctr.${domain}:3000";
+          proxyWebsockets = true;
+          extraConfig = proxyHeaders;
+        };
+        useACMEHost = pubDomain;
+      };
+      "pb.${pubDomain}" = {
+        locations."/".proxyPass = "http://object-ctr.${domain}:8088";
+        useACMEHost = pubDomain;
+      };
+      "photos.${pubDomain}" = {
+        locations."/" = {
+          proxyPass = "http://jackflix-ctr.${domain}:2342";
+          proxyWebsockets = true;
+          extraConfig = proxyHeaders;
+        };
+        useACMEHost = pubDomain;
+      };
+
+      "pront.${pubDomain}" = mkMerge [
+        {
+          locations."/" = mkMerge [
+            {
+              proxyPass = "http://stream-hi.${home.domain}:5000";
+              proxyWebsockets = true;
+              extraConfig = proxyHeaders;
+            }
+            (ssoLoc "generic")
+          ];
+          locations."~* ^/webcam/(.*)" = mkMerge [
+            {
+              proxyPass = "http://stream-hi.${home.domain}:5050/$1$is_args$args";
+              extraConfig = proxyHeaders;
+            }
+            (ssoLoc "generic")
+          ];
+          useACMEHost = pubDomain;
+        }
+        (ssoServer "generic")
+      ];
     };
 
     minio =
@@ -410,10 +424,13 @@ in
         ignore_invalid_headers off;
       '';
 
-      nixCacheableRegex = ''^\/(\S+\.narinfo|nar\/\S+\.nar\.\S+)$'';
+      nixCacheableRegex = ''^\/(\S+\.narinfo|nar\/\S+\.nar.*|serve\/.+)$'';
       nixCacheHeaders = ''
         add_header Cache-Control $nix_cache_control;
         add_header Expires $nix_expires;
+
+        brotli on;
+        brotli_types application/x-nix-archive;
       '';
     in
     {
@@ -434,15 +451,32 @@ in
       "s3.${pubDomain}" = {
         serverAliases = [ "*.s3.${pubDomain}" ];
         inherit extraConfig;
-        locations."/".proxyPass = s3Upstream;
+        locations = {
+          "/".proxyPass = s3Upstream;
+          "/gitea/packages/" = {
+            proxyPass = s3Upstream;
+            # HACK: Docker images need the MIME type to be correct for the manifest but Gitea
+            # doesn't tell S3... By hiding the header we can use add_header to set Content-Type
+            # (normally can't be set directly)
+            extraConfig = ''
+              proxy_hide_header Content-Type;
+              add_header Content-Type $upstream_http_content_type always;
+              if ($args ~ "response-content-disposition=.+filename%3D%22manifest\.json%22") {
+                add_header Content-Type "application/vnd.docker.distribution.manifest.v2+json";
+              }
+            '';
+          };
+        };
         useACMEHost = pubDomain;
       };
 
       "nix-cache.${pubDomain}" = {
         locations = {
-          "/".proxyPass = "http://${host}:8069";
+          "/" = {
+            proxyPass = "http://${host}:5000";
+          };
           "~ ${nixCacheableRegex}" = {
-            proxyPass = "http://${host}:8069";
+            proxyPass = "http://${host}:5000";
             extraConfig = nixCacheHeaders;
           };
         };

nixos/boxes/colony/vms/shill/containers/object.nix

@@ -31,6 +31,13 @@ in
     {
       config = mkMerge [
         {
+          fileSystems = {
+            "/var/lib/harmonia" = {
+              device = "/mnt/nix-cache";
+              options = [ "bind" ];
+            };
+          };
+
           my = {
             deploy.enable = false;
             server.enable = true;
@@ -48,11 +55,21 @@ in
                   group = config.my.user.config.group;
                 };
                 "object/atticd.env" = {};
+                "nix-cache.key" = {};
+                "object/hedgedoc.env" = {};
+                "object/wastebin.env" = {};
               };
             };
 
             firewall = {
-              tcp.allowed = [ 9000 9001 config.services.sharry.config.bind.port 8069 ];
+              tcp.allowed = [
+                9000 9001
+                config.services.sharry.config.bind.port
+                8069
+                5000
+                config.services.hedgedoc.settings.port
+                8088
+              ];
             };
 
             user.homeConfig = {
@@ -60,14 +77,26 @@ in
             };
           };
 
-          users = with lib.my.c.ids; let inherit (config.services.atticd) user group; in {
+          users = with lib.my.c.ids; mkMerge [
-            users."${user}" = {
+            (let inherit (config.services.atticd) user group; in {
-              isSystemUser = true;
+              users."${user}" = {
-              uid = uids.atticd;
+                isSystemUser = true;
-              group = group;
+                uid = uids.atticd;
-            };
+                group = group;
-            groups."${user}".gid = gids.atticd;
+              };
-          };
+              groups."${user}".gid = gids.atticd;
+            })
+            {
+              users = {
+                harmonia = {
+                  shell = pkgs.bashInteractive;
+                  openssh.authorizedKeys.keyFiles = [
+                    lib.my.c.sshKeyFiles.harmonia
+                  ];
+                };
+              };
+            }
+          ];
 
           systemd = {
             network.networks."80-container-host0" = networkdAssignment "host0" assignments.internal;
@@ -85,7 +114,9 @@ in
                   MINIO_BROWSER_REDIRECT_URL = "https://minio.nul.ie";
                 };
               };
+
               sharry = awaitPostgres;
+
               atticd = mkMerge [
                 awaitPostgres
                 {
@@ -96,6 +127,15 @@ in
                   };
                 }
               ];
+              harmonia = {
+                environment.NIX_REMOTE = "/var/lib/harmonia";
+                preStart = ''
+                  ${config.nix.package}/bin/nix store ping
+                '';
+                serviceConfig = {
+                  StateDirectory = "harmonia";
+                };
+              };
             };
           };
 
@@ -175,8 +215,8 @@ in
             };
 
             atticd = {
-              enable = true;
+              enable = false;
-              credentialsFile = config.age.secrets."object/atticd.env".path;
+              environmentFile = config.age.secrets."object/atticd.env".path;
               settings = {
                 listen = "[::]:8069";
                 allowed-hosts = [ "nix-cache.${pubDomain}" ];
@@ -194,6 +234,43 @@ in
                 };
               };
             };
+
+            harmonia = {
+              enable = true;
+              signKeyPaths = [ config.age.secrets."nix-cache.key".path ];
+              settings = {
+                priority = 30;
+              };
+            };
+
+            hedgedoc = {
+              enable = true;
+              environmentFile = config.age.secrets."object/hedgedoc.env".path;
+              settings = {
+                domain = "md.${pubDomain}";
+                protocolUseSSL = true;
+                db = {
+                  dialect = "postgresql";
+                  username = "hedgedoc";
+                  database = "hedgedoc";
+                  host = "colony-psql";
+                };
+                host = "::";
+                allowAnonymous = false;
+                allowAnonymousEdits = true;
+                email = true;
+                allowEmailRegister = false;
+              };
+            };
+
+            wastebin = {
+              enable = true;
+              settings = {
+                WASTEBIN_MAX_BODY_SIZE = 67108864; # 16 MiB
+                WASTEBIN_PASSWORD_SALT = "TeGhaemeer0Siez3";
+              };
+              secretFile = config.age.secrets."object/wastebin.env".path;
+            };
           };
         }
         (mkIf config.my.build.isDevVM {

nixos/boxes/colony/vms/shill/containers/qclk/default.nix

@@ -0,0 +1,115 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.colony) domain prefixes qclk;
+in
+{
+  nixos.systems.qclk = { config, ... }: {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    rendered = config.configuration.config.my.asContainer;
+
+    assignments = {
+      internal = {
+        name = "qclk-ctr";
+        inherit domain;
+        ipv4.address = net.cidr.host 10 prefixes.ctrs.v4;
+        ipv6 = {
+          iid = "::a";
+          address = net.cidr.host 10 prefixes.ctrs.v6;
+        };
+      };
+      qclk = {
+        ipv4 = {
+          address = net.cidr.host 1 prefixes.qclk.v4;
+          gateway = null;
+        };
+      };
+    };
+
+    configuration = { lib, pkgs, config, assignments, ... }:
+    let
+      inherit (lib) concatStringsSep mkMerge mkIf mkForce;
+      inherit (lib.my) networkdAssignment;
+
+      apiPort = 8080;
+
+      instances = [
+        {
+          host = 2;
+          wgKey = "D7z1FhcdxpnrGCE0wBW5PZb5BKuhCu6tcZ/5ZaYxdwQ=";
+        }
+      ];
+      ipFor = i: net.cidr.host i.host prefixes.qclk.v4;
+    in
+    {
+      config = {
+        environment = {
+          systemPackages = with pkgs; [
+            wireguard-tools
+          ];
+        };
+
+        my = {
+          deploy.enable = false;
+          server.enable = true;
+
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC1kcfvahYmSk8IJKaUIcGkhxf/8Yse2XnU7Qqgcglyq";
+            files = {
+              "qclk/wg.key" = {
+                group = "systemd-network";
+                mode = "440";
+              };
+            };
+          };
+
+          firewall = {
+            udp.allowed = [ qclk.wgPort ];
+            extraRules = ''
+              table inet filter {
+                chain input {
+                  iifname management tcp dport ${toString apiPort} accept
+                }
+                chain forward {
+                  iifname host0 oifname management ip saddr { ${concatStringsSep ", " lib.my.c.as211024.trusted.v4} } accept
+                }
+              }
+              table inet nat {
+                chain postrouting {
+                  iifname host0 oifname management snat ip to ${assignments.qclk.ipv4.address}
+                }
+              }
+            '';
+          };
+        };
+
+        systemd = {
+          network = {
+            netdevs."30-management" = {
+              netdevConfig = {
+                Name = "management";
+                Kind = "wireguard";
+              };
+              wireguardConfig = {
+                PrivateKeyFile = config.age.secrets."qclk/wg.key".path;
+                ListenPort = qclk.wgPort;
+              };
+              wireguardPeers = map (i: {
+                PublicKey = i.wgKey;
+                AllowedIPs = [ (ipFor i) ];
+              }) instances;
+            };
+            networks = {
+              "30-container-host0" = networkdAssignment "host0" assignments.internal;
+
+              "30-management" = networkdAssignment "management" assignments.qclk;
+            };
+          };
+        };
+
+        services = { };
+      };
+    };
+  };
+}

nixos/boxes/colony/vms/shill/containers/toot.nix

@@ -1,5 +1,6 @@
 { lib, ... }:
 let
+  inherit (lib) mkForce;
   inherit (lib.my) net;
   inherit (lib.my.c.colony) domain prefixes;
 in
@@ -54,8 +55,7 @@ in
               tcp.allowed = [
                 19999
 
-                config.services.mastodon.webPort
+                "http"
-                config.services.mastodon.streamingPort
               ];
             };
           };
@@ -78,10 +78,13 @@ in
           services = {
             netdata.enable = true;
             mastodon = mkMerge [
-              {
+              rec {
                 enable = true;
-                localDomain = "nul.ie";
+                localDomain = extraConfig.WEB_DOMAIN; # for nginx config
-                extraConfig.WEB_DOMAIN = "toot.nul.ie";
+                extraConfig = {
+                  LOCAL_DOMAIN = "nul.ie";
+                  WEB_DOMAIN = "toot.nul.ie";
+                };
 
                 secretKeyBaseFile = config.age.secrets."toot/secret-key.txt".path;
                 otpSecretFile = config.age.secrets."toot/otp-secret.txt".path;
@@ -90,9 +93,8 @@ in
                   "vapid-pubkey.txt"
                   "BAyRyD2pnLQtMHr3J5AzjNMll_HDC6ra1ilOLAUmKyhkEdbm7_OwKZUgw1UefY4CHEcv4OOX9TnnN2DOYYuPZu8=");
 
-                enableUnixSocket = false;
+                streamingProcesses = 4;
-                configureNginx = false;
+                configureNginx = true;
-                trustedProxy = allAssignments.middleman.internal.ipv6.address;
 
                 database = {
                   createLocally = false;
@@ -134,13 +136,31 @@ in
                 };
               }
             ];
+
+            # Override some stuff since we are proxying upstream
+            nginx = {
+              recommendedProxySettings = mkForce false;
+              virtualHosts."${config.services.mastodon.localDomain}" =
+              let
+                extraConfig = ''
+                  proxy_set_header Host $host;
+                '';
+              in
+              {
+                forceSSL = false;
+                enableACME = false;
+                locations = {
+                  "@proxy" = { inherit extraConfig; };
+                  "/api/v1/streaming/" = { inherit extraConfig; };
+                };
+              };
+            };
           };
         }
         (mkIf config.my.build.isDevVM {
           virtualisation = {
             forwardPorts = with config.services.mastodon; [
               { from = "host"; guest.port = webPort; }
-              { from = "host"; guest.port = streamingPort; }
             ];
           };
         })

nixos/boxes/colony/vms/shill/containers/vaultwarden.nix

@@ -83,7 +83,7 @@ in
                 DOMAIN = "https://pass.${lib.my.c.pubDomain}";
 
                 ROCKET_ADDRESS = "::";
-                ROCKET_PORT = 80;
+                ROCKET_PORT = 8080;
 
                 SMTP_HOST = "mail.nul.ie";
                 SMTP_FROM = "pass@nul.ie";
@@ -99,6 +99,8 @@ in
             };
 
             borgbackup.jobs.vaultwarden = {
+              readWritePaths = [ "/var/lib/borgbackup" "/var/cache/borgbackup" ];
+
               paths = [ vwData ];
               repo = "zh2855@zh2855.rsync.net:borg/vaultwarden2";
               doInit = true;

nixos/boxes/colony/vms/shill/containers/waffletail.nix

@@ -0,0 +1,100 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.colony) domain prefixes;
+in
+{
+  nixos.systems.waffletail = { config, ... }: {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    rendered = config.configuration.config.my.asContainer;
+
+    assignments = {
+      internal = {
+        name = "waffletail-ctr";
+        inherit domain;
+        ipv4.address = net.cidr.host 9 prefixes.ctrs.v4;
+        ipv6 = {
+          iid = "::9";
+          address = net.cidr.host 9 prefixes.ctrs.v6;
+        };
+      };
+      tailscale = with lib.my.c.tailscale; {
+        ipv4 = {
+          address = net.cidr.host 5 prefix.v4;
+          mask = 32;
+          gateway = null;
+        };
+        ipv6 = {
+          address = net.cidr.host 5 prefix.v6;
+          mask = 128;
+        };
+      };
+    };
+
+    configuration = { lib, config, assignments, ... }:
+    let
+      inherit (lib) concatStringsSep mkMerge mkIf mkForce;
+      inherit (lib.my) networkdAssignment;
+    in
+    {
+      config = {
+        my = {
+          deploy.enable = false;
+          server.enable = true;
+
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICZc88lcSQ9zzQzDITdE/T5ty++TxFQUAED7p9YfFBiR";
+            files = {
+              "tailscale-auth.key" = {};
+            };
+          };
+
+          firewall = {
+            trustedInterfaces = [ "tailscale0" ];
+            extraRules = ''
+              table inet filter {
+                chain forward {
+                  iifname host0 oifname tailscale0 ip saddr ${prefixes.all.v4} accept
+                  iifname host0 oifname tailscale0 ip6 saddr ${prefixes.all.v6} accept
+                }
+              }
+              table inet nat {
+                chain postrouting {
+                  iifname tailscale0 ip daddr != ${prefixes.all.v4} snat to ${assignments.internal.ipv4.address}
+                  iifname tailscale0 ip6 daddr != ${prefixes.all.v6} snat ip6 to ${assignments.internal.ipv6.address}
+                }
+              }
+            '';
+          };
+        };
+
+        systemd = {
+          network.networks."80-container-host0" = networkdAssignment "host0" assignments.internal;
+        };
+
+        services = {
+          tailscale =
+          let
+            advRoutes = concatStringsSep "," (with prefixes.all; [ v4 v6 ]);
+          in
+          {
+            enable = true;
+            authKeyFile = config.age.secrets."tailscale-auth.key".path;
+            port = 41641;
+            openFirewall = true;
+            interfaceName = "tailscale0";
+            extraUpFlags = [
+              "--operator=${config.my.user.config.name}"
+              "--login-server=https://hs.nul.ie"
+              "--netfilter-mode=off"
+              "--advertise-exit-node"
+              "--advertise-routes=${advRoutes}"
+              "--accept-routes=false"
+            ];
+          };
+        };
+      };
+    };
+  };
+}

nixos/boxes/colony/vms/shill/default.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 let
-  inherit (lib.my) net;
+  inherit (lib.my) net nft;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my.c.colony) domain prefixes firewallForwards;
 in
 {
   imports = [ ./containers ];
@@ -49,7 +49,11 @@ in
         inherit (lib.my) networkdAssignment;
       in
       {
-        imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./gitea.nix ];
+        imports = [
+          "${modulesPath}/profiles/qemu-guest.nix"
+
+          ./containers-ext.nix
+        ];
 
         config = mkMerge [
           {
@@ -90,8 +94,8 @@ in
                 device = "/dev/disk/by-label/minio";
                 fsType = "xfs";
               };
-              "/mnt/atticd" = {
+              "/mnt/nix-cache" = {
-                device = "/dev/disk/by-label/atticd";
+                device = "/dev/disk/by-label/nix-cache";
                 fsType = "ext4";
               };
             };
@@ -136,7 +140,22 @@ in
                     };
                     ipv6Prefixes = [
                       {
-                        ipv6PrefixConfig.Prefix = prefixes.ctrs.v6;
+                        Prefix = prefixes.ctrs.v6;
+                      }
+                    ];
+                    routes = [
+                      {
+                        Destination = lib.my.c.tailscale.prefix.v4;
+                        Gateway = allAssignments.waffletail.internal.ipv4.address;
+                      }
+                      {
+                        Destination = lib.my.c.tailscale.prefix.v6;
+                        Gateway = allAssignments.waffletail.internal.ipv6.address;
+                      }
+
+                      {
+                        Destination = prefixes.qclk.v4;
+                        Gateway = allAssignments.qclk.internal.ipv4.address;
                       }
                     ];
                   }
@@ -151,6 +170,7 @@ in
               firewall = {
                 tcp.allowed = [ 19999 ];
                 trustedInterfaces = [ "ctrs" ];
+                nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
                 extraRules = ''
                   table inet filter {
                     chain forward {
@@ -158,6 +178,17 @@ in
                       iifname vms oifname ctrs accept
                     }
                   }
+                  table inet nat {
+                    # Hack to fix our NAT situation with internal routing
+                    # We need to snat to our public IP, otherwise on the return path from e.g. middleman it will
+                    # try to forward packet directly with its own IP, bypassing our carefully crafted DNAT...
+                    chain ${nft.dnatChain allAssignments.estuary.internal.ipv4.address} {
+                      ct mark set 0x1337
+                    }
+                    chain postrouting {
+                      ct mark 0x1337 snat ip to ${assignments.internal.ipv4.address}
+                    }
+                  }
                 '';
               };
 
@@ -180,10 +211,12 @@ in
                   object = {
                     bindMounts = {
                       "/mnt/minio".readOnly = false;
-                      "/mnt/atticd".readOnly = false;
+                      "/mnt/nix-cache".readOnly = false;
                     };
                   };
                   toot = {};
+                  waffletail = {};
+                  qclk = {};
                 };
               in
               mkMerge [

nixos/boxes/colony/vms/whale2/default.nix

@@ -50,6 +50,9 @@ in
       };
     }) {
       valheim-oci = 2;
+      simpcraft-oci = 3;
+      simpcraft-staging-oci = 4;
+      enshrouded-oci = 5;
     };
 
     configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
@@ -63,7 +66,8 @@ in
           "${modulesPath}/profiles/qemu-guest.nix"
 
           ./valheim.nix
-          ./gitea-actions.nix
+          ./minecraft
+          # ./enshrouded.nix
         ];
 
         config = mkMerge [
@@ -106,45 +110,30 @@ in
               oci-containers = {
                 backend = "podman";
               };
-              # NixOS has switched to using netavark, which is native to podman. It's currently missing an option to
+              containers.containersConf.settings.network = {
-              # disable iptables rules generation, which is very annoying.
+                network_backend = "netavark";
-              containers.containersConf.settings.network.network_backend = mkForce "cni";
+                firewall_driver = "none";
+              };
             };
 
             environment = {
               etc = {
-                "cni/net.d/90-colony.conflist".text = toJSON {
+                "containers/networks/colony.json".text = toJSON {
-                  cniVersion = "0.4.0";
                   name = "colony";
-                  plugins = [
+                  id = "0000000000000000000000000000000000000000000000000000000000000001";
+                  driver = "bridge";
+                  network_interface = "oci";
+                  ipv6_enabled = true;
+                  internal = false;
+                  dns_enabled = false;
+                  subnets = [
                     {
-                      type = "bridge";
+                      subnet = prefixes.oci.v4;
-                      bridge = "oci";
+                      gateway = net.cidr.host 1 prefixes.oci.v4;
-                      isGateway = true;
+                    }
-                      ipMasq = false;
+                    {
-                      hairpinMode = true;
+                      subnet = prefixes.oci.v6;
-                      ipam = {
+                      gateway = net.cidr.host 1 prefixes.oci.v6;
-                        type = "host-local";
-                        routes = [
-                          { dst = "0.0.0.0/0"; }
-                          { dst = "::/0"; }
-                        ];
-                        ranges = [
-                          [
-                            {
-                              subnet = prefixes.oci.v4;
-                              gateway = net.cidr.host 1 prefixes.oci.v4;
-                            }
-                          ]
-                          [
-                            {
-                              subnet = prefixes.oci.v6;
-                              gateway = net.cidr.host 1 prefixes.oci.v6;
-                            }
-                          ]
-                        ];
-                      };
-                      capabilities.ips = true;
                     }
                   ];
                 };

nixos/boxes/colony/vms/whale2/enshrouded.nix

@@ -0,0 +1,35 @@
+{ lib, config, allAssignments, ... }:
+let
+  inherit (lib) concatStringsSep;
+  inherit (lib.my) dockerNetAssignment;
+in
+{
+  config = {
+    virtualisation.oci-containers.containers = {
+      enshrouded = {
+        image = "sknnr/enshrouded-dedicated-server@sha256:f163e8ba9caa2115d8a0a7b16c3696968242fb6fba82706d9a77a882df083497";
+
+        environment = {
+          SERVER_NAME = "UWUshrouded";
+          # SERVER_IP = "::"; # no IPv6?? :(
+          TZ = "Europe/Dublin";
+        };
+        environmentFiles = [ config.age.secrets."whale2/enshrouded.env".path ];
+
+        volumes = [
+          "enshrouded:/home/steam/enshrouded/savegame"
+        ];
+
+        extraOptions = [
+          ''--network=colony:${dockerNetAssignment allAssignments "enshrouded-oci"}''
+        ];
+      };
+    };
+
+    my = {
+      secrets.files = {
+        "whale2/enshrouded.env" = {};
+      };
+    };
+  };
+}

nixos/boxes/colony/vms/whale2/minecraft/default.nix

@@ -0,0 +1,146 @@
+{ lib, pkgs, config, allAssignments, ... }:
+let
+  inherit (lib) concatStringsSep;
+  inherit (lib.my) dockerNetAssignment;
+
+  # devplayer0
+  op = "6d7d971b-ce10-435b-85c5-c99c0d8d288c";
+  whitelist = concatStringsSep "," [
+    op
+    "dcd2ecb9-2b5e-49cb-9d4f-f5a76162df56" # Elderlypug
+    "fcb26db2-c3ce-41aa-b588-efec79d37a8a" # Jesthral_
+    "1d366062-12c0-4e29-aba7-6ab5d8c6bb05" # shr3kas0ras
+    "703b378a-09f9-4c1d-9876-1c9305728c49" # OROURKEIRE
+    "f105bbe6-eda6-4a13-a8cf-894e77cab77b" # Adzerq
+    "1fc94979-41fb-497a-81e9-34ae24ca537a" # johnnyscrims
+    "d53c91df-b6e6-4463-b106-e8427d7a8d01" # BossLonus
+    "f439f64d-91c9-4c74-9ce5-df4d24cd8e05" # hynge_
+    "d6ec4c91-5da2-44eb-b89d-71dc8fe017a0" # Eefah98
+    "096a7348-fabe-4b2d-93fc-fd1fd5608fb0" # ToTheMoonStar
+  ];
+
+  fastback = {
+    gitConfig = pkgs.writeText "git-config" ''
+      [user]
+      	email = "simpcraft@nul.ie"
+      	name = "Simpcraft bot"
+    '';
+  };
+in
+{
+  config = {
+    virtualisation.oci-containers.containers = {
+      simpcraft = {
+        image = "git.nul.ie/dev/craftblock:2024.1.0-java17-alpine";
+
+        environment = {
+          TYPE = "MODRINTH";
+
+          EULA = "true";
+          ENABLE_QUERY = "true";
+          ENABLE_RCON = "true";
+          MOTD = "§4§k----- §9S§ai§bm§cp§dc§er§fa§6f§5t §4§k-----";
+          ICON = "/ext/icon.png";
+
+          EXISTING_WHITELIST_FILE = "SYNCHRONIZE";
+          WHITELIST = whitelist;
+          EXISTING_OPS_FILE = "SYNCHRONIZE";
+          OPS = op;
+          DIFFICULTY = "normal";
+          SPAWN_PROTECTION = "0";
+          VIEW_DISTANCE = "20";
+
+          MAX_MEMORY = "8G";
+          MODRINTH_MODPACK = "https://cdn.modrinth.com/data/CIYf3Hk8/versions/NGutsQSd/Simpcraft-0.2.1.mrpack";
+
+          TZ = "Europe/Dublin";
+        };
+        environmentFiles = [ config.age.secrets."whale2/simpcraft.env".path ];
+
+        volumes = [
+          "minecraft_data:/data"
+          "${./icon.png}:/ext/icon.png:ro"
+          "${fastback.gitConfig}:/data/.config/git/config:ro"
+        ];
+
+        extraOptions = [
+          ''--network=colony:${dockerNetAssignment allAssignments "simpcraft-oci"}''
+        ];
+      };
+
+      # simpcraft-staging = {
+      #   image = "git.nul.ie/dev/craftblock:2024.1.0-java17-alpine";
+
+      #   environment = {
+      #     TYPE = "MODRINTH";
+
+      #     EULA = "true";
+      #     ENABLE_QUERY = "true";
+      #     ENABLE_RCON = "true";
+      #     MOTD = "§4§k----- §9S§ai§bm§cp§dc§er§fa§6f§5t [staging] §4§k-----";
+      #     ICON = "/ext/icon.png";
+
+      #     EXISTING_WHITELIST_FILE = "SYNCHRONIZE";
+      #     WHITELIST = whitelist;
+      #     EXISTING_OPS_FILE = "SYNCHRONIZE";
+      #     OPS = op;
+      #     DIFFICULTY = "normal";
+      #     SPAWN_PROTECTION = "0";
+      #     VIEW_DISTANCE = "20";
+
+      #     MAX_MEMORY = "4G";
+      #     MODRINTH_MODPACK = "https://cdn.modrinth.com/data/CIYf3Hk8/versions/Ym3sIi6H/Simpcraft-0.2.0.mrpack";
+
+      #     TZ = "Europe/Dublin";
+      #   };
+      #   environmentFiles = [ config.age.secrets."whale2/simpcraft.env".path ];
+
+      #   volumes = [
+      #     "minecraft_staging_data:/data"
+      #     "${./icon.png}:/ext/icon.png:ro"
+      #   ];
+
+      #   extraOptions = [
+      #     ''--network=colony:${dockerNetAssignment allAssignments "simpcraft-staging-oci"}''
+      #   ];
+      # };
+    };
+
+    services = {
+      borgbackup.jobs.simpcraft =
+      let
+        rconCommand = cmd: ''${pkgs.mcrcon}/bin/mcrcon -H simpcraft-oci -p "$RCON_PASSWORD" "${cmd}"'';
+      in
+      {
+        paths = [ "/var/lib/containers/storage/volumes/minecraft_data/_data/world" ];
+        repo = "/var/lib/containers/backup/simpcraft";
+        doInit = true;
+        encryption.mode = "none";
+        compression = "zstd,10";
+        # every ~15 minutes offset from 5 minute intervals (Minecraft seems to save at precise times?)
+        startAt = "*:03,17,33,47";
+        prune.keep = {
+          within = "12H";
+          hourly = 48;
+        };
+        readWritePaths = [ "/var/lib/borgbackup" "/var/cache/borgbackup" ];
+
+        # Avoid Minecraft poking the files while we back up
+        preHook = rconCommand "save-off";
+        postHook = rconCommand "save-on";
+      };
+    };
+
+    systemd = {
+      services = {
+        borgbackup-job-simpcraft.serviceConfig.EnvironmentFile = [ config.age.secrets."whale2/simpcraft.env".path ];
+      };
+    };
+
+    my = {
+      secrets.files = {
+        "whale2/simpcraft.env" = {};
+      };
+    };
+  };
+}

nixos/boxes/home/castle/default.nix

@@ -1,12 +1,34 @@
-{ lib, ... }: {
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c) networkd;
+  inherit (lib.my.c.home) domain vlans prefixes vips roceBootModules;
+in
+{
   nixos.systems.castle = {
     system = "x86_64-linux";
     nixpkgs = "mine";
     home-manager = "mine";
 
+    assignments = {
+      hi = {
+        inherit domain;
+        ipv4 = {
+          address = net.cidr.host 40 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::3:1";
+          address = net.cidr.host (65536*3+1) prefixes.hi.v6;
+        };
+      };
+    };
+
     configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
       let
         inherit (lib) mkIf mkMerge mkForce;
+        inherit (lib.my) mkVLAN networkdAssignment;
       in
       {
         hardware = {
@@ -14,7 +36,7 @@
           cpu = {
             amd.updateMicrocode = true;
           };
-          opengl.extraPackages = with pkgs; [
+          graphics.extraPackages = with pkgs; [
             intel-media-driver
           ];
           bluetooth.enable = true;
@@ -25,8 +47,8 @@
             efi.canTouchEfiVariables = false;
             timeout = 10;
           };
-          kernelPackages = pkgs.linuxKernel.packages.linux_6_5;
+          kernelPackages = lib.my.c.kernel.latest pkgs;
-          kernelModules = [ "kvm-amd" ];
+          kernelModules = [ "kvm-amd" "dm-snapshot" ];
           kernelParams = [ "amd_iommu=on" "amd_pstate=passive" ];
           kernelPatches = [
             # {
@@ -36,27 +58,40 @@
             # }
           ];
           initrd = {
-            availableKernelModules = [ "thunderbolt" "xhci_pci" "nvme" "ahci" "usbhid" "usb_storage" "sd_mod" ];
+            availableKernelModules = [
+              "thunderbolt" "xhci_pci" "nvme" "ahci" "usbhid" "usb_storage" "sd_mod"
+              "8021q"
+            ] ++ roceBootModules;
+            systemd.network = {
+              netdevs = mkVLAN "lan-hi" vlans.hi;
+              networks = {
+                "10-et100g" = {
+                  matchConfig.Name = "et100g";
+                  vlan = [ "lan-hi" ];
+                  linkConfig.RequiredForOnline = "no";
+                  networkConfig = networkd.noL3;
+                };
+                "20-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+              };
+            };
           };
+
+          binfmt.emulatedSystems = [ "aarch64-linux" "armv7l-linux" ];
         };
 
         fileSystems = {
-          "/boot" = {
-            device = "/dev/disk/by-partuuid/8ce4248a-3ee4-f44f-801f-064a628b4d6e";
-            fsType = "vfat";
-          };
           "/nix" = {
-            device = "/dev/disk/by-partuuid/2da23a1d-2daf-d943-b91e-fc175f3dad07";
+            device = "/dev/nvmeof/nix";
             fsType = "ext4";
           };
-
           "/persist" = {
-            device = "/dev/disk/by-partuuid/f4c80d4f-a022-e941-b5d1-fe2e65e444b9";
+            device = "/dev/nvmeof/persist";
             fsType = "ext4";
             neededForBoot = true;
           };
+
           "/home" = {
-            device = "/dev/disk/by-partuuid/992a93cf-6c9c-324b-b0ce-f8eb2d1ce10d";
+            device = "/dev/nvmeof/home";
             fsType = "ext4";
           };
         };
@@ -80,11 +115,26 @@
             dnssec = "false";
           };
 
+          pipewire.extraConfig.pipewire = {
+            "10-buffer"."context.properties" = {
+              "default.clock.quantum" = 128;
+              "default.clock.max-quantum" = 128;
+            };
+          };
           blueman.enable = true;
         };
 
+        programs = {
+          virt-manager.enable = true;
+          wireshark = {
+            enable = true;
+            package = pkgs.wireshark-qt;
+          };
+        };
+        virtualisation.libvirtd.enable = true;
+
         networking = {
-          domain = "h.${lib.my.c.pubDomain}";
+          inherit domain;
           firewall.enable = false;
         };
 
@@ -101,22 +151,20 @@
           qperf
           ethtool
         ];
-        environment.etc = {
-          "pipewire/pipewire.conf.d/sample-size.conf".text = ''
-            context.properties = {
-              default.clock.quantum = 128
-              default.clock.max-quantum = 128
-            }
-          '';
-        };
 
         nix = {
           gc.automatic = false;
+          settings = {
+            experimental-features = [ "recursive-nix" ];
+            system-features = [ "nixos-test" "benchmark" "big-parallel" "kvm" "recursive-nix" ];
+          };
         };
 
         systemd = {
           network = {
-            wait-online.enable = false;
+            netdevs = mkMerge [
+              (mkVLAN "lan-hi" vlans.hi)
+            ];
             links = {
               "10-et2.5g" = {
                 matchConfig.MACAddress = "c8:7f:54:6e:17:0f";
@@ -127,15 +175,24 @@
                 linkConfig.Name = "et10g";
               };
               "12-et100g" = {
-                matchConfig.MACAddress = "24:8a:07:a8:fe:3a";
+                matchConfig.PermanentMACAddress = "24:8a:07:a8:fe:3a";
-                linkConfig.Name = "et100g";
+                linkConfig = {
+                  Name = "et100g";
+                  MTUBytes = toString lib.my.c.home.hiMTU;
+                };
               };
             };
             networks = {
-              "50-lan" = {
+              "30-et100g" = {
-                matchConfig.Name = "et2.5g";
+                matchConfig.Name = "et100g";
-                DHCP = "yes";
+                vlan = [ "lan-hi" ];
+                networkConfig.IPv6AcceptRA = false;
               };
+              "40-lan-hi" = mkMerge [
+                (networkdAssignment "lan-hi" assignments.hi)
+                # So we don't drop the IP we use to connect to NVMe-oF!
+                { networkConfig.KeepConfiguration = "static"; }
+              ];
             };
           };
         };
@@ -154,10 +211,7 @@
                 packages = with pkgs; [
                   jacktrip
                   qpwgraph
-                  # TODO: seems to be borked (infinite recursion???)
+                  boardie
-                  # (writeShellScriptBin "boardie" ''
-                  #   exec pw-jack ${boardie}/bin/boardie "$@"
-                  # '')
                 ];
               };
 
@@ -171,6 +225,7 @@
                     HDMI-A-1 = {
                       transform = "270";
                       position = "0 0";
+                      bg = "${./his-team-player.jpg} fill";
                     };
                     DP-1 = {
                       mode = "2560x1440@170Hz";
@@ -192,10 +247,19 @@
           };
 
           #deploy.generate.system.mode = "boot";
-          deploy.node.hostname = "castle.box.${config.networking.domain}";
           secrets = {
             key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlVuTzKObeaUuPocCF41IO/8X+443lzUJLuCIclt2vr";
           };
+          netboot.client = {
+            enable = true;
+          };
+          nvme = {
+            uuid = "2230b066-a674-4f45-a1dc-f7727b3a9e7b";
+            boot = {
+              nqn = "nqn.2016-06.io.spdk:castle";
+              address = "192.168.68.80";
+            };
+          };
 
           firewall = {
             enable = false;

nixos/boxes/home/oxbow.nix

@@ -1,56 +0,0 @@
-{
-  imports = [
-    (import ./routing-common {
-      index = 1;
-      name = "oxbow";
-    })
-  ];
-
-  config.nixos.systems.oxbow = {
-    system = "x86_64-linux";
-    nixpkgs = "mine";
-    home-manager = "mine";
-
-    configuration = { lib, pkgs, config, ... }:
-    let
-      inherit (lib);
-    in
-    {
-      config = {
-        boot = {
-          kernelParams = [ "intel_iommu=on" ];
-        };
-
-        hardware = {
-          enableRedistributableFirmware = true;
-          cpu = {
-            intel.updateMicrocode = true;
-          };
-        };
-
-        fileSystems = {
-          "/boot" = {
-            device = "/dev/disk/by-label/ESP";
-            fsType = "vfat";
-          };
-          "/nix" = {
-            device = "/dev/disk/by-label/nix";
-            fsType = "ext4";
-          };
-          "/persist" = {
-            device = "/dev/disk/by-label/persist";
-            fsType = "ext4";
-            neededForBoot = true;
-          };
-        };
-
-        my = {
-          secrets = {
-            # key = "ssh-ed25519 ";
-          };
-          server.enable = true;
-        };
-      };
-    };
-  };
-}

nixos/boxes/home/palace/default.nix

@@ -0,0 +1,227 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net mkVLAN;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.home) domain vlans prefixes vips hiMTU;
+in
+{
+  imports = [ ./vms ];
+
+  nixos.systems.palace = {
+    system = "x86_64-linux";
+    nixpkgs = "mine-stable";
+    home-manager = "mine-stable";
+
+    assignments = {
+      hi = {
+        inherit domain;
+        mtu = hiMTU;
+        ipv4 = {
+          address = net.cidr.host 22 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::2:1";
+          address = net.cidr.host (65536*2+1) prefixes.hi.v6;
+        };
+      };
+      core = {
+        inherit domain;
+        name = "palace-core";
+        mtu = 1500;
+        ipv4 = {
+          address = net.cidr.host 20 prefixes.core.v4;
+          gateway = null;
+        };
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
+      let
+        inherit (lib) mkForce mkMerge;
+        inherit (lib.my) networkdAssignment;
+        inherit (lib.my.c) networkd;
+      in
+      {
+        boot = {
+          kernelPackages = (lib.my.c.kernel.lts pkgs).extend (self: super: {
+            kernel = super.kernel.override {
+              structuredExtraConfig = with lib.kernel; {
+                ACPI_APEI_PCIEAER = yes;
+                PCIEAER = yes;
+              };
+            };
+          });
+          kernelModules = [ "kvm-amd" ];
+          kernelParams = [ "amd_iommu=on" ];
+          initrd = {
+            availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+          cpu = {
+            amd.updateMicrocode = true;
+          };
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/c06a8d24-2af9-4416-bf5e-cfe6defdbd47";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/disk/by-uuid/450e1f72-238a-4160-98b8-b5e6d0d6fdf6";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/disk/by-uuid/9d6d53a8-dff8-49e0-9bc3-fb5f7c6760d0";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          lvm = {
+            boot.thin.enable = true;
+            dmeventd.enable = true;
+          };
+          smartd = {
+            enable = true;
+            autodetect = true;
+            extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
+          };
+          udev.extraRules = ''
+            ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="4"
+          '';
+        };
+
+        environment.systemPackages = with pkgs; [
+          pciutils
+          usbutils
+          partclone
+          lm_sensors
+          linuxPackages.cpupower
+          smartmontools
+          mstflint
+          ethtool
+          hwloc
+        ];
+
+        networking = { inherit domain; };
+
+        systemd = {
+          tmpfiles.rules = [
+            "d /var/log/smartd 0755 root root"
+          ];
+
+          network = {
+            links = {
+              "10-et1g0" = {
+                matchConfig = {
+                  PermanentMACAddress = "e0:d5:5e:68:0c:6e";
+                  Driver = "igb";
+                };
+                linkConfig.Name = "et1g0";
+              };
+              "10-lan-core" = {
+                matchConfig.PermanentMACAddress = "e0:d5:5e:68:0c:70";
+                linkConfig.Name = "lan-core";
+              };
+              "10-et100g" = {
+                matchConfig = {
+                  PermanentMACAddress = "24:8a:07:ac:59:c0";
+                  Driver = "mlx5_core";
+                };
+                linkConfig = {
+                  Name = "et100g";
+                  MTUBytes = toString hiMTU;
+                };
+              };
+            };
+
+            netdevs = mkMerge [
+              (mkVLAN "lan-hi" vlans.hi)
+              (mkVLAN "lan-lo-phy" vlans.lo)
+              {
+                "25-lan-lo".netdevConfig = {
+                  Name = "lan-lo";
+                  Kind = "bridge";
+                };
+              }
+            ];
+
+            networks = {
+              "50-lan-core" = mkMerge [
+                (networkdAssignment "lan-core" assignments.core)
+                {
+                  matchConfig.Name = "lan-core";
+                  vlan = [ "lan-lo-phy" ];
+                  networkConfig.IPv6AcceptRA = mkForce false;
+                }
+              ];
+
+              "50-et100g" = {
+                matchConfig.Name = "et100g";
+                vlan = [ "lan-hi" ];
+                networkConfig = networkd.noL3;
+                linkConfig.RequiredForOnline = "no";
+                extraConfig = ''
+                  # cellar
+                  [SR-IOV]
+                  VirtualFunction=0
+                  VLANId=${toString vlans.hi}
+                  LinkState=yes
+                  MACAddress=52:54:00:cc:3e:70
+
+                  # river
+                  [SR-IOV]
+                  VirtualFunction=1
+                  LinkState=yes
+                  MACAddress=52:54:00:8a:8a:f2
+
+                  # sfh
+                  [SR-IOV]
+                  VirtualFunction=2
+                  VLANId=${toString vlans.hi}
+                  LinkState=yes
+                  MACAddress=52:54:00:ac:15:a9
+
+                  # sfh bridge
+                  [SR-IOV]
+                  VirtualFunction=3
+                  VLANId=${toString vlans.hi}
+                  LinkState=yes
+                  MACAddress=52:54:00:90:34:95
+                '';
+              };
+              "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+
+              "50-lan-lo-phy" = {
+                matchConfig.Name = "lan-lo-phy";
+                networkConfig = {
+                  Bridge = "lan-lo";
+                } // networkd.noL3;
+              };
+              "60-lan-lo" = {
+                matchConfig.Name = "lan-lo";
+                linkConfig.RequiredForOnline = "no";
+                networkConfig = networkd.noL3;
+              };
+            };
+          };
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHzVJpoDz/AAYLJGzU8t6DgZ2sY3oehRqrlSO7C+GWiK";
+          };
+
+          server.enable = true;
+          deploy.node.hostname = "192.168.68.22";
+        };
+      };
+  };
+}

nixos/boxes/home/palace/vms/cellar/default.nix

@@ -0,0 +1,99 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.home) domain prefixes vips hiMTU;
+in
+{
+  nixos.systems.cellar = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+
+    assignments = {
+      hi = {
+        inherit domain;
+        mtu = hiMTU;
+        ipv4 = {
+          address = net.cidr.host 80 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::4:1";
+          address = net.cidr.host (65536*4+1) prefixes.hi.v6;
+        };
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
+      let
+        inherit (lib) mkMerge;
+        inherit (lib.my) networkdAssignment;
+      in
+      {
+        imports = [
+          "${modulesPath}/profiles/qemu-guest.nix"
+          ./spdk.nix
+        ];
+
+        config = mkMerge [
+          {
+            boot = {
+              kernelParams = [ "console=ttyS0,115200n8" "intel_iommu=on" ];
+            };
+
+            fileSystems = {
+              "/boot" = {
+                device = "/dev/disk/by-partuuid/f7562ee6-34c1-4e94-8ae7-c6e71794d563";
+                fsType = "vfat";
+              };
+              "/nix" = {
+                device = "/dev/disk/by-uuid/f31f6abd-0832-4014-a761-f3c3126d5739";
+                fsType = "ext4";
+              };
+              "/persist" = {
+                device = "/dev/disk/by-uuid/620364e3-3a30-4704-be80-8593516e7482";
+                fsType = "ext4";
+                neededForBoot = true;
+              };
+            };
+
+            networking = { inherit domain; };
+
+            environment.systemPackages = with pkgs; [
+              pciutils
+              partclone
+            ];
+
+            services = {
+              fstrim.enable = true;
+              netdata.enable = true;
+            };
+
+            systemd.network = {
+              links = {
+                "10-lan-hi" = {
+                  matchConfig.PermanentMACAddress = "52:54:00:cc:3e:70";
+                  linkConfig.Name = "lan-hi";
+                };
+              };
+
+              networks = {
+                "80-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+              };
+            };
+
+            my = {
+              secrets.key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDcklmJp8xVRddNDU1DruKV+Ipim3Jtl6nE1oCWmpmZH";
+              server.enable = true;
+              deploy.node.hostname = "192.168.68.80";
+
+              firewall = {
+                tcp.allowed = [ 19999 ];
+              };
+            };
+          }
+        ];
+      };
+  };
+}

nixos/boxes/home/palace/vms/cellar/spdk.nix

@@ -0,0 +1,143 @@
+{ lib, pkgs, config, assignments, ... }:
+let
+  inherit (lib) mapAttrsToList;
+in
+{
+  config = {
+    boot.blacklistedKernelModules = [ "nvme" ];
+
+    systemd.services = {
+      spdk-tgt.after = [ "systemd-networkd-wait-online@lan-hi.service" ];
+    };
+
+    my = {
+      spdk = {
+        enable = true;
+        extraArgs = "--mem-channels 2 --cpumask 0xffff";
+        debugCommands = ''
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
+          spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'
+
+          spdk-rpc ublk_create_target
+          spdk-rpc ublk_start_disk NVMeRaid 1
+        '';
+        config.subsystems =
+        let
+          nvmeAttaches = mapAttrsToList (name: bdf: {
+            method = "bdev_nvme_attach_controller";
+            params = {
+              hostnqn =
+                "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775";
+              trtype = "PCIe";
+              inherit name;
+              traddr = bdf;
+            };
+          }) {
+            "NVMe0" = "02:00.0";
+            "NVMe1" = "03:00.0";
+            "NVMe2" = "04:00.0";
+          };
+
+          nvmfListener = nqn: {
+            method = "nvmf_subsystem_add_listener";
+            params = {
+              inherit nqn;
+              listen_address = {
+                adrfam = "IPv4";
+                traddr = assignments.hi.ipv4.address;
+                trsvcid = "4420";
+                trtype = "RDMA";
+              };
+              secure_channel = false;
+            };
+          };
+          nvmfBdev = { nqn, hostnqn, bdev, serial }: [
+            {
+              method = "nvmf_create_subsystem";
+              params = {
+                inherit nqn;
+                serial_number = serial;
+              };
+            }
+            (nvmfListener nqn)
+            {
+              method = "nvmf_subsystem_add_host";
+              params = {
+                inherit nqn;
+                host = hostnqn;
+              };
+            }
+            {
+              method = "nvmf_subsystem_add_ns";
+              params = {
+                inherit nqn;
+                namespace = {
+                  bdev_name = bdev;
+                  nsid = 1;
+                };
+              };
+            }
+          ];
+        in
+        {
+          scheduler = [
+            {
+              method = "framework_set_scheduler";
+              params.name = "dynamic";
+            }
+          ];
+
+          bdev = [
+            {
+              method = "bdev_set_options";
+              params.bdev_auto_examine = false;
+            }
+          ] ++ nvmeAttaches ++ [
+            {
+              method = "bdev_raid_create";
+              params = {
+                base_bdevs = [ "NVMe0n1" "NVMe1n1" "NVMe2n1" ];
+                name = "NVMeRaid";
+                raid_level = "raid0";
+                strip_size_kb = 64;
+              };
+            }
+            {
+              method = "bdev_examine";
+              params.name = "NVMeRaid";
+            }
+            { method = "bdev_wait_for_examine"; }
+          ];
+
+          nvmf = [
+            {
+              method = "nvmf_create_transport";
+              params.trtype = "RDMA";
+            }
+            (nvmfListener "nqn.2014-08.org.nvmexpress.discovery")
+          ] ++ (nvmfBdev {
+            bdev = "NVMeRaidp1";
+            nqn = "nqn.2016-06.io.spdk:river";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            serial = "SPDK00000000000001";
+          }) ++ (nvmfBdev {
+            bdev = "NVMeRaidp2";
+            nqn = "nqn.2016-06.io.spdk:castle";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
+            serial = "SPDK00000000000002";
+          }) ++ (nvmfBdev {
+            bdev = "NVMeRaidp3";
+            nqn = "nqn.2016-06.io.spdk:sfh";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:85d7df36-0de0-431b-b06e-51f7c0a455b4";
+            serial = "SPDK00000000000003";
+          });
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/palace/vms/default.nix

@@ -0,0 +1,196 @@
+{
+  imports = [
+    ./cellar
+    ./river.nix
+    ./sfh
+  ];
+
+  nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }:
+  let
+    inherit (lib) mkMerge;
+    inherit (lib.my) vm;
+    inherit (lib.my.c) networkd;
+
+    installerDisk = {
+      name = "installer";
+      backend = {
+        driver = "file";
+        filename = "/persist/home/dev/nixos-installer-devplayer0.iso";
+        read-only = "on";
+      };
+      format.driver = "raw";
+      frontend = "ide-cd";
+      frontendOpts = {
+        bootindex = 1;
+      };
+    };
+  in
+  {
+    systemd.network = {
+      netdevs = {
+        "25-vm-et1g0" = {
+           netdevConfig = {
+             Name = "vm-et1g0";
+             Kind = "macvtap";
+           };
+           # TODO: Upstream this missing section
+           extraConfig = ''
+             [MACVTAP]
+             Mode=passthru
+           '';
+        };
+      };
+      networks = {
+        "75-et1g0" = {
+          matchConfig.Name = "et1g0";
+          linkConfig.RequiredForOnline = "no";
+          networkConfig = {
+            MACVTAP = "vm-et1g0";
+          } // networkd.noL3;
+        };
+        "75-vm-et1g0" = {
+          matchConfig.Name = "vm-et1g0";
+          linkConfig.RequiredForOnline = "no";
+          networkConfig = networkd.noL3;
+        };
+      };
+    };
+
+    systemd.services =
+    let
+      awaitVM = system: {
+        after = [ "vm@${system}.service" ];
+        bindsTo = [ "vm@${system}.service" ];
+        preStart = ''
+          until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.${system}.hi.ipv4.address} 22; do
+            sleep 1
+          done
+        '';
+      };
+    in
+    {
+      "vm@cellar" = {
+        serviceConfig = {
+          CPUAffinity = "numa";
+          NUMAPolicy = "bind";
+          NUMAMask = "1";
+        };
+      };
+
+      "vm@river" =
+      let
+        vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
+      in
+      mkMerge [
+        (awaitVM "cellar")
+        {
+          requires = [ vtapUnit ];
+          after = [ vtapUnit ];
+        }
+      ];
+      "vm@sfh" = (awaitVM "river");
+    };
+
+    my = {
+      vms = {
+        instances = {
+          cellar = {
+            uuid = "b126d135-9fc1-415a-b675-aaf727bf2f38";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 8;
+              threads = 2;
+            };
+            memory = 16384;
+            cleanShutdown.timeout = 120;
+            drives = [
+              (mkMerge [ (vm.disk "cellar" "esp") { frontendOpts.bootindex = 0; } ])
+              (vm.disk "cellar" "nix")
+              (vm.disk "cellar" "persist")
+            ];
+            hostDevices = {
+              et100g0vf0 = {
+                index = 0;
+                hostBDF = "44:00.1";
+              };
+              nvme0 = {
+                index = 1;
+                hostBDF = "41:00.0";
+              };
+              nvme1 = {
+                index = 2;
+                hostBDF = "42:00.0";
+              };
+              nvme2 = {
+                index = 3;
+                hostBDF = "43:00.0";
+              };
+            };
+            qemuFlags = [
+              "machine kernel-irqchip=split"
+              "device intel-iommu,caching-mode=on,device-iotlb=on,intremap=on"
+            ];
+          };
+
+          river = {
+            uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 3;
+              threads = 2;
+            };
+            memory = 4096;
+            cleanShutdown.timeout = 60;
+            networks = {
+              et1g0 = {
+                ifname = "vm-et1g0";
+                bridge = null;
+                tapFD = 100;
+                # Real hardware MAC
+                mac = "e0:d5:5e:68:0c:6e";
+                waitOnline = false;
+              };
+            };
+            drives = [
+              installerDisk
+              (mkMerge [ (vm.disk "river" "esp") { frontendOpts.bootindex = 0; } ])
+            ];
+            hostDevices = {
+              et100g0vf1 = {
+                index = 0;
+                hostBDF = "44:00.2";
+              };
+            };
+          };
+
+          sfh = {
+            uuid = "82ec149d-577c-421a-93e2-a9307c756cd8";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 8;
+              threads = 2;
+            };
+            memory = 32768;
+            cleanShutdown.timeout = 120;
+            networks.netboot = {
+              bridge = "lan-lo";
+              waitOnline = "carrier";
+              mac = "52:54:00:a5:7e:93";
+              extraOptions.bootindex = 1;
+            };
+            hostDevices = {
+              et100g0vf2 = {
+                index = 0;
+                hostBDF = "44:00.3";
+              };
+              et100g0vf3 = {
+                index = 1;
+                hostBDF = "44:00.4";
+              };
+            };
+          };
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/palace/vms/river.nix

@@ -0,0 +1,134 @@
+{
+  imports = [ (import ../../routing-common 0) ];
+
+  config.nixos.systems.river = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    home-manager = "mine";
+
+    configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
+    let
+      inherit (lib.my) networkdAssignment mkVLAN;
+      inherit (lib.my.c) networkd;
+      inherit (lib.my.c.home) vlans domain prefixes roceBootModules;
+    in
+    {
+      imports = [
+        "${modulesPath}/profiles/qemu-guest.nix"
+      ];
+
+      config = {
+        boot = {
+          kernelModules = [ "kvm-amd" ];
+          kernelParams = [ "console=ttyS0,115200n8" ];
+          initrd = {
+            availableKernelModules = [
+              "virtio_pci" "ahci" "sr_mod" "virtio_blk"
+              "8021q"
+            ] ++ roceBootModules;
+            kernelModules = [ "dm-snapshot" ];
+            systemd = {
+              network = {
+                # Don't need to put the link config here, they're copied from main config
+                netdevs = mkVLAN "lan-hi" vlans.hi;
+                networks = {
+                  "20-lan" = {
+                    matchConfig.Name = "lan";
+                    vlan = [ "lan-hi" ];
+                    linkConfig.RequiredForOnline = "no";
+                    networkConfig = networkd.noL3;
+                  };
+                  "30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+                };
+              };
+            };
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/3ec6c49e-b485-40cb-8eff-315581ac6fe9";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/main/nix";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/main/persist";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          lvm = {
+            boot.thin.enable = true;
+            dmeventd.enable = true;
+          };
+          fstrim.enable = true;
+        };
+
+        systemd.network = {
+          links = {
+            "10-wan" = {
+              matchConfig = {
+                # Matching against MAC address seems to break VLAN interfaces
+                # (since they share the same MAC address)
+                Driver = "virtio_net";
+                PermanentMACAddress = "e0:d5:5e:68:0c:6e";
+              };
+              linkConfig = {
+                Name = "wan";
+                RxBufferSize = 4096;
+                TxBufferSize = 4096;
+              };
+            };
+
+            "10-lan" = {
+              matchConfig = {
+                Driver = "mlx5_core";
+                PermanentMACAddress = "52:54:00:8a:8a:f2";
+              };
+              linkConfig = {
+                Name = "lan";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
+          };
+
+          # So we don't drop the IP we use to connect to NVMe-oF!
+          networks."60-lan-hi".networkConfig.KeepConfiguration = "static";
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
+          };
+          server.enable = true;
+          nvme = {
+            uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            boot = {
+              nqn = "nqn.2016-06.io.spdk:river";
+              address = "192.168.68.80";
+            };
+          };
+
+          netboot.server = {
+            enable = true;
+            ip = assignments.lo.ipv4.address;
+            host = "boot.${domain}";
+            allowedPrefixes = with prefixes; [ hi.v4 hi.v6 lo.v4 lo.v6 ];
+            instances = [ "sfh" "castle" ];
+          };
+
+          deploy.node.hostname = "192.168.68.1";
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/palace/vms/sfh/containers/default.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ./unifi.nix
+  ];
+}

nixos/boxes/home/palace/vms/sfh/containers/unifi.nix

@@ -0,0 +1,65 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) domain prefixes vips hiMTU;
+in
+{
+  nixos.systems.unifi = { config, ... }: {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    rendered = config.configuration.config.my.asContainer;
+
+    assignments = {
+      hi = {
+        name = "unifi-ctr";
+        inherit domain;
+        mtu = hiMTU;
+        ipv4 = {
+          address = net.cidr.host 100 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::5:1";
+          address = net.cidr.host (65536*5+1) prefixes.hi.v6;
+        };
+      };
+    };
+
+    configuration = { lib, config, pkgs, assignments, ... }:
+    let
+      inherit (lib) mkMerge mkIf mkForce;
+      inherit (lib.my) networkdAssignment;
+    in
+    {
+      config = {
+        my = {
+          deploy.enable = false;
+          server.enable = true;
+
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdgcziQki/RH7E+NH2bYnzSVKaJ27905Yo5TcOjSh/U";
+            files = { };
+          };
+
+          firewall = {
+            tcp.allowed = [ 8443 ];
+          };
+        };
+
+        systemd = {
+          network.networks."80-container-host0" = networkdAssignment "host0" assignments.hi;
+        };
+
+        services = {
+          unifi = {
+            enable = true;
+            openFirewall = true;
+            unifiPackage = pkgs.unifi8;
+            mongodbPackage = pkgs.mongodb-6_0;
+          };
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/palace/vms/sfh/default.nix

@@ -0,0 +1,157 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) domain prefixes vips hiMTU roceBootModules;
+in
+{
+  imports = [ ./containers ];
+
+  config.nixos.systems.sfh = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    home-manager = "mine";
+
+    assignments = {
+      hi = {
+        inherit domain;
+        mtu = hiMTU;
+        ipv4 = {
+          address = net.cidr.host 81 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::4:2";
+          address = net.cidr.host (65536*4+2) prefixes.hi.v6;
+        };
+      };
+    };
+
+    configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
+    let
+      inherit (lib) mapAttrs mkMerge;
+      inherit (lib.my) networkdAssignment;
+      inherit (lib.my.c) networkd;
+      inherit (lib.my.c.home) domain;
+    in
+    {
+      imports = [
+        "${modulesPath}/profiles/qemu-guest.nix"
+      ];
+
+      config = {
+        boot = {
+          kernelModules = [ "kvm-amd" ];
+          kernelParams = [ "console=ttyS0,115200n8" ];
+          initrd = {
+            availableKernelModules = [
+              "virtio_pci" "ahci" "sr_mod" "virtio_blk"
+            ] ++ roceBootModules;
+            kernelModules = [ "dm-snapshot" ];
+            systemd = {
+              network = {
+                networks = {
+                  "20-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+                };
+              };
+            };
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+        };
+
+        fileSystems = {
+          "/nix" = {
+            device = "/dev/main/nix";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/main/persist";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        networking = { inherit domain; };
+
+        services = {
+          lvm = {
+            boot.thin.enable = true;
+            dmeventd.enable = true;
+          };
+        };
+
+        systemd.network = {
+          links = {
+            "10-lan-hi" = {
+              matchConfig = {
+                Driver = "mlx5_core";
+                PermanentMACAddress = "52:54:00:ac:15:a9";
+              };
+              linkConfig = {
+                Name = "lan-hi";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
+            "10-lan-hi-ctrs" = {
+              matchConfig = {
+                Driver = "mlx5_core";
+                PermanentMACAddress = "52:54:00:90:34:95";
+              };
+              linkConfig = {
+                Name = "lan-hi-ctrs";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
+          };
+
+          networks = {
+            "30-lan-hi" = mkMerge [
+              (networkdAssignment "lan-hi" assignments.hi)
+              # So we don't drop the IP we use to connect to NVMe-oF!
+              { networkConfig.KeepConfiguration = "static"; }
+            ];
+            "30-lan-hi-ctrs" = {
+              matchConfig.Name = "lan-hi-ctrs";
+              linkConfig.RequiredForOnline = "no";
+              networkConfig = networkd.noL3;
+            };
+          };
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAAaav5Se1E/AbqEXmADryVszYfNDscyP6jrWioN57R7";
+          };
+          server.enable = true;
+
+          netboot.client = {
+            enable = true;
+          };
+          nvme = {
+            uuid = "85d7df36-0de0-431b-b06e-51f7c0a455b4";
+            boot = {
+              nqn = "nqn.2016-06.io.spdk:sfh";
+              address = "192.168.68.80";
+            };
+          };
+
+          containers.instances =
+          let
+            instances = {
+              unifi = {};
+            };
+          in
+          mkMerge [
+            instances
+            (mapAttrs (n: i: {
+              networking.macVLAN = "lan-hi-ctrs";
+            }) instances)
+          ];
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/routing-common/default.nix

@@ -1,24 +1,35 @@
-{ index, name }: { lib, ... }:
+index: { lib, allAssignments, ... }:
 let
-  inherit (lib.my) net;
+  inherit (builtins) elemAt;
-  inherit (lib.my.c.home) domain vlans prefixes;
+  inherit (lib.my) net mkVLAN;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.home) domain vlans prefixes vips routers routersPubV4;
+
+  name = elemAt routers index;
+  otherIndex = 1 - index;
 in
 {
   nixos.systems."${name}" = {
     assignments = {
       modem = {
-        ipv4.address = net.cidr.host (254 - index) prefixes.modem.v4;
+        ipv4 = {
+          address = net.cidr.host (254 - index) prefixes.modem.v4;
+          gateway = null;
+        };
       };
       core = {
         name = "${name}-core";
         inherit domain;
+        mtu = 1500;
         ipv4 = {
           address = net.cidr.host (index + 1) prefixes.core.v4;
-          mask = 24;
+          gateway = null;
         };
       };
       hi = {
+        name = "${name}-hi";
         inherit domain;
+        mtu = 9000;
         ipv4 = {
           address = net.cidr.host (index + 1) prefixes.hi.v4;
           mask = 22;
@@ -29,6 +40,7 @@ in
       lo = {
         name = "${name}-lo";
         inherit domain;
+        mtu = 1500;
         ipv4 = {
           address = net.cidr.host (index + 1) prefixes.lo.v4;
           mask = 21;
@@ -39,6 +51,7 @@ in
       untrusted  = {
         name = "${name}-ut";
         inherit domain;
+        mtu = 1500;
         ipv4 = {
           address = net.cidr.host (index + 1) prefixes.untrusted.v4;
           mask = 24;
@@ -46,18 +59,64 @@ in
         };
         ipv6.address = net.cidr.host (index + 1) prefixes.untrusted.v6;
       };
+      as211024 = {
+        ipv4 = {
+          address = net.cidr.host (index + 2) prefixes.as211024.v4;
+          gateway = null;
+        };
+        ipv6 = {
+          address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
+          gateway = net.cidr.host ((2*65536*65536*65536) + 1) prefixes.as211024.v6;
+        };
+      };
+    };
+
+    extraAssignments = {
+      router-hi.hi = {
+        name = "router-hi";
+        inherit domain;
+        ipv4 = {
+          address = vips.hi.v4;
+          mask = 22;
+        };
+        ipv6.address = vips.hi.v6;
+      };
+      router-lo.lo = {
+        name = "router-lo";
+        inherit domain;
+        ipv4 = {
+          address = vips.lo.v4;
+          mask = 21;
+        };
+        ipv6.address = vips.lo.v6;
+      };
+      router-ut.untrusted = {
+        name = "router-ut";
+        inherit domain;
+        ipv4.address = vips.untrusted.v4;
+        ipv6.address = vips.untrusted.v6;
+      };
     };
 
     configuration = { lib, pkgs, config, assignments, allAssignments, ... }:
       let
         inherit (lib) mkIf mkMerge mkForce;
         inherit (lib.my) networkdAssignment;
+        inherit (lib.my.c) networkd;
       in
       {
+        imports = map (m: import m index) [
+          ./keepalived.nix
+          ./dns.nix
+          ./radvd.nix
+          ./kea.nix
+        ];
+
         config = {
           environment = {
             systemPackages = with pkgs; [
               ethtool
+              conntrack-tools
             ];
           };
 
@@ -73,6 +132,49 @@ in
               enable = true;
               openFirewall = true;
             };
+
+            networkd-dispatcher = {
+              enable = true;
+              rules = {
+                # tc filter hasn't been networkd-ified yet
+                setup-wan-mirror = {
+                  onState = [ "configured" ];
+                  script = ''
+                  #!${pkgs.runtimeShell}
+                  if [ $IFACE = "wan-ifb" ]; then
+                    ${pkgs.iproute2}/bin/tc filter add dev wan parent ffff: matchall action mirred egress redirect dev $IFACE
+                  fi
+                  '';
+                };
+              };
+            };
+
+            nginx.enable = true;
+          };
+
+          networking = { inherit domain; };
+
+          systemd.services =
+          let
+            waitOnline = "systemd-networkd-wait-online@wan.service";
+          in
+          {
+            ipsec = {
+              after = [ waitOnline ];
+              requires = [ waitOnline ];
+            };
+
+            ipv6-clear-default-route = {
+              description = "Clear IPv6 RA default route";
+              after = [ waitOnline ];
+              requires = [ waitOnline ];
+              script = ''
+                # Seems like we can sometimes pick up a default route somehow...
+                ${pkgs.iproute2}/bin/ip -6 route del default via fe80::1 || true
+              '';
+              serviceConfig.Type = "oneshot";
+              wantedBy = [ "multi-user.target" ];
+            };
           };
 
           systemd.network = {
@@ -83,169 +185,176 @@ in
               };
             };
 
-            netdevs =
+            netdevs = mkMerge [
-            let
-              mkVLAN = name: vid: {
-                "25-${name}" = {
-                  netdevConfig = {
-                    Name = name;
-                    Kind = "vlan";
-                  };
-                  vlanConfig.Id = vid;
-                };
-              };
-            in
-            mkMerge [
               {
-                "25-wan".netdevConfig = {
+                "25-wan-ifb".netdevConfig = {
-                  Name = "wan";
+                  Name = "wan-ifb";
-                  Kind = "bridge";
+                  Kind = "ifb";
                 };
-                "25-lan".netdevConfig = {
+                "30-lan-core".netdevConfig = {
-                  Name = "lan";
+                  Name = "lan-core";
-                  Kind = "bridge";
+                  Kind = "macvlan";
+                  MTUBytes = "1500";
                 };
               }
 
-              (mkVLAN "hi" vlans.hi)
+              (mkVLAN "lan-hi" vlans.hi)
-              (mkVLAN "lo" vlans.lo)
+              (mkVLAN "lan-lo" vlans.lo)
-              (mkVLAN "untrusted" vlans.untrusted)
+              (mkVLAN "lan-untrusted" vlans.untrusted)
-              (mkVLAN "wan-tunnel" vlans.wan)
             ];
 
-            links = {
-              "10-lan-jim" = {
-                matchConfig = {
-                  # Matching against MAC address seems to break VLAN interfaces
-                  # (since they share the same MAC address)
-                  Driver = "igb";
-                  Path = "pci-0000:01:00.0";
-                };
-                linkConfig = {
-                  Name = "lan-jim";
-                  RxBufferSize = 4096;
-                  TxBufferSize = 4096;
-                  MTUBytes = toString lib.my.c.home.hiMTU;
-                };
-              };
-            };
-
             networks =
             let
-              mkVLANConfig = name: {
+              mkVLANConfig = name:
-                "60-${name}" = mkMerge [
+              let
-                  (networkdAssignment name assignments.hi)
+                iface = "lan-${name}";
+              in
+              {
+                "60-${iface}" = mkMerge [
+                  (networkdAssignment iface assignments."${name}")
                   {
                     dns = [ "127.0.0.1" "::1" ];
                     domains = [ config.networking.domain ];
-                    networkConfig = {
+                    networkConfig.IPv6AcceptRA = mkForce false;
-                      IPv6AcceptRA = mkForce false;
-                      IPv6SendRA = true;
-                    };
-                    ipv6SendRAConfig = {
-                      DNS = [
-                        (net.cidr.host 1 prefixes."${name}".v4)
-                        (net.cidr.host 2 prefixes."${name}".v4)
-                        (net.cidr.host 1 prefixes."${name}".v6)
-                        (net.cidr.host 2 prefixes."${name}".v6)
-                      ];
-                      Domains = [ config.networking.domain ];
-                    };
-                    ipv6Prefixes = [
-                      {
-                        ipv6PrefixConfig.Prefix = prefixes."${name}".v6;
-                      }
-                    ];
                   }
                 ];
               };
             in
             mkMerge [
               {
-                "50-wan-phy" = {
+                "50-wan-ifb" = {
-                  matchConfig.Name = "wan-phy";
+                  matchConfig.Name = "wan-ifb";
-                  networkConfig.Bridge = "wan";
+                  networkConfig = networkd.noL3;
-                };
+                  extraConfig = ''
-                "50-wan-tunnel" = {
+                    [CAKE]
-                  matchConfig.Name = "wan-tunnel";
+                    Bandwidth=490M
-                  networkConfig.Bridge = "wan";
+                    RTTSec=50ms
+                    PriorityQueueingPreset=besteffort
+                    # DOCSIS preset
+                    OverheadBytes=18
+                    MPUBytes=64
+                    CompensationMode=none
+                  '';
                 };
                 "50-wan" = mkMerge [
                   (networkdAssignment "wan" assignments.modem)
                   {
                     matchConfig.Name = "wan";
                     DHCP = "ipv4";
+                    dns = [ "127.0.0.1" "::1" ];
                     dhcpV4Config.UseDNS = false;
-                    routes = map (r: { routeConfig = r; }) [
+
-                      # {
+                    qdiscConfig = {
-                      #   Destination = prefixes.ctrs.v4;
+                      Parent = "ingress";
-                      #   Gateway = allAssignments.shill.routing.ipv4.address;
+                      Handle = "0xffff";
-                      # }
+                    };
-                    ];
+                    extraConfig = ''
+                      [CAKE]
+                      Parent=root
+                      Bandwidth=48M
+                      RTTSec=50ms
+                    '';
                   }
                 ];
 
-                "50-lan-jim" = {
-                  matchConfig.Name = "lan-jim";
-                  networkConfig.Bridge = "lan";
-                };
-                "50-lan-dave" = {
-                  matchConfig.Name = "lan-dave";
-                  networkConfig.Bridge = "lan";
-                };
                 "55-lan" = {
                   matchConfig.Name = "lan";
-                  vlan = [ "hi" "lo" "untrusted" ];
+                  vlan = [ "lan-hi" "lan-lo" "lan-untrusted" "wan-tunnel" ];
+                  macvlan = [ "lan-core" ];
+                  networkConfig = networkd.noL3;
                 };
+                "60-lan-core" = mkMerge [
+                  (networkdAssignment "lan-core" assignments.core)
+                  {
+                    matchConfig.Name = "lan-core";
+                    networkConfig.IPv6AcceptRA = mkForce false;
+                  }
+                ];
+
+                "90-l2mesh-as211024" = mkMerge [
+                  (networkdAssignment "as211024" assignments.as211024)
+                  {
+                    matchConfig.Name = "as211024";
+                    networkConfig.IPv6AcceptRA = mkForce false;
+                    routes = [
+                      {
+                        Destination = lib.my.c.colony.prefixes.all.v4;
+                        Gateway = allAssignments.estuary.as211024.ipv4.address;
+                      }
+
+                      {
+                        Destination = lib.my.c.tailscale.prefix.v4;
+                        Gateway = allAssignments.britway.as211024.ipv4.address;
+                      }
+                      {
+                        Destination = lib.my.c.tailscale.prefix.v6;
+                        Gateway = allAssignments.britway.as211024.ipv6.address;
+                      }
+                    ];
+                  }
+                ];
               }
 
               (mkVLANConfig "hi")
               (mkVLANConfig "lo")
               (mkVLANConfig "untrusted")
+
+              {
+                "60-lan-hi" = {
+                  routes = [
+                    {
+                      Destination = elemAt routersPubV4 otherIndex;
+                      Gateway = net.cidr.host (otherIndex + 1) prefixes.hi.v4;
+                    }
+                  ];
+                };
+              }
             ];
           };
 
           my = {
             secrets = {
               files = {
-                # "estuary/kelder-wg.key" = {
+                "l2mesh/as211024.key" = {};
-                #   owner = "systemd-network";
-                # };
               };
             };
 
+            vpns = {
+              l2.pskFiles = {
+                as211024 = config.age.secrets."l2mesh/as211024.key".path;
+              };
+            };
             firewall = {
-              trustedInterfaces = [ "hi" "lo" ];
+              trustedInterfaces = [ "lan-hi" "lan-lo" ];
               udp.allowed = [ 5353 ];
               tcp.allowed = [ 5353 ];
               nat = {
                 enable = true;
                 externalInterface = "wan";
-                # externalIP = assignments.internal.ipv4.address;
-                forwardPorts = [
-                  # {
-                  #   port = "http";
-                  #   dst = allAssignments.middleman.internal.ipv4.address;
-                  # }
-                ];
               };
               extraRules =
               let
                 aa = allAssignments;
-                matchInet = rule: sys: ''
-                  ip daddr ${aa."${sys}".hi.ipv4.address} ${rule}
-                  ip6 daddr ${aa."${sys}".hi.ipv6.address} ${rule}
-                '';
               in
               ''
                 table inet filter {
                   chain input {
+                    ${lib.my.c.as211024.nftTrust}
                     iifname base meta l4proto { udp, tcp } th dport domain accept
+                    iifname lan-core meta l4proto vrrp accept
                   }
 
                   chain routing-tcp {
-                    # Safe enough to allow all SSH
+                    ip daddr {
-                    tcp dport ssh accept
+                      ${aa.castle.hi.ipv4.address},
+                      ${aa.cellar.hi.ipv4.address},
+                      ${aa.palace.hi.ipv4.address}
+                    } tcp dport ssh accept
+                    ip6 daddr {
+                      ${aa.castle.hi.ipv6.address},
+                      ${aa.cellar.hi.ipv6.address},
+                      ${aa.palace.hi.ipv6.address}
+                    } tcp dport ssh accept
 
                     return
                   }
@@ -263,15 +372,28 @@ in
                     return
                   }
 
+                  chain forward-early {
+                    type filter hook forward priority -1; policy accept;
+
+                    # MSS clamping to workaround IPv6 PMTUD being broken...
+                    tcp flags syn tcp option maxseg size set rt mtu counter
+
+                    # More Disney+ discrimination...
+                    # TODO: This prefix could change (random AWS block)
+                    ip6 daddr 2600:9000:2245::/48 drop
+                  }
                   chain forward {
-                    iifname untrusted jump filter-untrusted
+                    ${lib.my.c.as211024.nftTrust}
-                    iifname { wan, untrusted } oifname { hi, lo } jump filter-routing
+                    iifname lan-untrusted jump filter-untrusted
+                    iifname { wan, as211024, lan-untrusted } oifname { lan-hi, lan-lo } jump filter-routing
+                    oifname as211024 accept
                   }
                   chain output { }
                 }
                 table inet nat {
                   chain prerouting {
-                    ${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" name}
+                    ip daddr ${elemAt routersPubV4 index} meta l4proto { udp, tcp } th dport domain redirect to :5353
+                    ip6 daddr ${assignments.as211024.ipv6.address} meta l4proto { udp, tcp } th dport domain redirect to :5353
                   }
                   chain postrouting {
                     oifname wan masquerade

nixos/boxes/home/routing-common/dns.nix

@@ -0,0 +1,262 @@
+index: { lib, pkgs, config, assignments, allAssignments, ... }:
+let
+  inherit (builtins) attrNames elemAt;
+  inherit (lib.my) net;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.home) prefixes vips routers;
+
+  name = elemAt routers index;
+  otherIndex = 1 - index;
+  otherName = elemAt routers otherIndex;
+
+  authZones = attrNames config.my.pdns.auth.bind.zones;
+in
+{
+  config = {
+    my = {
+      secrets.files = {
+        "home/pdns/auth.conf" = {
+          owner = "pdns";
+          group = "pdns";
+        };
+        "home/pdns/recursor.conf" = {
+          owner = "pdns-recursor";
+          group = "pdns-recursor";
+        };
+        "home/ddclient-cloudflare.key" = {};
+      };
+
+      pdns.recursor = {
+        enable = true;
+        extraSettingsFile = config.age.secrets."home/pdns/recursor.conf".path;
+      };
+    };
+
+    services = {
+      pdns-recursor = {
+        dns = {
+          address = [
+            "127.0.0.1" "::1"
+            assignments.hi.ipv4.address assignments.hi.ipv6.address
+            assignments.lo.ipv4.address assignments.lo.ipv6.address
+          ];
+          allowFrom = [
+            "127.0.0.0/8" "::1/128"
+            prefixes.hi.v4 prefixes.hi.v6
+            prefixes.lo.v4 prefixes.lo.v6
+          ] ++ (with lib.my.c.tailscale.prefix; [ v4 v6 ]);
+        };
+
+        settings = {
+          query-local-address = [
+            "0.0.0.0"
+            "::"
+          ];
+          forward-zones = map (z: "${z}=127.0.0.1:5353") authZones;
+
+          # DNS NOTIFY messages override TTL
+          allow-notify-for = authZones;
+          allow-notify-from = [ "127.0.0.0/8" "::1/128" ];
+
+          webserver = true;
+          webserver-address = "::";
+          webserver-allow-from = [ "127.0.0.1" "::1" ];
+
+          lua-dns-script = pkgs.writeText "pdns-script.lua" ''
+            -- Disney+ doesn't like our IP space...
+            function preresolve(dq)
+              local name = dq.qname:toString()
+              if dq.qtype == pdns.AAAA and (string.find(name, "disneyplus") or string.find(name, "disney-plus") or string.find(name , "disney.api")) then
+                dq.rcode = 0
+                return true
+              end
+
+              return false
+            end
+          '';
+        };
+      };
+    };
+
+    systemd = {
+      services = {
+        # Add AF_NETLINK to allow pulling IP from network interfaces
+        pdns.serviceConfig.RestrictAddressFamilies = "AF_UNIX AF_INET AF_INET6 AF_NETLINK";
+        ddns-update = {
+          description = "DNS update script";
+          after = [ "network.target" ];
+          path = [
+            (pkgs.python3.withPackages (ps: [ ps.cloudflare ]))
+            pkgs.ldns
+          ];
+          serviceConfig = {
+            Type = "oneshot";
+            ExecStart =
+              ''${./dns_update.py} -k ${config.age.secrets."home/ddclient-cloudflare.key".path} '' +
+              ''${pubDomain} ns${toString (index + 1)}.${config.networking.domain}'';
+          };
+          wantedBy = [ "multi-user.target" ];
+        };
+      };
+      timers = {
+        ddns-update = {
+          description = "Periodically update DNS";
+          wantedBy = [ "timers.target" ];
+          timerConfig = {
+            OnBootSec = "5min";
+            OnUnitInactiveSec = "5min";
+          };
+        };
+      };
+    };
+
+    environment.systemPackages = with pkgs; [
+      # For rec_control
+      pdns-recursor
+      sqlite
+    ];
+
+    my.pdns.auth = {
+      enable = true;
+      extraSettingsFile = config.age.secrets."home/pdns/auth.conf".path;
+      settings = {
+        primary = true;
+        resolver = "127.0.0.1";
+        expand-alias = true;
+        local-address = [
+          "0.0.0.0:5353" "[::]:5353"
+        ];
+        also-notify = [ "127.0.0.1" ];
+        enable-lua-records = true;
+        # loglevel = 7;
+        # log-dns-queries = true;
+        # log-dns-details = true;
+
+        api = true;
+        webserver = true;
+        webserver-address = "::";
+        webserver-allow-from = [ "127.0.0.1" "::1" ];
+
+        dnsupdate = true;
+        launch = [ "gsqlite3" ];
+        gsqlite3-database = "/var/lib/pdns/dynamic.sqlite3";
+      };
+
+      bind.zones =
+      let
+        names = [ "core" "hi" "lo" "untrusted" ];
+        i = toString (index + 1);
+      in
+      {
+        "${config.networking.domain}" = {
+          type = "master";
+          text = ''
+            $TTL 60
+            @ IN SOA ns${i}.${config.networking.domain}. dev.nul.ie. (
+              @@SERIAL@@ ; serial
+              3h ; refresh
+              1h ; retry
+              1w ; expire
+              1h ; minimum
+            )
+
+            ${name} IN LUA ${lib.my.dns.ifaceA {
+              inherit pkgs;
+              iface = "wan";
+              skipBroadcasts = [ (lib.my.netBroadcast prefixes.modem.v4) ];
+            }}
+            ${otherName} IN LUA ${lib.my.dns.lookupIP {
+              inherit pkgs;
+              hostname = "${otherName}.${config.networking.domain}";
+              server = net.cidr.host (otherIndex + 1) prefixes.hi.v4;
+            }}
+            ${elemAt routers 0} IN AAAA ${allAssignments."${elemAt routers 0}".as211024.ipv6.address}
+            ${elemAt routers 1} IN AAAA ${allAssignments."${elemAt routers 1}".as211024.ipv6.address}
+            boot IN CNAME river-hi.${config.networking.domain}.
+
+            @ IN NS ns1
+            @ IN NS ns2
+            ns1 IN ALIAS ${elemAt routers 0}.${config.networking.domain}.
+            ns2 IN ALIAS ${elemAt routers 1}.${config.networking.domain}.
+
+            dyn IN NS ns1.dyn.h.nul.ie.
+            dyn IN NS ns2.dyn.h.nul.ie.
+            ns1.dyn.h.nul.ie. IN ALIAS ${elemAt routers 0}.${config.networking.domain}.
+            ns2.dyn.h.nul.ie. IN ALIAS ${elemAt routers 1}.${config.networking.domain}.
+
+            jim-core IN A ${net.cidr.host 10 prefixes.core.v4}
+            jim IN A ${net.cidr.host 10 prefixes.hi.v4}
+            jim IN AAAA ${net.cidr.host (65536+1) prefixes.hi.v6}
+            jim-lo IN A ${net.cidr.host 10 prefixes.lo.v4}
+            jim-lo IN AAAA ${net.cidr.host (65536+1) prefixes.lo.v6}
+
+            dave-core IN A ${net.cidr.host 11 prefixes.core.v4}
+            dave IN A ${net.cidr.host 11 prefixes.hi.v4}
+            dave IN AAAA ${net.cidr.host (65536+2) prefixes.hi.v6}
+            dave-lo IN A ${net.cidr.host 11 prefixes.lo.v4}
+            dave-lo IN AAAA ${net.cidr.host (65536+2) prefixes.lo.v6}
+
+            shytzel IN A ${net.cidr.host 12 prefixes.core.v4}
+
+            wave IN A ${net.cidr.host 12 prefixes.hi.v4}
+            ; wave IN AAAA ${net.cidr.host (65536+3) prefixes.hi.v6}
+            vibe IN A ${net.cidr.host 13 prefixes.hi.v4}
+            vibe IN AAAA ${net.cidr.host (65536+4) prefixes.hi.v6}
+
+            ups IN A ${net.cidr.host 20 prefixes.lo.v4}
+            palace-kvm IN A ${net.cidr.host 21 prefixes.lo.v4}
+
+            ${lib.my.dns.fwdRecords {
+              inherit allAssignments names;
+              domain = config.networking.domain;
+            }}
+          '';
+        };
+        "168.192.in-addr.arpa" = {
+          type = "master";
+          text = ''
+            $TTL 60
+            @ IN SOA ns${i}.${config.networking.domain}. dev.nul.ie. (
+              @@SERIAL@@ ; serial
+              3h ; refresh
+              1h ; retry
+              1w ; expire
+              1h ; minimum
+            )
+
+            @ IN NS ns1.${config.networking.domain}.
+            @ IN NS ns2.${config.networking.domain}.
+
+            ${lib.my.dns.ptrRecords {
+              inherit allAssignments names;
+              domain = config.networking.domain;
+              ndots = 2;
+            }}
+          '';
+        };
+        "0.d.4.0.0.c.7.9.e.0.a.2.ip6.arpa" = {
+          type = "master";
+          text = ''
+            $TTL 60
+            @ IN SOA ns${i}.${config.networking.domain}. dev.nul.ie. (
+              @@SERIAL@@ ; serial
+              3h ; refresh
+              1h ; retry
+              1w ; expire
+              1h ; minimum
+            )
+
+            @ IN NS ns1.${config.networking.domain}.
+            @ IN NS ns2.${config.networking.domain}.
+
+            ${lib.my.dns.ptr6Records {
+              inherit allAssignments names;
+              domain = config.networking.domain;
+              ndots = 20;
+            }}
+          '';
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/routing-common/dns_update.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import argparse
+import subprocess
+
+import CloudFlare
+
+def main():
+    parser = argparse.ArgumentParser(description='Cloudflare DNS update script')
+    parser.add_argument('-k', '--api-token-file', help='Cloudflare API token file')
+    parser.add_argument('zone', help='Cloudflare Zone')
+    parser.add_argument('record', help='Cloudflare record name')
+    args = parser.parse_args()
+
+    address = subprocess.check_output(
+        ['drill', '-Q', '-p5353', '@127.0.0.1', args.record, 'A'],
+        encoding='utf8').strip()
+
+    cf_token = None
+    if args.api_token_file:
+        with open(args.api_token_file) as f:
+            cf_token = f.readline().strip()
+
+    cf = CloudFlare.CloudFlare(token=cf_token)
+    zones = cf.zones.get(params={'name': args.zone})
+    assert zones, f'Zone {args.zone} not found'
+    records = cf.zones.dns_records.get(zones[0]['id'], params={'name': args.record})
+    assert records, f'Record {args.record} not found in zone {args.zone}'
+
+    print(f'Updating {args.record} -> {address}')
+    cf.zones.dns_records.patch(
+        zones[0]['id'], records[0]['id'],
+        data={'type': 'A', 'name': args.record, 'content': address})
+
+if __name__ == '__main__':
+    main()

nixos/boxes/home/routing-common/kea.nix

@@ -0,0 +1,161 @@
+index: { lib, pkgs, config, assignments, allAssignments, ... }:
+let
+  inherit (lib) mkForce;
+  inherit (lib.my) net netbootKeaClientClasses;
+  inherit (lib.my.c.home) domain prefixes vips hiMTU;
+
+  dns-servers = [
+    {
+      ip-address = net.cidr.host 1 prefixes.core.v4;
+      port = 5353;
+    }
+    {
+      ip-address = net.cidr.host 2 prefixes.core.v4;
+      port = 5353;
+    }
+  ];
+in
+{
+  users = with lib.my.c.ids; {
+    users.kea= {
+      isSystemUser = true;
+      uid = uids.kea;
+      group = "kea";
+    };
+    groups.kea.gid = gids.kea;
+  };
+
+  systemd.services = {
+    kea-dhcp4-server.serviceConfig = {
+      # Sometimes interfaces might not be ready in time and Kea doesn't like that
+      Restart = "on-failure";
+      DynamicUser = mkForce false;
+    };
+    kea-dhcp-ddns-server.serviceConfig.DynamicUser = mkForce false;
+  };
+
+  services = {
+    kea = {
+      dhcp4 = {
+        enable = true;
+        settings = {
+          interfaces-config = {
+            interfaces = [
+              "lan-hi/${assignments.hi.ipv4.address}"
+              "lan-lo/${assignments.lo.ipv4.address}"
+              "lan-untrusted/${assignments.untrusted.ipv4.address}"
+            ];
+          };
+          lease-database = {
+            type = "memfile";
+            persist = true;
+            name = "/var/lib/kea/dhcp.leases";
+          };
+
+          option-data = [
+            {
+              name = "domain-name";
+              data = domain;
+            }
+            {
+              name = "domain-search";
+              data = "${domain}, dyn.${domain}, ${lib.my.c.colony.domain}, ${lib.my.c.britway.domain}";
+              always-send = true;
+            }
+          ];
+          client-classes = netbootKeaClientClasses {
+            tftpIP = allAssignments.river.lo.ipv4.address;
+            hostname = "boot.${domain}";
+            systems = {
+              sfh = "52:54:00:a5:7e:93";
+              castle = "c8:7f:54:6e:17:0f";
+            };
+          };
+          subnet4 = [
+            {
+              id = 1;
+              subnet = prefixes.hi.v4;
+              interface = "lan-hi";
+              option-data = [
+                {
+                  name = "routers";
+                  data = vips.hi.v4;
+                }
+                {
+                  name = "domain-name-servers";
+                  data = "${net.cidr.host 1 prefixes.hi.v4}, ${net.cidr.host 2 prefixes.hi.v4}";
+                }
+                {
+                  name = "interface-mtu";
+                  data = toString hiMTU;
+                }
+              ];
+              pools = [
+                {
+                  pool = if index == 0
+                    then "192.168.68.120 - 192.168.69.255"
+                    else "192.168.70.0 - 192.168.71.240";
+                }
+              ];
+              reservations = [
+                {
+                  # castle
+                  hw-address = "24:8a:07:a8:fe:3a";
+                  ip-address = net.cidr.host 40 prefixes.hi.v4;
+                }
+              ];
+            }
+            {
+              id = 2;
+              subnet = prefixes.lo.v4;
+              interface = "lan-lo";
+              option-data = [
+                {
+                  name = "routers";
+                  data = vips.lo.v4;
+                }
+                {
+                  name = "domain-name-servers";
+                  data = "${net.cidr.host 1 prefixes.lo.v4}, ${net.cidr.host 2 prefixes.lo.v4}";
+                }
+              ];
+              pools = [
+                {
+                  pool = if index == 0
+                    then "192.168.72.120 - 192.168.75.255"
+                    else "192.168.76.0 - 192.168.79.240";
+                }
+              ];
+              reservations = [
+                {
+                  # castle
+                  hw-address = "24:8a:07:a8:fe:3a";
+                  ip-address = net.cidr.host 40 prefixes.lo.v4;
+                }
+              ];
+            }
+          ];
+          ddns-send-updates = true;
+          ddns-replace-client-name = "when-not-present";
+          ddns-qualifying-suffix = "dyn.${domain}";
+          ddns-generated-prefix = "ip";
+          ddns-update-on-renew = true;
+
+          dhcp-ddns.enable-updates = true;
+        };
+      };
+
+      dhcp-ddns = {
+        enable = true;
+        settings = {
+          forward-ddns.ddns-domains = [
+            {
+              name = "dyn.${domain}.";
+              inherit dns-servers;
+            }
+          ];
+        };
+      };
+    };
+  };
+}

nixos/boxes/home/routing-common/keepalived.nix

@@ -0,0 +1,83 @@
+index: { lib, pkgs, config, ... }:
+let
+  inherit (builtins) attrNames concatMap length;
+  inherit (lib) optional concatMapStringsSep;
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) prefixes vips;
+
+  pingScriptFor = name: ips:
+  let
+    script' = pkgs.writeShellScript
+      "keepalived-ping-${name}"
+      (concatMapStringsSep " || " (ip: "${pkgs.iputils}/bin/ping -qnc 1 -W 1 ${ip}") ips);
+  in
+  {
+    script = toString script';
+    interval = 1;
+    timeout = (length ips) + 1;
+    rise = 3;
+    fall = 3;
+  };
+
+  vlanIface = vlan: if vlan == "as211024" then vlan else "lan-${vlan}";
+  vrrpIPs = family: concatMap (vlan: (optional (family == "v6") {
+      addr = "fe80::1/64";
+      dev = vlanIface vlan;
+    }) ++ [
+    {
+      addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}";
+      dev = vlanIface vlan;
+    }
+  ]) (attrNames vips);
+  mkVRRP = family: routerId: {
+    state = if index == 0 then "MASTER" else "BACKUP";
+    interface = "lan-core";
+    priority = 255 - index;
+    virtualRouterId = routerId;
+    virtualIps = vrrpIPs family;
+    trackScripts = [ "${family}Alive" ];
+  };
+in
+{
+  users = with lib.my.c.ids; {
+    users.keepalived_script = {
+      uid = uids.keepalived_script;
+      isSystemUser = true;
+      group = "keepalived_script";
+    };
+    groups.keepalived_script.gid = gids.keepalived_script;
+  };
+
+  services = {
+    keepalived = {
+      enable = true;
+      enableScriptSecurity = true;
+      extraGlobalDefs = ''
+        vrrp_version 3
+        nftables keepalived
+      '';
+      vrrpScripts = {
+        v4Alive = pingScriptFor "v4" [ "1.1.1.1" "8.8.8.8" "216.218.236.2" ];
+        v6Alive = pingScriptFor "v6" [ "2606:4700:4700::1111" "2001:4860:4860::8888" "2600::" ];
+      };
+      vrrpInstances = {
+        v4 = mkVRRP "v4" 51;
+        v6 = (mkVRRP "v6" 52) // {
+          extraConfig = ''
+            notify_master "${config.systemd.package}/bin/systemctl start radvd.service" root
+            notify_backup "${config.systemd.package}/bin/systemctl stop radvd.service" root
+          '';
+        };
+      };
+      # Actually disable this for now, don't want to fault IPv4 just because IPv6 is broken...
+      # extraConfig = ''
+      #   vrrp_sync_group main {
+      #     group {
+      #       v4
+      #       v6
+      #     }
+      #   }
+      # '';
+    };
+  };
+}

nixos/boxes/home/routing-common/mstpd.nix

@@ -0,0 +1,53 @@
+{ lib, pkgs, ... }:
+let
+  # TODO: Move into nixpkgs
+  mstpd = pkgs.mstpd.overrideAttrs {
+    patches = [ ./mstpd.patch ];
+  };
+in
+{
+  environment = {
+    systemPackages = [
+      mstpd
+    ];
+    etc = {
+      "bridge-stp.conf".text = ''
+        MANAGE_MSTPD=n
+        MSTP_BRIDGES=lan
+      '';
+    };
+  };
+
+  services = {
+    networkd-dispatcher.rules = {
+      configure-mstpd = {
+        onState = [ "routable" ];
+        script = ''
+          #!${pkgs.runtimeShell}
+          if [ $IFACE = "lan" ]; then
+            ${mstpd}/sbin/mstpctl setforcevers $IFACE rstp
+          fi
+        '';
+      };
+    };
+  };
+
+  systemd = {
+    services = {
+      mstpd = {
+        description = "MSTP daemon";
+        before = [ "network-pre.target" ];
+        serviceConfig = {
+          Type = "forking";
+          ExecStart = "${mstpd}/sbin/bridge-stp restart";
+          ExecReload = "${mstpd}/sbin/bridge-stp restart_config";
+          PIDFile = "/run/mstpd.pid";
+          Restart = "always";
+          PrivateTmp = true;
+          ProtectHome = true;
+        };
+        wantedBy = [ "multi-user.target" ];
+      };
+    };
+  };
+}

nixos/boxes/home/routing-common/mstpd.patch

@@ -0,0 +1,26 @@
+diff --git a/bridge-stp.in b/bridge-stp.in
+index 3807873..9c73126 100755
+--- a/bridge-stp.in
++++ b/bridge-stp.in
+@@ -31,6 +31,10 @@
+ # bridge or any associated kernel network interfaces in any code paths that are
+ # used when this script is called by the kernel.
+ 
++# Ensure that we have a sane PATH.
++PATH='/run/current-system/sw/bin'
++export PATH
++
+ # Parse arguments.
+ CalledAs="$(basename "$0")"
+ if [ "$CalledAs" = 'mstpctl_restart_config' ]; then
+@@ -62,10 +66,6 @@ fi
+ # Ensure that we have a sane umask.
+ umask 022
+ 
+-# Ensure that we have a sane PATH.
+-PATH='/sbin:/usr/sbin:/bin:/usr/bin'
+-export PATH
+-
+ # Define some relevant paths.
+ mstpctl='@mstpctlfile@'
+ mstpd='@mstpdfile@'

nixos/boxes/home/routing-common/radvd.nix

@@ -0,0 +1,28 @@
+index: { lib, pkgs, ... }:
+let
+  inherit (lib) mkForce concatMapStringsSep;
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) domain prefixes;
+
+  mkInterface = name: ''
+    interface lan-${name} {
+      AdvSendAdvert on;
+      AdvRASrcAddress { fe80::1; };
+      AdvLinkMTU ${toString prefixes."${name}".mtu};
+      prefix ${prefixes."${name}".v6} {};
+      RDNSS ${net.cidr.host 1 prefixes."${name}".v6} ${net.cidr.host 2 prefixes."${name}".v6} {};
+      DNSSL ${domain} dyn.${domain} ${lib.my.c.colony.domain} ${lib.my.c.britway.domain} {};
+     };
+  '';
+in
+{
+  # To be started by keepalived
+  systemd.services.radvd.wantedBy = mkForce [ ];
+
+  services = {
+    radvd = {
+      enable = true;
+      config = concatMapStringsSep "\n" mkInterface [ "hi" "lo" "untrusted" ];
+    };
+  };
+}

nixos/boxes/home/stream.nix

@@ -0,0 +1,163 @@
+{
+  imports = [ (import ./routing-common 1) ];
+
+  config.nixos.systems.stream = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    home-manager = "mine";
+
+    configuration = { lib, pkgs, config, ... }:
+    let
+      inherit (lib);
+    in
+    {
+      imports = [ ./routing-common/mstpd.nix ];
+
+      config = {
+        boot = {
+          kernelModules = [ "kvm-intel" ];
+          kernelParams = [ "intel_iommu=on" ];
+          initrd.availableKernelModules = [ "xhci_pci" "usbhid" "usb_storage" "sd_mod" "sdhci_pci" ];
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+          cpu = {
+            intel.updateMicrocode = true;
+          };
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/fe081885-9157-46b5-be70-46ac6fcb4069";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/disk/by-partuuid/a195e55e-397f-440d-a190-59ffa63cdb3f";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/disk/by-partuuid/ad71fafd-2d26-49c8-b0cb-794a28e0beb7";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          mjpg-streamer = {
+            enable = true;
+            inputPlugin = "input_uvc.so";
+            outputPlugin = "output_http.so -w @www@ -n -p 5050";
+          };
+          octoprint = {
+            enable = true;
+            host = "::";
+            extraConfig = {
+              plugins = {
+                classicwebcam = {
+                  snapshot = "/webcam/?action=snapshot";
+                  stream = "/webcam/?action=stream";
+                  streamRatio = "4:3";
+                };
+              };
+              serial = {
+                port = "/dev/ttyACM0";
+                baudrate = 115200;
+              };
+              temperature.profiles = [
+                {
+                  bed = 60;
+                  extruder = 215;
+                  name = "PLA";
+                }
+              ];
+            };
+          };
+        };
+
+        systemd.network = {
+          netdevs = {
+            "25-lan" = {
+              netdevConfig = {
+                Name = "lan";
+                Kind = "bridge";
+              };
+              extraConfig = ''
+                [Bridge]
+                STP=true
+              '';
+            };
+          };
+          links = {
+            "10-wan" = {
+              matchConfig = {
+                # Matching against MAC address seems to break VLAN interfaces
+                # (since they share the same MAC address)
+                Driver = "igc";
+                PermanentMACAddress = "00:f0:cb:ee:ca:dd";
+              };
+              linkConfig = {
+                Name = "wan";
+                RxBufferSize = 4096;
+                TxBufferSize = 4096;
+              };
+            };
+            "10-lan-jim" = {
+              matchConfig = {
+                Driver = "igc";
+                PermanentMACAddress = "00:f0:cb:ee:ca:de";
+              };
+              linkConfig = {
+                Name = "lan-jim";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
+            "10-et2" = {
+              matchConfig = {
+                Driver = "igc";
+                PermanentMACAddress = "00:f0:cb:ee:ca:df";
+              };
+              linkConfig.Name = "et2";
+            };
+
+            "10-lan-dave" = {
+              matchConfig = {
+                Driver = "mlx4_en";
+                PermanentMACAddress = "00:02:c9:d5:b1:d6";
+              };
+              linkConfig = {
+                Name = "lan-dave";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
+            "10-et5" = {
+              matchConfig = {
+                Driver = "mlx4_en";
+                PermanentMACAddress = "00:02:c9:d5:b1:d7";
+              };
+              linkConfig.Name = "et5";
+            };
+          };
+          networks = {
+            "50-lan-jim" = {
+              matchConfig.Name = "lan-jim";
+              networkConfig.Bridge = "lan";
+            };
+            "50-lan-dave" = {
+              matchConfig.Name = "lan-dave";
+              networkConfig.Bridge = "lan";
+            };
+          };
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPYTB4zeAqotrEJ8M+AiGm/s9PFsWlAodz3hYSROGuDb";
+          };
+          server.enable = true;
+          # deploy.node.hostname = "192.168.68.2";
+        };
+      };
+    };
+  };
+}

nixos/boxes/kelder/containers/acquisition/default.nix

@@ -26,7 +26,7 @@ in
 
       config = {
         # Hardware acceleration for Jellyfin
-        hardware.opengl = {
+        hardware.graphics = {
           enable = true;
           extraPackages = with pkgs; [
             vaapiIntel
@@ -65,13 +65,27 @@ in
         systemd = {
           services = {
             jackett.bindsTo = [ "systemd-networkd-wait-online@vpn.service" ];
+
             transmission.bindsTo = [ "systemd-networkd-wait-online@vpn.service" ];
+            # https://github.com/NixOS/nixpkgs/issues/258793#issuecomment-1748168206
+            transmission.serviceConfig = {
+              RootDirectoryStartOnly = lib.mkForce false;
+              RootDirectory = lib.mkForce "";
+            };
 
             radarr.serviceConfig.UMask = "0002";
             sonarr.serviceConfig.UMask = "0002";
           };
         };
 
+        nixpkgs.config.permittedInsecurePackages = [
+          # FIXME: This is needed for Sonarr
+          "aspnetcore-runtime-wrapped-6.0.36"
+          "aspnetcore-runtime-6.0.36"
+          "dotnet-sdk-wrapped-6.0.428"
+          "dotnet-sdk-6.0.428"
+        ];
+
         services = {
           transmission = {
             enable = true;

nixos/boxes/kelder/containers/acquisition/networking.nix

@@ -2,6 +2,7 @@
 let
   inherit (lib) mkMerge mkIf;
   inherit (lib.my) networkdAssignment;
+  inherit (lib.my.c.kelder) ipv4MTU;
 
   wg = {
     keyFile = "kelder/acquisition/airvpn-privkey";
@@ -72,14 +73,12 @@ in
               RouteTable = routeTable;
             };
             wireguardPeers = [
+              # AirVPN IE
               {
-                # AirVPN IE
+                Endpoint = "146.70.94.2:1637";
-                wireguardPeerConfig = {
+                PublicKey = "PyLCXAQT8KkM4T+dUsOQfn+Ub3pGxfGlxkIApuig+hk=";
-                  Endpoint = "146.70.94.2:1637";
+                PresharedKeyFile = config.age.secrets."${pskFile}".path;
-                  PublicKey = "PyLCXAQT8KkM4T+dUsOQfn+Ub3pGxfGlxkIApuig+hk=";
+                AllowedIPs = [ "0.0.0.0/0" "::/0" ];
-                  PresharedKeyFile = config.age.secrets."${pskFile}".path;
-                  AllowedIPs = [ "0.0.0.0/0" "::/0" ];
-                };
               }
             ];
           };
@@ -89,13 +88,14 @@ in
               (networkdAssignment "host0" assignments.internal)
               {
                 networkConfig.DNSDefaultRoute = false;
+                linkConfig.MTUBytes = toString ipv4MTU;
               }
             ];
             "90-vpn" = with wg; {
               matchConfig.Name = "vpn";
               address = [ "10.161.170.28/32" "fd7d:76ee:e68f:a993:b12d:6d15:c80a:9516/128" ];
               dns = [ "10.128.0.1" "fd7d:76ee:e68f:a993::1" ];
-              routingPolicyRules = map (r: { routingPolicyRuleConfig = r; }) [
+              routingPolicyRules = [
                 {
                   Family = "both";
                   SuppressPrefixLength = 0;

nixos/boxes/kelder/containers/spoder/default.nix

@@ -92,17 +92,17 @@ in
 
           nextcloud = {
             enable = true;
-            package = pkgs.nextcloud27;
+            package = pkgs.nextcloud29;
             datadir = "/mnt/storage/nextcloud";
             hostName = "cloud.${domain}";
             https = true;
             config = {
-              extraTrustedDomains = [ "cloud-local.${domain}" ];
               adminpassFile = config.age.secrets."kelder/nextcloud-root.txt".path;
-              defaultPhoneRegion = "IE";
             };
-            extraOptions = {
+            settings = {
               updatechecker = false;
+              trusted_domains = [ "cloud-local.${domain}" ];
+              default_phone_region = "IE";
             };
           };
         };

nixos/boxes/kelder/containers/spoder/nginx.nix

@@ -2,7 +2,7 @@
 let
   inherit (builtins) mapAttrs;
   inherit (lib) mkMerge mkIf mkDefault;
-  inherit (lib.my.c.nginx) proxyHeaders;
+  inherit (lib.my.c.nginx) baseHttpConfig proxyHeaders;
   inherit (lib.my.c.kelder) domain;
 in
 {
@@ -39,43 +39,7 @@ in
 
         # Based on recommended*Settings, but probably better to be explicit about these
         appendHttpConfig = ''
-          # NixOS provides a logrotate config that auto-compresses :)
+          ${baseHttpConfig}
-          log_format main
-            '$remote_addr - $remote_user [$time_local] $scheme "$host" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"';
-          access_log /var/log/nginx/access.log main;
-
-          # optimisation
-          sendfile on;
-          tcp_nopush on;
-          tcp_nodelay on;
-          keepalive_timeout 65;
-
-          # gzip
-          gzip on;
-          gzip_proxied any;
-          gzip_comp_level 5;
-          gzip_types
-            application/atom+xml
-            application/javascript
-            application/json
-            application/xml
-            application/xml+rss
-            image/svg+xml
-            text/css
-            text/javascript
-            text/plain
-            text/xml;
-          gzip_vary on;
-
-          # proxying
-          proxy_buffering off;
-          proxy_redirect off;
-          proxy_connect_timeout 60s;
-          proxy_read_timeout 60s;
-          proxy_send_timeout 60s;
-          proxy_http_version 1.1;
-
-          ${proxyHeaders}
 
           # caching
           proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=CACHE:10m inactive=7d max_size=4g;
@@ -120,6 +84,7 @@ in
             c
           ];
           acquisition = "http://${allAssignments.kelder-acquisition.internal.ipv4.address}";
+          # This is kinda borked because Virgin Media filters DNS responses with local IPs...
           localRedirect = to: ''
             rewrite_by_lua_block {
               if ngx.var.remote_addr == pub_ip then
@@ -139,7 +104,7 @@ in
 
             "monitor.${domain}" = withAuth {
               serverAliases = [ "monitor-local.${domain}" ];
-              extraConfig = localRedirect "monitor-local.${domain}";
+              # extraConfig = localRedirect "monitor-local.${domain}";
               locations = {
                 "/" = {
                   proxyPass = "http://${allAssignments.kelder.ctrs.ipv4.address}:19999";
@@ -172,17 +137,17 @@ in
             };
             "torrents.${domain}" = withAuth {
               serverAliases = [ "torrents-local.${domain}" ];
-              extraConfig = localRedirect "torrents-local.${domain}";
+              # extraConfig = localRedirect "torrents-local.${domain}";
               locations."/".proxyPass = "${acquisition}:9091";
             };
             "jackett.${domain}" = withAuth {
               serverAliases = [ "jackett-local.${domain}" ];
-              extraConfig = localRedirect "jackett-local.${domain}";
+              # extraConfig = localRedirect "jackett-local.${domain}";
               locations."/".proxyPass = "${acquisition}:9117";
             };
             "radarr.${domain}" = withAuth {
               serverAliases = [ "radarr-local.${domain}" ];
-              extraConfig = localRedirect "radarr-local.${domain}";
+              # extraConfig = localRedirect "radarr-local.${domain}";
               locations."/" = {
                 proxyPass = "${acquisition}:7878";
                 proxyWebsockets = true;
@@ -191,7 +156,7 @@ in
             };
             "sonarr.${domain}" = withAuth {
               serverAliases = [ "sonarr-local.${domain}" ];
-              extraConfig = localRedirect "sonarr-local.${domain}";
+              # extraConfig = localRedirect "sonarr-local.${domain}";
               locations."/" = {
                 proxyPass = "${acquisition}:8989";
                 proxyWebsockets = true;

nixos/boxes/kelder/default.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 let
   inherit (lib.my) net;
-  inherit (lib.my.c.kelder) domain prefixes;
+  inherit (lib.my.c.kelder) domain prefixes ipv4MTU;
 in
 {
   imports = [ ./containers ];
@@ -54,7 +54,7 @@ in
               efi.canTouchEfiVariables = true;
               timeout = 5;
             };
-            kernelPackages = pkgs.linuxKernel.packages.linux_6_1;
+            kernelPackages = lib.my.c.kernel.lts pkgs;
             kernelModules = [ "kvm-intel" ];
             kernelParams = [ "intel_iommu=on" ];
             initrd = {
@@ -121,8 +121,7 @@ in
 
             samba = {
               enable = true;
-              enableNmbd = true;
+              settings = {
-              shares = {
                 storage = {
                   path = "/mnt/storage";
                   browseable = "yes";
@@ -131,16 +130,20 @@ in
                   "directory mask" = "0775";
                 };
               };
+
+              nmbd.enable = true;
             };
             samba-wsdd.enable = true;
 
             minecraft-server = {
-              enable = true;
+              enable = false;
-              package = pkgs.minecraftServers.vanilla-1-19;
+              package = pkgs.minecraftServers.vanilla-1-20;
               declarative = true;
               eula = true;
               whitelist = {
                 devplayer0 = "6d7d971b-ce10-435b-85c5-c99c0d8d288c";
+                Elderlypug = "dcd2ecb9-2b5e-49cb-9d4f-f5a76162df56";
+                shr3kas0ras = "1d366062-12c0-4e29-aba7-6ab5d8c6bb05";
               };
               serverProperties = {
                 motd = "Simpcraft";
@@ -178,12 +181,10 @@ in
                   };
                   wireguardPeers = [
                     {
-                      wireguardPeerConfig = {
+                      PublicKey = "bP1XUNxp9i8NLOXhgPaIaRzRwi5APbam44/xjvYcyjU=";
-                        PublicKey = "bP1XUNxp9i8NLOXhgPaIaRzRwi5APbam44/xjvYcyjU=";
+                      Endpoint = "${allAssignments.estuary.internal.ipv4.address}:${toString lib.my.c.kelder.vpn.port}";
-                        Endpoint = "estuary-vm.${lib.my.c.colony.domain}:${toString lib.my.c.kelder.vpn.port}";
+                      AllowedIPs = [ "0.0.0.0/0" ];
-                        AllowedIPs = [ "0.0.0.0/0" ];
+                      PersistentKeepalive = 25;
-                        PersistentKeepalive = 25;
-                      };
                     }
                   ];
                 };
@@ -198,6 +199,7 @@ in
                 "50-lan" = {
                   matchConfig.Name = "et1g0";
                   DHCP = "yes";
+                  linkConfig.MTUBytes = toString ipv4MTU;
                 };
                 "80-ctrs" = mkMerge [
                   (networkdAssignment "ctrs" assignments.ctrs)
@@ -210,7 +212,7 @@ in
                   address = with assignments.estuary; [
                     (with ipv4; "${address}/${toString mask}")
                   ];
-                  routingPolicyRules = map (r: { routingPolicyRuleConfig = r; }) [
+                  routingPolicyRules = [
                     {
                       Family = "both";
                       SuppressPrefixLength = 0;
@@ -270,7 +272,7 @@ in
               config.name = "kontent";
             };
 
-            #deploy.node.hostname = "10.16.9.21";
+            # deploy.node.hostname = "192.168.0.69";
             secrets = {
               key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOFvUdJshXkqmchEgkZDn5rgtZ1NO9vbd6Px+S6YioWi";
               files = {

nixos/boxes/tower/default.nix

@@ -14,7 +14,7 @@
           cpu = {
             intel.updateMicrocode = true;
           };
-          opengl.extraPackages = with pkgs; [
+          graphics.extraPackages = with pkgs; [
             intel-media-driver
           ];
           bluetooth.enable = true;
@@ -25,23 +25,20 @@
             efi.canTouchEfiVariables = true;
             timeout = 10;
           };
-          kernelPackages = pkgs.linuxKernel.packages.linux_6_5;
+          kernelPackages = lib.my.c.kernel.latest pkgs;
           kernelModules = [ "kvm-intel" ];
           kernelParams = [ "intel_iommu=on" ];
           initrd = {
             availableKernelModules = [ "nvme" "xhci_pci" "usb_storage" "usbhid" "thunderbolt" ];
             luks = {
-              reusePassphrases = true;
               devices = {
                 persist = {
                   device = "/dev/disk/by-uuid/27840c6f-445c-4b95-8c39-e69d07219f33";
                   allowDiscards = true;
-                  preLVM = false;
                 };
                 home = {
                   device = "/dev/disk/by-uuid/c16c5038-7883-42c3-960a-a085a99364eb";
                   allowDiscards = true;
-                  preLVM = false;
                 };
               };
             };
@@ -110,10 +107,19 @@
 
           fprintd.enable = true;
           blueman.enable = true;
+
+          tailscale = {
+            enable = true;
+            openFirewall = true;
+          };
         };
 
         programs = {
           steam.enable = true;
+          wireshark = {
+            enable = true;
+            package = pkgs.wireshark-qt;
+          };
         };
 
         networking = {
@@ -123,10 +129,9 @@
             wifi = {
               backend = "wpa_supplicant";
             };
-            extraConfig = ''
+            settings = {
-              [main]
+              main.no-auto-default = "*";
-              no-auto-default=*
+            };
-            '';
           };
         };
 
@@ -169,6 +174,14 @@
                 packages = with pkgs; [ ];
               };
 
+              programs = {
+                fish = {
+                  shellAbbrs = {
+                    tsup = "doas tailscale up --login-server=https://hs.nul.ie --accept-routes";
+                  };
+                };
+              };
+
               services = {
                 blueman-applet.enable = true;
               };

nixos/default.nix

@@ -1,4 +1,4 @@
-{ lib, pkgsFlakes, hmFlakes, inputs, pkgs', config, ... }:
+{ self, lib, pkgsFlakes, hmFlakes, inputs, pkgs', config, ... }:
 let
   inherit (builtins) attrValues mapAttrs;
   inherit (lib)
@@ -25,13 +25,17 @@ let
 
       modules' = [ hmFlakes.${config'.home-manager}.nixosModule ] ++ (attrValues cfg.modules);
     in
-    pkgsFlake.lib.nixosSystem {
+    # Import eval-config ourselves since the flake now force-sets lib
+    import "${pkgsFlake}/nixos/lib/eval-config.nix" {
       # Gotta override lib here unforunately, eval-config.nix likes to import its own (unextended) lib. We explicitly
       # don't pass pkgs so that it'll be imported with modularly applied config and overlays.
-      lib = pkgs.lib;
+      lib = pkgs.lib.extend (lib.my.versionOverlay { inherit self pkgsFlake; });
+
+      # Set to null since we pass modularly
+      system = null;
 
       # Put the inputs in specialArgs to avoid infinite recursion when modules try to do imports
-      specialArgs = { inherit inputs pkgsFlakes pkgsFlake allAssignments; inherit (cfg) systems; };
+      specialArgs = { inherit self inputs pkgsFlakes pkgsFlake allAssignments; inherit (cfg) systems; };
 
       # `baseModules` informs the manual which modules to document
       baseModules =
@@ -51,7 +55,7 @@ let
             pkgs' = allPkgs;
           };
 
-          system.name = name;
+          system = { inherit name; };
           networking = {
             domain = let d = config'.assignments.internal.domain or null; in mkIf (d != null) (mkDefault' d);
             hostName = mkDefault (config'.assignments.internal.name or name);
@@ -86,6 +90,8 @@ let
                   pkgsPath = toString pkgsFlakes.${config'.hmNixpkgs};
                   pkgs' = allPkgs;
                 };
+
+                home.enableNixpkgsReleaseCheck = false;
               }
               (homeStateVersion config'.home-manager)
             ];
@@ -100,6 +106,7 @@ let
       altNames = mkOpt' (listOf str) [ ] "Extra names to assign.";
       visible = mkBoolOpt' true "Whether or not this assignment should be visible.";
       domain = mkOpt' (nullOr str) null "Domain for this assignment.";
+      mtu = mkOpt' (nullOr ints.unsigned) null "Interface MTU.";
       ipv4 = {
         address = mkOpt' net.types.ipv4 null "IPv4 address.";
         mask = mkOpt' ints.u8 24 "Network mask.";
@@ -125,6 +132,10 @@ let
   l2MeshOpts = with lib.types; { name, ... }: {
     options = {
       interface = mkOpt' str name "Name of VXLAN interface.";
+      ipv6 = mkBoolOpt' false "Whether this mesh's underlay operates over IPv6.";
+      baseMTU = mkOpt' ints.unsigned 1500 "Base MTU to calculate VXLAN MTU with.";
+      l3Overhead = mkOpt' ints.unsigned 40 "Overhead of L3 header (to calculate MTU).";
+      udpEncapsulation = mkBoolOpt' false "Whether to encapsulate ESP frames in UDP.";
       firewall = mkBoolOpt' true "Whether to generate firewall rules.";
       vni = mkOpt' ints.unsigned 1 "VXLAN VNI.";
       peers = mkOpt' (attrsOf (submodule l2PeerOpts)) { } "Peers.";
@@ -181,6 +192,11 @@ in
       secretsPath = mkOpt' path null "Path to encrypted secret files.";
       modules = mkOpt' (attrsOf commonOpts.moduleType) { } "NixOS modules to be exported by nixfiles.";
       systems = mkOpt' (attrsOf (submodule systemOpts)) { } "NixOS systems to be exported by nixfiles.";
+      allAssignments = mkOption {
+        type = attrsOf (attrsOf (submodule assignmentOpts));
+        description = "All network assignments.";
+        readOnly = true;
+      };
       vpns = {
         l2 = mkOpt' (attrsOf (submodule l2MeshOpts)) { } "Layer 2 meshes.";
       };
@@ -206,5 +222,9 @@ in
         message = "Duplicate assignments: ${toString dupIPs}";
       }
     ];
+
+    nixos = {
+      inherit allAssignments;
+    };
   };
 }

nixos/installer.nix

@@ -1,7 +1,7 @@
 {
   nixos.systems.installer = { config, ... }: {
     system = "x86_64-linux";
-    nixpkgs = "unstable";
+    nixpkgs = "mine";
     docCustom = false;
     rendered = config.configuration.config.my.asISO;
 
@@ -32,7 +32,8 @@
           };
 
           isoImage = {
-            isoBaseName = "nixos-installer-devplayer0";
+            isoBaseName = "jackos-installer";
+            volumeID = "jackos-${config.system.nixos.release}-${pkgs.stdenv.hostPlatform.uname.processor}";
             edition = "devplayer0";
             appendToMenuLabel = " /dev/player0 Installer";
           };
@@ -51,6 +52,8 @@
             home.shellAliases = {
               show-hw-config = "nixos-generate-config --show-hardware-config --root $INSTALL_ROOT";
             };
+
+            my.gui.enable = false;
           };
 
           services = {
@@ -58,8 +61,8 @@
           };
 
           networking = {
-            # Will be set dynamically
+            # Will be set dynamically, but need something to satisfy `/etc/os-release` stuff
-            hostName = "";
+            hostName = "installer";
             useNetworkd = false;
           };
 
@@ -79,6 +82,8 @@
                 ${pkgs.gawk}/bin/awk '{ print $1 }')"
             '';
 
+          boot.supportedFilesystems.nfs = true;
+
           environment.systemPackages = with pkgs; [
             dhcpcd
             lm_sensors

nixos/modules/_list.nix

@@ -17,5 +17,9 @@
     gui = ./gui.nix;
     l2mesh = ./l2mesh.nix;
     borgthin = ./borgthin.nix;
+    nvme = ./nvme;
+    spdk = ./spdk.nix;
+    librespeed = ./librespeed;
+    netboot = ./netboot;
   };
 }

nixos/modules/build.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, extendModules, modulesPath, options, config, ... }:
 let
-  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional;
+  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional;
   inherit (lib.my) mkBoolOpt' dummyOption;
 
   cfg = config.my.build;
@@ -18,6 +18,9 @@ let
       "${modulesPath}/installer/cd-dvd/iso-image.nix"
       allHardware
       {
+        # Doesn't work right now... (missing /dev/root)
+        boot.initrd.systemd.enable = false;
+
         isoImage = {
           makeEfiBootable = true;
           makeUsbBootable = true;
@@ -43,6 +46,146 @@ let
     ];
   };
 
+  asNetboot = extendModules {
+    modules = flatten [
+      allHardware
+      ({ pkgs, config, ... }: {
+        boot = {
+          loader.grub.enable = false;
+          initrd = {
+            kernelModules = [ "nbd" ];
+            availableKernelModules = [ "igb" "igc" ];
+
+            systemd = {
+              storePaths = with pkgs; [
+                gnused
+                nbd
+                netcat
+              ];
+              extraBin = with pkgs; {
+                dmesg = "${util-linux}/bin/dmesg";
+                ip = "${iproute2}/bin/ip";
+                nbd-client = "${nbd}/bin/nbd-client";
+              };
+              extraConfig = ''
+                DefaultTimeoutStartSec=20
+                DefaultDeviceTimeoutSec=20
+              '';
+
+              network = {
+                enable = true;
+                wait-online.enable = true;
+
+                networks."10-netboot" = {
+                  matchConfig.Name = "et-boot";
+                  DHCP = "yes";
+                };
+              };
+
+              services = {
+                nbd = {
+                  description = "NBD Root FS";
+
+                  script = ''
+                    get_cmdline() {
+                      ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
+                    }
+
+                    s="$(get_cmdline nbd_server)"
+                    until ${pkgs.netcat}/bin/nc -zv "$s" 22; do
+                      sleep 0.1
+                    done
+
+                    exec ${pkgs.nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0
+                  '';
+                  unitConfig = {
+                    IgnoreOnIsolate = "yes";
+                    DefaultDependencies = "no";
+                  };
+                  serviceConfig = {
+                    Type = "forking";
+                    Restart = "on-failure";
+                    RestartSec = 10;
+                  };
+
+                  wantedBy = [ "initrd-root-device.target" ];
+                };
+              };
+            };
+          };
+
+          postBootCommands = ''
+            # After booting, register the contents of the Nix store
+            # in the Nix database in the COW root.
+            ${config.nix.package}/bin/nix-store --load-db < /nix-path-registration
+
+            # nixos-rebuild also requires a "system" profile and an
+            # /etc/NIXOS tag.
+            touch /etc/NIXOS
+            ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+          '';
+        };
+
+        programs.nbd.enable = true;
+
+        fileSystems = {
+          "/" = {
+            fsType = "ext4";
+            device = "/dev/nbd0";
+            noCheck = true;
+            autoResize = true;
+          };
+        };
+
+        networking.useNetworkd = mkForce true;
+
+        systemd = {
+          network.networks."10-boot" = {
+            matchConfig.Name = "et-boot";
+            DHCP = "yes";
+            networkConfig.KeepConfiguration = "yes";
+          };
+        };
+
+        system.build = {
+          rootImage = pkgs.callPackage "${modulesPath}/../lib/make-ext4-fs.nix" {
+            storePaths = [ config.system.build.toplevel ];
+            volumeLabel = "netboot-root";
+          };
+          netbootScript = pkgs.writeText "boot.ipxe" ''
+            #!ipxe
+            kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-boot:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline}
+            initrd initrd
+            boot
+          '';
+
+          netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [
+            {
+              name = config.system.boot.loader.kernelFile;
+              path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
+            }
+            {
+              name = "initrd";
+              path = "${config.system.build.initialRamdisk}/initrd";
+            }
+            {
+              name = "rootfs.ext4";
+              path = config.system.build.rootImage;
+            }
+            {
+              name = "boot.ipxe";
+              path = config.system.build.netbootScript;
+            }
+          ];
+          netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar.zst" { } ''
+            export PATH=${pkgs.zstd}/bin:$PATH
+            ${pkgs.gnutar}/bin/tar --dereference --zstd -cvC ${config.system.build.netbootTree} -f "$out" .
+          '';
+        };
+      })
+    ];
+  };
+
   mkAsOpt = ext: desc: with lib.types; mkOption {
     type = unspecified;
     default = ext;
@@ -64,6 +207,7 @@ in
       asISO = mkAsOpt asISO "a bootable .iso image";
       asContainer = mkAsOpt asContainer "a container";
       asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd";
+      asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script";
 
       buildAs = options.system.build;
     };
@@ -79,6 +223,7 @@ in
     };
     isoImage = {
       isoBaseName = dummyOption;
+      volumeID = dummyOption;
       edition = dummyOption;
       appendToMenuLabel = dummyOption;
     };
@@ -96,6 +241,8 @@ in
         iso = config.my.asISO.config.system.build.isoImage;
         container = config.my.asContainer.config.system.build.toplevel;
         kexecTree = config.my.asKexecTree.config.system.build.kexecTree;
+        netbootTree = config.my.asNetboot.config.system.build.netbootTree;
+        netbootArchive = config.my.asNetboot.config.system.build.netbootArchive;
       };
     };
   };

nixos/modules/common.nix

@@ -1,7 +1,7 @@
-{ lib, pkgs, pkgs', inputs, config, ... }:
+{ lib, pkgsFlake, pkgs, pkgs', self, inputs, config, ... }:
 let
   inherit (lib) mkIf mkDefault mkMerge;
-  inherit (lib.my) mkBoolOpt' dummyOption;
+  inherit (lib.my) mkDefault';
 in
 {
   options = with lib.types; {
@@ -12,7 +12,6 @@ in
     inputs.impermanence.nixosModule
     inputs.ragenix.nixosModules.age
     inputs.sharry.nixosModules.default
-    inputs.attic.nixosModules.atticd
   ];
 
   config = mkMerge [
@@ -41,6 +40,7 @@ in
 
       nix = {
         package = pkgs'.mine.nix;
+        channel.enable = false;
         settings = with lib.my.c.nix; {
           trusted-users = [ "@wheel" ];
           experimental-features = [ "nix-command" "flakes" "ca-derivations" ];
@@ -53,7 +53,7 @@ in
           pkgs = {
             to = {
               type = "path";
-              path = "${pkgs.path}";
+              path = "${pkgsFlake}";
             };
             exact = true;
           };
@@ -88,12 +88,13 @@ in
 
       boot = {
         # Use latest LTS release by default
-        kernelPackages = mkDefault pkgs.linuxKernel.packages.linux_6_1;
+        kernelPackages = mkDefault (lib.my.c.kernel.lts pkgs);
         kernel = {
           sysctl = {
             "net.ipv6.route.max_size" = mkDefault 16384;
           };
         };
+
         loader = {
           efi = {
             efiSysMountPoint = mkDefault "/boot";
@@ -111,12 +112,27 @@ in
             memtest86.enable = mkDefault true;
           };
         };
+
+        initrd = {
+          systemd = {
+            enable = mkDefault true;
+            emergencyAccess = mkDefault true;
+          };
+          services.lvm.enable = mkDefault true;
+        };
+      };
+      system = {
+        nixos = {
+          distroName = mkDefault' "JackOS";
+        };
       };
 
+      environment.etc = {
+        "nixos/flake.nix".source = "/run/nixfiles/flake.nix";
+      };
       environment.systemPackages = with pkgs; mkMerge [
         [
           bash-completion
-          vim
           git
           unzip
         ]
@@ -129,6 +145,10 @@ in
         fish.enable = mkDefault true;
         # TODO: This is expecting to look up the channel for the database...
         command-not-found.enable = mkDefault false;
+        vim = {
+          enable = true;
+          defaultEditor = true;
+        };
       };
 
       services = {
@@ -142,6 +162,7 @@ in
               font-name=SauceCodePro Nerd Font Mono
             '';
         };
+        getty.greetingLine = mkDefault' ''<<< Welcome to ${config.system.nixos.distroName} ${config.system.nixos.label} (\m) - \l >>>'';
 
         openssh = {
           enable = mkDefault true;
@@ -194,14 +215,35 @@ in
             # python.d plugin script does #!/usr/bin/env bash
             path = with pkgs; [ bash ];
           };
+
+          nixfiles-mutable = {
+            description = "Mutable nixfiles";
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+            };
+
+            path = with pkgs; [ util-linux ];
+            script = ''
+              nixfilesDir="${self}"
+
+              mkdir -p /run/nixfiles{,/.rw,/.work}
+              mount -t overlay overlay -o lowerdir="$nixfilesDir",upperdir=/run/nixfiles/.rw,workdir=/run/nixfiles/.work /run/nixfiles
+              chmod -R u+w /run/nixfiles
+            '';
+            preStop = ''
+              umount /run/nixfiles
+              rm -rf /run/nixfiles
+            '';
+
+            wantedBy = [ "multi-user.target" ];
+          };
         };
       };
     }
     (mkIf config.services.kmscon.enable {
       fonts.fonts = with pkgs; [
-        (nerdfonts.override {
+        nerd-fonts.sauce-code-pro
-          fonts = [ "SourceCodePro" ];
-        })
       ];
     })
   ];

nixos/modules/containers.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, options, config, systems, ... }:
 let
-  inherit (builtins) attrNames attrValues all hashString toJSON;
+  inherit (builtins) attrNames attrValues all hashString toJSON any;
   inherit (lib)
     groupBy' mapAttrsToList optionalString optional concatMapStringsSep filterAttrs mkOption mkDefault mkIf mkMerge;
   inherit (lib.my) mkOpt' mkBoolOpt';
@@ -98,6 +98,7 @@ let
       };
       networking = {
         bridge = mkOpt' (nullOr str) null "Network bridge to connect to.";
+        macVLAN = mkOpt' (nullOr str) null "Network interface to make MACVLAN interface from.";
       };
     };
   };
@@ -115,26 +116,19 @@ in
           assertion = config.systemd.network.enable;
           message = "Containers currently require systemd-networkd!";
         }
+        {
+          assertion = all (i: i.networking.bridge == null || i.networking.macVLAN == null) (attrValues cfg.instances);
+          message = "Only bridge OR MACVLAN can be set";
+        }
       ];
 
       # TODO: Better security
       my.firewall.trustedInterfaces =
         mapAttrsToList
           (n: _: "ve-${n}")
-          (filterAttrs (_: c: c.networking.bridge == null) cfg.instances);
+          (filterAttrs (_: c: c.networking.bridge == null && c.networking.macVLAN == null) cfg.instances);
 
-      systemd = mkMerge ([
+      systemd = mkMerge (mapAttrsToList (n: c: {
-        {
-          # By symlinking to the original systemd-nspawn@.service for every instance we force the unit generator to
-          # create overrides instead of replacing the unit entirely
-          packages = [
-            (pkgs.linkFarm "systemd-nspawn-containers" (map (n: {
-              name = "etc/systemd/system/systemd-nspawn@${n}.service";
-              path = "${pkgs.systemd}/example/systemd/system/systemd-nspawn@.service";
-            }) (attrNames cfg.instances)))
-          ];
-        }
-      ] ++ (mapAttrsToList (n: c: {
         nspawn."${n}" = {
           execConfig = {
             Boot = true;
@@ -165,6 +159,8 @@ in
           };
           networkConfig = if (c.networking.bridge != null) then {
             Bridge = c.networking.bridge;
+          } else if (c.networking.macVLAN != null) then {
+            MACVLAN = "${c.networking.macVLAN}:host0";
           } else {
             VirtualEthernet = true;
           };
@@ -182,6 +178,9 @@ in
             c.containerSystem;
         in
         {
+          # To prevent creating a whole new unit file
+          overrideStrategy = "asDropin";
+
           environment = {
             # systemd.nspawn units can't set the root directory directly, but /run/machines/${n} is one of the search paths
             root = "/run/machines/${n}";
@@ -208,7 +207,17 @@ in
 
             mkdir -p -m 0755 "$root"/sbin "$root"/etc
             touch "$root"/etc/os-release
-            ln -sf "${containerSystem}"/init "$root"/sbin/init
+
+            ${if system == sysProfile then ''
+              if [ -e "${sysProfile}"/prepare-root ]; then
+                initSource="${containerSystem}"/prepare-root
+              else
+                initSource="${containerSystem}"/init
+              fi
+              ln -sf "$initSource" "$root"/sbin/init
+            '' else ''
+              ln -sf "${containerSystem}/prepare-root" "$root"/sbin/init
+            ''}
           '';
           postStop =
           ''
@@ -237,7 +246,7 @@ in
             Bridge = c.networking.bridge;
           };
         };
-      }) cfg.instances));
+      }) cfg.instances);
     })
 
     # Inside container

nixos/modules/deploy-rs.nix

@@ -15,16 +15,20 @@ let
   # Based on https://github.com/serokell/deploy-rs/blob/master/flake.nix
   nixosActivate = cfg': base: (pkgs.deploy-rs.lib.activate.custom // {
     dryActivate = "$PROFILE/bin/switch-to-configuration dry-activate";
-    boot = "$PROFILE/bin/switch-to-configuration boot";
+    boot = ''
+      $PROFILE/bin/switch-to-configuration boot
+
+      ${keepGensSnippet "$PROFILE" cfg'.keepGenerations}
+    '';
   }) base.config.system.build.toplevel ''
     # work around https://github.com/NixOS/nixpkgs/issues/73404
     cd /tmp
 
-    "$PROFILE"/bin/switch-to-configuration ${cfg'.mode}
+    "$PROFILE"/bin/switch-to-configuration switch
 
     # https://github.com/serokell/deploy-rs/issues/31
     ${with base.config.boot.loader;
-    optionalString ((cfg'.mode == "switch" || cfg'.mode == "boot") && systemd-boot.enable)
+    optionalString systemd-boot.enable
     "sed -i '/^default /d' ${efi.efiSysMountPoint}/loader/loader.conf"}
 
     ${keepGensSnippet "$PROFILE" cfg'.keepGenerations}
@@ -59,7 +63,11 @@ let
   {
     name = "container-${n}";
     value = {
-      path = pkgs.deploy-rs.lib.activate.custom ctrConfig.my.buildAs.container ''
+      path = (pkgs.deploy-rs.lib.activate.custom // {
+        boot = ''
+          echo "Next systemd-nspawn@${n}.service restart / reload will load config"
+        '';
+      }) ctrConfig.my.buildAs.container ''
         source ${systemdUtil}/bin/systemd-util.sh
         ${if c.hotReload then ''
           if (! systemctl show -p ActiveState systemd-nspawn@${n} | grep -q "ActiveState=active") || \

nixos/modules/firewall.nix

@@ -1,6 +1,9 @@
 { lib, options, config, ... }:
 let
-  inherit (lib) optionalString concatStringsSep concatMapStringsSep optionalAttrs mkIf mkDefault mkMerge mkOverride;
+  inherit (builtins) typeOf attrNames;
+  inherit (lib)
+    optionalString concatStringsSep concatMapStringsSep mapAttrsToList optionalAttrs mkIf
+    mkDefault mkMerge mkOverride;
   inherit (lib.my) isIPv6 mkOpt' mkBoolOpt';
 
   allowICMP = ''
@@ -63,8 +66,8 @@ in
 
     nat = with options.networking.nat; {
       enable = mkBoolOpt' true "Whether to enable IP forwarding and NAT.";
-      inherit externalInterface externalIP;
+      inherit externalInterface;
-      forwardPorts = mkOpt' (listOf (submodule forwardOpts)) [ ] "List of port forwards.";
+      forwardPorts = mkOpt' (either (listOf (submodule forwardOpts)) (attrsOf (listOf (submodule forwardOpts)))) [ ] "IPv4 port forwards";
     };
   };
 
@@ -131,9 +134,15 @@ in
                 chain prerouting {
                   type nat hook prerouting priority dstnat;
                 }
+                chain output {
+                  type nat hook output priority dstnat;
+                }
                 chain postrouting {
                   type nat hook postrouting priority srcnat;
                 }
+                chain input {
+                  type nat hook input priority srcnat;
+                }
               }
 
               ${cfg.extraRules}
@@ -141,11 +150,16 @@ in
         };
       };
     }
-    (mkIf cfg.nat.enable {
+    (mkIf cfg.nat.enable (
+    let
+      iifForward = typeOf cfg.nat.forwardPorts == "list" && cfg.nat.forwardPorts != [ ];
+      dipForward = typeOf cfg.nat.forwardPorts == "set" && cfg.nat.forwardPorts != { };
+    in
+    {
       assertions = [
         {
-          assertion = with cfg.nat; (forwardPorts != [ ]) -> (externalInterface != null);
+          assertion = with cfg.nat; iifForward -> (externalInterface != null);
-          message = "my.firewall.nat.forwardPorts requires my.firewall.nat.external{Interface,IP}";
+          message = "my.firewall.nat.forwardPorts as list requires my.firewall.nat.externalInterface";
         }
       ];
 
@@ -168,43 +182,75 @@ in
 
       my.firewall.extraRules =
         let
+          inherit (lib.my.nft) natFilterChain dnatChain;
+          ipK = ip: "ip${optionalString (isIPv6 ip) "6"}";
+
           makeFilter = f:
-          let
+            "${ipK f.dst} daddr ${f.dst} ${f.proto} dport ${toString f.dstPort} accept";
-            v6 = isIPv6 f.dst;
-          in
-            "ip${optionalString v6 "6"} daddr ${f.dst} ${f.proto} dport ${toString f.dstPort} accept";
           makeForward = f:
-            let
+            "${f.proto} dport ${toString f.port} dnat ${ipK f.dst} to ${f.dst}:${toString f.dstPort}";
-              v6 = isIPv6 f.dst;
+
-            in
+          dnatJumps = ''
-              "${f.proto} dport ${toString f.port} dnat ip${optionalString v6 "6"} to ${f.dst}:${toString f.dstPort}";
+            ${optionalString
+              iifForward
+              "iifname ${cfg.nat.externalInterface} jump iif-port-forward"}
+            ${optionalString
+              dipForward
+              (concatMapStringsSep "\n    " (ip: "${ipK ip} daddr ${ip} jump ${dnatChain ip}") (attrNames cfg.nat.forwardPorts))}
+          '';
         in
         ''
           table inet filter {
-            chain filter-port-forwards {
+            ${optionalString iifForward ''
-              ${concatMapStringsSep "\n    " makeFilter cfg.nat.forwardPorts}
+              chain filter-iif-port-forwards {
-              return
+                ${concatMapStringsSep "\n    " makeFilter cfg.nat.forwardPorts}
-            }
+                return
+              }
+            ''}
+            ${optionalString
+              dipForward
+              (concatStringsSep "\n" (mapAttrsToList (ip: fs: ''
+                chain ${natFilterChain ip} {
+                  ${concatMapStringsSep "\n    " makeFilter fs}
+                  return
+                }
+              '') cfg.nat.forwardPorts))}
+
             chain forward {
               ${optionalString
-                (cfg.nat.externalInterface != null)
+                iifForward
-                "iifname ${cfg.nat.externalInterface} jump filter-port-forwards"}
+                "iifname ${cfg.nat.externalInterface} jump filter-iif-port-forwards"}
+              ${optionalString
+                dipForward
+                (concatMapStringsSep "\n    " (ip: "jump ${natFilterChain ip}") (attrNames cfg.nat.forwardPorts))}
             }
           }
 
           table inet nat {
-            chain port-forward {
+            ${optionalString iifForward ''
-              ${concatMapStringsSep "\n    " makeForward cfg.nat.forwardPorts}
+              chain iif-port-forward {
-              return
+                ${concatMapStringsSep "\n    " makeForward cfg.nat.forwardPorts}
-            }
+                return
+              }
+            ''}
+            ${optionalString
+              dipForward
+              (concatStringsSep "\n" (mapAttrsToList (ip: fs: ''
+                chain ${dnatChain ip} {
+                  ${concatMapStringsSep "\n    " makeForward fs}
+                  return
+                }
+              '') cfg.nat.forwardPorts))}
+
             chain prerouting {
-              ${optionalString
+              ${dnatJumps}
-                (cfg.nat.externalInterface != null)
+            }
-                "${if (cfg.nat.externalIP != null) then "ip daddr ${cfg.nat.externalIP}" else "iifname ${cfg.nat.externalInterface}"} jump port-forward"}
+            chain output {
+              ${dnatJumps}
             }
           }
         '';
-    })
+    }))
   ]);
 
   meta.buildDocsInSandbox = false;

nixos/modules/gui.nix

@@ -12,7 +12,7 @@ in
 
   config = mkIf cfg.enable {
     hardware = {
-      opengl.enable = mkDefault true;
+      graphics.enable = mkDefault true;
     };
 
     systemd = {
@@ -23,13 +23,13 @@ in
 
     security = {
       polkit.enable = true;
-      pam.services.swaylock = {};
+      pam.services.swaylock-plugin = {};
     };
 
     environment.systemPackages = with pkgs; [
       # for pw-jack
       pipewire.jack
-      swaylock
+      swaylock-plugin
     ];
     services = {
       pipewire = {
@@ -51,13 +51,17 @@ in
           SUBSYSTEM=="usb", ATTR{idVendor}=="0955", MODE="0664", GROUP="wheel"
           # Nintendo
           SUBSYSTEM=="usb", ATTR{idVendor}=="057e", MODE="0664", GROUP="wheel"
+          # FT
+          SUBSYSTEM=="usb", ATTR{idVendor}=="0403", MODE="0664", GROUP="wheel"
+          # /dev/player0
+          SUBSYSTEM=="usb", ATTR{idVendor}=="6969", MODE="0664", GROUP="wheel"
         '';
       };
     };
 
     programs.dconf.enable = true;
 
-    fonts.fonts = with pkgs; [
+    fonts.packages = with pkgs; [
       dejavu_fonts
       freefont_ttf
       gyre-fonts # TrueType substitutes for standard PostScript fonts
@@ -69,8 +73,19 @@ in
     xdg = {
       portal = {
         enable = true;
+        extraPortals = with pkgs; [
+          xdg-desktop-portal-gtk
+        ];
         # For sway
         wlr.enable = true;
+        configPackages = [
+          (pkgs.writeTextDir "share/xdg-desktop-portal/sway-portals.conf" ''
+            [preferred]
+            default=gtk
+            org.freedesktop.impl.portal.Screenshot=wlr
+            org.freedesktop.impl.portal.ScreenCast=wlr
+          '')
+        ];
       };
     };
   };

nixos/modules/l2mesh.nix

@@ -1,7 +1,8 @@
-{ lib, pkgs, config, vpns, ... }:
+{ lib, config, vpns, ... }:
 let
-  inherit (lib) optionalString mapAttrsToList concatStringsSep filterAttrs mkIf mkMerge;
+  inherit (builtins) any attrValues;
-  inherit (lib.my) isIPv6;
+  inherit (lib) optionalString mapAttrsToList concatStringsSep concatMapStringsSep filterAttrs mkIf mkMerge;
+  inherit (lib.my) isIPv6 mkOpt';
 
   vxlanPort = 4789;
 
@@ -24,38 +25,32 @@ let
         Local = ownAddr;
         MacLearning = true;
         DestinationPort = vxlanPort;
+        PortRange = "${toString vxlanPort}-${toString (vxlanPort + 1)}";
         Independent = true;
       };
     };
-    links."20-l2mesh-${name}" = {
-      matchConfig.Name = mesh.interface;
-      # TODO: ipv6? ipsec?
-      linkConfig.MTUBytes = "1450";
-    };
     networks."90-l2mesh-${name}" = {
       matchConfig.Name = mesh.interface;
-      extraConfig = concatStringsSep "\n" (mapAttrsToList (n: peer: ''
+      linkConfig.MTUBytes =
-        [BridgeFDB]
+      let
-        MACAddress=00:00:00:00:00:00
+        espOverhead =
-        Destination=${peer.addr}
+          if (!mesh.security.enable) then 0
-      '') otherPeers);
+          else
+            # UDP encap + SPI + seq + IV + pad / header + ICV
+            (if mesh.udpEncapsulation then 8 else 0) + 4 + 4 + (if mesh.security.encrypt then 8 else 0) + 2 + 16;
+        # UDP + VXLAN + Ethernet + L3 (IPv4/IPv6)
+        overhead = espOverhead + 8 + 8 + 14 + mesh.l3Overhead;
+      in
+      toString (mesh.baseMTU - overhead);
+
+      bridgeFDBs = mapAttrsToList (n: peer: {
+        MACAddress = "00:00:00:00:00:00";
+        Destination = peer.addr;
+      }) otherPeers;
     };
   };
 
-  mkLibreswanConfig = name: mesh: with info mesh; {
+  vxlanAllow = vni: "udp dport ${toString vxlanPort} @th,96,24 ${toString vni} accept";
-    enable = true;
-    # TODO: finish this...
-    connections."l2mesh-${name}" = ''
-      keyexchange=ike
-      type=transport
-      left=${ownAddr}
-
-      auto=start
-      phase2=esp
-      ikev2=yes
-    '';
-  };
-
   mkFirewallConfig = name: mesh: with info mesh;
   let
     netProto = if (isIPv6 ownAddr) then "ip6" else "ip";
@@ -63,8 +58,15 @@ let
   ''
     table inet filter {
       chain l2mesh-${name} {
-        ${optionalString mesh.security.enable "meta l4proto esp accept"}
+        ${optionalString mesh.security.enable ''
-        udp dport ${toString vxlanPort} @th,96,24 ${toString mesh.vni} accept
+          udp dport isakmp accept
+          ${if mesh.udpEncapsulation then ''
+            udp dport ipsec-nat-t accept
+          '' else ''
+            meta l4proto esp accept
+          ''}
+        ''}
+        ${optionalString (!mesh.security.enable) (vxlanAllow mesh.vni)}
         return
       }
       chain input {
@@ -72,12 +74,64 @@ let
       }
     }
   '';
+
+  mkLibreswanConfig = name: mesh: with info mesh; {
+    enable = true;
+    connections = mkMerge (mapAttrsToList
+      (pName: peer: {
+        "l2mesh-${name}-${pName}" = ''
+          keyexchange=ike
+          hostaddrfamily=ipv${if mesh.ipv6 then "6" else "4"}
+          type=transport
+
+          left=${ownAddr}
+          leftprotoport=udp/${toString vxlanPort}
+          right=${peer.addr}
+          rightprotoport=udp/${toString vxlanPort}
+          rightupdown=
+
+          auto=start
+          authby=secret
+          phase2=esp
+          esp=${if mesh.security.encrypt then "aes_gcm256" else "null-sha256"}
+          ikev2=yes
+          modecfgpull=no
+          encapsulation=${if mesh.udpEncapsulation then "yes" else "no"}
+        '';
+      })
+    otherPeers);
+  };
+  genSecrets = name: mesh: with info mesh; concatMapStringsSep "\n" (p: ''
+    echo "${ownAddr} ${p.addr} : PSK \"$(< "${config.my.vpns.l2.pskFiles.${name}}")\"" >> /run/l2mesh.secrets
+  '') (attrValues otherPeers);
+  anySecurity = any (c: c.security.enable) (attrValues memberMeshes);
 in
 {
+  options = {
+    my.vpns.l2 = with lib.types; {
+      pskFiles = mkOpt' (attrsOf str) { } "PSK files for secured meshes.";
+    };
+  };
+
   config = {
     systemd.network = mkMerge (mapAttrsToList mkNetConfig memberMeshes);
-    # TODO: finish this...
+
-    #services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
+    environment.etc."ipsec.d/l2mesh.secrets" = mkIf anySecurity {
+      source = "/run/l2mesh.secrets";
+    };
+    systemd.services.ipsec = mkIf anySecurity {
+      preStart = ''
+        oldUmask="$(umask)"
+        umask 006
+
+        > /run/l2mesh.secrets
+        ${concatStringsSep "\n" (mapAttrsToList genSecrets memberMeshes)}
+
+        umask "$oldUmask"
+      '';
+    };
+
+    services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
     my.firewall.extraRules = concatStringsSep "\n" (mapAttrsToList mkFirewallConfig (filterAttrs (_: c: c.firewall) memberMeshes));
   };
 }

nixos/modules/librespeed/default.nix

@@ -0,0 +1,76 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (builtins) toJSON;
+  inherit (lib) mkOption mkMerge mkIf mkDefault;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  cfg = config.my.librespeed;
+
+  serversConf = map (s: s // {
+    dlURL = "backend/garbage";
+    ulURL = "backend/empty";
+    pingURL = "backend/empty";
+    getIpURL = "backend/getIP";
+  }) cfg.frontend.servers;
+  frontendTree = pkgs.runCommand "librespeed-frontend" {
+    speedtestServers = toJSON serversConf;
+  } ''
+    mkdir "$out"
+    cp "${pkgs.librespeed-go}"/assets/* "$out"/
+    substitute ${./index.html} "$out"/index.html --subst-var speedtestServers
+  '';
+
+  backendConf = pkgs.writers.writeTOML "librespeed.toml" cfg.backend.settings;
+  generateBackendSettings = base: dst: if (cfg.backend.extraSettingsFile != null) then ''
+    oldUmask="$(umask)"
+    umask 006
+    cat "${base}" "${cfg.backend.extraSettingsFile}" > "${dst}"
+    umask "$oldUmask"
+  '' else ''
+    cp "${base}" "${dst}"
+  '';
+in
+{
+  options.my.librespeed = with lib.types; {
+    frontend = {
+      servers = mkOpt' (listOf (attrsOf unspecified)) { } "Server configs.";
+      webroot = mkOption {
+        description = "Frontend webroot.";
+        type = package;
+        readOnly = true;
+      };
+    };
+    backend = {
+      enable = mkBoolOpt' false "Whether to enable librespeed backend.";
+      settings = mkOpt' (attrsOf unspecified) { } "Backend settings.";
+      extraSettingsFile = mkOpt' (nullOr str) null "Extra settings file.";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf (cfg.frontend.servers != { }) {
+      my.librespeed.frontend.webroot = frontendTree;
+    })
+    (mkIf cfg.backend.enable {
+      my.librespeed.backend.settings = {
+        assets_path = frontendTree;
+        database_type = mkDefault "bolt";
+        database_file = mkDefault "/var/lib/librespeed-go/speedtest.db";
+      };
+
+      systemd.services.librespeed = {
+        description = "LibreSpeed Go backend";
+        wants = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+
+        preStart = generateBackendSettings backendConf "/run/librespeed-go/settings.toml";
+        serviceConfig = {
+          ExecStart = "${pkgs.librespeed-go}/bin/speedtest -c /run/librespeed-go/settings.toml";
+          RuntimeDirectory = "librespeed-go";
+          StateDirectory = "librespeed-go";
+        };
+        wantedBy = [ "multi-user.target" ];
+      };
+    })
+  ];
+}

nixos/modules/librespeed/index.html

@@ -0,0 +1,491 @@
+<!DOCTYPE html>
+<html>
+<head>
+<link rel="shortcut icon" href="favicon.ico">
+<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no, user-scalable=no" />
+<meta charset="UTF-8" />
+<script type="text/javascript" src="speedtest.js"></script>
+<script type="text/javascript">
+function I(i){return document.getElementById(i);}
+
+//LIST OF TEST SERVERS. See documentation for details if needed
+var SPEEDTEST_SERVERS=@speedtestServers@;
+// var SPEEDTEST_SERVERS=[
+// 	{	//this server doesn't actually exist, remove it
+// 		name:"Example Server 1", //user friendly name for the server
+// 		server:"//test1.mydomain.com/", //URL to the server. // at the beginning will be replaced with http:// or https:// automatically
+// 		dlURL:"backend/garbage.php",  //path to download test on this server (garbage.php or replacement)
+// 		ulURL:"backend/empty.php",  //path to upload test on this server (empty.php or replacement)
+// 		pingURL:"backend/empty.php",  //path to ping/jitter test on this server (empty.php or replacement)
+// 		getIpURL:"backend/getIP.php"  //path to getIP on this server (getIP.php or replacement)
+// 	},
+// 	{	//this server doesn't actually exist, remove it
+// 		name:"Example Server 2", //user friendly name for the server
+// 		server:"//test2.example.com/", //URL to the server. // at the beginning will be replaced with http:// or https:// automatically
+// 		dlURL:"garbage.php",  //path to download test on this server (garbage.php or replacement)
+// 		ulURL:"empty.php",  //path to upload test on this server (empty.php or replacement)
+// 		pingURL:"empty.php",  //path to ping/jitter test on this server (empty.php or replacement)
+// 		getIpURL:"getIP.php"  //path to getIP on this server (getIP.php or replacement)
+// 	}
+// 	//add other servers here, comma separated
+// ];
+
+//INITIALIZE SPEEDTEST
+var s=new Speedtest(); //create speed test object
+s.setParameter("telemetry_level","basic"); //enable telemetry
+
+//SERVER AUTO SELECTION
+function initServers(){
+    var noServersAvailable=function(){
+        I("message").innerHTML="No servers available";
+    }
+    var runServerSelect=function(){
+        s.selectServer(function(server){
+            if(server!=null){ //at least 1 server is available
+                I("loading").className="hidden"; //hide loading message
+                //populate server list for manual selection
+                for(var i=0;i<SPEEDTEST_SERVERS.length;i++){
+                    if(SPEEDTEST_SERVERS[i].pingT==-1) continue;
+                    var option=document.createElement("option");
+                    option.value=i;
+                    option.textContent=SPEEDTEST_SERVERS[i].name;
+                    if(SPEEDTEST_SERVERS[i]===server) option.selected=true;
+                    I("server").appendChild(option);
+                }
+                //show test UI
+                I("testWrapper").className="visible";
+                initUI();
+            }else{ //no servers are available, the test cannot proceed
+                noServersAvailable();
+            }
+        });
+    }
+    if(typeof SPEEDTEST_SERVERS === "string"){
+        //need to fetch list of servers from specified URL
+        s.loadServerList(SPEEDTEST_SERVERS,function(servers){
+            if(servers==null){ //failed to load server list
+                noServersAvailable();
+            }else{ //server list loaded
+                SPEEDTEST_SERVERS=servers;
+                runServerSelect();
+            }
+        });
+    }else{
+        //hardcoded server list
+        s.addTestPoints(SPEEDTEST_SERVERS);
+        runServerSelect();
+    }
+}
+
+var meterBk=/Trident.*rv:(\d+\.\d+)/i.test(navigator.userAgent)?"#EAEAEA":"#80808040";
+var dlColor="#6060AA",
+	ulColor="#616161";
+var progColor=meterBk;
+
+//CODE FOR GAUGES
+function drawMeter(c,amount,bk,fg,progress,prog){
+	var ctx=c.getContext("2d");
+	var dp=window.devicePixelRatio||1;
+	var cw=c.clientWidth*dp, ch=c.clientHeight*dp;
+	var sizScale=ch*0.0055;
+	if(c.width==cw&&c.height==ch){
+		ctx.clearRect(0,0,cw,ch);
+	}else{
+		c.width=cw;
+		c.height=ch;
+	}
+	ctx.beginPath();
+	ctx.strokeStyle=bk;
+	ctx.lineWidth=12*sizScale;
+	ctx.arc(c.width/2,c.height-58*sizScale,c.height/1.8-ctx.lineWidth,-Math.PI*1.1,Math.PI*0.1);
+	ctx.stroke();
+	ctx.beginPath();
+	ctx.strokeStyle=fg;
+	ctx.lineWidth=12*sizScale;
+	ctx.arc(c.width/2,c.height-58*sizScale,c.height/1.8-ctx.lineWidth,-Math.PI*1.1,amount*Math.PI*1.2-Math.PI*1.1);
+	ctx.stroke();
+	if(typeof progress !== "undefined"){
+		ctx.fillStyle=prog;
+		ctx.fillRect(c.width*0.3,c.height-16*sizScale,c.width*0.4*progress,4*sizScale);
+	}
+}
+function mbpsToAmount(s){
+	return 1-(1/(Math.pow(1.3,Math.sqrt(s))));
+}
+function format(d){
+    d=Number(d);
+    if(d<10) return d.toFixed(2);
+    if(d<100) return d.toFixed(1);
+    return d.toFixed(0);
+}
+
+//UI CODE
+var uiData=null;
+function startStop(){
+    if(s.getState()==3){
+		//speed test is running, abort
+		s.abort();
+		data=null;
+		I("startStopBtn").className="";
+		I("server").disabled=false;
+		initUI();
+	}else{
+		//test is not running, begin
+		I("startStopBtn").className="running";
+		I("shareArea").style.display="none";
+		I("server").disabled=true;
+		s.onupdate=function(data){
+            uiData=data;
+		};
+		s.onend=function(aborted){
+            I("startStopBtn").className="";
+            I("server").disabled=false;
+            updateUI(true);
+            if(!aborted){
+                //if testId is present, show sharing panel, otherwise do nothing
+                try{
+                    var testId=uiData.testId;
+                    if(testId!=null){
+                        var shareURL=window.location.href.substring(0,window.location.href.lastIndexOf("/"))+"/results/?id="+testId;
+                        I("resultsImg").src=shareURL;
+                        I("resultsURL").value=shareURL;
+                        I("testId").innerHTML=testId;
+                        I("shareArea").style.display="";
+                    }
+                }catch(e){}
+            }
+		};
+		s.start();
+	}
+}
+//this function reads the data sent back by the test and updates the UI
+function updateUI(forced){
+	if(!forced&&s.getState()!=3) return;
+	if(uiData==null) return;
+	var status=uiData.testState;
+	I("ip").textContent=uiData.clientIp;
+	I("dlText").textContent=(status==1&&uiData.dlStatus==0)?"...":format(uiData.dlStatus);
+	drawMeter(I("dlMeter"),mbpsToAmount(Number(uiData.dlStatus*(status==1?oscillate():1))),meterBk,dlColor,Number(uiData.dlProgress),progColor);
+	I("ulText").textContent=(status==3&&uiData.ulStatus==0)?"...":format(uiData.ulStatus);
+	drawMeter(I("ulMeter"),mbpsToAmount(Number(uiData.ulStatus*(status==3?oscillate():1))),meterBk,ulColor,Number(uiData.ulProgress),progColor);
+	I("pingText").textContent=format(uiData.pingStatus);
+	I("jitText").textContent=format(uiData.jitterStatus);
+}
+function oscillate(){
+	return 1+0.02*Math.sin(Date.now()/100);
+}
+//update the UI every frame
+window.requestAnimationFrame=window.requestAnimationFrame||window.webkitRequestAnimationFrame||window.mozRequestAnimationFrame||window.msRequestAnimationFrame||(function(callback,element){setTimeout(callback,1000/60);});
+function frame(){
+	requestAnimationFrame(frame);
+	updateUI();
+}
+frame(); //start frame loop
+//function to (re)initialize UI
+function initUI(){
+	drawMeter(I("dlMeter"),0,meterBk,dlColor,0);
+	drawMeter(I("ulMeter"),0,meterBk,ulColor,0);
+	I("dlText").textContent="";
+	I("ulText").textContent="";
+	I("pingText").textContent="";
+	I("jitText").textContent="";
+	I("ip").textContent="";
+}
+</script>
+<style type="text/css">
+	html,body{
+		border:none; padding:0; margin:0;
+		background:#FFFFFF;
+		color:#202020;
+	}
+	body{
+		text-align:center;
+		font-family:"Roboto",sans-serif;
+	}
+	h1{
+		color:#404040;
+	}
+	#loading{
+		background-color:#FFFFFF;
+		color:#404040;
+		text-align:center;
+	}
+	span.loadCircle{
+		display:inline-block;
+		width:2em;
+		height:2em;
+		vertical-align:middle;
+		background:url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAMAAAD04JH5AAAAP1BMVEUAAAB2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZ2dnZyFzwnAAAAFHRSTlMAEvRFvX406baecwbf0casimhSHyiwmqgAAADpSURBVHja7dbJbQMxAENRahnN5lkc//5rDRAkDeRgHszXgACJoKiIiIiIiIiIiIiIiIiIiIj4HHspsrpAVhdVVguzrA4OWc10WcEqpwKbnBo0OU1Q5NSpsoJFTgOecrrdEag85DRgktNqfoEdTjnd7hrEHMEJvmRUYJbTYk5Agy6nau6Abp5Cm7mDBtRdPi9gyKdU7w4p1fsLvyqs8hl4z9/w3n/Hmr9WoQ65lAU4d7lMYOz//QboRR5jBZibLMZdAR6O/Vfa1PlxNr3XdS3HzK/HVPRu/KnLs8iAOh993VpRRERERMT/fAN60wwWaVyWwAAAAABJRU5ErkJggg==');
+		background-size:2em 2em;
+		margin-right:0.5em;
+		animation: spin 0.6s linear infinite;
+	}
+	@keyframes spin{
+		0%{transform:rotate(0deg);}
+		100%{transform:rotate(359deg);}
+	}
+	#startStopBtn{
+		display:inline-block;
+		margin:0 auto;
+		color:#6060AA;
+		background-color:rgba(0,0,0,0);
+		border:0.15em solid #6060FF;
+		border-radius:0.3em;
+		transition:all 0.3s;
+		box-sizing:border-box;
+		width:8em; height:3em;
+		line-height:2.7em;
+		cursor:pointer;
+		box-shadow: 0 0 0 rgba(0,0,0,0.1), inset 0 0 0 rgba(0,0,0,0.1);
+	}
+	#startStopBtn:hover{
+		box-shadow: 0 0 2em rgba(0,0,0,0.1), inset 0 0 1em rgba(0,0,0,0.1);
+	}
+	#startStopBtn.running{
+		background-color:#FF3030;
+		border-color:#FF6060;
+		color:#FFFFFF;
+	}
+	#startStopBtn:before{
+		content:"Start";
+	}
+	#startStopBtn.running:before{
+		content:"Abort";
+	}
+	#serverArea{
+		margin-top:1em;
+	}
+	#server{
+		font-size:1em;
+		padding:0.2em;
+	}
+	#test{
+		margin-top:2em;
+		margin-bottom:12em;
+	}
+	div.testArea{
+		display:inline-block;
+		width:16em;
+		height:12.5em;
+		position:relative;
+		box-sizing:border-box;
+	}
+	div.testArea2{
+		display:inline-block;
+		width:14em;
+		height:7em;
+		position:relative;
+		box-sizing:border-box;
+		text-align:center;
+	}
+	div.testArea div.testName{
+		position:absolute;
+		top:0.1em; left:0;
+		width:100%;
+		font-size:1.4em;
+		z-index:9;
+	}
+	div.testArea2 div.testName{
+        display:block;
+        text-align:center;
+        font-size:1.4em;
+	}
+	div.testArea div.meterText{
+		position:absolute;
+		bottom:1.55em; left:0;
+		width:100%;
+		font-size:2.5em;
+		z-index:9;
+	}
+	div.testArea2 div.meterText{
+        display:inline-block;
+        font-size:2.5em;
+	}
+	div.meterText:empty:before{
+		content:"0.00";
+	}
+	div.testArea div.unit{
+		position:absolute;
+		bottom:2em; left:0;
+		width:100%;
+		z-index:9;
+	}
+	div.testArea2 div.unit{
+		display:inline-block;
+	}
+	div.testArea canvas{
+		position:absolute;
+		top:0; left:0; width:100%; height:100%;
+		z-index:1;
+	}
+	div.testGroup{
+		display:block;
+        margin: 0 auto;
+	}
+	#shareArea{
+		width:95%;
+		max-width:40em;
+		margin:0 auto;
+		margin-top:2em;
+	}
+	#shareArea > *{
+		display:block;
+		width:100%;
+		height:auto;
+		margin: 0.25em 0;
+	}
+	#privacyPolicy{
+        position:fixed;
+        top:2em;
+        bottom:2em;
+        left:2em;
+        right:2em;
+        overflow-y:auto;
+        width:auto;
+        height:auto;
+        box-shadow:0 0 3em 1em #000000;
+        z-index:999999;
+        text-align:left;
+        background-color:#FFFFFF;
+        padding:1em;
+	}
+	a.privacy{
+        text-align:center;
+        font-size:0.8em;
+        color:#808080;
+        padding: 0 3em;
+	}
+    div.closePrivacyPolicy {
+        width: 100%;
+        text-align: center;
+    }
+    div.closePrivacyPolicy a.privacy {
+        padding: 1em 3em;
+    }
+	@media all and (max-width:40em){
+		body{
+			font-size:0.8em;
+		}
+	}
+	div.visible{
+		animation: fadeIn 0.4s;
+		display:block;
+	}
+	div.hidden{
+		animation: fadeOut 0.4s;
+		display:none;
+	}
+	@keyframes fadeIn{
+		0%{
+			opacity:0;
+		}
+		100%{
+			opacity:1;
+		}
+	}
+	@keyframes fadeOut{
+		0%{
+			display:block;
+			opacity:1;
+		}
+		100%{
+			display:block;
+			opacity:0;
+		}
+	}
+</style>
+<title>/dev/player0's speedtest</title>
+</head>
+<body onload="initServers()">
+<h1>/dev/player0's speedtest</h1>
+<div id="loading" class="visible">
+	<p id="message"><span class="loadCircle"></span>Selecting a server...</p>
+</div>
+<div id="testWrapper" class="hidden">
+	<div id="startStopBtn" onclick="startStop()"></div><br/>
+	<a class="privacy" href="#" onclick="I('privacyPolicy').style.display=''">Privacy</a>
+	<div id="serverArea">
+		Server: <select id="server" onchange="s.setSelectedServer(SPEEDTEST_SERVERS[this.value])"></select>
+	</div>
+	<div id="test">
+		<div class="testGroup">
+            <div class="testArea2">
+				<div class="testName">Ping</div>
+				<div id="pingText" class="meterText" style="color:#AA6060"></div>
+				<div class="unit">ms</div>
+			</div>
+			<div class="testArea2">
+				<div class="testName">Jitter</div>
+				<div id="jitText" class="meterText" style="color:#AA6060"></div>
+				<div class="unit">ms</div>
+			</div>
+		</div>
+		<div class="testGroup">
+			<div class="testArea">
+				<div class="testName">Download</div>
+				<canvas id="dlMeter" class="meter"></canvas>
+				<div id="dlText" class="meterText"></div>
+				<div class="unit">Mbit/s</div>
+			</div>
+			<div class="testArea">
+				<div class="testName">Upload</div>
+				<canvas id="ulMeter" class="meter"></canvas>
+				<div id="ulText" class="meterText"></div>
+				<div class="unit">Mbit/s</div>
+			</div>
+		</div>
+		<div id="ipArea">
+			<span id="ip"></span>
+		</div>
+		<div id="shareArea" style="display:none">
+			<h3>Share results</h3>
+			<p>Test ID: <span id="testId"></span></p>
+			<input type="text" value="" id="resultsURL" readonly="readonly" onclick="this.select();this.focus();this.select();document.execCommand('copy');alert('Link copied')"/>
+			<img src="" id="resultsImg" />
+		</div>
+	</div>
+	<a href="https://github.com/librespeed/speedtest">Source code</a>
+</div>
+<div id="privacyPolicy" style="display:none">
+    <h2>Privacy Policy</h2>
+    <p>This HTML5 speed test server is configured with telemetry enabled.</p>
+    <h4>What data we collect</h4>
+    <p>
+        At the end of the test, the following data is collected and stored:
+        <ul>
+            <li>Test ID</li>
+            <li>Time of testing</li>
+            <li>Test results (download and upload speed, ping and jitter)</li>
+            <li>IP address</li>
+            <li>ISP information</li>
+            <li>Approximate location (inferred from IP address, not GPS)</li>
+            <li>User agent and browser locale</li>
+            <li>Test log (contains no personal information)</li>
+        </ul>
+    </p>
+    <h4>How we use the data</h4>
+    <p>
+        Data collected through this service is used to:
+        <ul>
+            <li>Allow sharing of test results (sharable image for forums, etc.)</li>
+            <li>To improve the service offered to you (for instance, to detect problems on our side)</li>
+        </ul>
+        No personal information is disclosed to third parties.
+    </p>
+    <h4>Your consent</h4>
+    <p>
+        By starting the test, you consent to the terms of this privacy policy.
+    </p>
+    <h4>Data removal</h4>
+    <p>
+        If you want to have your information deleted, you need to provide either the ID of the test or your IP address. This is the only way to identify your data, without this information we won't be able to comply with your request.<br/><br/>
+        Contact this email address for all deletion requests: <a href="mailto:dev@nul.ie">dev@nul.ie</a>.
+    </p>
+    <br/><br/>
+    <div class="closePrivacyPolicy">
+        <a class="privacy" href="#" onclick="I('privacyPolicy').style.display='none'">Close</a>
+    </div>
+    <br/>
+</div>
+</body>
+</html>

nixos/modules/netboot/default.nix

@@ -0,0 +1,232 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (lib) mkMerge mkIf mkForce genAttrs concatMapStringsSep;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  cfg = config.my.netboot;
+
+  ipxe = pkgs.ipxe.overrideAttrs (o: rec {
+    version = "1.21.1-unstable-2024-06-27";
+    src = pkgs.fetchFromGitHub {
+      owner = "ipxe";
+      repo = "ipxe";
+      rev = "b66e27d9b29a172a097c737ab4d378d60fe01b05";
+      hash = "sha256-TKZ4WjNV2oZIYNefch7E7m1JpeoC/d7O1kofoNv8G40=";
+    };
+  });
+  tftpRoot = pkgs.linkFarm "tftp-root" [
+    {
+      name = "ipxe-x86_64.efi";
+      path = "${ipxe}/ipxe.efi";
+    }
+  ];
+  menuFile = pkgs.runCommand "menu.ipxe" {
+    bootHost = cfg.server.host;
+  } ''
+    substituteAll ${./menu.ipxe} "$out"
+  '';
+
+  bootBuilder = pkgs.substituteAll {
+    src = ./netboot-loader-builder.py;
+    isExecutable = true;
+
+    inherit (pkgs) python3;
+    bootspecTools = pkgs.bootspec;
+    nix = config.nix.package.out;
+
+    inherit (config.system.nixos) distroName;
+    systemName = config.system.name;
+    inherit (cfg.client) configurationLimit;
+    checkMountpoints = pkgs.writeShellScript "check-mountpoints" ''
+      if ! ${pkgs.util-linuxMinimal}/bin/findmnt /boot > /dev/null; then
+        echo "/boot is not a mounted partition. Is the path configured correctly?" >&2
+        exit 1
+      fi
+    '';
+  };
+in
+{
+  options.my.netboot = with lib.types; {
+    client = {
+      enable = mkBoolOpt' false "Whether network booting should be enabled.";
+      configurationLimit = mkOpt' ints.unsigned 10 "Max generations to show in boot menu.";
+    };
+    server = {
+      enable = mkBoolOpt' false "Whether a netboot server should be enabled.";
+      ip = mkOpt' str null "IP clients should connect to via TFTP.";
+      host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP / NFS.";
+      allowedPrefixes = mkOpt' (listOf str) null "Prefixes clients should be allowed to connect from (NFS).";
+      installer = {
+        storeSize = mkOpt' str "16GiB" "Total allowed writable size of store.";
+      };
+      instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for.";
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.client.enable {
+      systemd = {
+        services = {
+          mount-boot = {
+            description = "Mount /boot";
+            after = [ "systemd-networkd-wait-online.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+            };
+            path = with pkgs; [ gnused ldns nfs-utils ];
+            script = ''
+              get_cmdline() {
+                sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
+              }
+
+              host="$(get_cmdline boothost)"
+              if [ -z "$host" ]; then
+                echo "boothost kernel parameter not found!" >&2
+                exit 1
+              fi
+
+              until [ -n "$(drill -Q $host)" ]; do
+                sleep 0.1
+              done
+
+              mkdir -p /boot
+              mount.nfs $host:/srv/netboot/systems/${config.system.name} /boot
+            '';
+
+            wantedBy = [ "remote-fs.target" ];
+          };
+        };
+      };
+
+      boot.supportedFilesystems.nfs = true;
+      boot.loader = {
+        grub.enable = false;
+        systemd-boot.enable = false;
+      };
+      system = {
+        build.installBootLoader = bootBuilder;
+        boot.loader.id = "ipxe-netboot";
+      };
+    })
+    (mkIf cfg.server.enable {
+      environment = {
+        etc = {
+          "netboot/menu.ipxe".source = menuFile;
+          "netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi";
+        };
+      };
+
+      systemd = {
+        tmpfiles.settings."10-netboot" = genAttrs
+          (map (i: "/srv/netboot/systems/${i}") cfg.server.instances)
+          (p: {
+            d = {
+              user = "root";
+              group = "root";
+              mode = "0777";
+            };
+          });
+
+        services = {
+          netboot-update = {
+            description = "Update netboot images";
+            after = [ "systemd-networkd-wait-online.service" ];
+            serviceConfig.Type = "oneshot";
+            path = with pkgs; [
+              coreutils curl jq zstd gnutar
+            ];
+            script = ''
+              update_nixos() {
+                latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \
+                             | jq -r .commit.sha | cut -c -7)"
+                if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then
+                  echo "NixOS installer is up to date"
+                  return
+                fi
+
+                echo "Updating NixOS installer to $latestShort"
+                mkdir -p nixos-installer
+                fname="jackos-installer-netboot-$latestShort.tar.zst"
+                downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \
+                               jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")"
+                curl -Lo /tmp/nixos-installer-netboot.tar.zst "$downloadUrl"
+                tar -C nixos-installer --zstd -xf /tmp/nixos-installer-netboot.tar.zst
+                truncate -s "${cfg.server.installer.storeSize}" nixos-installer/rootfs.ext4
+                rm /tmp/nixos-installer-netboot.tar.zst
+                echo "$latestShort" > nixos-installer/tag.txt
+              }
+
+              mkdir -p /srv/netboot
+              cd /srv/netboot
+
+              ln -sf ${menuFile} boot.ipxe
+              ln -sf "${pkgs.edk2-uefi-shell}/shell.efi" "efi-shell-${config.nixpkgs.localSystem.linuxArch}.efi"
+              update_nixos
+            '';
+            startAt = "06:00";
+            wantedBy = [ "network-online.target" ];
+          };
+
+          nbd-server = {
+            serviceConfig = {
+              PrivateUsers = mkForce false;
+              CacheDirectory = "netboot";
+            };
+          };
+        };
+      };
+
+      services = {
+        atftpd = {
+          enable = true;
+          root = tftpRoot;
+        };
+
+        nginx = {
+          virtualHosts."${cfg.server.host}" = {
+            locations."/" = {
+              root = "/srv/netboot";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+
+        nbd.server = {
+          enable = true;
+          extraOptions = {
+            allowlist = true;
+          };
+          exports = {
+            nixos-installer = {
+              path = "/srv/netboot/nixos-installer/rootfs.ext4";
+              extraOptions = {
+                copyonwrite = true;
+                cowdir = "/var/cache/netboot";
+                sparse_cow = true;
+              };
+            };
+          };
+        };
+
+        nfs = {
+          server = {
+            enable = true;
+            exports = ''
+              /srv/netboot/systems ${concatMapStringsSep " " (p: "${p}(rw,all_squash)") cfg.server.allowedPrefixes}
+            '';
+          };
+        };
+      };
+
+      my = {
+        tmproot.persistence.config.directories = [
+          "/srv/netboot"
+          { directory = "/var/cache/netboot"; mode = "0700"; }
+        ];
+      };
+    })
+  ];
+}

nixos/modules/netboot/menu.ipxe

@@ -0,0 +1,68 @@
+#!ipxe
+
+set server http://@bootHost@
+
+# Figure out if client is 64-bit capable
+cpuid --ext 29 && set arch x86_64 || set arch i386
+
+isset ${menu-default} || set menu-default exit
+
+:start
+menu Welcome to /dev/player0's humble iPXE boot menu
+item --gap --           Operating Systems
+iseq ${arch} x86_64 &&
+item --key n nixos      NixOS installer
+# iseq ${arch} x86_64 &&
+# item --key a archlinux Arch Linux (archiso x86_64)
+# iseq ${arch} x86_64 &&
+# item --key p alpine   Alpine Linux
+item --gap --           Other Options
+item --key e efi_shell  UEFI Shell
+item --key x xyz        netboot.xyz
+item --key c config     iPXE settings
+item --key s shell      Drop to iPXE shell
+item --key r reboot     Reboot
+item --key q exit       Exit (and continue to next boot device)
+choose --timeout 0 --default ${menu-default} selected || goto cancel
+goto ${selected}
+
+:cancel
+echo You cancelled the menu, dropping you to an iPXE shell
+
+:shell
+echo Type 'exit' to go back to the menu
+shell
+set menu-default nixos
+goto start
+
+:failed
+echo Booting failed, dropping to shell
+goto shell
+
+:reboot
+reboot
+
+:exit
+exit
+
+:config
+config
+set menu-default config
+goto start
+
+:efi_shell
+chain ${server}/efi-shell-${arch}.efi || goto failed
+
+:xyz
+chain --autofree https://boot.netboot.xyz || goto failed
+
+:nixos
+set cmdline nbd_export=nixos-installer
+chain ${server}/nixos-installer/boot.ipxe || goto failed
+
+:archlinux
+# set mirrorurl https://arch.nul.ie/
+chain ${server}/arch.ipxe || goto failed
+
+:alpine
+chain ${server}/alpine.ipxe || goto failed

nixos/modules/netboot/netboot-loader-builder.py

@@ -0,0 +1,280 @@
+#! @python3@/bin/python3 -B
+# Based on `nixos/modules/system/boot/loader/systemd-boot/systemd-boot-builder.py`
+import argparse
+import datetime
+import glob
+import os
+import os.path
+import shutil
+import subprocess
+import sys
+import json
+from typing import NamedTuple, Dict, List
+from dataclasses import dataclass
+
+BOOT_MOUNT_POINT = '/boot'
+STORE_DIR = 'nix'
+
+# These values will be replaced with actual values during the package build
+BOOTSPEC_TOOLS = '@bootspecTools@'
+NIX = '@nix@'
+DISTRO_NAME = '@distroName@'
+SYSTEM_NAME = '@systemName@'
+CONFIGURATION_LIMIT = int('@configurationLimit@')
+CHECK_MOUNTPOINTS = "@checkMountpoints@"
+
+@dataclass
+class BootSpec:
+  init: str
+  initrd: str
+  kernel: str
+  kernelParams: List[str]
+  label: str
+  system: str
+  toplevel: str
+  specialisations: Dict[str, 'BootSpec']
+  sortKey: str
+  initrdSecrets: str | None = None
+
+class SystemIdentifier(NamedTuple):
+  profile: str | None
+  generation: int
+  specialisation: str | None
+
+def copy_if_not_exists(source: str, dest: str) -> None:
+  if not os.path.exists(dest):
+    shutil.copyfile(source, dest)
+
+def generation_dir(profile: str | None, generation: int) -> str:
+  if profile:
+    return f'/nix/var/nix/profiles/system-profiles/{profile}-{generation}-link'
+  else:
+    return f'/nix/var/nix/profiles/system-{generation}-link'
+
+def system_dir(i: SystemIdentifier) -> str:
+  d = generation_dir(i.profile, i.generation)
+  if i.specialisation:
+    return os.path.join(d, 'specialisation', i.specialisation)
+  else:
+    return d
+
+def entry_key(i: SystemIdentifier) -> str:
+  pieces = [
+    'nixos',
+    i.profile or None,
+    'generation',
+    str(i.generation),
+    f'specialisation-{i.specialisation}' if i.specialisation else None,
+  ]
+  return '-'.join(p for p in pieces if p)
+
+def bootspec_from_json(bootspec_json: Dict) -> BootSpec:
+  specialisations = bootspec_json['org.nixos.specialisation.v1']
+  specialisations = {k: bootspec_from_json(v) for k, v in specialisations.items()}
+  systemdBootExtension = bootspec_json.get('org.nixos.systemd-boot', {})
+  sortKey = systemdBootExtension.get('sortKey', 'nixos')
+  return BootSpec(
+    **bootspec_json['org.nixos.bootspec.v1'],
+    specialisations=specialisations,
+    sortKey=sortKey
+  )
+
+bootspecs = {}
+def get_bootspec(profile: str | None, generation: int) -> BootSpec:
+  k = (profile, generation)
+  if k in bootspecs:
+    return bootspecs[k]
+
+  system_directory = system_dir(SystemIdentifier(profile, generation, None))
+  boot_json_path = os.path.realpath(f'{system_directory}/boot.json')
+  if os.path.isfile(boot_json_path):
+    boot_json_f = open(boot_json_path, 'r')
+    bootspec_json = json.load(boot_json_f)
+  else:
+    boot_json_str = subprocess.check_output([
+      f'{BOOTSPEC_TOOLS}/bin/synthesize',
+      '--version',
+      '1',
+      system_directory,
+      '/dev/stdout',
+    ],
+    universal_newlines=True)
+    bootspec_json = json.loads(boot_json_str)
+
+  bs = bootspec_from_json(bootspec_json)
+  bootspecs[k] = bs
+  return bs
+
+def copy_from_file(file: str, dry_run: bool = False) -> str:
+  store_file_path = os.path.realpath(file)
+  suffix = os.path.basename(store_file_path)
+  store_dir = os.path.basename(os.path.dirname(store_file_path))
+  dst_path = f'/{STORE_DIR}/{store_dir}-{suffix}'
+  if not dry_run:
+    copy_if_not_exists(store_file_path, f'{BOOT_MOUNT_POINT}{dst_path}')
+  return dst_path
+
+MENU_ITEM = 'item {gen_key} {title} Generation {generation} {description}'
+
+BOOT_ENTRY = ''':{gen_key}
+kernel ${{server}}/systems/{system_name}{kernel} {kernel_params} boothost=${{boothost}}
+initrd ${{server}}/systems/{system_name}{initrd}
+boot
+'''
+
+def gen_entry(i: SystemIdentifier) -> (str, str):
+  bootspec = get_bootspec(i.profile, i.generation)
+  if i.specialisation:
+    bootspec = bootspec.specialisations[i.specialisation]
+  kernel = copy_from_file(bootspec.kernel)
+  initrd = copy_from_file(bootspec.initrd)
+
+  gen_key = entry_key(i)
+  title = '{name}{profile}{specialisation}'.format(
+    name=DISTRO_NAME,
+    profile=' [' + i.profile + ']' if i.profile else '',
+    specialisation=f' ({i.specialisation})' if i.specialisation else '')
+
+  kernel_params = f'init={bootspec.init} '
+
+  kernel_params = kernel_params + ' '.join(bootspec.kernelParams)
+  build_time = int(os.path.getctime(system_dir(i)))
+  build_date = datetime.datetime.fromtimestamp(build_time).strftime('%F')
+
+  return MENU_ITEM.format(
+    gen_key=gen_key,
+    title=title,
+    description=f'{bootspec.label}, built on {build_date}',
+    generation=i.generation,
+  ), BOOT_ENTRY.format(
+    gen_key=gen_key,
+    generation=i.generation,
+    system_name=SYSTEM_NAME,
+    kernel=kernel,
+    kernel_params=kernel_params,
+    initrd=initrd,
+  )
+
+def get_generations(profile: str | None = None) -> list[SystemIdentifier]:
+  gen_list = subprocess.check_output([
+    f'{NIX}/bin/nix-env',
+    '--list-generations',
+    '-p',
+    '/nix/var/nix/profiles/' + ('system-profiles/' + profile if profile else 'system')],
+    universal_newlines=True)
+  gen_lines = gen_list.split('\n')
+  gen_lines.pop()
+
+  configurationLimit = CONFIGURATION_LIMIT
+  configurations = [
+    SystemIdentifier(
+      profile=profile,
+      generation=int(line.split()[0]),
+      specialisation=None
+    )
+    for line in gen_lines
+  ]
+  return configurations[-configurationLimit:]
+
+def remove_old_files(gens: list[SystemIdentifier]) -> None:
+  known_paths = []
+  for gen in gens:
+    bootspec = get_bootspec(gen.profile, gen.generation)
+    known_paths.append(copy_from_file(bootspec.kernel, True))
+    known_paths.append(copy_from_file(bootspec.initrd, True))
+  for path in glob.iglob(f'{BOOT_MOUNT_POINT}/{STORE_DIR}/*'):
+    if not path in known_paths and not os.path.isdir(path):
+      os.unlink(path)
+
+def get_profiles() -> list[str]:
+  if os.path.isdir('/nix/var/nix/profiles/system-profiles/'):
+    return [x
+      for x in os.listdir('/nix/var/nix/profiles/system-profiles/')
+      if not x.endswith('-link')]
+  else:
+    return []
+
+MENU = '''#!ipxe
+# Server hostname option
+set boothost ${{66:string}}
+set server http://${{boothost}}
+
+:start
+menu {distro} boot menu
+item --gap -- Generations
+{generation_items}
+item --gap -- Other
+item --key m main Main netboot menu
+choose --timeout 5000 --default {menu_default} selected || goto cancel
+goto ${{selected}}
+
+:cancel
+shell
+goto start
+
+:error
+echo Booting failed, dropping to shell
+shell
+goto start
+
+:main
+chain ${{server}}/boot.ipxe || goto error
+'''
+
+def write_menu(gens: list[SystemIdentifier], default: SystemIdentifier) -> None:
+  gen_menu_items = []
+  gen_cmds = []
+
+  for g in gens:
+    bootspec = get_bootspec(g.profile, g.generation)
+    specialisations = [
+      SystemIdentifier(profile=g.profile, generation=g.generation, specialisation=s) for s in bootspec.specialisations]
+    for i in [g] + specialisations:
+      mi, cmds = gen_entry(i)
+      gen_menu_items.append(mi)
+      gen_cmds.append(cmds)
+
+  menu_file = f'{BOOT_MOUNT_POINT}/menu.ipxe'
+  with open(f'{menu_file}.tmp', 'w') as f:
+    f.write(MENU.format(
+      distro=DISTRO_NAME,
+      generation_items='\n'.join(gen_menu_items),
+      menu_default=entry_key(default),
+    ))
+
+    print(file=f)
+    print('\n\n'.join(gen_cmds), file=f)
+
+  os.rename(f'{menu_file}.tmp', menu_file)
+
+def install_bootloader(args: argparse.Namespace) -> None:
+  os.makedirs(f'{BOOT_MOUNT_POINT}/{STORE_DIR}', exist_ok=True)
+
+  gens = get_generations()
+  for profile in get_profiles():
+    gens += get_generations(profile)
+
+  gens = sorted(gens, key=lambda g: entry_key(g), reverse=True)
+
+  remove_old_files(gens)
+
+  for g in gens:
+    if os.path.dirname(get_bootspec(g.profile, g.generation).init) == os.path.realpath(args.default_config):
+      default = g
+      break
+  else:
+    assert False, 'No default generation found'
+
+  write_menu(gens, default)
+
+def main() -> None:
+  parser = argparse.ArgumentParser(description=f'Update {DISTRO_NAME}-related netboot files')
+  parser.add_argument('default_config', metavar='DEFAULT-CONFIG', help=f'The default {DISTRO_NAME} config to boot')
+  args = parser.parse_args()
+
+  subprocess.check_call(CHECK_MOUNTPOINTS)
+
+  install_bootloader(args)
+
+if __name__ == '__main__':
+  main()

nixos/modules/nvme/default.nix

@@ -0,0 +1,85 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (lib) mkIf;
+  inherit (lib.my) mkOpt';
+
+  cfg = config.my.nvme;
+  nvme-cli = pkgs.nvme-cli.override {
+    libnvme = pkgs.libnvme.overrideAttrs (o: rec {
+      # TODO: Remove when 1.11.1 releases (see https://github.com/linux-nvme/libnvme/pull/914)
+      version = "1.11.1";
+      src = pkgs.fetchFromGitHub {
+        owner = "linux-nvme";
+        repo = "libnvme";
+        rev = "v${version}";
+        hash = "sha256-CEGr7PDOVRi210XvICH8iLYDKn8S9bGruBO4tycvsT8=";
+      };
+      patches = (if (o ? patches) then o.patches else [ ]) ++ [ ./libnvme-hostconf.patch ];
+    });
+  };
+
+  hostNQN = "nqn.2014-08.org.nvmexpress:uuid:${cfg.uuid}";
+  etc = prefix: {
+    "${prefix}nvme/hostnqn".text = hostNQN;
+    "${prefix}nvme/hostid".text = cfg.uuid;
+  };
+in
+{
+  options.my.nvme = with lib.types; {
+    uuid = mkOpt' (nullOr str) null "NVMe host ID";
+    boot = {
+      nqn = mkOpt' (nullOr str) null "NQN to connect to on boot";
+      address = mkOpt' str null "Address of NVMe-oF target.";
+    };
+  };
+
+  config = mkIf (cfg.uuid != null) {
+    environment = {
+      systemPackages = [
+        nvme-cli
+      ];
+      etc = etc "";
+    };
+
+    boot = mkIf (cfg.boot.nqn != null) {
+      initrd = {
+        availableKernelModules = [ "rdma_cm" "iw_cm" "ib_cm" "nvme_core" "nvme_rdma" ];
+        kernelModules = [ "nvme-fabrics" ];
+        systemd = {
+          contents = etc "/etc/";
+          extraBin = with pkgs; {
+            dmesg = "${util-linux}/bin/dmesg";
+            ip = "${iproute2}/bin/ip";
+            nvme = "${nvme-cli}/bin/nvme";
+          };
+          extraConfig = ''
+            DefaultTimeoutStartSec=20
+            DefaultDeviceTimeoutSec=20
+          '';
+
+          network = {
+            enable = true;
+            wait-online.enable = true;
+          };
+
+          services.connect-nvme = {
+            description = "Connect NVMe-oF";
+            before = [ "initrd-root-device.target" ];
+            after = [ "systemd-networkd-wait-online.service" ];
+            requires = [ "systemd-networkd-wait-online.service" ];
+
+            serviceConfig = {
+              Type = "oneshot";
+              ExecStart = "${nvme-cli}/bin/nvme connect -t rdma -a ${cfg.boot.address} -n ${cfg.boot.nqn}";
+              Restart = "on-failure";
+              RestartSec = 10;
+            };
+
+            wantedBy = [ "initrd-root-device.target" ];
+          };
+        };
+      };
+    };
+  };
+}
+

nixos/modules/nvme/libnvme-hostconf.patch

@@ -0,0 +1,15 @@
+diff --git a/src/nvme/fabrics.c b/src/nvme/fabrics.c
+index 21fb292..f9090d1 100644
+--- a/src/nvme/fabrics.c
++++ b/src/nvme/fabrics.c
+@@ -41,8 +41,8 @@
+ 
+ #define NVMF_HOSTID_SIZE	37
+ 
+-#define NVMF_HOSTNQN_FILE	SYSCONFDIR "/nvme/hostnqn"
+-#define NVMF_HOSTID_FILE	SYSCONFDIR "/nvme/hostid"
++#define NVMF_HOSTNQN_FILE	"/etc/nvme/hostnqn"
++#define NVMF_HOSTID_FILE	"/etc/nvme/hostid"
+ 
+ const char *nvmf_dev = "/dev/nvme-fabrics";
+ 

nixos/modules/pdns.nix

@@ -268,6 +268,9 @@ in
         # For pdns_control etc
         systemPackages = with pkgs; [
           pdns
+          (pkgs.writeShellScriptBin "pu" ''
+            ${pdns}/bin/pdnsutil --config-dir /run/pdns "$@"
+          '')
           pdns-file-record
         ];
 

nixos/modules/spdk.nix

@@ -0,0 +1,94 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (builtins) toJSON;
+  inherit (lib) optional optionalAttrs mapAttrsToList mkIf withFeature;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  rpcOpts = with lib.types; {
+    options = {
+      method = mkOpt' str null "RPC method name.";
+      params = mkOpt' (attrsOf unspecified) { } "RPC params";
+    };
+  };
+
+  cfg = config.my.spdk;
+  config' = {
+    subsystems = mapAttrsToList (subsystem: c: {
+      inherit subsystem;
+      config = map (rpc: {
+        inherit (rpc) method;
+      } // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c;
+    }) cfg.config.subsystems;
+  };
+  configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
+
+  spdk = pkgs.spdk.overrideAttrs (o: {
+    configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
+    buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
+  });
+  spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
+    exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
+  '');
+  spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
+    exec ${spdk.src}/scripts/setup.sh "$@"
+  '');
+  spdk-debug = pkgs.writeShellApplication {
+    name = "spdk-debug";
+    runtimeInputs = [ spdk ];
+    text = ''
+      set -m
+      if [ "$(id -u)" -ne 0 ]; then
+        echo "I need to be root!"
+        exit 1
+      fi
+
+      spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
+      until spdk-rpc spdk_get_version > /dev/null; do
+        sleep 0.5
+      done
+
+      spdk-rpc bdev_set_options --disable-auto-examine
+      spdk-rpc framework_start_init
+
+      ${cfg.debugCommands}
+
+      fg %1
+    '';
+  };
+in
+{
+  options.my.spdk = with lib.types; {
+    enable = mkBoolOpt' false "Whether to enable SPDK target.";
+    extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt.";
+    debugCommands = mkOpt' lines "" "Commands to run with the spdk-debug script.";
+    config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs.";
+  };
+
+  config = mkIf cfg.enable {
+    boot.kernelModules = [ "ublk_drv" ];
+
+    environment.systemPackages = [
+      spdk
+      spdk-setup
+      spdk-rpc
+    ] ++ (optional (cfg.debugCommands != "") spdk-debug);
+
+    systemd.services = {
+      spdk-tgt = {
+        description = "SPDK target";
+        path = with pkgs; [
+          bash
+          python3
+          kmod
+          gawk
+          util-linux
+        ];
+        serviceConfig = {
+          ExecStartPre = "${spdk.src}/scripts/setup.sh";
+          ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
+        };
+        wantedBy = [ "multi-user.target" ];
+      };
+    };
+  };
+}

nixos/modules/tmproot.nix

@@ -2,7 +2,7 @@
 let
   inherit (lib)
     optionalString concatStringsSep concatMap concatMapStringsSep mkIf mkDefault mkMerge mkForce mkVMOverride
-    mkAliasDefinitions;
+    mkAliasDefinitions mapAttrsToList filterAttrs;
   inherit (lib.my) mkOpt' mkBoolOpt' mkVMOverride';
 
   cfg = config.my.tmproot;
@@ -147,6 +147,15 @@ in
             "/var/lib/systemd"
 
             { directory = "/root/.cache/nix"; mode = "0700"; }
+            # Including these unconditionally due to infinite recursion problems...
+            {
+              directory = "/etc/lvm/archive";
+              mode = "0700";
+            }
+            {
+              directory = "/etc/lvm/backup";
+              mode = "0700";
+            }
           ];
           files = [
             "/etc/machine-id"
@@ -187,6 +196,9 @@ in
     (mkIf config.networking.resolvconf.enable {
       my.tmproot.unsaved.ignore = [ "/etc/resolv.conf" ];
     })
+    (mkIf config.networking.nftables.enable {
+      my.tmproot.unsaved.ignore = [ "/var/lib/nftables/deletions.nft" ];
+    })
     (mkIf config.security.doas.enable {
       my.tmproot.unsaved.ignore = [ "/etc/doas.conf" ];
     })
@@ -221,8 +233,9 @@ in
 
         # Catch non-existent source directories that are needed for boot (see `pathsNeededForBoot` in
         # nixos/lib/util.nix). We do this by monkey-patching the `waitDevice` function that would otherwise hang.
-        boot.initrd.postDeviceCommands =
+        # Seems like systemd initrd doesn't care because it uses the systemd.mount units
-          ''
+        # ("If this mount is a bind mount and the specified path does not exist yet it is created as directory.")
+        boot.initrd.postDeviceCommands = mkIf (!config.boot.initrd.systemd.enable) ''
             ensurePersistSource() {
               [ -e "/mnt-root$1" ] && return
               echo "Persistent source directory $1 does not exist, creating..."
@@ -256,18 +269,6 @@ in
         my.tmproot.persistence.config.files =
           concatMap (k: [ k.path "${k.path}.pub" ]) config.services.openssh.hostKeys;
       })
-      (mkIf config.services.lvm.enable {
-        my.tmproot.persistence.config.directories = [
-          {
-            directory = "/etc/lvm/archive";
-            mode = "0700";
-          }
-          {
-            directory = "/etc/lvm/backup";
-            mode = "0700";
-          }
-        ];
-      })
       (mkIf (config.security.acme.certs != { }) {
         my.tmproot.persistence.config.directories = [
           {
@@ -459,6 +460,97 @@ in
           }
         ];
       })
+      (mkIf config.virtualisation.libvirtd.enable {
+        my.tmproot.persistence.config.directories = [ "/var/lib/libvirt" ];
+      })
+      (mkIf (with config.services.kea; (dhcp4.enable || dhcp6.enable || dhcp-ddns.enable)) {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = "/var/lib/kea";
+            mode = "0750";
+            user = "kea";
+            group = "kea";
+          }
+        ];
+      })
+      (persistSimpleSvc "headscale")
+      (mkIf config.services.tailscale.enable {
+        my.tmproot.persistence.config.directories = [ "/var/lib/tailscale" ];
+      })
+      (mkIf config.my.librespeed.backend.enable {
+        my.tmproot.persistence.config.directories = [ "/var/lib/librespeed-go" ];
+      })
+      (mkIf config.services.hedgedoc.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = "/var/lib/hedgedoc";
+            user = "hedgedoc";
+            group = "hedgedoc";
+          }
+        ];
+      })
+      (mkIf config.services.wastebin.enable {
+        my.tmproot.persistence.config.directories = [ "/var/lib/private/wastebin" ];
+      })
+      (mkIf config.services.photoprism.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = config.services.photoprism.storagePath;
+            mode = "0750";
+            user = "photoprism";
+            group = "photoprism";
+          }
+        ];
+      })
+      (mkIf config.services.mautrix-whatsapp.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = "/var/lib/mautrix-whatsapp";
+            mode = "0750";
+            user = "mautrix-whatsapp";
+            group = "mautrix-whatsapp";
+          }
+        ];
+      })
+      {
+        my.tmproot.persistence.config.directories = mapAttrsToList (n: i: {
+          directory = "/var/lib/${i.dataDir}";
+          mode = "0750";
+          user = "mautrix-meta-${n}";
+          group = "mautrix-meta";
+        }) (filterAttrs (_: i: i.enable) config.services.mautrix-meta.instances);
+      }
+      (mkIf config.services.unifi.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = "/var/lib/unifi";
+            mode = "0750";
+            user = "unifi";
+            group = "unifi";
+          }
+          {
+            directory = "/var/cache/unifi";
+            mode = "0750";
+            user = "unifi";
+            group = "unifi";
+          }
+        ];
+      })
+      (persistSimpleSvc "octoprint")
+      (mkIf (config.services.borgbackup.jobs != { }) {
+        my.tmproot.persistence.config.directories = [
+          "/var/lib/borgbackup"
+          "/var/cache/borgbackup"
+        ];
+
+        services.borgbackup.package = pkgs.borgbackup.overrideAttrs (o: {
+          makeWrapperArgs = o.makeWrapperArgs ++ [
+            "--set-default BORG_BASE_DIR /var/lib/borgbackup"
+            "--set-default BORG_CONFIG_DIR /var/lib/borgbackup/config"
+            "--set-default BORG_CACHE_DIR /var/cache/borgbackup"
+          ];
+        });
+      })
     ]))
   ]);
 

nixos/modules/user.nix

@@ -35,8 +35,11 @@ in
             isNormalUser = true;
             uid = mkDefault 1000;
             extraGroups =
-              [ "wheel" "kvm" ] ++
+              [ "wheel" "kvm" "dialout" ] ++
-              (optional config.networking.networkmanager.enable "networkmanager");
+              (optional config.networking.networkmanager.enable "networkmanager") ++
+              (optional config.virtualisation.libvirtd.enable "libvirtd") ++
+              (optional config.programs.wireshark.enable "wireshark") ++
+              (with config.services.headscale; (optional enable group));
             password = mkIf (cfg.passwordSecret == null) (mkDefault "hunter2");
             shell =
               let shell = cfg.homeConfig.my.shell;
@@ -57,33 +60,17 @@ in
 
             "/home/${user'.name}/.config/fish/fish_variables"
           ];
-          persistence.config =
+          persistence.config.users."${user'.name}" = {
-          let
+            files = [
-            perms = {
+              ".bash_history"
-              mode = "0700";
+              ".lesshst"
-              user = user.name;
-              group = user.group;
-            };
-          in
-          {
-            files = (map (file: {
-              inherit file;
-              parentDirectory = perms;
-            }) [
-              "/home/${user'.name}/.bash_history"
-              "/home/${user'.name}/.lesshst"
-            ]) ++ [
-              # Just to make sure we get correct default perms
-              "/home/.tmproot.dummy"
             ];
-            directories = map (directory: {
+            directories = [
-              inherit directory;
-            } // perms) [
               # Persist all of fish; it's not easy to persist just the history fish won't let you move it to a different
               # directory. Also it does some funny stuff and can't really be a symlink it seems.
-              "/home/${user'.name}/.local/share/fish"
+              ".local/share/fish"
 
-              "/home/${user'.name}/.cache/nix"
+              ".cache/nix"
             ];
           };
         };
@@ -95,11 +82,15 @@ in
       # NOTE: As the "outermost" module is still being evaluated in NixOS land, special params (e.g. pkgs) won't be
       # passed to it
       home-manager.users.${user'.name} = mkAliasDefinitions options.my.user.homeConfig;
+
+      systemd.services.nixfiles-mutable.script = ''
+        chown -R ${user'.name} /run/nixfiles
+      '';
     }
     (mkIf (cfg.passwordSecret != null) {
       my = {
         secrets.files."${cfg.passwordSecret}" = {};
-        user.config.passwordFile = config.age.secrets."${cfg.passwordSecret}".path;
+        user.config.hashedPasswordFile = config.age.secrets."${cfg.passwordSecret}".path;
       };
     })
   ]);