Hack hack hack

devshell/commands.nix

@@ -106,8 +106,8 @@ in
     {
       name = "build-netboot";
       category = "tasks";
-      help = "Build NixOS configuration as netboot archive";
+      help = "Build NixOS configuration as netboot tree";
-      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootArchive"'';
+      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"'';
     }
     {
       name = "build-home";

nixos/boxes/home/routing-common/default.nix

@@ -148,9 +148,11 @@ in
                 };
               };
             };
+
+            nginx.enable = true;
           };
 
-          networking.domain = "h.${pubDomain}";
+          networking = { inherit domain; };
 
           systemd.services = {
             ipsec =
@@ -376,6 +378,11 @@ in
                 }
               '';
             };
+            netboot.server = {
+              enable = true;
+              ip = vips.lo.v4;
+              host = "boot.${domain}";
+            };
           };
         };
       };

nixos/boxes/home/routing-common/dns.nix

@@ -130,6 +130,7 @@ in
             }}
             ${elemAt routers 0} IN AAAA ${net.cidr.host 1 prefixes.hi.v6}
             ${elemAt routers 1} IN AAAA ${net.cidr.host 2 prefixes.hi.v6}
+            boot IN CNAME router-hi.${config.networking.domain}.
 
             @ IN NS ns1
             @ IN NS ns2

nixos/boxes/home/routing-common/kea.nix

@@ -1,4 +1,4 @@
-index: { lib, pkgs, assignments, ... }:
+index: { lib, pkgs, config, assignments, ... }:
 let
   inherit (lib) mkForce;
   inherit (lib.my) net;
@@ -59,6 +59,7 @@ in
               always-send = true;
             }
           ];
+          client-classes = config.my.netboot.server.keaClientClasses;
           subnet4 = [
             {
               id = 1;

nixos/modules/_list.nix

@@ -19,5 +19,6 @@
     borgthin = ./borgthin.nix;
     nvme = ./nvme;
     spdk = ./spdk.nix;
+    netboot = ./netboot;
   };
 }

nixos/modules/build.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, extendModules, modulesPath, options, config, ... }:
 let
-  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional;
+  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional;
   inherit (lib.my) mkBoolOpt' dummyOption;
 
   cfg = config.my.build;
@@ -43,15 +43,206 @@ let
     modules = flatten [
       "${modulesPath}/installer/netboot/netboot.nix"
       allHardware
-      ({ pkgs, config, ... }: {
+      (mkIf config.boot.initrd.systemd.enable {
-        system.build.netbootArchive = pkgs.runCommand "netboot-${config.system.name}-archive.tar" { } ''
+        boot.initrd.systemd.services.setup-overlay-dirs = {
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.kernel}" \
+          description = "Create overlayfs dirs";
-            -f "$out" "${config.system.boot.loader.kernelFile}"
+          after = [ "sysroot-nix-.rw\\x2dstore.mount" ];
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootRamdisk}" \
+          before = [ "sysroot-nix-store.mount" ];
-            -f "$out" initrd
+          script = ''
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootIpxeScript}" \
+            mkdir /sysroot/nix/.rw-store/{store,work}
-            -f "$out" netboot.ipxe
+          '';
+          wantedBy = [ "initrd-fs.target" ];
+        };
+
+        fileSystems."/nix/store" = mkForce {
+          fsType = "overlay";
+          device = "overlay";
+          options = [
+            "lowerdir=/sysroot/nix/.ro-store"
+            "upperdir=/sysroot/nix/.rw-store/store"
+            "workdir=/sysroot/nix/.rw-store/work"
+          ];
+        };
+
+        systemd.units."nix-store.mount".enable = false;
+      })
+    ];
+  };
+
+  asNetboot = extendModules {
+    modules = flatten [
+      allHardware
+      ({ pkgs, config, ... }:
+      let 
+        initrdNbdWrapper = pkgs.writeCBin "nbd-wrapper" ''
+          #include <stdio.h>
+          #include <unistd.h>
+
+          int main(int argc, char **argv) {
+            if (argc < 3) {
+              fprintf(stderr, "usage: %s <export> <server>\n", argv[0]);
+              return -1;
+            }
+
+            argv[0][0] = '@';
+            char* args[] = {
+              "@", "-nofork", "-N", argv[1], argv[2], "/dev/nbd0", NULL
+            };
+            execv("${pkgs.nbd}/bin/nbd-client", args);
+            return 0;
+          };
         '';
+        nbd = pkgs.nbd.overrideAttrs (o: {
+          # TODO: Remove once this makes it to us
+          # https://github.com/NixOS/nixpkgs/commit/52f1d9b03ae38126e7f648634fcad35897f464ed
+          configureFlags = [ "--sysconfdir=/etc" ];
+        });
+      in
+      {
+        boot = {
+          loader.grub.enable = false;
+          kernelParams = [ "console=ttyS0,115200n8" ];
+          initrd = {
+            kernelModules = [ "nbd" ];
+            
+            systemd = {
+              storePaths = with pkgs; [
+                gnused
+                nbd
+                initrdNbdWrapper
+                netcat
+              ];
+              extraBin = with pkgs; {
+                dmesg = "${util-linux}/bin/dmesg";
+                ip = "${iproute2}/bin/ip";
+                nbd-client = "${nbd}/bin/nbd-client";
+              };
+              extraConfig = ''
+                DefaultTimeoutStartSec=10
+                DefaultDeviceTimeoutSec=10
+              '';
+
+              network = {
+                enable = true;
+                wait-online.enable = true;
+
+                networks."10-nbd" = {
+                  matchConfig.Name = "et-nbd";
+                  DHCP = "yes";
+                };
+              };
+
+              services = {
+                # gennbdtab = {
+                #   description = "Generate nbdtab";
+                #   script = ''
+                #     get_cmdline() {
+                #       ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
+                #     }
+
+                #     e="$(get_cmdline nbd_export)"
+                #     s="$(get_cmdline nbd_server)"
+                #     echo "Setting nbdtab for $e @ $s"
+                #     echo "nbd0 $s $e persist" > /etc/nbdtab
+                #   '';
+                #   serviceConfig.Type = "oneshot";
+                #   # wantedBy = [ "initrd-root-device.target" ];
+                # };
+                nbd = {
+                  description = "NBD Nix store";
+                  # before = [ "initrd-root-device.target" ];
+                  # after = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ];
+                  # wants = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ];
+                  # after = [ "systemd-networkd-wait-online.service" ];
+                  # wants = [ "systemd-networkd-wait-online.service" ];
+
+                  script = ''
+                    get_cmdline() {
+                      ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
+                    }
+                    s="$(get_cmdline nbd_server)"
+                    until ${pkgs.netcat}/bin/nc -zv "$s" 22; do
+                      sleep 0.1
+                    done
+                    exec ${nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0
+                    # exec ${initrdNbdWrapper}/bin/nbd-wrapper "$(get_cmdline nbd_export)" "$(get_cmdline nbd_server)"
+                  '';
+                  unitConfig = {
+                    IgnoreOnIsolate = "yes";
+                    DefaultDependencies = "no";
+                  };
+                  serviceConfig = {
+                    Type = "forking";
+                    # ExecStart = "${nbd}/bin/nbd-client -nofork -systemd-mark nbd0";
+                    Restart = "on-failure";
+                    RestartSec = 10;
+                  };
+
+                  # wantedBy = [ "initrd-root-device.target" ];
+                };
+              };
+            };
+          };
+        };
+
+        programs.nbd.enable = true;
+
+        fileSystems = {
+          "/" = {
+            fsType = "tmpfs";
+            options = [ "mode=0755" ];
+          };
+          "/nix/store" = {
+            fsType = "squashfs";
+            device = "/dev/nbd0";
+            options = [ "x-systemd.requires=nbd.service" ];
+          };
+        };
+
+        system.build = {
+          squashfsStore = pkgs.callPackage "${modulesPath}/../lib/make-squashfs.nix" {
+            storeContents = [ config.system.build.toplevel ];
+            comp = "zstd";
+          };
+          netbootScript = pkgs.writeText "boot.ipxe" ''
+            #!ipxe
+            kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-nbd:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline}
+            initrd initrd
+            boot
+          '';
+          netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [
+            {
+              name = config.system.boot.loader.kernelFile;
+              path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
+            }
+            {
+              name = "initrd";
+              path = "${config.system.build.initialRamdisk}/initrd";
+            }
+            {
+              name = "nix-store.sfs";
+              path = config.system.build.squashfsStore;
+            }
+            {
+              name = "boot.ipxe";
+              path = config.system.build.netbootScript;
+            }
+          ];
+          netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar" { } ''
+            add() {
+              ${pkgs.gnutar}/bin/tar --dereference -rvC "$1" -f "$out" "$2"
+            }
+
+            add "${config.system.build.kernel}" "${config.system.boot.loader.kernelFile}"
+            add "${config.system.build.initialRamdisk}" initrd
+
+            tmpdir="$(mktemp -d sfsStore.XXXXXX)"
+            ln -s "${config.system.build.squashfsStore}" "$tmpdir"/nix-store.sfs
+            add "$tmpdir" nix-store.sfs
+
+            add "${config.system.build.netbootScript}" boot.ipxe
+          '';
+        };
       })
     ];
   };
@@ -77,6 +268,7 @@ in
       asISO = mkAsOpt asISO "a bootable .iso image";
       asContainer = mkAsOpt asContainer "a container";
       asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd";
+      asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script";
 
       buildAs = options.system.build;
     };
@@ -109,7 +301,7 @@ in
         iso = config.my.asISO.config.system.build.isoImage;
         container = config.my.asContainer.config.system.build.toplevel;
         kexecTree = config.my.asKexecTree.config.system.build.kexecTree;
-        netbootArchive = config.my.asKexecTree.config.system.build.netbootArchive;
+        netbootTree = config.my.asNetboot.config.system.build.netbootTree;
       };
     };
   };

nixos/modules/netboot/default.nix

@@ -0,0 +1,228 @@
+{ lib, pkgs, config, systems, ... }:
+let
+  inherit (builtins) toJSON;
+  inherit (lib) optional optionalAttrs mapAttrsToList mkMerge mkIf withFeature mkOption;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  rpcOpts = with lib.types; {
+    options = {
+      method = mkOpt' str null "RPC method name.";
+      params = mkOpt' (attrsOf unspecified) { } "RPC params";
+    };
+  };
+
+  cfg = config.my.netboot;
+  config' = {
+    subsystems = mapAttrsToList (subsystem: c: {
+      inherit subsystem;
+      config = map (rpc: {
+        inherit (rpc) method;
+      } // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c;
+    }) cfg.config.subsystems;
+  };
+  configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
+
+  spdk = pkgs.spdk.overrideAttrs (o: {
+    configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
+    buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
+  });
+  spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
+    exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
+  '');
+  spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
+    exec ${spdk.src}/scripts/setup.sh "$@"
+  '');
+  spdk-debug = pkgs.writeShellApplication {
+    name = "spdk-debug";
+    runtimeInputs = [ spdk ];
+    text = ''
+      set -m
+      if [ "$(id -u)" -ne 0 ]; then
+        echo "I need to be root!"
+        exit 1
+      fi
+
+      spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
+      until spdk-rpc spdk_get_version > /dev/null; do
+        sleep 0.5
+      done
+
+      spdk-rpc bdev_set_options --disable-auto-examine
+      spdk-rpc framework_start_init
+
+      ${cfg.debugCommands}
+
+      fg %1
+    '';
+  };
+
+  tftpRoot = pkgs.linkFarm "tftp-root" [
+    {
+      name = "ipxe-x86_64.efi";
+      path = "${pkgs.ipxe}/ipxe.efi";
+    }
+  ];
+  menuFile = pkgs.runCommand "menu.ipxe" {
+    bootHost = cfg.server.host;
+  } ''
+    substituteAll ${./menu.ipxe} "$out"
+  '';
+in
+{
+  options.my.netboot = with lib.types; {
+    client = {
+      enable = mkBoolOpt' false "Whether network booting should be enabled.";
+    };
+    server = {
+      enable = mkBoolOpt' false "Whether a netboot server should be enabled.";
+      ip = mkOpt' str null "IP clients should connect to via TFTP.";
+      host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP.";
+      instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for.";
+      keaClientClasses = mkOption {
+        type = listOf (attrsOf str);
+        description = "Kea client classes for PXE boot.";
+        readOnly = true;
+      };
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.client.enable {
+      environment.systemPackages = [
+        spdk
+        spdk-setup
+        spdk-rpc
+      ] ++ (optional (cfg.debugCommands != "") spdk-debug);
+
+      systemd.services = {
+        spdk-tgt = {
+          description = "SPDK target";
+          path = with pkgs; [
+            bash
+            python3
+            kmod
+            gawk
+            util-linux
+          ];
+          serviceConfig = {
+            ExecStartPre = "${spdk.src}/scripts/setup.sh";
+            ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
+          };
+          wantedBy = [ "multi-user.target" ];
+        };
+      };
+    })
+    (mkIf cfg.server.enable {
+      environment = {
+        etc = {
+          "netboot/menu.ipxe".source = menuFile;  
+          "netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi";
+        };
+      };
+    
+      systemd = {
+        services = {
+          netboot-update = {
+            description = "Update netboot images";
+            after = [ "systemd-networkd-wait-online.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+            };
+            path = with pkgs; [
+              coreutils curl jq gnutar
+            ];
+            script = ''
+              update_nixos() {
+                latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \
+                             | jq -r .commit.sha | cut -c -7)"
+                if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then
+                  echo "NixOS installer is up to date"
+                  return
+                fi
+
+                echo "Updating NixOS installer to $latestShort"
+                mkdir -p nixos-installer
+                fname="nixos-installer-devplayer0-netboot-$latestShort.tar"
+                downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \
+                               jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")"
+                curl -Lo /tmp/nixos-installer-netboot.tar "$downloadUrl"
+                tar -C nixos-installer -xf /tmp/nixos-installer-netboot.tar
+                rm /tmp/nixos-installer-netboot.tar
+                echo "$latestShort" > nixos-installer/tag.txt
+              }
+
+              mkdir -p /srv/netboot
+              cd /srv/netboot
+
+              ln -sf ${menuFile} boot.ipxe
+              ln -sf "${pkgs.edk2-uefi-shell}/shell.efi"
+              update_nixos
+            '';
+            startAt = "06:00";
+            wantedBy = [ "network-online.target" ];
+          };
+
+          nbd-server.preStart = ''
+            mkdir /tmp/nbdcow
+          '';
+        };
+      };
+
+      services = {
+        atftpd = {
+          enable = true;
+          root = tftpRoot;
+        };
+
+        nginx = {
+          virtualHosts."${cfg.server.host}" = {
+            locations."/" = {
+              root = "/srv/netboot";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+
+        nbd.server = {
+          enable = true;
+          extraOptions = {
+            allowlist = true;
+          };
+          exports = {
+            nixos-installer = {
+              path = "/srv/netboot/nixos-installer/nix-store.sfs";
+              extraOptions = {
+                copyonwrite = true;
+                cowdir = "/tmp/nbdcow";
+                sparse_cow = true;
+              };
+            };
+          };
+        };
+      };
+
+      my = {
+        tmproot.persistence.config.directories = [ "/srv/netboot" ];
+        netboot.server.keaClientClasses = [
+          {
+            name = "ipxe";
+            test = "substring(option[user-class].hex, 0, 4) == 'iPXE'";
+            next-server = cfg.server.ip;
+            server-hostname = cfg.server.host;
+            boot-file-name = "http://${cfg.server.host}/boot.ipxe";
+          }
+          {
+            name = "efi-x86_64";
+            test = "option[client-system].hex == 0x0007";
+            next-server = cfg.server.ip;
+            server-hostname = cfg.server.host;
+            boot-file-name = "ipxe-x86_64.efi";
+          }
+        ];
+      };
+    })
+  ];
+}

nixos/modules/netboot/menu.ipxe

@@ -0,0 +1,68 @@
+#!ipxe
+
+set server http://@bootHost@
+
+# Figure out if client is 64-bit capable
+cpuid --ext 29 && set arch x86_64 || set arch i386
+
+isset ${menu-default} || set menu-default exit
+
+:start
+menu Welcome to /dev/player0's humble iPXE boot menu
+item --gap --           Operating Systems
+iseq ${arch} x86_64 &&
+item --key n nixos      NixOS installer
+# iseq ${arch} x86_64 &&
+# item --key a archlinux Arch Linux (archiso x86_64)
+# iseq ${arch} x86_64 &&
+# item --key p alpine   Alpine Linux
+item --gap --           Other Options
+item --key e efi_shell  UEFI Shell
+item --key x xyz        netboot.xyz
+item --key c config     iPXE settings
+item --key s shell      Drop to iPXE shell
+item --key r reboot     Reboot
+item --key q exit       Exit (and continue to next boot device)
+choose --timeout 0 --default ${menu-default} selected || goto cancel
+goto ${selected}
+
+:cancel
+echo You cancelled the menu, dropping you to an iPXE shell
+
+:shell
+echo Type 'exit' to go back to the menu
+shell
+set menu-default nixos
+goto start
+
+:failed
+echo Booting failed, dropping to shell
+goto shell
+
+:reboot
+reboot
+
+:exit
+exit
+
+:config
+config
+set menu-default config
+goto start
+
+:efi_shell
+chain ${server}/efi-shell-${arch}.efi || goto failed
+
+:xyz
+chain --autofree https://boot.netboot.xyz || goto failed
+
+:nixos
+set cmdline nbd_export=nixos-installer
+chain ${server}/nixos-installer/boot.ipxe || goto failed
+
+:archlinux
+# set mirrorurl https://arch.nul.ie/
+chain ${server}/arch.ipxe || goto failed
+
+:alpine
+chain ${server}/alpine.ipxe || goto failed