nixos: Initial netbooting installer

.gitea/workflows/installer.yaml

@@ -39,7 +39,7 @@ jobs:
         run: |
           nix build .#nixfiles.config.nixos.systems.installer.configuration.config.my.buildAs.netbootArchive
           ln -s "$(readlink result)" \
-            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar
+            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst
 
       - name: Create release
         uses: https://gitea.com/actions/release-action@main
@@ -48,4 +48,4 @@ jobs:
           api_key: '${{ secrets.RELEASE_TOKEN }}'
           files: |
             jackos-installer-${{ steps.setup.outputs.short_rev }}.iso
-            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar
+            jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst

devshell/commands.nix

@@ -106,8 +106,8 @@ in
     {
       name = "build-netboot";
       category = "tasks";
-      help = "Build NixOS configuration as netboot archive";
+      help = "Build NixOS configuration as netboot tree";
-      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootArchive"'';
+      command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"'';
     }
     {
       name = "build-home";

nixos/boxes/home/routing-common/default.nix

@@ -148,9 +148,11 @@ in
                 };
               };
             };
+
+            nginx.enable = true;
           };
 
-          networking.domain = "h.${pubDomain}";
+          networking = { inherit domain; };
 
           systemd.services =
           let
@@ -399,6 +401,11 @@ in
                 }
               '';
             };
+            netboot.server = {
+              enable = true;
+              ip = vips.lo.v4;
+              host = "boot.${domain}";
+            };
           };
         };
       };

nixos/boxes/home/routing-common/dns.nix

@@ -172,6 +172,7 @@ in
             }}
             ${elemAt routers 0} IN AAAA ${net.cidr.host 1 prefixes.hi.v6}
             ${elemAt routers 1} IN AAAA ${net.cidr.host 2 prefixes.hi.v6}
+            boot IN CNAME router-hi.${config.networking.domain}.
 
             @ IN NS ns1
             @ IN NS ns2

nixos/boxes/home/routing-common/kea.nix

@@ -1,4 +1,4 @@
-index: { lib, pkgs, assignments, ... }:
+index: { lib, pkgs, config, assignments, ... }:
 let
   inherit (lib) mkForce;
   inherit (lib.my) net;
@@ -63,6 +63,7 @@ in
               always-send = true;
             }
           ];
+          client-classes = config.my.netboot.server.keaClientClasses;
           subnet4 = [
             {
               id = 1;

nixos/modules/_list.nix

@@ -20,5 +20,6 @@
     nvme = ./nvme;
     spdk = ./spdk.nix;
     librespeed = ./librespeed;
+    netboot = ./netboot;
   };
 }

nixos/modules/build.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, extendModules, modulesPath, options, config, ... }:
 let
-  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional;
+  inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional;
   inherit (lib.my) mkBoolOpt' dummyOption;
 
   cfg = config.my.build;
@@ -43,15 +43,145 @@ let
     modules = flatten [
       "${modulesPath}/installer/netboot/netboot.nix"
       allHardware
+    ];
+  };
+
+  asNetboot = extendModules {
+    modules = flatten [
+      allHardware
       ({ pkgs, config, ... }: {
-        system.build.netbootArchive = pkgs.runCommand "netboot-${config.system.name}-archive.tar" { } ''
+        boot = {
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.kernel}" \
+          loader.grub.enable = false;
-            -f "$out" "${config.system.boot.loader.kernelFile}"
+          kernelParams = [ "console=ttyS0,115200n8" ];
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootRamdisk}" \
+          initrd = {
-            -f "$out" initrd
+            kernelModules = [ "nbd" ];
-          ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootIpxeScript}" \
+
-            -f "$out" netboot.ipxe
+            systemd = {
-        '';
+              storePaths = with pkgs; [
+                gnused
+                nbd
+                netcat
+              ];
+              extraBin = with pkgs; {
+                dmesg = "${util-linux}/bin/dmesg";
+                ip = "${iproute2}/bin/ip";
+                nbd-client = "${nbd}/bin/nbd-client";
+              };
+              extraConfig = ''
+                DefaultTimeoutStartSec=10
+                DefaultDeviceTimeoutSec=10
+              '';
+
+              network = {
+                enable = true;
+                wait-online.enable = true;
+
+                networks."10-netboot" = {
+                  matchConfig.Name = "et-boot";
+                  DHCP = "yes";
+                };
+              };
+
+              services = {
+                nbd = {
+                  description = "NBD Root FS";
+
+                  script = ''
+                    get_cmdline() {
+                      ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
+                    }
+
+                    s="$(get_cmdline nbd_server)"
+                    until ${pkgs.netcat}/bin/nc -zv "$s" 22; do
+                      sleep 0.1
+                    done
+
+                    exec ${pkgs.nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0
+                  '';
+                  unitConfig = {
+                    IgnoreOnIsolate = "yes";
+                    DefaultDependencies = "no";
+                  };
+                  serviceConfig = {
+                    Type = "forking";
+                    Restart = "on-failure";
+                    RestartSec = 10;
+                  };
+
+                  wantedBy = [ "initrd-root-device.target" ];
+                };
+              };
+            };
+          };
+
+          postBootCommands = ''
+            # After booting, register the contents of the Nix store
+            # in the Nix database in the COW root.
+            ${config.nix.package}/bin/nix-store --load-db < /nix-path-registration
+
+            # nixos-rebuild also requires a "system" profile and an
+            # /etc/NIXOS tag.
+            touch /etc/NIXOS
+            ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system
+          '';
+        };
+
+        programs.nbd.enable = true;
+
+        fileSystems = {
+          "/" = {
+            fsType = "ext4";
+            device = "/dev/nbd0";
+            noCheck = true;
+            autoResize = true;
+          };
+        };
+
+        networking.useNetworkd = mkForce true;
+
+        systemd = {
+          network.networks."10-boot" = {
+            matchConfig.Name = "et-boot";
+            DHCP = "yes";
+            networkConfig.KeepConfiguration = "yes";
+          };
+        };
+
+        system.build = {
+          rootImage = pkgs.callPackage "${modulesPath}/../lib/make-ext4-fs.nix" {
+            storePaths = [ config.system.build.toplevel ];
+            volumeLabel = "netboot-root";
+          };
+          netbootScript = pkgs.writeText "boot.ipxe" ''
+            #!ipxe
+            kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-boot:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline}
+            initrd initrd
+            boot
+          '';
+
+          netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [
+            {
+              name = config.system.boot.loader.kernelFile;
+              path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
+            }
+            {
+              name = "initrd";
+              path = "${config.system.build.initialRamdisk}/initrd";
+            }
+            {
+              name = "rootfs.ext4";
+              path = config.system.build.rootImage;
+            }
+            {
+              name = "boot.ipxe";
+              path = config.system.build.netbootScript;
+            }
+          ];
+          netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar.zst" { } ''
+            export PATH=${pkgs.zstd}/bin:$PATH
+            ${pkgs.gnutar}/bin/tar --dereference --zstd -cvC ${config.system.build.netbootTree} -f "$out" .
+          '';
+        };
       })
     ];
   };
@@ -77,6 +207,7 @@ in
       asISO = mkAsOpt asISO "a bootable .iso image";
       asContainer = mkAsOpt asContainer "a container";
       asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd";
+      asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script";
 
       buildAs = options.system.build;
     };
@@ -110,7 +241,8 @@ in
         iso = config.my.asISO.config.system.build.isoImage;
         container = config.my.asContainer.config.system.build.toplevel;
         kexecTree = config.my.asKexecTree.config.system.build.kexecTree;
-        netbootArchive = config.my.asKexecTree.config.system.build.netbootArchive;
+        netbootTree = config.my.asNetboot.config.system.build.netbootTree;
+        netbootArchive = config.my.asNetboot.config.system.build.netbootArchive;
       };
     };
   };

nixos/modules/netboot/default.nix

@@ -0,0 +1,165 @@
+{ lib, pkgs, config, systems, ... }:
+let
+  inherit (lib) mkMerge mkIf mkForce mkOption;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  cfg = config.my.netboot;
+
+  tftpRoot = pkgs.linkFarm "tftp-root" [
+    {
+      name = "ipxe-x86_64.efi";
+      path = "${pkgs.ipxe}/ipxe.efi";
+    }
+  ];
+  menuFile = pkgs.runCommand "menu.ipxe" {
+    bootHost = cfg.server.host;
+  } ''
+    substituteAll ${./menu.ipxe} "$out"
+  '';
+in
+{
+  options.my.netboot = with lib.types; {
+    client = {
+      enable = mkBoolOpt' false "Whether network booting should be enabled.";
+    };
+    server = {
+      enable = mkBoolOpt' false "Whether a netboot server should be enabled.";
+      ip = mkOpt' str null "IP clients should connect to via TFTP.";
+      host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP.";
+      installer = {
+        storeSize = mkOpt' str "16GiB" "Total allowed writable size of store.";
+      };
+      instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for.";
+      keaClientClasses = mkOption {
+        type = listOf (attrsOf str);
+        description = "Kea client classes for PXE boot.";
+        readOnly = true;
+      };
+    };
+  };
+
+  config = mkMerge [
+    (mkIf cfg.client.enable {
+      # TODO: Implement!
+    })
+    (mkIf cfg.server.enable {
+      environment = {
+        etc = {
+          "netboot/menu.ipxe".source = menuFile;
+          "netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi";
+        };
+      };
+
+      systemd = {
+        services = {
+          netboot-update = {
+            description = "Update netboot images";
+            after = [ "systemd-networkd-wait-online.service" ];
+            serviceConfig = {
+              Type = "oneshot";
+              RemainAfterExit = true;
+            };
+            path = with pkgs; [
+              coreutils curl jq zstd gnutar
+            ];
+            script = ''
+              update_nixos() {
+                latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \
+                             | jq -r .commit.sha | cut -c -7)"
+                if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then
+                  echo "NixOS installer is up to date"
+                  return
+                fi
+
+                echo "Updating NixOS installer to $latestShort"
+                mkdir -p nixos-installer
+                fname="jackos-installer-netboot-$latestShort.tar.zst"
+                downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \
+                               jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")"
+                curl -Lo /tmp/nixos-installer-netboot.tar.zst "$downloadUrl"
+                tar -C nixos-installer --zstd -xf /tmp/nixos-installer-netboot.tar.zst
+                truncate -s "${cfg.server.installer.storeSize}" nixos-installer/rootfs.ext4
+                rm /tmp/nixos-installer-netboot.tar.zst
+                echo "$latestShort" > nixos-installer/tag.txt
+              }
+
+              mkdir -p /srv/netboot
+              cd /srv/netboot
+
+              ln -sf ${menuFile} boot.ipxe
+              ln -sf "${pkgs.edk2-uefi-shell}/efi-shell-${config.nixpkgs.localSystem.linuxArch}.efi"
+              update_nixos
+            '';
+            startAt = "06:00";
+            wantedBy = [ "network-online.target" ];
+          };
+
+          nbd-server = {
+            serviceConfig = {
+              PrivateUsers = mkForce false;
+              CacheDirectory = "netboot";
+            };
+          };
+        };
+      };
+
+      services = {
+        atftpd = {
+          enable = true;
+          root = tftpRoot;
+        };
+
+        nginx = {
+          virtualHosts."${cfg.server.host}" = {
+            locations."/" = {
+              root = "/srv/netboot";
+              extraConfig = ''
+                autoindex on;
+              '';
+            };
+          };
+        };
+
+        nbd.server = {
+          enable = true;
+          extraOptions = {
+            allowlist = true;
+          };
+          exports = {
+            nixos-installer = {
+              path = "/srv/netboot/nixos-installer/rootfs.ext4";
+              extraOptions = {
+                copyonwrite = true;
+                cowdir = "/var/cache/netboot";
+                sparse_cow = true;
+              };
+            };
+          };
+        };
+      };
+
+      my = {
+        tmproot.persistence.config.directories = [
+          "/srv/netboot"
+          { directory = "/var/cache/netboot"; mode = "0700"; }
+        ];
+        netboot.server.keaClientClasses = [
+          {
+            name = "ipxe";
+            test = "substring(option[user-class].hex, 0, 4) == 'iPXE'";
+            next-server = cfg.server.ip;
+            server-hostname = cfg.server.host;
+            boot-file-name = "http://${cfg.server.host}/boot.ipxe";
+          }
+          {
+            name = "efi-x86_64";
+            test = "option[client-system].hex == 0x0007";
+            next-server = cfg.server.ip;
+            server-hostname = cfg.server.host;
+            boot-file-name = "ipxe-x86_64.efi";
+          }
+        ];
+      };
+    })
+  ];
+}

nixos/modules/netboot/menu.ipxe

@@ -0,0 +1,68 @@
+#!ipxe
+
+set server http://@bootHost@
+
+# Figure out if client is 64-bit capable
+cpuid --ext 29 && set arch x86_64 || set arch i386
+
+isset ${menu-default} || set menu-default exit
+
+:start
+menu Welcome to /dev/player0's humble iPXE boot menu
+item --gap --           Operating Systems
+iseq ${arch} x86_64 &&
+item --key n nixos      NixOS installer
+# iseq ${arch} x86_64 &&
+# item --key a archlinux Arch Linux (archiso x86_64)
+# iseq ${arch} x86_64 &&
+# item --key p alpine   Alpine Linux
+item --gap --           Other Options
+item --key e efi_shell  UEFI Shell
+item --key x xyz        netboot.xyz
+item --key c config     iPXE settings
+item --key s shell      Drop to iPXE shell
+item --key r reboot     Reboot
+item --key q exit       Exit (and continue to next boot device)
+choose --timeout 0 --default ${menu-default} selected || goto cancel
+goto ${selected}
+
+:cancel
+echo You cancelled the menu, dropping you to an iPXE shell
+
+:shell
+echo Type 'exit' to go back to the menu
+shell
+set menu-default nixos
+goto start
+
+:failed
+echo Booting failed, dropping to shell
+goto shell
+
+:reboot
+reboot
+
+:exit
+exit
+
+:config
+config
+set menu-default config
+goto start
+
+:efi_shell
+chain ${server}/efi-shell-${arch}.efi || goto failed
+
+:xyz
+chain --autofree https://boot.netboot.xyz || goto failed
+
+:nixos
+set cmdline nbd_export=nixos-installer
+chain ${server}/nixos-installer/boot.ipxe || goto failed
+
+:archlinux
+# set mirrorurl https://arch.nul.ie/
+chain ${server}/arch.ipxe || goto failed
+
+:alpine
+chain ${server}/alpine.ipxe || goto failed