diff --git a/devshell/commands.nix b/devshell/commands.nix index a3fa286..215d1c0 100644 --- a/devshell/commands.nix +++ b/devshell/commands.nix @@ -106,8 +106,8 @@ in { name = "build-netboot"; category = "tasks"; - help = "Build NixOS configuration as netboot archive"; - command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootArchive"''; + help = "Build NixOS configuration as netboot tree"; + command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"''; } { name = "build-home"; diff --git a/nixos/boxes/home/routing-common/default.nix b/nixos/boxes/home/routing-common/default.nix index 3bbcdc7..7b5544a 100644 --- a/nixos/boxes/home/routing-common/default.nix +++ b/nixos/boxes/home/routing-common/default.nix @@ -148,9 +148,11 @@ in }; }; }; + + nginx.enable = true; }; - networking.domain = "h.${pubDomain}"; + networking = { inherit domain; }; systemd.services = { ipsec = @@ -376,6 +378,11 @@ in } ''; }; + netboot.server = { + enable = true; + ip = vips.lo.v4; + host = "boot.${domain}"; + }; }; }; }; diff --git a/nixos/boxes/home/routing-common/dns.nix b/nixos/boxes/home/routing-common/dns.nix index b08cfff..2a81a07 100644 --- a/nixos/boxes/home/routing-common/dns.nix +++ b/nixos/boxes/home/routing-common/dns.nix @@ -130,6 +130,7 @@ in }} ${elemAt routers 0} IN AAAA ${net.cidr.host 1 prefixes.hi.v6} ${elemAt routers 1} IN AAAA ${net.cidr.host 2 prefixes.hi.v6} + boot IN CNAME router-hi.${config.networking.domain}. @ IN NS ns1 @ IN NS ns2 diff --git a/nixos/boxes/home/routing-common/kea.nix b/nixos/boxes/home/routing-common/kea.nix index a92a4e7..9eee1d8 100644 --- a/nixos/boxes/home/routing-common/kea.nix +++ b/nixos/boxes/home/routing-common/kea.nix @@ -1,4 +1,4 @@ -index: { lib, pkgs, assignments, ... }: +index: { lib, pkgs, config, assignments, ... }: let inherit (lib) mkForce; inherit (lib.my) net; @@ -59,6 +59,7 @@ in always-send = true; } ]; + client-classes = config.my.netboot.server.keaClientClasses; subnet4 = [ { id = 1; diff --git a/nixos/modules/_list.nix b/nixos/modules/_list.nix index 899bc1d..4901630 100644 --- a/nixos/modules/_list.nix +++ b/nixos/modules/_list.nix @@ -19,5 +19,6 @@ borgthin = ./borgthin.nix; nvme = ./nvme; spdk = ./spdk.nix; + netboot = ./netboot; }; } diff --git a/nixos/modules/build.nix b/nixos/modules/build.nix index aad2a11..03ccae0 100644 --- a/nixos/modules/build.nix +++ b/nixos/modules/build.nix @@ -1,6 +1,6 @@ { lib, pkgs, extendModules, modulesPath, options, config, ... }: let - inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional; + inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional; inherit (lib.my) mkBoolOpt' dummyOption; cfg = config.my.build; @@ -43,15 +43,206 @@ let modules = flatten [ "${modulesPath}/installer/netboot/netboot.nix" allHardware - ({ pkgs, config, ... }: { - system.build.netbootArchive = pkgs.runCommand "netboot-${config.system.name}-archive.tar" { } '' - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.kernel}" \ - -f "$out" "${config.system.boot.loader.kernelFile}" - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootRamdisk}" \ - -f "$out" initrd - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootIpxeScript}" \ - -f "$out" netboot.ipxe + (mkIf config.boot.initrd.systemd.enable { + boot.initrd.systemd.services.setup-overlay-dirs = { + description = "Create overlayfs dirs"; + after = [ "sysroot-nix-.rw\\x2dstore.mount" ]; + before = [ "sysroot-nix-store.mount" ]; + script = '' + mkdir /sysroot/nix/.rw-store/{store,work} + ''; + wantedBy = [ "initrd-fs.target" ]; + }; + + fileSystems."/nix/store" = mkForce { + fsType = "overlay"; + device = "overlay"; + options = [ + "lowerdir=/sysroot/nix/.ro-store" + "upperdir=/sysroot/nix/.rw-store/store" + "workdir=/sysroot/nix/.rw-store/work" + ]; + }; + + systemd.units."nix-store.mount".enable = false; + }) + ]; + }; + + asNetboot = extendModules { + modules = flatten [ + allHardware + ({ pkgs, config, ... }: + let + initrdNbdWrapper = pkgs.writeCBin "nbd-wrapper" '' + #include + #include + + int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s \n", argv[0]); + return -1; + } + + argv[0][0] = '@'; + char* args[] = { + "@", "-nofork", "-N", argv[1], argv[2], "/dev/nbd0", NULL + }; + execv("${pkgs.nbd}/bin/nbd-client", args); + return 0; + }; ''; + nbd = pkgs.nbd.overrideAttrs (o: { + # TODO: Remove once this makes it to us + # https://github.com/NixOS/nixpkgs/commit/52f1d9b03ae38126e7f648634fcad35897f464ed + configureFlags = [ "--sysconfdir=/etc" ]; + }); + in + { + boot = { + loader.grub.enable = false; + kernelParams = [ "console=ttyS0,115200n8" ]; + initrd = { + kernelModules = [ "nbd" ]; + + systemd = { + storePaths = with pkgs; [ + gnused + nbd + initrdNbdWrapper + netcat + ]; + extraBin = with pkgs; { + dmesg = "${util-linux}/bin/dmesg"; + ip = "${iproute2}/bin/ip"; + nbd-client = "${nbd}/bin/nbd-client"; + }; + extraConfig = '' + DefaultTimeoutStartSec=10 + DefaultDeviceTimeoutSec=10 + ''; + + network = { + enable = true; + wait-online.enable = true; + + networks."10-nbd" = { + matchConfig.Name = "et-nbd"; + DHCP = "yes"; + }; + }; + + services = { + # gennbdtab = { + # description = "Generate nbdtab"; + # script = '' + # get_cmdline() { + # ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline + # } + + # e="$(get_cmdline nbd_export)" + # s="$(get_cmdline nbd_server)" + # echo "Setting nbdtab for $e @ $s" + # echo "nbd0 $s $e persist" > /etc/nbdtab + # ''; + # serviceConfig.Type = "oneshot"; + # # wantedBy = [ "initrd-root-device.target" ]; + # }; + nbd = { + description = "NBD Nix store"; + # before = [ "initrd-root-device.target" ]; + # after = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ]; + # wants = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ]; + # after = [ "systemd-networkd-wait-online.service" ]; + # wants = [ "systemd-networkd-wait-online.service" ]; + + script = '' + get_cmdline() { + ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline + } + s="$(get_cmdline nbd_server)" + until ${pkgs.netcat}/bin/nc -zv "$s" 22; do + sleep 0.1 + done + exec ${nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0 + # exec ${initrdNbdWrapper}/bin/nbd-wrapper "$(get_cmdline nbd_export)" "$(get_cmdline nbd_server)" + ''; + unitConfig = { + IgnoreOnIsolate = "yes"; + DefaultDependencies = "no"; + }; + serviceConfig = { + Type = "forking"; + # ExecStart = "${nbd}/bin/nbd-client -nofork -systemd-mark nbd0"; + Restart = "on-failure"; + RestartSec = 10; + }; + + # wantedBy = [ "initrd-root-device.target" ]; + }; + }; + }; + }; + }; + + programs.nbd.enable = true; + + fileSystems = { + "/" = { + fsType = "tmpfs"; + options = [ "mode=0755" ]; + }; + "/nix/store" = { + fsType = "squashfs"; + device = "/dev/nbd0"; + options = [ "x-systemd.requires=nbd.service" ]; + }; + }; + + system.build = { + squashfsStore = pkgs.callPackage "${modulesPath}/../lib/make-squashfs.nix" { + storeContents = [ config.system.build.toplevel ]; + comp = "zstd"; + }; + netbootScript = pkgs.writeText "boot.ipxe" '' + #!ipxe + kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-nbd:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline} + initrd initrd + boot + ''; + netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [ + { + name = config.system.boot.loader.kernelFile; + path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}"; + } + { + name = "initrd"; + path = "${config.system.build.initialRamdisk}/initrd"; + } + { + name = "nix-store.sfs"; + path = config.system.build.squashfsStore; + } + { + name = "boot.ipxe"; + path = config.system.build.netbootScript; + } + ]; + netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar" { } '' + add() { + ${pkgs.gnutar}/bin/tar --dereference -rvC "$1" -f "$out" "$2" + } + + add "${config.system.build.kernel}" "${config.system.boot.loader.kernelFile}" + add "${config.system.build.initialRamdisk}" initrd + + tmpdir="$(mktemp -d sfsStore.XXXXXX)" + ln -s "${config.system.build.squashfsStore}" "$tmpdir"/nix-store.sfs + add "$tmpdir" nix-store.sfs + + add "${config.system.build.netbootScript}" boot.ipxe + ''; + }; }) ]; }; @@ -77,6 +268,7 @@ in asISO = mkAsOpt asISO "a bootable .iso image"; asContainer = mkAsOpt asContainer "a container"; asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd"; + asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script"; buildAs = options.system.build; }; @@ -109,7 +301,7 @@ in iso = config.my.asISO.config.system.build.isoImage; container = config.my.asContainer.config.system.build.toplevel; kexecTree = config.my.asKexecTree.config.system.build.kexecTree; - netbootArchive = config.my.asKexecTree.config.system.build.netbootArchive; + netbootTree = config.my.asNetboot.config.system.build.netbootTree; }; }; }; diff --git a/nixos/modules/netboot/default.nix b/nixos/modules/netboot/default.nix new file mode 100644 index 0000000..7bc2206 --- /dev/null +++ b/nixos/modules/netboot/default.nix @@ -0,0 +1,228 @@ +{ lib, pkgs, config, systems, ... }: +let + inherit (builtins) toJSON; + inherit (lib) optional optionalAttrs mapAttrsToList mkMerge mkIf withFeature mkOption; + inherit (lib.my) mkOpt' mkBoolOpt'; + + rpcOpts = with lib.types; { + options = { + method = mkOpt' str null "RPC method name."; + params = mkOpt' (attrsOf unspecified) { } "RPC params"; + }; + }; + + cfg = config.my.netboot; + config' = { + subsystems = mapAttrsToList (subsystem: c: { + inherit subsystem; + config = map (rpc: { + inherit (rpc) method; + } // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c; + }) cfg.config.subsystems; + }; + configJSON = pkgs.writeText "spdk-config.json" (toJSON config'); + + spdk = pkgs.spdk.overrideAttrs (o: { + configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]); + buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]); + }); + spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" '' + exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@" + ''); + spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" '' + exec ${spdk.src}/scripts/setup.sh "$@" + ''); + spdk-debug = pkgs.writeShellApplication { + name = "spdk-debug"; + runtimeInputs = [ spdk ]; + text = '' + set -m + if [ "$(id -u)" -ne 0 ]; then + echo "I need to be root!" + exit 1 + fi + + spdk_tgt ${cfg.extraArgs} --wait-for-rpc & + until spdk-rpc spdk_get_version > /dev/null; do + sleep 0.5 + done + + spdk-rpc bdev_set_options --disable-auto-examine + spdk-rpc framework_start_init + + ${cfg.debugCommands} + + fg %1 + ''; + }; + + tftpRoot = pkgs.linkFarm "tftp-root" [ + { + name = "ipxe-x86_64.efi"; + path = "${pkgs.ipxe}/ipxe.efi"; + } + ]; + menuFile = pkgs.runCommand "menu.ipxe" { + bootHost = cfg.server.host; + } '' + substituteAll ${./menu.ipxe} "$out" + ''; +in +{ + options.my.netboot = with lib.types; { + client = { + enable = mkBoolOpt' false "Whether network booting should be enabled."; + }; + server = { + enable = mkBoolOpt' false "Whether a netboot server should be enabled."; + ip = mkOpt' str null "IP clients should connect to via TFTP."; + host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP."; + instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for."; + keaClientClasses = mkOption { + type = listOf (attrsOf str); + description = "Kea client classes for PXE boot."; + readOnly = true; + }; + }; + }; + + config = mkMerge [ + (mkIf cfg.client.enable { + environment.systemPackages = [ + spdk + spdk-setup + spdk-rpc + ] ++ (optional (cfg.debugCommands != "") spdk-debug); + + systemd.services = { + spdk-tgt = { + description = "SPDK target"; + path = with pkgs; [ + bash + python3 + kmod + gawk + util-linux + ]; + serviceConfig = { + ExecStartPre = "${spdk.src}/scripts/setup.sh"; + ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}"; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; + }) + (mkIf cfg.server.enable { + environment = { + etc = { + "netboot/menu.ipxe".source = menuFile; + "netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi"; + }; + }; + + systemd = { + services = { + netboot-update = { + description = "Update netboot images"; + after = [ "systemd-networkd-wait-online.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + path = with pkgs; [ + coreutils curl jq gnutar + ]; + script = '' + update_nixos() { + latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \ + | jq -r .commit.sha | cut -c -7)" + if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then + echo "NixOS installer is up to date" + return + fi + + echo "Updating NixOS installer to $latestShort" + mkdir -p nixos-installer + fname="nixos-installer-devplayer0-netboot-$latestShort.tar" + downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \ + jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")" + curl -Lo /tmp/nixos-installer-netboot.tar "$downloadUrl" + tar -C nixos-installer -xf /tmp/nixos-installer-netboot.tar + rm /tmp/nixos-installer-netboot.tar + echo "$latestShort" > nixos-installer/tag.txt + } + + mkdir -p /srv/netboot + cd /srv/netboot + + ln -sf ${menuFile} boot.ipxe + ln -sf "${pkgs.edk2-uefi-shell}/shell.efi" + update_nixos + ''; + startAt = "06:00"; + wantedBy = [ "network-online.target" ]; + }; + + nbd-server.preStart = '' + mkdir /tmp/nbdcow + ''; + }; + }; + + services = { + atftpd = { + enable = true; + root = tftpRoot; + }; + + nginx = { + virtualHosts."${cfg.server.host}" = { + locations."/" = { + root = "/srv/netboot"; + extraConfig = '' + autoindex on; + ''; + }; + }; + }; + + nbd.server = { + enable = true; + extraOptions = { + allowlist = true; + }; + exports = { + nixos-installer = { + path = "/srv/netboot/nixos-installer/nix-store.sfs"; + extraOptions = { + copyonwrite = true; + cowdir = "/tmp/nbdcow"; + sparse_cow = true; + }; + }; + }; + }; + }; + + my = { + tmproot.persistence.config.directories = [ "/srv/netboot" ]; + netboot.server.keaClientClasses = [ + { + name = "ipxe"; + test = "substring(option[user-class].hex, 0, 4) == 'iPXE'"; + next-server = cfg.server.ip; + server-hostname = cfg.server.host; + boot-file-name = "http://${cfg.server.host}/boot.ipxe"; + } + { + name = "efi-x86_64"; + test = "option[client-system].hex == 0x0007"; + next-server = cfg.server.ip; + server-hostname = cfg.server.host; + boot-file-name = "ipxe-x86_64.efi"; + } + ]; + }; + }) + ]; +} diff --git a/nixos/modules/netboot/menu.ipxe b/nixos/modules/netboot/menu.ipxe new file mode 100644 index 0000000..dc2e8df --- /dev/null +++ b/nixos/modules/netboot/menu.ipxe @@ -0,0 +1,68 @@ +#!ipxe + +set server http://@bootHost@ + +# Figure out if client is 64-bit capable +cpuid --ext 29 && set arch x86_64 || set arch i386 + +isset ${menu-default} || set menu-default exit + +:start +menu Welcome to /dev/player0's humble iPXE boot menu +item --gap -- Operating Systems +iseq ${arch} x86_64 && +item --key n nixos NixOS installer +# iseq ${arch} x86_64 && +# item --key a archlinux Arch Linux (archiso x86_64) +# iseq ${arch} x86_64 && +# item --key p alpine Alpine Linux +item --gap -- Other Options +item --key e efi_shell UEFI Shell +item --key x xyz netboot.xyz +item --key c config iPXE settings +item --key s shell Drop to iPXE shell +item --key r reboot Reboot +item --key q exit Exit (and continue to next boot device) +choose --timeout 0 --default ${menu-default} selected || goto cancel +goto ${selected} + +:cancel +echo You cancelled the menu, dropping you to an iPXE shell + +:shell +echo Type 'exit' to go back to the menu +shell +set menu-default nixos +goto start + +:failed +echo Booting failed, dropping to shell +goto shell + +:reboot +reboot + +:exit +exit + +:config +config +set menu-default config +goto start + +:efi_shell +chain ${server}/efi-shell-${arch}.efi || goto failed + +:xyz +chain --autofree https://boot.netboot.xyz || goto failed + +:nixos +set cmdline nbd_export=nixos-installer +chain ${server}/nixos-installer/boot.ipxe || goto failed + +:archlinux +# set mirrorurl https://arch.nul.ie/ +chain ${server}/arch.ipxe || goto failed + +:alpine +chain ${server}/alpine.ipxe || goto failed