diff --git a/.gitea/workflows/installer.yaml b/.gitea/workflows/installer.yaml index 615750e..226f9f1 100644 --- a/.gitea/workflows/installer.yaml +++ b/.gitea/workflows/installer.yaml @@ -39,7 +39,7 @@ jobs: run: | nix build .#nixfiles.config.nixos.systems.installer.configuration.config.my.buildAs.netbootArchive ln -s "$(readlink result)" \ - jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar + jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst - name: Create release uses: https://gitea.com/actions/release-action@main @@ -48,4 +48,4 @@ jobs: api_key: '${{ secrets.RELEASE_TOKEN }}' files: | jackos-installer-${{ steps.setup.outputs.short_rev }}.iso - jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar + jackos-installer-netboot-${{ steps.setup.outputs.short_rev }}.tar.zst diff --git a/devshell/commands.nix b/devshell/commands.nix index b4f8937..5f54fac 100644 --- a/devshell/commands.nix +++ b/devshell/commands.nix @@ -106,8 +106,8 @@ in { name = "build-netboot"; category = "tasks"; - help = "Build NixOS configuration as netboot archive"; - command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootArchive"''; + help = "Build NixOS configuration as netboot tree"; + command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"''; } { name = "build-home"; diff --git a/nixos/boxes/home/routing-common/default.nix b/nixos/boxes/home/routing-common/default.nix index b8e2a22..338b8da 100644 --- a/nixos/boxes/home/routing-common/default.nix +++ b/nixos/boxes/home/routing-common/default.nix @@ -148,9 +148,11 @@ in }; }; }; + + nginx.enable = true; }; - networking.domain = "h.${pubDomain}"; + networking = { inherit domain; }; systemd.services = let @@ -399,6 +401,11 @@ in } ''; }; + netboot.server = { + enable = true; + ip = vips.lo.v4; + host = "boot.${domain}"; + }; }; }; }; diff --git a/nixos/boxes/home/routing-common/dns.nix b/nixos/boxes/home/routing-common/dns.nix index 45e6c95..17ea74d 100644 --- a/nixos/boxes/home/routing-common/dns.nix +++ b/nixos/boxes/home/routing-common/dns.nix @@ -172,6 +172,7 @@ in }} ${elemAt routers 0} IN AAAA ${net.cidr.host 1 prefixes.hi.v6} ${elemAt routers 1} IN AAAA ${net.cidr.host 2 prefixes.hi.v6} + boot IN CNAME router-hi.${config.networking.domain}. @ IN NS ns1 @ IN NS ns2 diff --git a/nixos/boxes/home/routing-common/kea.nix b/nixos/boxes/home/routing-common/kea.nix index f63f952..c073c91 100644 --- a/nixos/boxes/home/routing-common/kea.nix +++ b/nixos/boxes/home/routing-common/kea.nix @@ -1,4 +1,4 @@ -index: { lib, pkgs, assignments, ... }: +index: { lib, pkgs, config, assignments, ... }: let inherit (lib) mkForce; inherit (lib.my) net; @@ -63,6 +63,7 @@ in always-send = true; } ]; + client-classes = config.my.netboot.server.keaClientClasses; subnet4 = [ { id = 1; diff --git a/nixos/modules/_list.nix b/nixos/modules/_list.nix index 8fbf0f3..dc2ee46 100644 --- a/nixos/modules/_list.nix +++ b/nixos/modules/_list.nix @@ -20,5 +20,6 @@ nvme = ./nvme; spdk = ./spdk.nix; librespeed = ./librespeed; + netboot = ./netboot; }; } diff --git a/nixos/modules/build.nix b/nixos/modules/build.nix index efb5d33..6cb2973 100644 --- a/nixos/modules/build.nix +++ b/nixos/modules/build.nix @@ -1,6 +1,6 @@ { lib, pkgs, extendModules, modulesPath, options, config, ... }: let - inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional; + inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional; inherit (lib.my) mkBoolOpt' dummyOption; cfg = config.my.build; @@ -43,15 +43,144 @@ let modules = flatten [ "${modulesPath}/installer/netboot/netboot.nix" allHardware + ]; + }; + + asNetboot = extendModules { + modules = flatten [ + allHardware ({ pkgs, config, ... }: { - system.build.netbootArchive = pkgs.runCommand "netboot-${config.system.name}-archive.tar" { } '' - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.kernel}" \ - -f "$out" "${config.system.boot.loader.kernelFile}" - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootRamdisk}" \ - -f "$out" initrd - ${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootIpxeScript}" \ - -f "$out" netboot.ipxe - ''; + boot = { + loader.grub.enable = false; + initrd = { + kernelModules = [ "nbd" ]; + + systemd = { + storePaths = with pkgs; [ + gnused + nbd + netcat + ]; + extraBin = with pkgs; { + dmesg = "${util-linux}/bin/dmesg"; + ip = "${iproute2}/bin/ip"; + nbd-client = "${nbd}/bin/nbd-client"; + }; + extraConfig = '' + DefaultTimeoutStartSec=10 + DefaultDeviceTimeoutSec=10 + ''; + + network = { + enable = true; + wait-online.enable = true; + + networks."10-netboot" = { + matchConfig.Name = "et-boot"; + DHCP = "yes"; + }; + }; + + services = { + nbd = { + description = "NBD Root FS"; + + script = '' + get_cmdline() { + ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline + } + + s="$(get_cmdline nbd_server)" + until ${pkgs.netcat}/bin/nc -zv "$s" 22; do + sleep 0.1 + done + + exec ${pkgs.nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0 + ''; + unitConfig = { + IgnoreOnIsolate = "yes"; + DefaultDependencies = "no"; + }; + serviceConfig = { + Type = "forking"; + Restart = "on-failure"; + RestartSec = 10; + }; + + wantedBy = [ "initrd-root-device.target" ]; + }; + }; + }; + }; + + postBootCommands = '' + # After booting, register the contents of the Nix store + # in the Nix database in the COW root. + ${config.nix.package}/bin/nix-store --load-db < /nix-path-registration + + # nixos-rebuild also requires a "system" profile and an + # /etc/NIXOS tag. + touch /etc/NIXOS + ${config.nix.package.out}/bin/nix-env -p /nix/var/nix/profiles/system --set /run/current-system + ''; + }; + + programs.nbd.enable = true; + + fileSystems = { + "/" = { + fsType = "ext4"; + device = "/dev/nbd0"; + noCheck = true; + autoResize = true; + }; + }; + + networking.useNetworkd = mkForce true; + + systemd = { + network.networks."10-boot" = { + matchConfig.Name = "et-boot"; + DHCP = "yes"; + networkConfig.KeepConfiguration = "yes"; + }; + }; + + system.build = { + rootImage = pkgs.callPackage "${modulesPath}/../lib/make-ext4-fs.nix" { + storePaths = [ config.system.build.toplevel ]; + volumeLabel = "netboot-root"; + }; + netbootScript = pkgs.writeText "boot.ipxe" '' + #!ipxe + kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-boot:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline} + initrd initrd + boot + ''; + + netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [ + { + name = config.system.boot.loader.kernelFile; + path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}"; + } + { + name = "initrd"; + path = "${config.system.build.initialRamdisk}/initrd"; + } + { + name = "rootfs.ext4"; + path = config.system.build.rootImage; + } + { + name = "boot.ipxe"; + path = config.system.build.netbootScript; + } + ]; + netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar.zst" { } '' + export PATH=${pkgs.zstd}/bin:$PATH + ${pkgs.gnutar}/bin/tar --dereference --zstd -cvC ${config.system.build.netbootTree} -f "$out" . + ''; + }; }) ]; }; @@ -77,6 +206,7 @@ in asISO = mkAsOpt asISO "a bootable .iso image"; asContainer = mkAsOpt asContainer "a container"; asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd"; + asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script"; buildAs = options.system.build; }; @@ -110,7 +240,8 @@ in iso = config.my.asISO.config.system.build.isoImage; container = config.my.asContainer.config.system.build.toplevel; kexecTree = config.my.asKexecTree.config.system.build.kexecTree; - netbootArchive = config.my.asKexecTree.config.system.build.netbootArchive; + netbootTree = config.my.asNetboot.config.system.build.netbootTree; + netbootArchive = config.my.asNetboot.config.system.build.netbootArchive; }; }; }; diff --git a/nixos/modules/netboot/default.nix b/nixos/modules/netboot/default.nix new file mode 100644 index 0000000..9b33f26 --- /dev/null +++ b/nixos/modules/netboot/default.nix @@ -0,0 +1,165 @@ +{ lib, pkgs, config, systems, ... }: +let + inherit (lib) mkMerge mkIf mkForce mkOption; + inherit (lib.my) mkOpt' mkBoolOpt'; + + cfg = config.my.netboot; + + tftpRoot = pkgs.linkFarm "tftp-root" [ + { + name = "ipxe-x86_64.efi"; + path = "${pkgs.ipxe}/ipxe.efi"; + } + ]; + menuFile = pkgs.runCommand "menu.ipxe" { + bootHost = cfg.server.host; + } '' + substituteAll ${./menu.ipxe} "$out" + ''; +in +{ + options.my.netboot = with lib.types; { + client = { + enable = mkBoolOpt' false "Whether network booting should be enabled."; + }; + server = { + enable = mkBoolOpt' false "Whether a netboot server should be enabled."; + ip = mkOpt' str null "IP clients should connect to via TFTP."; + host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP."; + installer = { + storeSize = mkOpt' str "16GiB" "Total allowed writable size of store."; + }; + instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for."; + keaClientClasses = mkOption { + type = listOf (attrsOf str); + description = "Kea client classes for PXE boot."; + readOnly = true; + }; + }; + }; + + config = mkMerge [ + (mkIf cfg.client.enable { + # TODO: Implement! + }) + (mkIf cfg.server.enable { + environment = { + etc = { + "netboot/menu.ipxe".source = menuFile; + "netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi"; + }; + }; + + systemd = { + services = { + netboot-update = { + description = "Update netboot images"; + after = [ "systemd-networkd-wait-online.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + path = with pkgs; [ + coreutils curl jq zstd gnutar + ]; + script = '' + update_nixos() { + latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \ + | jq -r .commit.sha | cut -c -7)" + if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then + echo "NixOS installer is up to date" + return + fi + + echo "Updating NixOS installer to $latestShort" + mkdir -p nixos-installer + fname="jackos-installer-netboot-$latestShort.tar.zst" + downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \ + jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")" + curl -Lo /tmp/nixos-installer-netboot.tar.zst "$downloadUrl" + tar -C nixos-installer --zstd -xf /tmp/nixos-installer-netboot.tar.zst + truncate -s "${cfg.server.installer.storeSize}" nixos-installer/rootfs.ext4 + rm /tmp/nixos-installer-netboot.tar.zst + echo "$latestShort" > nixos-installer/tag.txt + } + + mkdir -p /srv/netboot + cd /srv/netboot + + ln -sf ${menuFile} boot.ipxe + ln -sf "${pkgs.edk2-uefi-shell}/shell.efi" "efi-shell-${config.nixpkgs.localSystem.linuxArch}.efi" + update_nixos + ''; + startAt = "06:00"; + wantedBy = [ "network-online.target" ]; + }; + + nbd-server = { + serviceConfig = { + PrivateUsers = mkForce false; + CacheDirectory = "netboot"; + }; + }; + }; + }; + + services = { + atftpd = { + enable = true; + root = tftpRoot; + }; + + nginx = { + virtualHosts."${cfg.server.host}" = { + locations."/" = { + root = "/srv/netboot"; + extraConfig = '' + autoindex on; + ''; + }; + }; + }; + + nbd.server = { + enable = true; + extraOptions = { + allowlist = true; + }; + exports = { + nixos-installer = { + path = "/srv/netboot/nixos-installer/rootfs.ext4"; + extraOptions = { + copyonwrite = true; + cowdir = "/var/cache/netboot"; + sparse_cow = true; + }; + }; + }; + }; + }; + + my = { + tmproot.persistence.config.directories = [ + "/srv/netboot" + { directory = "/var/cache/netboot"; mode = "0700"; } + ]; + netboot.server.keaClientClasses = [ + { + name = "ipxe"; + test = "substring(option[user-class].hex, 0, 4) == 'iPXE'"; + next-server = cfg.server.ip; + server-hostname = cfg.server.host; + boot-file-name = "http://${cfg.server.host}/boot.ipxe"; + } + { + name = "efi-x86_64"; + test = "option[client-system].hex == 0x0007"; + next-server = cfg.server.ip; + server-hostname = cfg.server.host; + boot-file-name = "ipxe-x86_64.efi"; + } + ]; + }; + }) + ]; +} diff --git a/nixos/modules/netboot/menu.ipxe b/nixos/modules/netboot/menu.ipxe new file mode 100644 index 0000000..dc2e8df --- /dev/null +++ b/nixos/modules/netboot/menu.ipxe @@ -0,0 +1,68 @@ +#!ipxe + +set server http://@bootHost@ + +# Figure out if client is 64-bit capable +cpuid --ext 29 && set arch x86_64 || set arch i386 + +isset ${menu-default} || set menu-default exit + +:start +menu Welcome to /dev/player0's humble iPXE boot menu +item --gap -- Operating Systems +iseq ${arch} x86_64 && +item --key n nixos NixOS installer +# iseq ${arch} x86_64 && +# item --key a archlinux Arch Linux (archiso x86_64) +# iseq ${arch} x86_64 && +# item --key p alpine Alpine Linux +item --gap -- Other Options +item --key e efi_shell UEFI Shell +item --key x xyz netboot.xyz +item --key c config iPXE settings +item --key s shell Drop to iPXE shell +item --key r reboot Reboot +item --key q exit Exit (and continue to next boot device) +choose --timeout 0 --default ${menu-default} selected || goto cancel +goto ${selected} + +:cancel +echo You cancelled the menu, dropping you to an iPXE shell + +:shell +echo Type 'exit' to go back to the menu +shell +set menu-default nixos +goto start + +:failed +echo Booting failed, dropping to shell +goto shell + +:reboot +reboot + +:exit +exit + +:config +config +set menu-default config +goto start + +:efi_shell +chain ${server}/efi-shell-${arch}.efi || goto failed + +:xyz +chain --autofree https://boot.netboot.xyz || goto failed + +:nixos +set cmdline nbd_export=nixos-installer +chain ${server}/nixos-installer/boot.ipxe || goto failed + +:archlinux +# set mirrorurl https://arch.nul.ie/ +chain ${server}/arch.ipxe || goto failed + +:alpine +chain ${server}/alpine.ipxe || goto failed