Hack hack hack

This commit is contained in:
Jack O'Sullivan 2023-12-22 17:31:33 +00:00
parent 0a86a649a6
commit c4df21da3b
8 changed files with 512 additions and 14 deletions

View File

@ -106,8 +106,8 @@ in
{
name = "build-netboot";
category = "tasks";
help = "Build NixOS configuration as netboot archive";
command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootArchive"'';
help = "Build NixOS configuration as netboot tree";
command = ''nix build "''${@:2}" ".#nixfiles.config.nixos.systems.\"$1\".configuration.config.my.buildAs.netbootTree"'';
}
{
name = "build-home";

View File

@ -148,9 +148,11 @@ in
};
};
};
nginx.enable = true;
};
networking.domain = "h.${pubDomain}";
networking = { inherit domain; };
systemd.services = {
ipsec =
@ -376,6 +378,11 @@ in
}
'';
};
netboot.server = {
enable = true;
ip = vips.lo.v4;
host = "boot.${domain}";
};
};
};
};

View File

@ -130,6 +130,7 @@ in
}}
${elemAt routers 0} IN AAAA ${net.cidr.host 1 prefixes.hi.v6}
${elemAt routers 1} IN AAAA ${net.cidr.host 2 prefixes.hi.v6}
boot IN CNAME router-hi.${config.networking.domain}.
@ IN NS ns1
@ IN NS ns2

View File

@ -1,4 +1,4 @@
index: { lib, pkgs, assignments, ... }:
index: { lib, pkgs, config, assignments, ... }:
let
inherit (lib) mkForce;
inherit (lib.my) net;
@ -59,6 +59,7 @@ in
always-send = true;
}
];
client-classes = config.my.netboot.server.keaClientClasses;
subnet4 = [
{
id = 1;

View File

@ -19,5 +19,6 @@
borgthin = ./borgthin.nix;
nvme = ./nvme;
spdk = ./spdk.nix;
netboot = ./netboot;
};
}

View File

@ -1,6 +1,6 @@
{ lib, pkgs, extendModules, modulesPath, options, config, ... }:
let
inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge flatten optional;
inherit (lib) recursiveUpdate mkOption mkDefault mkIf mkMerge mkForce flatten optional;
inherit (lib.my) mkBoolOpt' dummyOption;
cfg = config.my.build;
@ -43,15 +43,206 @@ let
modules = flatten [
"${modulesPath}/installer/netboot/netboot.nix"
allHardware
({ pkgs, config, ... }: {
system.build.netbootArchive = pkgs.runCommand "netboot-${config.system.name}-archive.tar" { } ''
${pkgs.gnutar}/bin/tar -rvC "${config.system.build.kernel}" \
-f "$out" "${config.system.boot.loader.kernelFile}"
${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootRamdisk}" \
-f "$out" initrd
${pkgs.gnutar}/bin/tar -rvC "${config.system.build.netbootIpxeScript}" \
-f "$out" netboot.ipxe
(mkIf config.boot.initrd.systemd.enable {
boot.initrd.systemd.services.setup-overlay-dirs = {
description = "Create overlayfs dirs";
after = [ "sysroot-nix-.rw\\x2dstore.mount" ];
before = [ "sysroot-nix-store.mount" ];
script = ''
mkdir /sysroot/nix/.rw-store/{store,work}
'';
wantedBy = [ "initrd-fs.target" ];
};
fileSystems."/nix/store" = mkForce {
fsType = "overlay";
device = "overlay";
options = [
"lowerdir=/sysroot/nix/.ro-store"
"upperdir=/sysroot/nix/.rw-store/store"
"workdir=/sysroot/nix/.rw-store/work"
];
};
systemd.units."nix-store.mount".enable = false;
})
];
};
asNetboot = extendModules {
modules = flatten [
allHardware
({ pkgs, config, ... }:
let
initrdNbdWrapper = pkgs.writeCBin "nbd-wrapper" ''
#include <stdio.h>
#include <unistd.h>
int main(int argc, char **argv) {
if (argc < 3) {
fprintf(stderr, "usage: %s <export> <server>\n", argv[0]);
return -1;
}
argv[0][0] = '@';
char* args[] = {
"@", "-nofork", "-N", argv[1], argv[2], "/dev/nbd0", NULL
};
execv("${pkgs.nbd}/bin/nbd-client", args);
return 0;
};
'';
nbd = pkgs.nbd.overrideAttrs (o: {
# TODO: Remove once this makes it to us
# https://github.com/NixOS/nixpkgs/commit/52f1d9b03ae38126e7f648634fcad35897f464ed
configureFlags = [ "--sysconfdir=/etc" ];
});
in
{
boot = {
loader.grub.enable = false;
kernelParams = [ "console=ttyS0,115200n8" ];
initrd = {
kernelModules = [ "nbd" ];
systemd = {
storePaths = with pkgs; [
gnused
nbd
initrdNbdWrapper
netcat
];
extraBin = with pkgs; {
dmesg = "${util-linux}/bin/dmesg";
ip = "${iproute2}/bin/ip";
nbd-client = "${nbd}/bin/nbd-client";
};
extraConfig = ''
DefaultTimeoutStartSec=10
DefaultDeviceTimeoutSec=10
'';
network = {
enable = true;
wait-online.enable = true;
networks."10-nbd" = {
matchConfig.Name = "et-nbd";
DHCP = "yes";
};
};
services = {
# gennbdtab = {
# description = "Generate nbdtab";
# script = ''
# get_cmdline() {
# ${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
# }
# e="$(get_cmdline nbd_export)"
# s="$(get_cmdline nbd_server)"
# echo "Setting nbdtab for $e @ $s"
# echo "nbd0 $s $e persist" > /etc/nbdtab
# '';
# serviceConfig.Type = "oneshot";
# # wantedBy = [ "initrd-root-device.target" ];
# };
nbd = {
description = "NBD Nix store";
# before = [ "initrd-root-device.target" ];
# after = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ];
# wants = [ "gennbdtab.service" "systemd-networkd-wait-online.service" ];
# after = [ "systemd-networkd-wait-online.service" ];
# wants = [ "systemd-networkd-wait-online.service" ];
script = ''
get_cmdline() {
${pkgs.gnused}/bin/sed -rn "s/^.*$1=(\\S+).*\$/\\1/p" < /proc/cmdline
}
s="$(get_cmdline nbd_server)"
until ${pkgs.netcat}/bin/nc -zv "$s" 22; do
sleep 0.1
done
exec ${nbd}/bin/nbd-client -systemd-mark -N "$(get_cmdline nbd_export)" "$s" /dev/nbd0
# exec ${initrdNbdWrapper}/bin/nbd-wrapper "$(get_cmdline nbd_export)" "$(get_cmdline nbd_server)"
'';
unitConfig = {
IgnoreOnIsolate = "yes";
DefaultDependencies = "no";
};
serviceConfig = {
Type = "forking";
# ExecStart = "${nbd}/bin/nbd-client -nofork -systemd-mark nbd0";
Restart = "on-failure";
RestartSec = 10;
};
# wantedBy = [ "initrd-root-device.target" ];
};
};
};
};
};
programs.nbd.enable = true;
fileSystems = {
"/" = {
fsType = "tmpfs";
options = [ "mode=0755" ];
};
"/nix/store" = {
fsType = "squashfs";
device = "/dev/nbd0";
options = [ "x-systemd.requires=nbd.service" ];
};
};
system.build = {
squashfsStore = pkgs.callPackage "${modulesPath}/../lib/make-squashfs.nix" {
storeContents = [ config.system.build.toplevel ];
comp = "zstd";
};
netbootScript = pkgs.writeText "boot.ipxe" ''
#!ipxe
kernel ${pkgs.stdenv.hostPlatform.linux-kernel.target} init=${config.system.build.toplevel}/init initrd=initrd ifname=et-nbd:''${mac} nbd_server=''${next-server} ${toString config.boot.kernelParams} ''${cmdline}
initrd initrd
boot
'';
netbootTree = pkgs.linkFarm "netboot-${config.system.name}" [
{
name = config.system.boot.loader.kernelFile;
path = "${config.system.build.kernel}/${config.system.boot.loader.kernelFile}";
}
{
name = "initrd";
path = "${config.system.build.initialRamdisk}/initrd";
}
{
name = "nix-store.sfs";
path = config.system.build.squashfsStore;
}
{
name = "boot.ipxe";
path = config.system.build.netbootScript;
}
];
netbootArchive = pkgs.runCommand "netboot-${config.system.name}.tar" { } ''
add() {
${pkgs.gnutar}/bin/tar --dereference -rvC "$1" -f "$out" "$2"
}
add "${config.system.build.kernel}" "${config.system.boot.loader.kernelFile}"
add "${config.system.build.initialRamdisk}" initrd
tmpdir="$(mktemp -d sfsStore.XXXXXX)"
ln -s "${config.system.build.squashfsStore}" "$tmpdir"/nix-store.sfs
add "$tmpdir" nix-store.sfs
add "${config.system.build.netbootScript}" boot.ipxe
'';
};
})
];
};
@ -77,6 +268,7 @@ in
asISO = mkAsOpt asISO "a bootable .iso image";
asContainer = mkAsOpt asContainer "a container";
asKexecTree = mkAsOpt asKexecTree "a kexec-able kernel and initrd";
asNetboot = mkAsOpt asNetboot "a netboot-able kernel initrd, and iPXE script";
buildAs = options.system.build;
};
@ -109,7 +301,7 @@ in
iso = config.my.asISO.config.system.build.isoImage;
container = config.my.asContainer.config.system.build.toplevel;
kexecTree = config.my.asKexecTree.config.system.build.kexecTree;
netbootArchive = config.my.asKexecTree.config.system.build.netbootArchive;
netbootTree = config.my.asNetboot.config.system.build.netbootTree;
};
};
};

View File

@ -0,0 +1,228 @@
{ lib, pkgs, config, systems, ... }:
let
inherit (builtins) toJSON;
inherit (lib) optional optionalAttrs mapAttrsToList mkMerge mkIf withFeature mkOption;
inherit (lib.my) mkOpt' mkBoolOpt';
rpcOpts = with lib.types; {
options = {
method = mkOpt' str null "RPC method name.";
params = mkOpt' (attrsOf unspecified) { } "RPC params";
};
};
cfg = config.my.netboot;
config' = {
subsystems = mapAttrsToList (subsystem: c: {
inherit subsystem;
config = map (rpc: {
inherit (rpc) method;
} // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c;
}) cfg.config.subsystems;
};
configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
spdk = pkgs.spdk.overrideAttrs (o: {
configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
});
spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
'');
spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
exec ${spdk.src}/scripts/setup.sh "$@"
'');
spdk-debug = pkgs.writeShellApplication {
name = "spdk-debug";
runtimeInputs = [ spdk ];
text = ''
set -m
if [ "$(id -u)" -ne 0 ]; then
echo "I need to be root!"
exit 1
fi
spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
until spdk-rpc spdk_get_version > /dev/null; do
sleep 0.5
done
spdk-rpc bdev_set_options --disable-auto-examine
spdk-rpc framework_start_init
${cfg.debugCommands}
fg %1
'';
};
tftpRoot = pkgs.linkFarm "tftp-root" [
{
name = "ipxe-x86_64.efi";
path = "${pkgs.ipxe}/ipxe.efi";
}
];
menuFile = pkgs.runCommand "menu.ipxe" {
bootHost = cfg.server.host;
} ''
substituteAll ${./menu.ipxe} "$out"
'';
in
{
options.my.netboot = with lib.types; {
client = {
enable = mkBoolOpt' false "Whether network booting should be enabled.";
};
server = {
enable = mkBoolOpt' false "Whether a netboot server should be enabled.";
ip = mkOpt' str null "IP clients should connect to via TFTP.";
host = mkOpt' str config.networking.fqdn "Hostname clients should connect to over HTTP.";
instances = mkOpt' (listOf str) [ ] "Systems to hold boot files for.";
keaClientClasses = mkOption {
type = listOf (attrsOf str);
description = "Kea client classes for PXE boot.";
readOnly = true;
};
};
};
config = mkMerge [
(mkIf cfg.client.enable {
environment.systemPackages = [
spdk
spdk-setup
spdk-rpc
] ++ (optional (cfg.debugCommands != "") spdk-debug);
systemd.services = {
spdk-tgt = {
description = "SPDK target";
path = with pkgs; [
bash
python3
kmod
gawk
util-linux
];
serviceConfig = {
ExecStartPre = "${spdk.src}/scripts/setup.sh";
ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
};
wantedBy = [ "multi-user.target" ];
};
};
})
(mkIf cfg.server.enable {
environment = {
etc = {
"netboot/menu.ipxe".source = menuFile;
"netboot/shell.efi".source = "${pkgs.edk2-uefi-shell}/shell.efi";
};
};
systemd = {
services = {
netboot-update = {
description = "Update netboot images";
after = [ "systemd-networkd-wait-online.service" ];
serviceConfig = {
Type = "oneshot";
RemainAfterExit = true;
};
path = with pkgs; [
coreutils curl jq gnutar
];
script = ''
update_nixos() {
latestShort="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/tags/installer \
| jq -r .commit.sha | cut -c -7)"
if [ -f nixos-installer/tag.txt ] && [ "$(< nixos-installer/tag.txt)" = "$latestShort" ]; then
echo "NixOS installer is up to date"
return
fi
echo "Updating NixOS installer to $latestShort"
mkdir -p nixos-installer
fname="nixos-installer-devplayer0-netboot-$latestShort.tar"
downloadUrl="$(curl -s https://git.nul.ie/api/v1/repos/dev/nixfiles/releases/tags/installer | \
jq -r ".assets[] | select(.name == \"$fname\").browser_download_url")"
curl -Lo /tmp/nixos-installer-netboot.tar "$downloadUrl"
tar -C nixos-installer -xf /tmp/nixos-installer-netboot.tar
rm /tmp/nixos-installer-netboot.tar
echo "$latestShort" > nixos-installer/tag.txt
}
mkdir -p /srv/netboot
cd /srv/netboot
ln -sf ${menuFile} boot.ipxe
ln -sf "${pkgs.edk2-uefi-shell}/shell.efi"
update_nixos
'';
startAt = "06:00";
wantedBy = [ "network-online.target" ];
};
nbd-server.preStart = ''
mkdir /tmp/nbdcow
'';
};
};
services = {
atftpd = {
enable = true;
root = tftpRoot;
};
nginx = {
virtualHosts."${cfg.server.host}" = {
locations."/" = {
root = "/srv/netboot";
extraConfig = ''
autoindex on;
'';
};
};
};
nbd.server = {
enable = true;
extraOptions = {
allowlist = true;
};
exports = {
nixos-installer = {
path = "/srv/netboot/nixos-installer/nix-store.sfs";
extraOptions = {
copyonwrite = true;
cowdir = "/tmp/nbdcow";
sparse_cow = true;
};
};
};
};
};
my = {
tmproot.persistence.config.directories = [ "/srv/netboot" ];
netboot.server.keaClientClasses = [
{
name = "ipxe";
test = "substring(option[user-class].hex, 0, 4) == 'iPXE'";
next-server = cfg.server.ip;
server-hostname = cfg.server.host;
boot-file-name = "http://${cfg.server.host}/boot.ipxe";
}
{
name = "efi-x86_64";
test = "option[client-system].hex == 0x0007";
next-server = cfg.server.ip;
server-hostname = cfg.server.host;
boot-file-name = "ipxe-x86_64.efi";
}
];
};
})
];
}

View File

@ -0,0 +1,68 @@
#!ipxe
set server http://@bootHost@
# Figure out if client is 64-bit capable
cpuid --ext 29 && set arch x86_64 || set arch i386
isset ${menu-default} || set menu-default exit
:start
menu Welcome to /dev/player0's humble iPXE boot menu
item --gap -- Operating Systems
iseq ${arch} x86_64 &&
item --key n nixos NixOS installer
# iseq ${arch} x86_64 &&
# item --key a archlinux Arch Linux (archiso x86_64)
# iseq ${arch} x86_64 &&
# item --key p alpine Alpine Linux
item --gap -- Other Options
item --key e efi_shell UEFI Shell
item --key x xyz netboot.xyz
item --key c config iPXE settings
item --key s shell Drop to iPXE shell
item --key r reboot Reboot
item --key q exit Exit (and continue to next boot device)
choose --timeout 0 --default ${menu-default} selected || goto cancel
goto ${selected}
:cancel
echo You cancelled the menu, dropping you to an iPXE shell
:shell
echo Type 'exit' to go back to the menu
shell
set menu-default nixos
goto start
:failed
echo Booting failed, dropping to shell
goto shell
:reboot
reboot
:exit
exit
:config
config
set menu-default config
goto start
:efi_shell
chain ${server}/efi-shell-${arch}.efi || goto failed
:xyz
chain --autofree https://boot.netboot.xyz || goto failed
:nixos
set cmdline nbd_export=nixos-installer
chain ${server}/nixos-installer/boot.ipxe || goto failed
:archlinux
# set mirrorurl https://arch.nul.ie/
chain ${server}/arch.ipxe || goto failed
:alpine
chain ${server}/alpine.ipxe || goto failed