From 5058dec3f11843fa099c087d158670d4b56e9077 Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Mon, 16 Nov 2026 10:33:17 +0000 Subject: [PATCH] nixos/cellar: Keep spdk-tgt running on deploy Restarting `spdk-tgt` takes every NVMe-oF root disk away (river, castle, sfh), so don't do it on deploy. `spdk-debug` repeated the RAID definition by hand. Generate its config from the real one instead (no examine, no exports, ublk disks from `my.spdk.debug.ublkDisks`), so it can't drift and runs without a terminal. Document adding a partition export with it. Co-Authored-By: Claude Opus 5.5 --- docs/sites/home/cellar.md | 65 +++++++++++++++++++++ nixos/boxes/home/palace/vms/cellar/spdk.nix | 11 +--- nixos/modules/spdk.nix | 52 +++++++++++------ 3 files changed, 102 insertions(+), 26 deletions(-) diff --git a/docs/sites/home/cellar.md b/docs/sites/home/cellar.md index 9f74d69..d016535 100644 --- a/docs/sites/home/cellar.md +++ b/docs/sites/home/cellar.md @@ -24,6 +24,8 @@ it. | `NVMeRaidp3` | `nqn.2016-06.io.spdk:sfh` | [`sfh`](sfh/README.md) | Each subsystem is pinned to its consumer's `hostnqn` (the `my.nvme.uuid` on the client side). +- `spdk-tgt` doesn't restart on deploy (`restartIfChanged = false`): restarting it takes every + consumer's disk away, so restart it by hand when they can do without. - `spdk-tgt` is ordered after `lan-hi` is online; the RDMA listener binds the `hi` assignment on port 4420. The VM itself is pinned to NUMA node 1 on `palace` and gets SR-IOV VF 0. - `netdata` (port 19999 allowed in the firewall) and `fstrim`. @@ -32,6 +34,69 @@ it. See the consolidated [network assignments](../../networking.md#box-assignments) table (this box: `cellar`). +## Adding an export + +Partitions on `NVMeRaid` are ordinary GPT partitions. SPDK's `gpt` module finds them when it +examines the RAID (`bdev_examine` in `spdk.nix`; auto-examine is off) and exposes those whose type +GUID is SPDK's, as `NVMeRaidp`. `p1`–`p3` use the current type GUID, +`6527994e-2c5a-4eec-9613-8f5944074e8b`. Use it for new partitions and leave existing partition +types as they are. SPDK still supports its deprecated type, `7c5222bd-8f5d-4087-9c00-bf9843c7b58c`, +and exposes one fewer block for that type to preserve compatibility with existing on-disk metadata. +[SPDK deprecation notice](https://github.com/spdk/spdk/blob/master/deprecation.md#gpt) + +SPDK has no RPC to edit a partition table, so a new partition is written from Linux through ublk, +using `spdk-debug`. +Once examined, the `gpt` module holds an exclusive write claim on `NVMeRaid`, so this can't be done +while the exports are live: every consumer has to be down. Start from a box that stays online +when the exports are offline. + +1. Shut down every consumer. Power off `castle` and run + `doas systemctl stop vm@sfh vm@river` on `palace`. `stream` takes over the router VIPs. On + `cellar`, verify that `doas spdk-rpc nvmf_subsystem_get_controllers ` returns `[]` for each + exported subsystem in the table above. +2. On `cellar`, stop the target and start `spdk-debug` in its place. It runs `spdk_tgt` on a + config generated from the real one: the same controllers and RAID, but nothing examined (so + the `gpt` module never claims the RAID), nothing exported, and the RAID exposed through ublk + (`my.spdk.debug.ublkDisks`): + + ```sh + doas systemctl stop spdk-tgt + doas systemd-run --unit=spdk-debug --setenv=PATH=/run/wrappers/bin:/run/current-system/sw/bin "$(command -v spdk-debug)" + doas spdk-rpc ublk_get_disks # NVMeRaid as ublk 1 + lsblk /dev/ublkb1 + ``` + + Run plain `doas spdk-debug` instead to keep it in the foreground (`Ctrl-C` stops it). +3. Check the existing table, then add the partition. `sgdisk` isn't installed on `cellar`; a start + of `0` selects the start of the largest free region. Substitute the new partition number for + `` and the desired capacity for ``. Verify that all existing entries match the + expected layout before writing, and check that they remain intact afterward. + + ```sh + nix shell pkgs#gptfdisk -c doas sgdisk -p /dev/ublkb1 + nix shell pkgs#gptfdisk -c doas sgdisk \ + -n :0:+ -t :6527994e-2c5a-4eec-9613-8f5944074e8b /dev/ublkb1 + nix shell pkgs#gptfdisk -c doas sgdisk -p /dev/ublkb1 + ``` +4. `doas systemctl stop spdk-debug`, and check `/dev/ublkb1` is gone. +5. If `spdk-debug` fails to start, `journalctl -u spdk-debug` shows why. The same steps can be + done by hand: `spdk_tgt --wait-for-rpc` under `systemd-run`, then `spdk-rpc bdev_set_options + --no-auto-examine`, `framework_start_init`, the three `bdev_nvme_attach_controller` calls and + `bdev_raid_create` exactly as in `spdk.nix`, `ublk_create_target` and + `ublk_start_disk NVMeRaid 1`. +6. Add an `nvmfBdev` entry for `NVMeRaidp` in `spdk.nix` (its consumer's host NQN and the next + serial), deploy `cellar` (the unit doesn't restart) and `doas systemctl start spdk-tgt`. Check + `doas spdk-rpc bdev_get_bdevs` lists the new partition and `doas spdk-rpc nvmf_get_subsystems` + the new subsystem. +7. Start the consumers again. + +If `spdk-tgt` won't start in step 6, the usual cause is a config naming a bdev that doesn't exist. +Switch `cellar` back to its previous generation +(`doas /nix/var/nix/profiles/system--link/bin/switch-to-configuration switch`), start +`spdk-tgt` and check the existing exports before trying again. + +`ublk_drv` is loaded on `cellar`. + ## Notes - The `ublk_*` calls in `my.spdk.debugCommands` are only a debugging aid — they create a local diff --git a/nixos/boxes/home/palace/vms/cellar/spdk.nix b/nixos/boxes/home/palace/vms/cellar/spdk.nix index fc4805b..53dd85e 100644 --- a/nixos/boxes/home/palace/vms/cellar/spdk.nix +++ b/nixos/boxes/home/palace/vms/cellar/spdk.nix @@ -14,15 +14,8 @@ in spdk = { enable = true; extraArgs = "--mem-channels 2 --cpumask 0xffff"; - debugCommands = '' - spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0 - spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1 - spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2 - spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1' - - spdk-rpc ublk_create_target - spdk-rpc ublk_start_disk NVMeRaid 1 - ''; + # `spdk-debug` exposes the RAID as /dev/ublkb1 (e.g. to edit its partition table) + debug.ublkDisks = [ "NVMeRaid" ]; config.subsystems = let nvmeAttaches = mapAttrsToList (name: bdf: { diff --git a/nixos/modules/spdk.nix b/nixos/modules/spdk.nix index 57546ce..a8c681a 100644 --- a/nixos/modules/spdk.nix +++ b/nixos/modules/spdk.nix @@ -1,7 +1,7 @@ { lib, pkgs, config, ... }: let - inherit (builtins) toJSON; - inherit (lib) optional optionalAttrs mapAttrsToList mkIf withFeature; + inherit (builtins) toJSON elem filter; + inherit (lib) optional optionalAttrs mapAttrsToList imap1 mkIf withFeature; inherit (lib.my) mkOpt' mkBoolOpt'; rpcOpts = with lib.types; { @@ -22,6 +22,30 @@ let }; configJSON = pkgs.writeText "spdk-config.json" (toJSON config'); + # The same bdevs, but never examined (so the `gpt` module doesn't claim them) and not exported; + # `debug.ublkDisks` are exposed locally instead + debugConfig = { + subsystems = + map + (s: if s.subsystem != "bdev" then s else s // { + config = filter (c: !(elem c.method [ "bdev_examine" "bdev_wait_for_examine" ])) s.config; + }) + (filter (s: s.subsystem != "nvmf") config'.subsystems) + ++ optional (cfg.debug.ublkDisks != [ ]) { + subsystem = "ublk"; + config = [ + { method = "ublk_create_target"; } + ] ++ imap1 (i: bdev: { + method = "ublk_start_disk"; + params = { + bdev_name = bdev; + ublk_id = i; + }; + }) cfg.debug.ublkDisks; + }; + }; + debugConfigJSON = pkgs.writeText "spdk-debug-config.json" (toJSON debugConfig); + spdk = pkgs.spdk.overrideAttrs (o: { configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]); buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]); @@ -43,25 +67,15 @@ let ''); spdk-debug = pkgs.writeShellApplication { name = "spdk-debug"; - runtimeInputs = [ spdk ]; + runtimeInputs = with pkgs; [ bash python3 kmod gawk util-linux ]; text = '' - set -m if [ "$(id -u)" -ne 0 ]; then echo "I need to be root!" exit 1 fi - spdk_tgt ${cfg.extraArgs} --wait-for-rpc & - until spdk-rpc spdk_get_version > /dev/null; do - sleep 0.5 - done - - spdk-rpc bdev_set_options --disable-auto-examine - spdk-rpc framework_start_init - - ${cfg.debugCommands} - - fg %1 + ${spdk.src}/scripts/setup.sh + exec ${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${debugConfigJSON} ''; }; in @@ -69,7 +83,8 @@ in options.my.spdk = with lib.types; { enable = mkBoolOpt' false "Whether to enable SPDK target."; extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt."; - debugCommands = mkOpt' lines "" "Commands to run with the spdk-debug script."; + debug.ublkDisks = mkOpt' (listOf str) [ ] + "Bdevs `spdk-debug` exposes as `/dev/ublkb`, numbered from 1, instead of serving the config."; config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs."; }; @@ -80,7 +95,8 @@ in spdk spdk-setup spdk-rpc - ] ++ (optional (cfg.debugCommands != "") spdk-debug); + spdk-debug + ]; systemd.services = { spdk-tgt = { @@ -97,6 +113,8 @@ in ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}"; }; wantedBy = [ "multi-user.target" ]; + # Other boxes' root disks; restart by hand when they can do without them + restartIfChanged = false; }; }; };