nixos/cellar: Keep spdk-tgt running on deploy
Restarting `spdk-tgt` takes every NVMe-oF root disk away (river, castle, sfh), so don't do it on deploy. `spdk-debug` repeated the RAID definition by hand. Generate its config from the real one instead (no examine, no exports, ublk disks from `my.spdk.debug.ublkDisks`), so it can't drift and runs without a terminal. Document adding a partition export with it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -14,15 +14,8 @@ in
|
||||
spdk = {
|
||||
enable = true;
|
||||
extraArgs = "--mem-channels 2 --cpumask 0xffff";
|
||||
debugCommands = ''
|
||||
spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
|
||||
spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
|
||||
spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
|
||||
spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'
|
||||
|
||||
spdk-rpc ublk_create_target
|
||||
spdk-rpc ublk_start_disk NVMeRaid 1
|
||||
'';
|
||||
# `spdk-debug` exposes the RAID as /dev/ublkb1 (e.g. to edit its partition table)
|
||||
debug.ublkDisks = [ "NVMeRaid" ];
|
||||
config.subsystems =
|
||||
let
|
||||
nvmeAttaches = mapAttrsToList (name: bdf: {
|
||||
|
||||
+35
-17
@@ -1,7 +1,7 @@
|
||||
{ lib, pkgs, config, ... }:
|
||||
let
|
||||
inherit (builtins) toJSON;
|
||||
inherit (lib) optional optionalAttrs mapAttrsToList mkIf withFeature;
|
||||
inherit (builtins) toJSON elem filter;
|
||||
inherit (lib) optional optionalAttrs mapAttrsToList imap1 mkIf withFeature;
|
||||
inherit (lib.my) mkOpt' mkBoolOpt';
|
||||
|
||||
rpcOpts = with lib.types; {
|
||||
@@ -22,6 +22,30 @@ let
|
||||
};
|
||||
configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
|
||||
|
||||
# The same bdevs, but never examined (so the `gpt` module doesn't claim them) and not exported;
|
||||
# `debug.ublkDisks` are exposed locally instead
|
||||
debugConfig = {
|
||||
subsystems =
|
||||
map
|
||||
(s: if s.subsystem != "bdev" then s else s // {
|
||||
config = filter (c: !(elem c.method [ "bdev_examine" "bdev_wait_for_examine" ])) s.config;
|
||||
})
|
||||
(filter (s: s.subsystem != "nvmf") config'.subsystems)
|
||||
++ optional (cfg.debug.ublkDisks != [ ]) {
|
||||
subsystem = "ublk";
|
||||
config = [
|
||||
{ method = "ublk_create_target"; }
|
||||
] ++ imap1 (i: bdev: {
|
||||
method = "ublk_start_disk";
|
||||
params = {
|
||||
bdev_name = bdev;
|
||||
ublk_id = i;
|
||||
};
|
||||
}) cfg.debug.ublkDisks;
|
||||
};
|
||||
};
|
||||
debugConfigJSON = pkgs.writeText "spdk-debug-config.json" (toJSON debugConfig);
|
||||
|
||||
spdk = pkgs.spdk.overrideAttrs (o: {
|
||||
configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
|
||||
buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
|
||||
@@ -43,25 +67,15 @@ let
|
||||
'');
|
||||
spdk-debug = pkgs.writeShellApplication {
|
||||
name = "spdk-debug";
|
||||
runtimeInputs = [ spdk ];
|
||||
runtimeInputs = with pkgs; [ bash python3 kmod gawk util-linux ];
|
||||
text = ''
|
||||
set -m
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
echo "I need to be root!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
|
||||
until spdk-rpc spdk_get_version > /dev/null; do
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
spdk-rpc bdev_set_options --disable-auto-examine
|
||||
spdk-rpc framework_start_init
|
||||
|
||||
${cfg.debugCommands}
|
||||
|
||||
fg %1
|
||||
${spdk.src}/scripts/setup.sh
|
||||
exec ${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${debugConfigJSON}
|
||||
'';
|
||||
};
|
||||
in
|
||||
@@ -69,7 +83,8 @@ in
|
||||
options.my.spdk = with lib.types; {
|
||||
enable = mkBoolOpt' false "Whether to enable SPDK target.";
|
||||
extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt.";
|
||||
debugCommands = mkOpt' lines "" "Commands to run with the spdk-debug script.";
|
||||
debug.ublkDisks = mkOpt' (listOf str) [ ]
|
||||
"Bdevs `spdk-debug` exposes as `/dev/ublkb<n>`, numbered from 1, instead of serving the config.";
|
||||
config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs.";
|
||||
};
|
||||
|
||||
@@ -80,7 +95,8 @@ in
|
||||
spdk
|
||||
spdk-setup
|
||||
spdk-rpc
|
||||
] ++ (optional (cfg.debugCommands != "") spdk-debug);
|
||||
spdk-debug
|
||||
];
|
||||
|
||||
systemd.services = {
|
||||
spdk-tgt = {
|
||||
@@ -97,6 +113,8 @@ in
|
||||
ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
|
||||
};
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
# Other boxes' root disks; restart by hand when they can do without them
|
||||
restartIfChanged = false;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user