Files
nixfiles/nixos/modules/spdk.nix
T
jackos1998andClaude Opus 5.5 5058dec3f1 nixos/cellar: Keep spdk-tgt running on deploy
Restarting `spdk-tgt` takes every NVMe-oF root disk away (river,
castle, sfh), so don't do it on deploy.

`spdk-debug` repeated the RAID definition by hand. Generate its config
from the real one instead (no examine, no exports, ublk disks from
`my.spdk.debug.ublkDisks`), so it can't drift and runs without a
terminal. Document adding a partition export with it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 22:01:54 +00:00

122 lines
4.0 KiB
Nix

{ lib, pkgs, config, ... }:
let
inherit (builtins) toJSON elem filter;
inherit (lib) optional optionalAttrs mapAttrsToList imap1 mkIf withFeature;
inherit (lib.my) mkOpt' mkBoolOpt';
rpcOpts = with lib.types; {
options = {
method = mkOpt' str null "RPC method name.";
params = mkOpt' (attrsOf unspecified) { } "RPC params";
};
};
cfg = config.my.spdk;
config' = {
subsystems = mapAttrsToList (subsystem: c: {
inherit subsystem;
config = map (rpc: {
inherit (rpc) method;
} // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c;
}) cfg.config.subsystems;
};
configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
# The same bdevs, but never examined (so the `gpt` module doesn't claim them) and not exported;
# `debug.ublkDisks` are exposed locally instead
debugConfig = {
subsystems =
map
(s: if s.subsystem != "bdev" then s else s // {
config = filter (c: !(elem c.method [ "bdev_examine" "bdev_wait_for_examine" ])) s.config;
})
(filter (s: s.subsystem != "nvmf") config'.subsystems)
++ optional (cfg.debug.ublkDisks != [ ]) {
subsystem = "ublk";
config = [
{ method = "ublk_create_target"; }
] ++ imap1 (i: bdev: {
method = "ublk_start_disk";
params = {
bdev_name = bdev;
ublk_id = i;
};
}) cfg.debug.ublkDisks;
};
};
debugConfigJSON = pkgs.writeText "spdk-debug-config.json" (toJSON debugConfig);
spdk = pkgs.spdk.overrideAttrs (o: {
configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
# SPDK 26.01 recognizes prerelease DPDK 26.03, but selects the old PCI ABI for its release.
# TODO: Remove after upgrading to SPDK >= 26.05, which handles released DPDK 26.03:
# https://github.com/spdk/spdk/blob/v26.05/lib/env_dpdk/pci_dpdk.c
# nixos-unstable still packages 26.01 without this fix (checked 2026-11-15).
postPatch = (o.postPatch or "") + lib.optionalString (o.version == "26.01") ''
substituteInPlace lib/env_dpdk/pci_dpdk.c \
--replace-fail 'else if (year == 25) {' \
'else if (year == 25 || (year == 26 && month == 3 && minor == 0)) {'
'';
});
spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
'');
spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
exec ${spdk.src}/scripts/setup.sh "$@"
'');
spdk-debug = pkgs.writeShellApplication {
name = "spdk-debug";
runtimeInputs = with pkgs; [ bash python3 kmod gawk util-linux ];
text = ''
if [ "$(id -u)" -ne 0 ]; then
echo "I need to be root!"
exit 1
fi
${spdk.src}/scripts/setup.sh
exec ${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${debugConfigJSON}
'';
};
in
{
options.my.spdk = with lib.types; {
enable = mkBoolOpt' false "Whether to enable SPDK target.";
extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt.";
debug.ublkDisks = mkOpt' (listOf str) [ ]
"Bdevs `spdk-debug` exposes as `/dev/ublkb<n>`, numbered from 1, instead of serving the config.";
config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs.";
};
config = mkIf cfg.enable {
boot.kernelModules = [ "ublk_drv" ];
environment.systemPackages = [
spdk
spdk-setup
spdk-rpc
spdk-debug
];
systemd.services = {
spdk-tgt = {
description = "SPDK target";
path = with pkgs; [
bash
python3
kmod
gawk
util-linux
];
serviceConfig = {
ExecStartPre = "${spdk.src}/scripts/setup.sh";
ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
};
wantedBy = [ "multi-user.target" ];
# Other boxes' root disks; restart by hand when they can do without them
restartIfChanged = false;
};
};
};
}