dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/cellar: Move SPDK config to separate module
All checks were successful
CI / Check, build and cache Nix flake (push) Successful in 17m56s
Details

Browse Source
This commit is contained in:
Jack O'Sullivan 2023-12-13 02:19:04 +00:00
parent 2f2764a364
commit 82b24c3c55
5 changed files with 230 additions and 408 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
nixos/boxes/home/palace/vms/cellar/default.nix
View File

@ -24,21 +24,17 @@ in
let
inherit (lib) mkMerge;
inherit (lib.my) networkdAssignment;
spdk = pkgs.spdk.overrideAttrs (o: {
configureFlags = o.configureFlags ++ [ "--with-rdma" ];
});
in
{
imports = [
"${modulesPath}/profiles/qemu-guest.nix"
./spdk.nix
];
config = mkMerge [
{
boot = {
kernelParams = [ "console=ttyS0,115200n8" ];
blacklistedKernelModules = [ "nvme" ];
};
fileSystems = {
@ -57,37 +53,15 @@ in
};
};
environment.systemPackages = [
pkgs.pciutils
spdk
(pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
'')
environment.systemPackages = with pkgs; [
pciutils
partclone
];
services = {
netdata.enable = true;
};
systemd.services = {
spdk-nvmf = {
description = "SPDK NVMe-oF target";
path = with pkgs; [
bash
python3
kmod
gawk
util-linux
];
after = [ "systemd-networkd-wait-online@lan-hi.service" ];
preStart = ''
${spdk.src}/scripts/setup.sh
'';
serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt --cpumask 0xffff -c ${./spdk_nvmf.json}";
wantedBy = [ "multi-user.target" ];
};
};
systemd.network = {
links = {
"10-lan-hi" = {
@ -105,7 +79,7 @@ in
{
networkConfig.DNS = [
(allAssignments.stream.hi.ipv4.address)
# (allAssignments.river.hi.ipv4.address)
(allAssignments.river.hi.ipv4.address)
];
}
];

130
nixos/boxes/home/palace/vms/cellar/spdk.nix Normal file
View File

@ -0,0 +1,130 @@
{ lib, pkgs, config, assignments, ... }:
let
inherit (lib) mapAttrsToList;
in
{
config = {
boot.blacklistedKernelModules = [ "nvme" ];
systemd.services = {
spdk-tgt.after = [ "systemd-networkd-wait-online@lan-hi.service" ];
};
my = {
spdk = {
enable = true;
extraArgs = "--mem-channels 2 --cpumask 0xffff";
debugCommands = ''
spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'
spdk-rpc ublk_create_target
spdk-rpc ublk_start_disk NVMeRaid 1
'';
config.subsystems =
let
nvmeAttaches = mapAttrsToList (name: bdf: {
method = "bdev_nvme_attach_controller";
params = {
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775";
trtype = "PCIe";
inherit name;
traddr = bdf;
};
}) {
"NVMe0" = "02:00.0";
"NVMe1" = "03:00.0";
"NVMe2" = "04:00.0";
};
nvmfListener = nqn: {
method = "nvmf_subsystem_add_listener";
params = {
inherit nqn;
listen_address = {
adrfam = "IPv4";
traddr = assignments.hi.ipv4.address;
trsvcid = "4420";
trtype = "RDMA";
};
secure_channel = false;
};
};
nvmfBdev = { nqn, hostnqn, bdev, serial }: [
{
method = "nvmf_create_subsystem";
params = {
inherit nqn;
serial_number = serial;
};
}
(nvmfListener nqn)
{
method = "nvmf_subsystem_add_host";
params = {
inherit nqn;
host = hostnqn;
};
}
{
method = "nvmf_subsystem_add_ns";
params = {
inherit nqn;
namespace = {
bdev_name = bdev;
nsid = 1;
};
};
}
];
in
{
bdev = [
{
method = "bdev_set_options";
params.bdev_auto_examine = false;
}
] ++ nvmeAttaches ++ [
{
method = "bdev_raid_create";
params = {
base_bdevs = [ "NVMe0n1" "NVMe1n1" "NVMe2n1" ];
name = "NVMeRaid";
raid_level = "raid0";
strip_size_kb = 64;
};
}
{
method = "bdev_examine";
params.name = "NVMeRaid";
}
{ method = "bdev_wait_for_examine"; }
];
nvmf = [
{
method = "nvmf_create_transport";
params.trtype = "RDMA";
}
(nvmfListener "nqn.2014-08.org.nvmexpress.discovery")
] ++ (nvmfBdev {
bdev = "NVMeRaidp1";
nqn = "nqn.2016-06.io.spdk:river";
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
serial = "SPDK00000000000001";
}) ++ (nvmfBdev {
bdev = "NVMeRaidp2";
nqn = "nqn.2016-06.io.spdk:castle";
hostnqn =
"nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
serial = "SPDK00000000000002";
});
};
};
};
};
}

377
nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json
View File

@ -1,377 +0,0 @@
{
"subsystems": [
{
"subsystem": "scheduler",
"config": [
{
"method": "framework_set_scheduler",
"params": {
"name": "static"
}
}
]
},
{
"subsystem": "vmd",
"config": []
},
{
"subsystem": "sock",
"config": [
{
"method": "sock_impl_set_options",
"params": {
"impl_name": "posix",
"recv_buf_size": 2097152,
"send_buf_size": 2097152,
"enable_recv_pipe": true,
"enable_quickack": false,
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 0,
"tls_version": 0,
"enable_ktls": false
}
},
{
"method": "sock_impl_set_options",
"params": {
"impl_name": "ssl",
"recv_buf_size": 4096,
"send_buf_size": 4096,
"enable_recv_pipe": true,
"enable_quickack": false,
"enable_placement_id": 0,
"enable_zerocopy_send_server": true,
"enable_zerocopy_send_client": false,
"zerocopy_threshold": 0,
"tls_version": 0,
"enable_ktls": false
}
}
]
},
{
"subsystem": "iobuf",
"config": [
{
"method": "iobuf_set_options",
"params": {
"small_pool_count": 8192,
"large_pool_count": 1024,
"small_bufsize": 8192,
"large_bufsize": 135168
}
}
]
},
{
"subsystem": "accel",
"config": [
{
"method": "accel_set_options",
"params": {
"small_cache_size": 128,
"large_cache_size": 16,
"task_count": 2048,
"sequence_count": 2048,
"buf_count": 2048
}
}
]
},
{
"subsystem": "bdev",
"config": [
{
"method": "bdev_set_options",
"params": {
"bdev_io_pool_size": 65535,
"bdev_io_cache_size": 256,
"bdev_auto_examine": true
}
},
{
"method": "bdev_nvme_set_options",
"params": {
"action_on_timeout": "none",
"timeout_us": 0,
"timeout_admin_us": 0,
"keep_alive_timeout_ms": 10000,
"transport_retry_count": 4,
"arbitration_burst": 0,
"low_priority_weight": 0,
"medium_priority_weight": 0,
"high_priority_weight": 0,
"nvme_adminq_poll_period_us": 10000,
"nvme_ioq_poll_period_us": 0,
"io_queue_requests": 512,
"delay_cmd_submit": true,
"bdev_retry_count": 3,
"transport_ack_timeout": 0,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"generate_uuids": false,
"transport_tos": 0,
"io_path_stat": false,
"allow_accel_sequence": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe0",
"trtype": "PCIe",
"traddr": "02:00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe1",
"trtype": "PCIe",
"traddr": "03.00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_attach_controller",
"params": {
"name": "NVMe2",
"trtype": "PCIe",
"traddr": "04.00.0",
"prchk_reftag": false,
"prchk_guard": false,
"ctrlr_loss_timeout_sec": 0,
"reconnect_delay_sec": 0,
"fast_io_fail_timeout_sec": 0,
"hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
"hdgst": false,
"ddgst": false
}
},
{
"method": "bdev_nvme_set_hotplug",
"params": {
"period_us": 100000,
"enable": false
}
},
{
"method": "bdev_raid_create",
"params": {
"name": "Raid0",
"strip_size_kb": 64,
"raid_level": "raid0",
"base_bdevs": [
"NVMe0n1",
"NVMe1n1",
"NVMe2n1"
]
}
},
{
"method": "bdev_wait_for_examine"
}
]
},
{
"subsystem": "nvmf",
"config": [
{
"method": "nvmf_set_config",
"params": {
"discovery_filter": "match_any",
"admin_cmd_passthru": {
"identify_ctrlr": false
}
}
},
{
"method": "nvmf_set_max_subsystems",
"params": {
"max_subsystems": 1024
}
},
{
"method": "nvmf_set_crdt",
"params": {
"crdt1": 0,
"crdt2": 0,
"crdt3": 0
}
},
{
"method": "nvmf_create_transport",
"params": {
"trtype": "RDMA",
"max_queue_depth": 128,
"max_io_qpairs_per_ctrlr": 127,
"in_capsule_data_size": 4096,
"max_io_size": 131072,
"io_unit_size": 8192,
"max_aq_depth": 128,
"num_shared_buffers": 4095,
"buf_cache_size": 4294967295,
"dif_insert_or_strip": false,
"zcopy": false,
"max_srq_depth": 4096,
"no_srq": false,
"acceptor_backlog": 100,
"no_wr_batching": false,
"abort_timeout_sec": 1
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"allow_any_host": false,
"serial_number": "SPDK00000000000003",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_host",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"host": "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b"
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"namespace": {
"nsid": 1,
"bdev_name": "70f5af98-d685-42bd-9024-ca8c498a0ec2",
"nguid": "70F5AF98D68542BD9024CA8C498A0EC2",
"uuid": "70f5af98-d685-42bd-9024-ca8c498a0ec2"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:castle",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:river",
"allow_any_host": false,
"serial_number": "SPDK00000000000001",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_host",
"params": {
"nqn": "nqn.2016-06.io.spdk:river",
"host": "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9"
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:river",
"namespace": {
"nsid": 1,
"bdev_name": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc",
"nguid": "9C545B0B5B9C4A32B1EB483ADF5369FC",
"uuid": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:river",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
},
{
"method": "nvmf_create_subsystem",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"allow_any_host": false,
"serial_number": "SPDK00000000000002",
"model_number": "SPDK bdev Controller",
"max_namespaces": 32,
"min_cntlid": 1,
"max_cntlid": 65519,
"ana_reporting": false
}
},
{
"method": "nvmf_subsystem_add_ns",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"namespace": {
"nsid": 1,
"bdev_name": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e",
"nguid": "780DDEB9646D4331A0D53B0AECD3BF3E",
"uuid": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e"
}
}
},
{
"method": "nvmf_subsystem_add_listener",
"params": {
"nqn": "nqn.2016-06.io.spdk:sfh",
"listen_address": {
"trtype": "RDMA",
"adrfam": "IPv4",
"traddr": "192.168.68.80",
"trsvcid": "4420"
},
"secure_channel": false
}
}
]
},
{
"subsystem": "nbd",
"config": []
}
]
}

1
nixos/modules/_list.nix
View File

@ -18,5 +18,6 @@
l2mesh = ./l2mesh.nix;
borgthin = ./borgthin.nix;
nvme = ./nvme;
spdk = ./spdk.nix;
};
}

94
nixos/modules/spdk.nix Normal file
View File

@ -0,0 +1,94 @@
{ lib, pkgs, config, ... }:
let
inherit (builtins) toJSON;
inherit (lib) optional mapAttrsToList mkIf withFeature;
inherit (lib.my) mkOpt' mkBoolOpt';
rpcOpts = with lib.types; {
options = {
method = mkOpt' str null "RPC method name.";
params = mkOpt' (attrsOf unspecified) { } "RPC params";
};
};
cfg = config.my.spdk;
config' = {
subsystems = mapAttrsToList (subsystem: c: {
inherit subsystem;
config = map (rpc: {
inherit (rpc) method;
} // (if rpc.params != { } then { inherit (rpc) params; } else { })) c;
}) cfg.config.subsystems;
};
configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
spdk = pkgs.spdk.overrideAttrs (o: {
configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
});
spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
'');
spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
exec ${spdk.src}/scripts/setup.sh "$@"
'');
spdk-debug = pkgs.writeShellApplication {
name = "spdk-debug";
runtimeInputs = [ spdk ];
text = ''
set -m
if [ "$(id -u)" -ne 0 ]; then
echo "I need to be root!"
exit 1
fi
spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
until spdk-rpc spdk_get_version > /dev/null; do
sleep 0.5
done
spdk-rpc bdev_set_options --disable-auto-examine
spdk-rpc framework_start_init
${cfg.debugCommands}
fg %1
'';
};
in
{
options.my.spdk = with lib.types; {
enable = mkBoolOpt' false "Whether to enable SPDK target.";
extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt.";
debugCommands = mkOpt' lines "" "Commands to run with the spdk-debug script.";
config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs.";
};
config = mkIf cfg.enable {
boot.kernelModules = [ "ublk_drv" ];
environment.systemPackages = [
spdk
spdk-setup
spdk-rpc
] ++ (optional (cfg.debugCommands != "") spdk-debug);
systemd.services = {
spdk-tgt = {
description = "SPDK target";
path = with pkgs; [
bash
python3
kmod
gawk
util-linux
];
serviceConfig = {
ExecStartPre = "${spdk.src}/scripts/setup.sh";
ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
};
wantedBy = [ "multi-user.target" ];
};
};
};
}