nixos/cellar: Move SPDK config to separate module

2023-12-13 02:19:04 +00:00
parent 2f2764a364
commit 82b24c3c55
5 changed files with 230 additions and 408 deletions
--- a/nixos/boxes/home/palace/vms/cellar/spdk.nix
+++ b/nixos/boxes/home/palace/vms/cellar/spdk.nix
@@ -0,0 +1,130 @@
+{ lib, pkgs, config, assignments, ... }:
+let
+  inherit (lib) mapAttrsToList;
+in
+{
+  config = {
+    boot.blacklistedKernelModules = [ "nvme" ];
+
+    systemd.services = {
+      spdk-tgt.after = [ "systemd-networkd-wait-online@lan-hi.service" ];
+    };
+
+    my = {
+      spdk = {
+        enable = true;
+        extraArgs = "--mem-channels 2 --cpumask 0xffff";
+        debugCommands = ''
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
+          spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'
+
+          spdk-rpc ublk_create_target
+          spdk-rpc ublk_start_disk NVMeRaid 1
+        '';
+        config.subsystems =
+        let
+          nvmeAttaches = mapAttrsToList (name: bdf: {
+            method = "bdev_nvme_attach_controller";
+            params = {
+              hostnqn =
+                "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775";
+              trtype = "PCIe";
+              inherit name;
+              traddr = bdf;
+            };
+          }) {
+            "NVMe0" = "02:00.0";
+            "NVMe1" = "03:00.0";
+            "NVMe2" = "04:00.0";
+          };
+
+          nvmfListener = nqn: {
+            method = "nvmf_subsystem_add_listener";
+            params = {
+              inherit nqn;
+              listen_address = {
+                adrfam = "IPv4";
+                traddr = assignments.hi.ipv4.address;
+                trsvcid = "4420";
+                trtype = "RDMA";
+              };
+              secure_channel = false;
+            };
+          };
+          nvmfBdev = { nqn, hostnqn, bdev, serial }: [
+            {
+              method = "nvmf_create_subsystem";
+              params = {
+                inherit nqn;
+                serial_number = serial;
+              };
+            }
+            (nvmfListener nqn)
+            {
+              method = "nvmf_subsystem_add_host";
+              params = {
+                inherit nqn;
+                host = hostnqn;
+              };
+            }
+            {
+              method = "nvmf_subsystem_add_ns";
+              params = {
+                inherit nqn;
+                namespace = {
+                  bdev_name = bdev;
+                  nsid = 1;
+                };
+              };
+            }
+          ];
+        in
+        {
+          bdev = [
+            {
+              method = "bdev_set_options";
+              params.bdev_auto_examine = false;
+            }
+          ] ++ nvmeAttaches ++ [
+            {
+              method = "bdev_raid_create";
+              params = {
+                base_bdevs = [ "NVMe0n1" "NVMe1n1" "NVMe2n1" ];
+                name = "NVMeRaid";
+                raid_level = "raid0";
+                strip_size_kb = 64;
+              };
+            }
+            {
+              method = "bdev_examine";
+              params.name = "NVMeRaid";
+            }
+            { method = "bdev_wait_for_examine"; }
+          ];
+
+          nvmf = [
+            {
+              method = "nvmf_create_transport";
+              params.trtype = "RDMA";
+            }
+            (nvmfListener "nqn.2014-08.org.nvmexpress.discovery")
+          ] ++ (nvmfBdev {
+            bdev = "NVMeRaidp1";
+            nqn = "nqn.2016-06.io.spdk:river";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            serial = "SPDK00000000000001";
+          }) ++ (nvmfBdev {
+            bdev = "NVMeRaidp2";
+            nqn = "nqn.2016-06.io.spdk:castle";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
+            serial = "SPDK00000000000002";
+          });
+        };
+      };
+    };
+  };
+}