nixfiles/nixos/boxes/home/palace/vms/cellar/spdk.nix

{ lib, pkgs, config, assignments, ... }:
let
  inherit (lib) mapAttrsToList;
in
{
  config = {
    boot.blacklistedKernelModules = [ "nvme" ];

    systemd.services = {
      spdk-tgt.after = [ "systemd-networkd-wait-online@lan-hi.service" ];
    };

    my = {
      spdk = {
        enable = true;
        extraArgs = "--mem-channels 2 --cpumask 0xffff";
        debugCommands = ''
          spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
          spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
          spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
          spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'

          spdk-rpc ublk_create_target
          spdk-rpc ublk_start_disk NVMeRaid 1
        '';
        config.subsystems =
        let
          nvmeAttaches = mapAttrsToList (name: bdf: {
            method = "bdev_nvme_attach_controller";
            params = {
              hostnqn =
                "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775";
              trtype = "PCIe";
              inherit name;
              traddr = bdf;
            };
          }) {
            "NVMe0" = "02:00.0";
            "NVMe1" = "03:00.0";
            "NVMe2" = "04:00.0";
          };

          nvmfListener = nqn: {
            method = "nvmf_subsystem_add_listener";
            params = {
              inherit nqn;
              listen_address = {
                adrfam = "IPv4";
                traddr = assignments.hi.ipv4.address;
                trsvcid = "4420";
                trtype = "RDMA";
              };
              secure_channel = false;
            };
          };
          nvmfBdev = { nqn, hostnqn, bdev, serial }: [
            {
              method = "nvmf_create_subsystem";
              params = {
                inherit nqn;
                serial_number = serial;
              };
            }
            (nvmfListener nqn)
            {
              method = "nvmf_subsystem_add_host";
              params = {
                inherit nqn;
                host = hostnqn;
              };
            }
            {
              method = "nvmf_subsystem_add_ns";
              params = {
                inherit nqn;
                namespace = {
                  bdev_name = bdev;
                  nsid = 1;
                };
              };
            }
          ];
        in
        {
          scheduler = [
            {
              method = "framework_set_scheduler";
              params.name = "dynamic";
            }
          ];

          bdev = [
            {
              method = "bdev_set_options";
              params.bdev_auto_examine = false;
            }
          ] ++ nvmeAttaches ++ [
            {
              method = "bdev_raid_create";
              params = {
                base_bdevs = [ "NVMe0n1" "NVMe1n1" "NVMe2n1" ];
                name = "NVMeRaid";
                raid_level = "raid0";
                strip_size_kb = 64;
              };
            }
            {
              method = "bdev_examine";
              params.name = "NVMeRaid";
            }
            { method = "bdev_wait_for_examine"; }
          ];

          nvmf = [
            {
              method = "nvmf_create_transport";
              params.trtype = "RDMA";
            }
            (nvmfListener "nqn.2014-08.org.nvmexpress.discovery")
          ] ++ (nvmfBdev {
            bdev = "NVMeRaidp1";
            nqn = "nqn.2016-06.io.spdk:river";
            hostnqn =
              "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
            serial = "SPDK00000000000001";
          }) ++ (nvmfBdev {
            bdev = "NVMeRaidp2";
            nqn = "nqn.2016-06.io.spdk:castle";
            hostnqn =
              "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
            serial = "SPDK00000000000002";
          });
        };
      };
    };
  };
}