diff --git a/nixos/boxes/home/palace/vms/cellar/default.nix b/nixos/boxes/home/palace/vms/cellar/default.nix
index 58df50a..5512821 100644
--- a/nixos/boxes/home/palace/vms/cellar/default.nix
+++ b/nixos/boxes/home/palace/vms/cellar/default.nix
@@ -24,21 +24,17 @@ in
       let
         inherit (lib) mkMerge;
         inherit (lib.my) networkdAssignment;
-
-        spdk = pkgs.spdk.overrideAttrs (o: {
-          configureFlags = o.configureFlags ++ [ "--with-rdma" ];
-        });
       in
       {
         imports = [
           "${modulesPath}/profiles/qemu-guest.nix"
+          ./spdk.nix
         ];
 
         config = mkMerge [
           {
             boot = {
               kernelParams = [ "console=ttyS0,115200n8" ];
-              blacklistedKernelModules = [ "nvme" ];
             };
 
             fileSystems = {
@@ -57,37 +53,15 @@ in
               };
             };
 
-            environment.systemPackages = [
-              pkgs.pciutils
-              spdk
-              (pkgs.writeShellScriptBin "spdk-rpc" ''
-                exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
-              '')
+            environment.systemPackages = with pkgs; [
+              pciutils
+              partclone
             ];
 
             services = {
               netdata.enable = true;
             };
 
-            systemd.services = {
-              spdk-nvmf = {
-                description = "SPDK NVMe-oF target";
-                path = with pkgs; [
-                  bash
-                  python3
-                  kmod
-                  gawk
-                  util-linux
-                ];
-                after = [ "systemd-networkd-wait-online@lan-hi.service" ];
-                preStart = ''
-                  ${spdk.src}/scripts/setup.sh
-                '';
-                serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt --cpumask 0xffff -c ${./spdk_nvmf.json}";
-                wantedBy = [ "multi-user.target" ];
-              };
-            };
-
             systemd.network = {
               links = {
                 "10-lan-hi" = {
@@ -105,7 +79,7 @@ in
                   {
                     networkConfig.DNS = [
                       (allAssignments.stream.hi.ipv4.address)
-                      # (allAssignments.river.hi.ipv4.address)
+                      (allAssignments.river.hi.ipv4.address)
                     ];
                   }
                 ];
diff --git a/nixos/boxes/home/palace/vms/cellar/spdk.nix b/nixos/boxes/home/palace/vms/cellar/spdk.nix
new file mode 100644
index 0000000..aec7da9
--- /dev/null
+++ b/nixos/boxes/home/palace/vms/cellar/spdk.nix
@@ -0,0 +1,130 @@
+{ lib, pkgs, config, assignments, ... }:
+let
+  inherit (lib) mapAttrsToList;
+in
+{
+  config = {
+    boot.blacklistedKernelModules = [ "nvme" ];
+
+    systemd.services = {
+      spdk-tgt.after = [ "systemd-networkd-wait-online@lan-hi.service" ];
+    };
+
+    my = {
+      spdk = {
+        enable = true;
+        extraArgs = "--mem-channels 2 --cpumask 0xffff";
+        debugCommands = ''
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 02:00.0 -b NVMe0
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 03:00.0 -b NVMe1
+          spdk-rpc bdev_nvme_attach_controller -t pcie -a 04:00.0 -b NVMe2
+          spdk-rpc bdev_raid_create -n NVMeRaid -z 64 -r 0 -b 'NVMe0n1 NVMe1n1 NVMe2n1'
+
+          spdk-rpc ublk_create_target
+          spdk-rpc ublk_start_disk NVMeRaid 1
+        '';
+        config.subsystems =
+        let
+          nvmeAttaches = mapAttrsToList (name: bdf: {
+            method = "bdev_nvme_attach_controller";
+            params = {
+              hostnqn =
+                "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775";
+              trtype = "PCIe";
+              inherit name;
+              traddr = bdf;
+            };
+          }) {
+            "NVMe0" = "02:00.0";
+            "NVMe1" = "03:00.0";
+            "NVMe2" = "04:00.0";
+          };
+
+          nvmfListener = nqn: {
+            method = "nvmf_subsystem_add_listener";
+            params = {
+              inherit nqn;
+              listen_address = {
+                adrfam = "IPv4";
+                traddr = assignments.hi.ipv4.address;
+                trsvcid = "4420";
+                trtype = "RDMA";
+              };
+              secure_channel = false;
+            };
+          };
+          nvmfBdev = { nqn, hostnqn, bdev, serial }: [
+            {
+              method = "nvmf_create_subsystem";
+              params = {
+                inherit nqn;
+                serial_number = serial;
+              };
+            }
+            (nvmfListener nqn)
+            {
+              method = "nvmf_subsystem_add_host";
+              params = {
+                inherit nqn;
+                host = hostnqn;
+              };
+            }
+            {
+              method = "nvmf_subsystem_add_ns";
+              params = {
+                inherit nqn;
+                namespace = {
+                  bdev_name = bdev;
+                  nsid = 1;
+                };
+              };
+            }
+          ];
+        in
+        {
+          bdev = [
+            {
+              method = "bdev_set_options";
+              params.bdev_auto_examine = false;
+            }
+          ] ++ nvmeAttaches ++ [
+            {
+              method = "bdev_raid_create";
+              params = {
+                base_bdevs = [ "NVMe0n1" "NVMe1n1" "NVMe2n1" ];
+                name = "NVMeRaid";
+                raid_level = "raid0";
+                strip_size_kb = 64;
+              };
+            }
+            {
+              method = "bdev_examine";
+              params.name = "NVMeRaid";
+            }
+            { method = "bdev_wait_for_examine"; }
+          ];
+
+          nvmf = [
+            {
+              method = "nvmf_create_transport";
+              params.trtype = "RDMA";
+            }
+            (nvmfListener "nqn.2014-08.org.nvmexpress.discovery")
+          ] ++ (nvmfBdev {
+            bdev = "NVMeRaidp1";
+            nqn = "nqn.2016-06.io.spdk:river";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            serial = "SPDK00000000000001";
+          }) ++ (nvmfBdev {
+            bdev = "NVMeRaidp2";
+            nqn = "nqn.2016-06.io.spdk:castle";
+            hostnqn =
+              "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b";
+            serial = "SPDK00000000000002";
+          });
+        };
+      };
+    };
+  };
+}
diff --git a/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json b/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json
deleted file mode 100644
index 6942a64..0000000
--- a/nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json
+++ /dev/null
@@ -1,377 +0,0 @@
-{
-  "subsystems": [
-    {
-      "subsystem": "scheduler",
-      "config": [
-        {
-          "method": "framework_set_scheduler",
-          "params": {
-            "name": "static"
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "vmd",
-      "config": []
-    },
-    {
-      "subsystem": "sock",
-      "config": [
-        {
-          "method": "sock_impl_set_options",
-          "params": {
-            "impl_name": "posix",
-            "recv_buf_size": 2097152,
-            "send_buf_size": 2097152,
-            "enable_recv_pipe": true,
-            "enable_quickack": false,
-            "enable_placement_id": 0,
-            "enable_zerocopy_send_server": true,
-            "enable_zerocopy_send_client": false,
-            "zerocopy_threshold": 0,
-            "tls_version": 0,
-            "enable_ktls": false
-          }
-        },
-        {
-          "method": "sock_impl_set_options",
-          "params": {
-            "impl_name": "ssl",
-            "recv_buf_size": 4096,
-            "send_buf_size": 4096,
-            "enable_recv_pipe": true,
-            "enable_quickack": false,
-            "enable_placement_id": 0,
-            "enable_zerocopy_send_server": true,
-            "enable_zerocopy_send_client": false,
-            "zerocopy_threshold": 0,
-            "tls_version": 0,
-            "enable_ktls": false
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "iobuf",
-      "config": [
-        {
-          "method": "iobuf_set_options",
-          "params": {
-            "small_pool_count": 8192,
-            "large_pool_count": 1024,
-            "small_bufsize": 8192,
-            "large_bufsize": 135168
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "accel",
-      "config": [
-        {
-          "method": "accel_set_options",
-          "params": {
-            "small_cache_size": 128,
-            "large_cache_size": 16,
-            "task_count": 2048,
-            "sequence_count": 2048,
-            "buf_count": 2048
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "bdev",
-      "config": [
-        {
-          "method": "bdev_set_options",
-          "params": {
-            "bdev_io_pool_size": 65535,
-            "bdev_io_cache_size": 256,
-            "bdev_auto_examine": true
-          }
-        },
-        {
-          "method": "bdev_nvme_set_options",
-          "params": {
-            "action_on_timeout": "none",
-            "timeout_us": 0,
-            "timeout_admin_us": 0,
-            "keep_alive_timeout_ms": 10000,
-            "transport_retry_count": 4,
-            "arbitration_burst": 0,
-            "low_priority_weight": 0,
-            "medium_priority_weight": 0,
-            "high_priority_weight": 0,
-            "nvme_adminq_poll_period_us": 10000,
-            "nvme_ioq_poll_period_us": 0,
-            "io_queue_requests": 512,
-            "delay_cmd_submit": true,
-            "bdev_retry_count": 3,
-            "transport_ack_timeout": 0,
-            "ctrlr_loss_timeout_sec": 0,
-            "reconnect_delay_sec": 0,
-            "fast_io_fail_timeout_sec": 0,
-            "generate_uuids": false,
-            "transport_tos": 0,
-            "io_path_stat": false,
-            "allow_accel_sequence": false
-          }
-        },
-        {
-          "method": "bdev_nvme_attach_controller",
-          "params": {
-            "name": "NVMe0",
-            "trtype": "PCIe",
-            "traddr": "02:00.0",
-            "prchk_reftag": false,
-            "prchk_guard": false,
-            "ctrlr_loss_timeout_sec": 0,
-            "reconnect_delay_sec": 0,
-            "fast_io_fail_timeout_sec": 0,
-            "hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
-            "hdgst": false,
-            "ddgst": false
-          }
-        },
-        {
-          "method": "bdev_nvme_attach_controller",
-          "params": {
-            "name": "NVMe1",
-            "trtype": "PCIe",
-            "traddr": "03.00.0",
-            "prchk_reftag": false,
-            "prchk_guard": false,
-            "ctrlr_loss_timeout_sec": 0,
-            "reconnect_delay_sec": 0,
-            "fast_io_fail_timeout_sec": 0,
-            "hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
-            "hdgst": false,
-            "ddgst": false
-          }
-        },
-        {
-          "method": "bdev_nvme_attach_controller",
-          "params": {
-            "name": "NVMe2",
-            "trtype": "PCIe",
-            "traddr": "04.00.0",
-            "prchk_reftag": false,
-            "prchk_guard": false,
-            "ctrlr_loss_timeout_sec": 0,
-            "reconnect_delay_sec": 0,
-            "fast_io_fail_timeout_sec": 0,
-            "hostnqn": "nqn.2014-08.org.nvmexpress:uuid:2b16606f-b82c-49f8-9b20-a589dac8b775",
-            "hdgst": false,
-            "ddgst": false
-          }
-        },
-        {
-          "method": "bdev_nvme_set_hotplug",
-          "params": {
-            "period_us": 100000,
-            "enable": false
-          }
-        },
-        {
-          "method": "bdev_raid_create",
-          "params": {
-            "name": "Raid0",
-            "strip_size_kb": 64,
-            "raid_level": "raid0",
-            "base_bdevs": [
-              "NVMe0n1",
-              "NVMe1n1",
-              "NVMe2n1"
-            ]
-          }
-        },
-        {
-          "method": "bdev_wait_for_examine"
-        }
-      ]
-    },
-    {
-      "subsystem": "nvmf",
-      "config": [
-        {
-          "method": "nvmf_set_config",
-          "params": {
-            "discovery_filter": "match_any",
-            "admin_cmd_passthru": {
-              "identify_ctrlr": false
-            }
-          }
-        },
-        {
-          "method": "nvmf_set_max_subsystems",
-          "params": {
-            "max_subsystems": 1024
-          }
-        },
-        {
-          "method": "nvmf_set_crdt",
-          "params": {
-            "crdt1": 0,
-            "crdt2": 0,
-            "crdt3": 0
-          }
-        },
-        {
-          "method": "nvmf_create_transport",
-          "params": {
-            "trtype": "RDMA",
-            "max_queue_depth": 128,
-            "max_io_qpairs_per_ctrlr": 127,
-            "in_capsule_data_size": 4096,
-            "max_io_size": 131072,
-            "io_unit_size": 8192,
-            "max_aq_depth": 128,
-            "num_shared_buffers": 4095,
-            "buf_cache_size": 4294967295,
-            "dif_insert_or_strip": false,
-            "zcopy": false,
-            "max_srq_depth": 4096,
-            "no_srq": false,
-            "acceptor_backlog": 100,
-            "no_wr_batching": false,
-            "abort_timeout_sec": 1
-          }
-        },
-        {
-          "method": "nvmf_create_subsystem",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:castle",
-            "allow_any_host": false,
-            "serial_number": "SPDK00000000000003",
-            "model_number": "SPDK bdev Controller",
-            "max_namespaces": 32,
-            "min_cntlid": 1,
-            "max_cntlid": 65519,
-            "ana_reporting": false
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_host",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:castle",
-            "host": "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b"
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_ns",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:castle",
-            "namespace": {
-              "nsid": 1,
-              "bdev_name": "70f5af98-d685-42bd-9024-ca8c498a0ec2",
-              "nguid": "70F5AF98D68542BD9024CA8C498A0EC2",
-              "uuid": "70f5af98-d685-42bd-9024-ca8c498a0ec2"
-            }
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_listener",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:castle",
-            "listen_address": {
-              "trtype": "RDMA",
-              "adrfam": "IPv4",
-              "traddr": "192.168.68.80",
-              "trsvcid": "4420"
-            },
-            "secure_channel": false
-          }
-        },
-        {
-          "method": "nvmf_create_subsystem",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:river",
-            "allow_any_host": false,
-            "serial_number": "SPDK00000000000001",
-            "model_number": "SPDK bdev Controller",
-            "max_namespaces": 32,
-            "min_cntlid": 1,
-            "max_cntlid": 65519,
-            "ana_reporting": false
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_host",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:river",
-            "host": "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9"
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_ns",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:river",
-            "namespace": {
-              "nsid": 1,
-              "bdev_name": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc",
-              "nguid": "9C545B0B5B9C4A32B1EB483ADF5369FC",
-              "uuid": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc"
-            }
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_listener",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:river",
-            "listen_address": {
-              "trtype": "RDMA",
-              "adrfam": "IPv4",
-              "traddr": "192.168.68.80",
-              "trsvcid": "4420"
-            },
-            "secure_channel": false
-          }
-        },
-        {
-          "method": "nvmf_create_subsystem",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:sfh",
-            "allow_any_host": false,
-            "serial_number": "SPDK00000000000002",
-            "model_number": "SPDK bdev Controller",
-            "max_namespaces": 32,
-            "min_cntlid": 1,
-            "max_cntlid": 65519,
-            "ana_reporting": false
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_ns",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:sfh",
-            "namespace": {
-              "nsid": 1,
-              "bdev_name": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e",
-              "nguid": "780DDEB9646D4331A0D53B0AECD3BF3E",
-              "uuid": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e"
-            }
-          }
-        },
-        {
-          "method": "nvmf_subsystem_add_listener",
-          "params": {
-            "nqn": "nqn.2016-06.io.spdk:sfh",
-            "listen_address": {
-              "trtype": "RDMA",
-              "adrfam": "IPv4",
-              "traddr": "192.168.68.80",
-              "trsvcid": "4420"
-            },
-            "secure_channel": false
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "nbd",
-      "config": []
-    }
-  ]
-}
diff --git a/nixos/modules/_list.nix b/nixos/modules/_list.nix
index 37cd76c..899bc1d 100644
--- a/nixos/modules/_list.nix
+++ b/nixos/modules/_list.nix
@@ -18,5 +18,6 @@
     l2mesh = ./l2mesh.nix;
     borgthin = ./borgthin.nix;
     nvme = ./nvme;
+    spdk = ./spdk.nix;
   };
 }
diff --git a/nixos/modules/spdk.nix b/nixos/modules/spdk.nix
new file mode 100644
index 0000000..6da6d7e
--- /dev/null
+++ b/nixos/modules/spdk.nix
@@ -0,0 +1,94 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (builtins) toJSON;
+  inherit (lib) optional mapAttrsToList mkIf withFeature;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  rpcOpts = with lib.types; {
+    options = {
+      method = mkOpt' str null "RPC method name.";
+      params = mkOpt' (attrsOf unspecified) { } "RPC params";
+    };
+  };
+
+  cfg = config.my.spdk;
+  config' = {
+    subsystems = mapAttrsToList (subsystem: c: {
+      inherit subsystem;
+      config = map (rpc: {
+        inherit (rpc) method;
+      } // (if rpc.params != { } then { inherit (rpc) params; } else { })) c;
+    }) cfg.config.subsystems;
+  };
+  configJSON = pkgs.writeText "spdk-config.json" (toJSON config');
+
+  spdk = pkgs.spdk.overrideAttrs (o: {
+    configureFlags = o.configureFlags ++ (map (withFeature true) [ "rdma" "ublk" ]);
+    buildInputs = o.buildInputs ++ (with pkgs; [ liburing ]);
+  });
+  spdk-rpc = (pkgs.writeShellScriptBin "spdk-rpc" ''
+    exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
+  '');
+  spdk-setup = (pkgs.writeShellScriptBin "spdk-setup" ''
+    exec ${spdk.src}/scripts/setup.sh "$@"
+  '');
+  spdk-debug = pkgs.writeShellApplication {
+    name = "spdk-debug";
+    runtimeInputs = [ spdk ];
+    text = ''
+      set -m
+      if [ "$(id -u)" -ne 0 ]; then
+        echo "I need to be root!"
+        exit 1
+      fi
+
+      spdk_tgt ${cfg.extraArgs} --wait-for-rpc &
+      until spdk-rpc spdk_get_version > /dev/null; do
+        sleep 0.5
+      done
+
+      spdk-rpc bdev_set_options --disable-auto-examine
+      spdk-rpc framework_start_init
+
+      ${cfg.debugCommands}
+
+      fg %1
+    '';
+  };
+in
+{
+  options.my.spdk = with lib.types; {
+    enable = mkBoolOpt' false "Whether to enable SPDK target.";
+    extraArgs = mkOpt' str "" "Extra arguments to pass to spdk_tgt.";
+    debugCommands = mkOpt' lines "" "Commands to run with the spdk-debug script.";
+    config.subsystems = mkOpt' (attrsOf (listOf (submodule rpcOpts))) { } "Subsystem config / RPCs.";
+  };
+
+  config = mkIf cfg.enable {
+    boot.kernelModules = [ "ublk_drv" ];
+
+    environment.systemPackages = [
+      spdk
+      spdk-setup
+      spdk-rpc
+    ] ++ (optional (cfg.debugCommands != "") spdk-debug);
+
+    systemd.services = {
+      spdk-tgt = {
+        description = "SPDK target";
+        path = with pkgs; [
+          bash
+          python3
+          kmod
+          gawk
+          util-linux
+        ];
+        serviceConfig = {
+          ExecStartPre = "${spdk.src}/scripts/setup.sh";
+          ExecStart = "${spdk}/bin/spdk_tgt ${cfg.extraArgs} -c ${configJSON}";
+        };
+        wantedBy = [ "multi-user.target" ];
+      };
+    };
+  };
+}