nixos/nvme: Add module

nixos/boxes/castle/default.nix

@@ -251,6 +251,7 @@ in
           secrets = {
             key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMlVuTzKObeaUuPocCF41IO/8X+443lzUJLuCIclt2vr";
           };
+          nvme.uuid = "2230b066-a674-4f45-a1dc-f7727b3a9e7b";
 
           firewall = {
             enable = false;

nixos/boxes/home/palace/vms/cellar/default.nix

@@ -57,8 +57,8 @@ in
               };
             };
 
-            environment.systemPackages = with pkgs; [
+            environment.systemPackages = [
-              pciutils
+              pkgs.pciutils
               spdk
               (pkgs.writeShellScriptBin "spdk-rpc" ''
                 exec ${pkgs.python3}/bin/python3 ${spdk.src}/scripts/rpc.py "$@"
@@ -83,7 +83,7 @@ in
                 preStart = ''
                   ${spdk.src}/scripts/setup.sh
                 '';
-                serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt -c ${./spdk_nvmf.json}";
+                serviceConfig.ExecStart = "${spdk}/bin/spdk_tgt --cpumask 0xffff -c ${./spdk_nvmf.json}";
                 wantedBy = [ "multi-user.target" ];
               };
             };

nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json

@@ -239,6 +239,51 @@
             "abort_timeout_sec": 1
           }
         },
+        {
+          "method": "nvmf_create_subsystem",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:castle",
+            "allow_any_host": false,
+            "serial_number": "SPDK00000000000003",
+            "model_number": "SPDK bdev Controller",
+            "max_namespaces": 32,
+            "min_cntlid": 1,
+            "max_cntlid": 65519,
+            "ana_reporting": false
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_host",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:castle",
+            "host": "nqn.2014-08.org.nvmexpress:uuid:2230b066-a674-4f45-a1dc-f7727b3a9e7b"
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:castle",
+            "namespace": {
+              "nsid": 1,
+              "bdev_name": "70f5af98-d685-42bd-9024-ca8c498a0ec2",
+              "nguid": "70F5AF98D68542BD9024CA8C498A0EC2",
+              "uuid": "70f5af98-d685-42bd-9024-ca8c498a0ec2"
+            }
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_listener",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:castle",
+            "listen_address": {
+              "trtype": "RDMA",
+              "adrfam": "IPv4",
+              "traddr": "192.168.68.80",
+              "trsvcid": "4420"
+            },
+            "secure_channel": false
+          }
+        },
         {
           "method": "nvmf_create_subsystem",
           "params": {
@@ -327,49 +372,6 @@
     {
       "subsystem": "nbd",
       "config": []
-    },
-    {
-      "subsystem": "vhost_blk",
-      "config": []
-    },
-    {
-      "subsystem": "scsi",
-      "config": null
-    },
-    {
-      "subsystem": "iscsi",
-      "config": [
-        {
-          "method": "iscsi_set_options",
-          "params": {
-            "node_base": "iqn.2016-06.io.spdk",
-            "max_sessions": 128,
-            "max_connections_per_session": 2,
-            "max_queue_depth": 64,
-            "default_time2wait": 2,
-            "default_time2retain": 20,
-            "first_burst_length": 8192,
-            "immediate_data": true,
-            "allow_duplicated_isid": false,
-            "error_recovery_level": 0,
-            "nop_timeout": 60,
-            "nop_in_interval": 30,
-            "disable_chap": false,
-            "require_chap": false,
-            "mutual_chap": false,
-            "chap_group": 0,
-            "max_large_datain_per_connection": 64,
-            "max_r2t_per_connection": 4,
-            "pdu_pool_size": 36864,
-            "immediate_data_pool_size": 16384,
-            "data_out_pool_size": 2048
-          }
-        }
-      ]
-    },
-    {
-      "subsystem": "vhost_scsi",
-      "config": []
     }
   ]
 }

nixos/boxes/home/palace/vms/default.nix

@@ -4,7 +4,7 @@
     ./river.nix
   ];
 
-  nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }:
+  nixos.systems.palace.configuration = { lib, pkgs, config, systems, allAssignments, ... }:
   let
     inherit (lib) mkMerge;
     inherit (lib.my) vm;
@@ -70,6 +70,11 @@
       {
         requires = [ vtapUnit ];
         after = [ vtapUnit ];
+        preStart = ''
+          until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do
+            sleep 1
+          done
+        '';
       };
     };
 
@@ -83,7 +88,8 @@
               cpus = 8;
               threads = 2;
             };
-            memory = 32768;
+            memory = 16384;
+            cleanShutdown.timeout = 120;
             drives = [
               (mkMerge [ (vm.disk "cellar" "esp") { frontendOpts.bootindex = 0; } ])
               (vm.disk "cellar" "nix")
@@ -117,6 +123,7 @@
               threads = 2;
             };
             memory = 4096;
+            cleanShutdown.timeout = 120;
             networks = {
               et1g0 = {
                 ifname = "vm-et1g0";

nixos/boxes/home/palace/vms/river.nix

@@ -43,7 +43,6 @@
               extraBin = with pkgs; {
                 dmesg = "${util-linux}/bin/dmesg";
                 ip = "${iproute2}/bin/ip";
-                nvme = "${nvme-cli}/bin/nvme";
               };
               extraConfig = ''
                 DefaultTimeoutStartSec=50
@@ -65,25 +64,6 @@
                   "30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
                 };
               };
-
-              services.connect-nvme = {
-                description = "Connect NVMe-oF";
-                before = [ "initrd-root-device.target" ];
-                after = [ "systemd-networkd-wait-online.service" ];
-                requires = [ "systemd-networkd-wait-online.service" ];
-
-                serviceConfig = {
-                  Type = "oneshot";
-                  Restart = "on-failure";
-                  RestartSec = 10;
-                };
-                script = ''
-                  ${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \
-                    -n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9
-                '';
-
-                wantedBy = [ "initrd-root-device.target" ];
-              };
             };
           };
         };
@@ -146,6 +126,14 @@
             key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
           };
           server.enable = true;
+          nvme = {
+            uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            boot = {
+              nqn = "nqn.2016-06.io.spdk:river";
+              address = "192.168.68.80";
+            };
+          };
+
           deploy.node.hostname = "192.168.68.1";
         };
       };

nixos/modules/_list.nix

@@ -17,5 +17,6 @@
     gui = ./gui.nix;
     l2mesh = ./l2mesh.nix;
     borgthin = ./borgthin.nix;
+    nvme = ./nvme;
   };
 }

nixos/modules/nvme/default.nix

@@ -0,0 +1,58 @@
+{ lib, pkgs, config, ... }:
+let
+  inherit (lib) mkIf;
+  inherit (lib.my) mkOpt';
+
+  cfg = config.my.nvme;
+  nvme-cli = pkgs.nvme-cli.override {
+    libnvme = pkgs.libnvme.overrideAttrs (o: {
+      patches = o.patches ++ [ ./libnvme-hostconf.patch ];
+    });
+  };
+
+  hostNQN = "nqn.2014-08.org.nvmexpress:uuid:${cfg.uuid}";
+  etc = prefix: {
+    "${prefix}nvme/hostnqn".text = hostNQN;
+    "${prefix}nvme/hostid".text = cfg.uuid;
+  };
+in
+{
+  options.my.nvme = with lib.types; {
+    uuid = mkOpt' (nullOr str) null "NVMe host ID";
+    boot = {
+      nqn = mkOpt' (nullOr str) null "NQN to connect to on boot";
+      address = mkOpt' str null "Address of NVMe-oF target.";
+    };
+  };
+
+  config = mkIf (cfg.uuid != null) {
+    environment = {
+      systemPackages = [
+        nvme-cli
+      ];
+      etc = etc "";
+    };
+
+    boot.initrd.systemd = mkIf (cfg.boot.nqn != null) {
+      contents = etc "/etc/";
+      extraBin.nvme = "${nvme-cli}/bin/nvme";
+
+      services.connect-nvme = {
+        description = "Connect NVMe-oF";
+        before = [ "initrd-root-device.target" ];
+        after = [ "systemd-networkd-wait-online.service" ];
+        requires = [ "systemd-networkd-wait-online.service" ];
+
+        serviceConfig = {
+          Type = "oneshot";
+          ExecStart = "${nvme-cli}/bin/nvme connect -t rdma -a ${cfg.boot.address} -n ${cfg.boot.nqn}";
+          Restart = "on-failure";
+          RestartSec = 10;
+        };
+
+        wantedBy = [ "initrd-root-device.target" ];
+      };
+    };
+  };
+}
+

nixos/modules/nvme/libnvme-hostconf.patch

@@ -0,0 +1,15 @@
+diff --git a/src/nvme/fabrics.c b/src/nvme/fabrics.c
+index 21fb292..f9090d1 100644
+--- a/src/nvme/fabrics.c
++++ b/src/nvme/fabrics.c
+@@ -41,8 +41,8 @@
+ 
+ #define NVMF_HOSTID_SIZE	37
+ 
+-#define NVMF_HOSTNQN_FILE	SYSCONFDIR "/nvme/hostnqn"
+-#define NVMF_HOSTID_FILE	SYSCONFDIR "/nvme/hostid"
++#define NVMF_HOSTNQN_FILE	"/etc/nvme/hostnqn"
++#define NVMF_HOSTID_FILE	"/etc/nvme/hostid"
+ 
+ const char *nvmf_dev = "/dev/nvme-fabrics";
+