nixos/river: Initial NVMe-oF booting river :)

nixos/boxes/home/palace/vms/cellar/spdk_nvmf.json

@@ -242,7 +242,7 @@
         {
           "method": "nvmf_create_subsystem",
           "params": {
-            "nqn": "nqn.2016-06.io.spdk:cellar",
+            "nqn": "nqn.2016-06.io.spdk:river",
             "allow_any_host": false,
             "serial_number": "SPDK00000000000001",
             "model_number": "SPDK bdev Controller",
@@ -255,26 +255,64 @@
         {
           "method": "nvmf_subsystem_add_host",
           "params": {
-            "nqn": "nqn.2016-06.io.spdk:cellar",
-            "host": "nqn.2014-08.org.nvmexpress:uuid:de712327-7fc8-6e54-15af-c87f546e170f"
+            "nqn": "nqn.2016-06.io.spdk:river",
+            "host": "nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9"
           }
         },
         {
           "method": "nvmf_subsystem_add_ns",
           "params": {
-            "nqn": "nqn.2016-06.io.spdk:cellar",
+            "nqn": "nqn.2016-06.io.spdk:river",
             "namespace": {
               "nsid": 1,
-              "bdev_name": "Raid0",
-              "nguid": "FA9D43CFD8D54885BB73F038BDC31B25",
-              "uuid": "fa9d43cf-d8d5-4885-bb73-f038bdc31b25"
+              "bdev_name": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc",
+              "nguid": "9C545B0B5B9C4A32B1EB483ADF5369FC",
+              "uuid": "9c545b0b-5b9c-4a32-b1eb-483adf5369fc"
             }
           }
         },
         {
           "method": "nvmf_subsystem_add_listener",
           "params": {
-            "nqn": "nqn.2016-06.io.spdk:cellar",
+            "nqn": "nqn.2016-06.io.spdk:river",
+            "listen_address": {
+              "trtype": "RDMA",
+              "adrfam": "IPv4",
+              "traddr": "192.168.68.80",
+              "trsvcid": "4420"
+            },
+            "secure_channel": false
+          }
+        },
+        {
+          "method": "nvmf_create_subsystem",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:sfh",
+            "allow_any_host": false,
+            "serial_number": "SPDK00000000000002",
+            "model_number": "SPDK bdev Controller",
+            "max_namespaces": 32,
+            "min_cntlid": 1,
+            "max_cntlid": 65519,
+            "ana_reporting": false
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_ns",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:sfh",
+            "namespace": {
+              "nsid": 1,
+              "bdev_name": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e",
+              "nguid": "780DDEB9646D4331A0D53B0AECD3BF3E",
+              "uuid": "780ddeb9-646d-4331-a0d5-3b0aecd3bf3e"
+            }
+          }
+        },
+        {
+          "method": "nvmf_subsystem_add_listener",
+          "params": {
+            "nqn": "nqn.2016-06.io.spdk:sfh",
             "listen_address": {
               "trtype": "RDMA",
               "adrfam": "IPv4",

nixos/boxes/home/palace/vms/default.nix

@@ -1,12 +1,14 @@
 {
   imports = [
     ./cellar
+    ./river.nix
   ];
 
   nixos.systems.palace.configuration = { lib, pkgs, config, systems, ... }:
   let
     inherit (lib) mkMerge;
     inherit (lib.my) vm;
+    inherit (lib.my.c) networkd;
 
     installerDisk = {
       name = "installer";
@@ -23,6 +25,36 @@
     };
   in
   {
+    systemd.network = {
+      netdevs = {
+        "25-vm-et1g0" = {
+           netdevConfig = {
+             Name = "vm-et1g0";
+             Kind = "macvtap";
+           };
+           # TODO: Upstream this missing section
+           extraConfig = ''
+             [MACVTAP]
+             Mode=passthru
+           '';
+        };
+      };
+      networks = {
+        "75-et1g0" = {
+          matchConfig.Name = "et1g0";
+          linkConfig.RequiredForOnline = "no";
+          networkConfig = {
+            MACVTAP = "vm-et1g0";
+          } // networkd.noL3;
+        };
+        "75-vm-et1g0" = {
+          matchConfig.Name = "vm-et1g0";
+          linkConfig.RequiredForOnline = "no";
+          networkConfig = networkd.noL3;
+        };
+      };
+    };
+
     systemd.services = {
       "vm@cellar" = {
         serviceConfig = {
@@ -31,6 +63,14 @@
           NUMAMask = "1";
         };
       };
+      "vm@river" =
+      let
+        vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
+      in
+      {
+        requires = [ vtapUnit ];
+        after = [ vtapUnit ];
+      };
     };
 
     my = {
@@ -68,6 +108,36 @@
               };
             };
           };
+
+          river = {
+            uuid = "12b52d80-ccb6-418d-9b2e-2be34bff3cd9";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 3;
+              threads = 2;
+            };
+            memory = 4096;
+            networks = {
+              et1g0 = {
+                ifname = "vm-et1g0";
+                bridge = null;
+                tapFD = 100;
+                # Real hardware MAC
+                mac = "e0:d5:5e:68:0c:6e";
+                waitOnline = false;
+              };
+            };
+            drives = [
+              installerDisk
+              (mkMerge [ (vm.disk "river" "esp") { frontendOpts.bootindex = 0; } ])
+            ];
+            hostDevices = {
+              et100g0vf1 = {
+                index = 0;
+                hostBDF = "44:00.2";
+              };
+            };
+          };
         };
       };
     };

nixos/boxes/home/palace/vms/river.nix

@@ -0,0 +1,154 @@
+{
+  imports = [ (import ../../routing-common 0) ];
+
+  config.nixos.systems.river = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    home-manager = "mine";
+
+    configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
+    let
+      inherit (lib.my) networkdAssignment mkVLAN;
+      inherit (lib.my.c) networkd;
+      inherit (lib.my.c.home) vlans;
+
+      lanLink = {
+        matchConfig = {
+          Driver = "mlx5_core";
+          PermanentMACAddress = "52:54:00:8a:8a:f2";
+        };
+        linkConfig = {
+          Name = "lan";
+          MTUBytes = toString lib.my.c.home.hiMTU;
+        };
+      };
+    in
+    {
+      imports = [
+        "${modulesPath}/profiles/qemu-guest.nix"
+      ];
+
+      config = {
+        boot = {
+          kernelModules = [ "kvm-intel" ];
+          kernelParams = [ "console=ttyS0,115200n8" ];
+          initrd = {
+            availableKernelModules = [
+              "virtio_pci" "ahci" "sr_mod" "virtio_blk"
+              "ib_core" "ib_uverbs" "mlx5_core" "mlx5_ib" "8021q"
+              "rdma_cm" "iw_cm" "ib_cm" "nvme_core" "nvme_rdma"
+            ];
+            kernelModules = [ "dm-snapshot" "nvme-fabrics" ];
+            systemd = {
+              extraBin = with pkgs; {
+                dmesg = "${util-linux}/bin/dmesg";
+                ip = "${iproute2}/bin/ip";
+                nvme = "${nvme-cli}/bin/nvme";
+              };
+              extraConfig = ''
+                DefaultTimeoutStartSec=50
+                DefaultDeviceTimeoutSec=50
+              '';
+              network = {
+                enable = true;
+                wait-online.enable = true;
+
+                links."10-lan" = lanLink;
+                netdevs = mkVLAN "lan-hi" vlans.hi;
+                networks = {
+                  "20-lan" = {
+                    matchConfig.Name = "lan";
+                    vlan = [ "lan-hi" ];
+                    linkConfig.RequiredForOnline = "no";
+                    networkConfig = networkd.noL3;
+                  };
+                  "30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+                };
+              };
+
+              services.connect-nvme = {
+                description = "Connect NVMe-oF";
+                before = [ "initrd-root-device.target" ];
+                after = [ "systemd-networkd-wait-online.service" ];
+                requires = [ "systemd-networkd-wait-online.service" ];
+
+                serviceConfig = {
+                  Type = "oneshot";
+                  Restart = "on-failure";
+                  RestartSec = 10;
+                };
+                script = ''
+                  ${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \
+                    -n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9
+                '';
+
+                wantedBy = [ "initrd-root-device.target" ];
+              };
+            };
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+          cpu = {
+            intel.updateMicrocode = true;
+          };
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/3ec6c49e-b485-40cb-8eff-315581ac6fe9";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/main/nix";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/main/persist";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          lvm = {
+            boot.thin.enable = true;
+            dmeventd.enable = true;
+          };
+        };
+
+        systemd.network = {
+          links = {
+            "10-wan" = {
+              matchConfig = {
+                # Matching against MAC address seems to break VLAN interfaces
+                # (since they share the same MAC address)
+                Driver = "virtio_net";
+                PermanentMACAddress = "e0:d5:5e:68:0c:6e";
+              };
+              linkConfig = {
+                Name = "wan";
+                RxBufferSize = 4096;
+                TxBufferSize = 4096;
+              };
+            };
+
+            "10-lan" = lanLink;
+          };
+
+          # So we don't drop the IP we use to connect to NVMe-oF!
+          networks."60-lan-hi".networkConfig.KeepConfiguration = "static";
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
+          };
+          server.enable = true;
+          deploy.node.hostname = "192.168.68.1";
+        };
+      };
+    };
+  };
+}