nixos/river: Initial NVMe-oF booting river :)

2023-12-11 01:55:02 +00:00
parent 33eded0626
commit 5e5f70501c
55 changed files with 1116 additions and 862 deletions
--- a/nixos/boxes/home/palace/vms/river.nix
+++ b/nixos/boxes/home/palace/vms/river.nix
@@ -0,0 +1,154 @@
+{
+  imports = [ (import ../../routing-common 0) ];
+
+  config.nixos.systems.river = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    home-manager = "mine";
+
+    configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
+    let
+      inherit (lib.my) networkdAssignment mkVLAN;
+      inherit (lib.my.c) networkd;
+      inherit (lib.my.c.home) vlans;
+
+      lanLink = {
+        matchConfig = {
+          Driver = "mlx5_core";
+          PermanentMACAddress = "52:54:00:8a:8a:f2";
+        };
+        linkConfig = {
+          Name = "lan";
+          MTUBytes = toString lib.my.c.home.hiMTU;
+        };
+      };
+    in
+    {
+      imports = [
+        "${modulesPath}/profiles/qemu-guest.nix"
+      ];
+
+      config = {
+        boot = {
+          kernelModules = [ "kvm-intel" ];
+          kernelParams = [ "console=ttyS0,115200n8" ];
+          initrd = {
+            availableKernelModules = [
+              "virtio_pci" "ahci" "sr_mod" "virtio_blk"
+              "ib_core" "ib_uverbs" "mlx5_core" "mlx5_ib" "8021q"
+              "rdma_cm" "iw_cm" "ib_cm" "nvme_core" "nvme_rdma"
+            ];
+            kernelModules = [ "dm-snapshot" "nvme-fabrics" ];
+            systemd = {
+              extraBin = with pkgs; {
+                dmesg = "${util-linux}/bin/dmesg";
+                ip = "${iproute2}/bin/ip";
+                nvme = "${nvme-cli}/bin/nvme";
+              };
+              extraConfig = ''
+                DefaultTimeoutStartSec=50
+                DefaultDeviceTimeoutSec=50
+              '';
+              network = {
+                enable = true;
+                wait-online.enable = true;
+
+                links."10-lan" = lanLink;
+                netdevs = mkVLAN "lan-hi" vlans.hi;
+                networks = {
+                  "20-lan" = {
+                    matchConfig.Name = "lan";
+                    vlan = [ "lan-hi" ];
+                    linkConfig.RequiredForOnline = "no";
+                    networkConfig = networkd.noL3;
+                  };
+                  "30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+                };
+              };
+
+              services.connect-nvme = {
+                description = "Connect NVMe-oF";
+                before = [ "initrd-root-device.target" ];
+                after = [ "systemd-networkd-wait-online.service" ];
+                requires = [ "systemd-networkd-wait-online.service" ];
+
+                serviceConfig = {
+                  Type = "oneshot";
+                  Restart = "on-failure";
+                  RestartSec = 10;
+                };
+                script = ''
+                  ${pkgs.nvme-cli}/bin/nvme connect -t rdma -a 192.168.68.80 \
+                    -n nqn.2016-06.io.spdk:river -q nqn.2014-08.org.nvmexpress:uuid:12b52d80-ccb6-418d-9b2e-2be34bff3cd9
+                '';
+
+                wantedBy = [ "initrd-root-device.target" ];
+              };
+            };
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+          cpu = {
+            intel.updateMicrocode = true;
+          };
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/3ec6c49e-b485-40cb-8eff-315581ac6fe9";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/main/nix";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/main/persist";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          lvm = {
+            boot.thin.enable = true;
+            dmeventd.enable = true;
+          };
+        };
+
+        systemd.network = {
+          links = {
+            "10-wan" = {
+              matchConfig = {
+                # Matching against MAC address seems to break VLAN interfaces
+                # (since they share the same MAC address)
+                Driver = "virtio_net";
+                PermanentMACAddress = "e0:d5:5e:68:0c:6e";
+              };
+              linkConfig = {
+                Name = "wan";
+                RxBufferSize = 4096;
+                TxBufferSize = 4096;
+              };
+            };
+
+            "10-lan" = lanLink;
+          };
+
+          # So we don't drop the IP we use to connect to NVMe-oF!
+          networks."60-lan-hi".networkConfig.KeepConfiguration = "static";
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9uFa4z9WPuXRFVA+PClQSitQCSPckhKTxo1Hq585Oa";
+          };
+          server.enable = true;
+          deploy.node.hostname = "192.168.68.1";
+        };
+      };
+    };
+  };
+}