nixos: Add SFH VM config

nixos/boxes/home/palace/default.nix

@@ -94,7 +94,7 @@ in
             extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
           };
           udev.extraRules = ''
-            ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="2"
+            ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3"
           '';
         };
 
@@ -110,7 +110,7 @@ in
           hwloc
         ];
 
-        networking.domain = "h.${pubDomain}";
+        networking = { inherit domain; };
 
         systemd = {
           tmpfiles.rules = [
@@ -144,6 +144,13 @@ in
 
             netdevs = mkMerge [
               (mkVLAN "lan-hi" vlans.hi)
+              (mkVLAN "lan-lo-phy" vlans.lo)
+              {
+                "25-lan-lo".netdevConfig = {
+                  Name = "lan-lo";
+                  Kind = "bridge";
+                };
+              }
             ];
 
             networks = {
@@ -151,6 +158,7 @@ in
                 (networkdAssignment "lan-core" assignments.core)
                 {
                   matchConfig.Name = "lan-core";
+                  vlan = [ "lan-lo-phy" ];
                   networkConfig.IPv6AcceptRA = mkForce false;
                 }
               ];
@@ -173,9 +181,28 @@ in
                   VirtualFunction=1
                   LinkState=yes
                   MACAddress=52:54:00:8a:8a:f2
+
+                  # sfh
+                  [SR-IOV]
+                  VirtualFunction=2
+                  VLANId=${toString vlans.hi}
+                  LinkState=yes
+                  MACAddress=52:54:00:ac:15:a9
                 '';
               };
               "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
+
+              "50-lan-lo-phy" = {
+                matchConfig.Name = "lan-lo-phy";
+                networkConfig = {
+                  Bridge = "lan-lo";
+                } // networkd.noL3;
+              };
+              "60-lan-lo" = {
+                matchConfig.Name = "lan-lo";
+                linkConfig.RequiredForOnline = "no";
+                networkConfig = networkd.noL3;
+              };
             };
           };
         };

nixos/boxes/home/palace/vms/cellar/default.nix

@@ -58,6 +58,8 @@ in
               };
             };
 
+            networking = { inherit domain; };
+
             environment.systemPackages = with pkgs; [
               pciutils
               partclone

nixos/boxes/home/palace/vms/default.nix

@@ -55,21 +55,10 @@
       };
     };
 
-    systemd.services = {
+    systemd.services =
-      "vm@cellar" = {
+    let
-        serviceConfig = {
+      awaitCellar = {
-          CPUAffinity = "numa";
+        after = [ "vm@cellar.service" ];
-          NUMAPolicy = "bind";
-          NUMAMask = "1";
-        };
-      };
-      "vm@river" =
-      let
-        vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
-      in
-      {
-        requires = [ vtapUnit ];
-        after = [ vtapUnit "vm@cellar.service" ];
         bindsTo = [ "vm@cellar.service" ];
         preStart = ''
           until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do
@@ -77,6 +66,28 @@
           done
         '';
       };
+    in
+    {
+      "vm@cellar" = {
+        serviceConfig = {
+          CPUAffinity = "numa";
+          NUMAPolicy = "bind";
+          NUMAMask = "1";
+        };
+      };
+
+      "vm@river" =
+      let
+        vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
+      in
+      mkMerge [
+        awaitCellar
+        {
+          requires = [ vtapUnit ];
+          after = [ vtapUnit ];
+        }
+      ];
+      "vm@sfh" = awaitCellar;
     };
 
     my = {
@@ -128,7 +139,7 @@
               threads = 2;
             };
             memory = 4096;
-            cleanShutdown.timeout = 120;
+            cleanShutdown.timeout = 60;
             networks = {
               et1g0 = {
                 ifname = "vm-et1g0";
@@ -150,6 +161,29 @@
               };
             };
           };
+
+          sfh = {
+            uuid = "82ec149d-577c-421a-93e2-a9307c756cd8";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 8;
+              threads = 2;
+            };
+            memory = 32768;
+            cleanShutdown.timeout = 120;
+            networks.netboot = {
+              bridge = "lan-lo";
+              waitOnline = "carrier";
+              mac = "52:54:00:a5:7e:93";
+              extraOptions.bootindex = 1;
+            };
+            hostDevices = {
+              et100g0vf2 = {
+                index = 0;
+                hostBDF = "44:00.3";
+              };
+            };
+          };
         };
       };
     };

nixos/modules/vms.nix

@@ -116,7 +116,7 @@ let
         });
         default = { };
       };
-      drives = mkOpt' (listOf (submodule driveOpts)) { } "Drives to attach to VM.";
+      drives = mkOpt' (listOf (submodule driveOpts)) [ ] "Drives to attach to VM.";
       hostDevices = mkOpt' (attrsOf (submodule hostDevOpts)) { } "Host PCI devices to pass to the VM.";
     };
   };
@@ -126,8 +126,8 @@ let
       (map
         (i: mapAttrsToList (name: c: c // { inherit name; }) i.hostDevices)
         (attrValues cfg.instances));
-  anyVfioDevs = any (d: d.bindVFIO) allHostDevs;
+  anyVfioDevs = any (d: d.bindVFIO);
-  vfioHostDevs = filter (d: d.bindVFIO) allHostDevs;
+  vfioHostDevs = filter (d: d.bindVFIO);
 
   mkQemuScript = n: i:
   let
@@ -204,7 +204,7 @@ in
     services.udev = {
       packages =
         optionals
-          anyVfioDevs
+          (anyVfioDevs allHostDevs)
           [
             pkgs.vfio-pci-bind
             (pkgs.writeTextDir
@@ -212,7 +212,7 @@ in
               (concatMapStringsSep
                 "\n"
                 (d: ''ACTION=="add", SUBSYSTEM=="pci", KERNEL=="0000:${d.hostBDF}", TAG="vfio-pci-bind"'')
-                vfioHostDevs))
+                (vfioHostDevs allHostDevs)))
           ];
     };
 
@@ -261,12 +261,15 @@ in
           };
 
           preStart =
+          let
+            hostDevs = attrValues i.hostDevices;
+          in
             ''
               if [ ! -e "$STATE_DIRECTORY"/ovmf_vars.bin ]; then
                 cp "${cfg.ovmfPackage.fd}"/FV/OVMF_VARS.fd "$STATE_DIRECTORY"/ovmf_vars.bin
               fi
 
-              ${optionalString anyVfioDevs ''
+              ${optionalString (anyVfioDevs hostDevs) ''
                 iommu_group() {
                   g=/sys/bus/pci/devices/0000:$1/iommu_group
                   until [ -e $g ]; do
@@ -280,7 +283,7 @@ in
                   done
                 }
 
-                ${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") vfioHostDevs}
+                ${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") (vfioHostDevs hostDevs) }
               ''}
             '';
           script = mkQemuScript n i;