nixos/sfh: Working containers

nixos/boxes/home/palace/default.nix

@@ -94,7 +94,7 @@ in
             extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
           };
           udev.extraRules = ''
-            ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3"
+            ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="4"
           '';
         };
 
@@ -188,6 +188,13 @@ in
                   VLANId=${toString vlans.hi}
                   LinkState=yes
                   MACAddress=52:54:00:ac:15:a9
+
+                  # sfh bridge
+                  [SR-IOV]
+                  VirtualFunction=3
+                  VLANId=${toString vlans.hi}
+                  LinkState=yes
+                  MACAddress=52:54:00:90:34:95
                 '';
               };
               "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi;

nixos/boxes/home/palace/vms/default.nix

@@ -183,6 +183,10 @@
                 index = 0;
                 hostBDF = "44:00.3";
               };
+              et100g0vf3 = {
+                index = 1;
+                hostBDF = "44:00.4";
+              };
             };
           };
         };

nixos/boxes/home/palace/vms/river.nix

@@ -24,6 +24,7 @@
           initrd = {
             availableKernelModules = [
               "virtio_pci" "ahci" "sr_mod" "virtio_blk"
+              "8021q"
             ] ++ roceBootModules;
             kernelModules = [ "dm-snapshot" ];
             systemd = {

nixos/boxes/home/palace/vms/sfh/containers/default.nix

@@ -0,0 +1,5 @@
+{
+  imports = [
+    ./unifi.nix
+  ];
+}

nixos/boxes/home/palace/vms/sfh/containers/unifi.nix

@@ -0,0 +1,54 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) domain prefixes vips hiMTU;
+in
+{
+  nixos.systems.unifi = { config, ... }: {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+    rendered = config.configuration.config.my.asContainer;
+
+    assignments = {
+      hi = {
+        name = "unifi-ctr";
+        inherit domain;
+        mtu = hiMTU;
+        ipv4 = {
+          address = net.cidr.host 100 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+        ipv6 = {
+          iid = "::5:1";
+          address = net.cidr.host (65536*5+1) prefixes.hi.v6;
+        };
+      };
+    };
+
+    configuration = { lib, config, assignments, ... }:
+    let
+      inherit (lib) mkMerge mkIf mkForce;
+      inherit (lib.my) networkdAssignment;
+    in
+    {
+      config = {
+        my = {
+          deploy.enable = false;
+          server.enable = true;
+
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKdgcziQki/RH7E+NH2bYnzSVKaJ27905Yo5TcOjSh/U";
+            files = { };
+          };
+        };
+
+        systemd = {
+          network.networks."80-container-host0" = networkdAssignment "host0" assignments.hi;
+        };
+
+        services = { };
+      };
+    };
+  };
+}

nixos/boxes/home/palace/vms/sfh/default.nix

@@ -1,14 +1,16 @@
 { lib, ... }:
 let
   inherit (lib.my) net;
-  inherit (lib.my.c) pubDomain;
   inherit (lib.my.c.home) domain prefixes vips hiMTU roceBootModules;
 in
 {
+  imports = [ ./containers ];
+
   config.nixos.systems.sfh = {
     system = "x86_64-linux";
     nixpkgs = "mine";
     home-manager = "mine";
+
     assignments = {
       hi = {
         inherit domain;
@@ -27,8 +29,9 @@ in
 
     configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
     let
-      inherit (lib) mkMerge;
+      inherit (lib) mapAttrs mkMerge;
       inherit (lib.my) networkdAssignment;
+      inherit (lib.my.c) networkd;
       inherit (lib.my.c.home) domain;
     in
     {
@@ -92,13 +95,30 @@ in
                 MTUBytes = toString lib.my.c.home.hiMTU;
               };
             };
+            "10-lan-hi-ctrs" = {
+              matchConfig = {
+                Driver = "mlx5_core";
+                PermanentMACAddress = "52:54:00:90:34:95";
+              };
+              linkConfig = {
+                Name = "lan-hi-ctrs";
+                MTUBytes = toString lib.my.c.home.hiMTU;
+              };
+            };
           };
 
-          networks."30-lan-hi" = mkMerge [
-            (networkdAssignment "lan-hi" assignments.hi)
-            # So we don't drop the IP we use to connect to NVMe-oF!
-            { networkConfig.KeepConfiguration = "static"; }
-          ];
+          networks = {
+            "30-lan-hi" = mkMerge [
+              (networkdAssignment "lan-hi" assignments.hi)
+              # So we don't drop the IP we use to connect to NVMe-oF!
+              { networkConfig.KeepConfiguration = "static"; }
+            ];
+            "30-lan-hi-ctrs" = {
+              matchConfig.Name = "lan-hi-ctrs";
+              linkConfig.RequiredForOnline = "no";
+              networkConfig = networkd.noL3;
+            };
+          };
         };
 
         my = {
@@ -117,6 +137,19 @@ in
               address = "192.168.68.80";
             };
           };
+
+          containers.instances =
+          let
+            instances = {
+              unifi = {};
+            };
+          in
+          mkMerge [
+            instances
+            (mapAttrs (n: i: {
+              networking.macVLAN = "lan-hi-ctrs";
+            }) instances)
+          ];
         };
       };
     };

nixos/modules/containers.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, options, config, systems, ... }:
 let
-  inherit (builtins) attrNames attrValues all hashString toJSON;
+  inherit (builtins) attrNames attrValues all hashString toJSON any;
   inherit (lib)
     groupBy' mapAttrsToList optionalString optional concatMapStringsSep filterAttrs mkOption mkDefault mkIf mkMerge;
   inherit (lib.my) mkOpt' mkBoolOpt';
@@ -98,6 +98,7 @@ let
       };
       networking = {
         bridge = mkOpt' (nullOr str) null "Network bridge to connect to.";
+        macVLAN = mkOpt' (nullOr str) null "Network interface to make MACVLAN interface from.";
       };
     };
   };
@@ -115,13 +116,17 @@ in
           assertion = config.systemd.network.enable;
           message = "Containers currently require systemd-networkd!";
         }
+        {
+          assertion = all (i: i.networking.bridge == null || i.networking.macVLAN == null) (attrValues cfg.instances);
+          message = "Only bridge OR MACVLAN can be set";
+        }
       ];
 
       # TODO: Better security
       my.firewall.trustedInterfaces =
         mapAttrsToList
           (n: _: "ve-${n}")
-          (filterAttrs (_: c: c.networking.bridge == null) cfg.instances);
+          (filterAttrs (_: c: c.networking.bridge == null && c.networking.macVLAN == null) cfg.instances);
 
       systemd = mkMerge (mapAttrsToList (n: c: {
         nspawn."${n}" = {
@@ -154,6 +159,8 @@ in
           };
           networkConfig = if (c.networking.bridge != null) then {
             Bridge = c.networking.bridge;
+          } else if (c.networking.macVLAN != null) then {
+            MACVLAN = "${c.networking.macVLAN}:host0";
           } else {
             VirtualEthernet = true;
           };