Initial networking VM

Also general improvements around VMs

nixos/modules/_list.nix

@@ -11,5 +11,6 @@
     secrets = ./secrets.nix;
     containers = ./containers.nix;
     vms = ./vms.nix;
+    network = ./network.nix;
   };
 }

nixos/modules/build.nix

@@ -80,6 +80,7 @@ in
       sharedDirectories = dummyOption;
       cores = dummyOption;
       memorySize = dummyOption;
+      qemu.options = dummyOption;
     };
   };
 

nixos/modules/common.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, pkgs', inputs, config, ... }:
 let
-  inherit (lib) flatten optional mkIf mkDefault mkMerge;
+  inherit (lib) mkIf mkDefault mkMerge;
   inherit (lib.my) mkBoolOpt' dummyOption;
 in
 {
@@ -95,17 +95,11 @@ in
         };
       };
 
-      networking = {
-        domain = mkDefault "int.nul.ie";
-        useDHCP = false;
-        enableIPv6 = mkDefault true;
-        useNetworkd = mkDefault true;
-      };
-
       environment.systemPackages = with pkgs; [
         bash-completion
         vim
         ldns
+        minicom
       ];
 
       programs = {
@@ -141,14 +135,6 @@ in
         })
       ];
     })
-    (mkIf config.my.build.isDevVM {
-      networking.interfaces.eth0.useDHCP = mkDefault true;
-      virtualisation = {
-        forwardPorts = flatten [
-          (optional config.services.openssh.openFirewall { from = "host"; host.port = 2222; guest.port = 22; })
-        ];
-      };
-    })
   ];
 
   meta.buildDocsInSandbox = false;

nixos/modules/deploy-rs.nix

@@ -1,11 +1,24 @@
 { lib, pkgs, config, systems, ... }:
 let
   inherit (builtins) head attrNames;
-  inherit (lib) mkMerge mkIf mkDefault optionalAttrs mapAttrs';
+  inherit (lib) mkMerge mkIf mkDefault optionalAttrs mapAttrs' optionalString;
   inherit (lib.my) mkOpt' mkBoolOpt';
 
   cfg = config.my.deploy;
 
+  # Based on https://github.com/serokell/deploy-rs/blob/master/flake.nix
+  nixosActivate = mode: base: (pkgs.deploy-rs.lib.activate.custom // { dryActivate = "$PROFILE/bin/switch-to-configuration dry-activate"; }) base.config.system.build.toplevel ''
+    # work around https://github.com/NixOS/nixpkgs/issues/73404
+    cd /tmp
+
+    $PROFILE/bin/switch-to-configuration ${mode}
+
+    # https://github.com/serokell/deploy-rs/issues/31
+    ${with base.config.boot.loader;
+    optionalString ((mode == "switch" || mode == "boot") && systemd-boot.enable)
+    "sed -i '/^default /d' ${efi.efiSysMountPoint}/loader/loader.conf"}
+  '';
+
   ctrProfiles = optionalAttrs cfg.generate.containers.enable (mapAttrs' (n: c:
   let
     ctrConfig = systems."${n}".configuration.config;
@@ -34,7 +47,10 @@ in
     inherit (lib.my.deploy-rs) node;
 
     generate = {
-      system.enable = mkBoolOpt' true "Whether to generate a deploy-rs profile for this system's config.";
+      system = {
+        enable = mkBoolOpt' true "Whether to generate a deploy-rs profile for this system's config.";
+        mode = mkOpt' str "switch" "switch-to-configuration mode.";
+      };
       containers.enable = mkBoolOpt' true "Whether to generate deploy-rs profiles for this system's containers.";
     };
   };
@@ -49,7 +65,7 @@ in
         profilesOrder = [ "system" ] ++ (attrNames ctrProfiles);
         profiles = {
           system = mkIf cfg.generate.system.enable {
-            path = pkgs.deploy-rs.lib.activate.nixos { inherit config; };
+            path = nixosActivate cfg.generate.system.mode { inherit config; };
 
             user = "root";
           };

nixos/modules/network.nix

@@ -0,0 +1,35 @@
+{ lib, config, ... }:
+let
+  inherit (lib) flatten optional mkIf mkDefault mkMerge;
+  inherit (lib.my) mkOpt' mkBoolOpt';
+
+  cfg = config.my.network;
+in
+{
+  options = with lib.types; {
+    my.network = {
+      ipv4 = mkOpt' str null "Internal network IPv4 address.";
+      ipv6 = mkOpt' str null "Internal network IPv6 address.";
+    };
+  };
+
+  config = mkMerge [
+    {
+      networking = {
+        domain = mkDefault "int.nul.ie";
+        useDHCP = false;
+        enableIPv6 = mkDefault true;
+        useNetworkd = mkDefault true;
+      };
+    }
+
+    (mkIf config.my.build.isDevVM {
+      networking.interfaces.eth0.useDHCP = mkDefault true;
+      virtualisation = {
+        forwardPorts = flatten [
+          (optional config.services.openssh.openFirewall { from = "host"; host.port = 2222; guest.port = 22; })
+        ];
+      };
+    })
+  ];
+}

nixos/modules/secrets.nix

@@ -22,15 +22,14 @@ in
         # agenix sets this as a default but adding any custom extras will _replace_ the list (different priority)
         identityPaths =
           mkIf config.services.openssh.enable
-          (map (e: e.path) (lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys));
+          (map
+            # Use the persit dir to grab the keys instead, otherwise they might not be ready. We can't really make
+            # agenix depend on impermanence, since users depends on agenix (to decrypt passwords) and impermanence
+            # depends on users
+            (e: let pDir = config.my.tmproot.persistence.dir; in if pDir != null then "${pDir}/${e.path}" else e.path)
+            (lib.filter (e: e.type == "rsa" || e.type == "ed25519") config.services.openssh.hostKeys));
       };
     }
-    (mkIf (config.age.secrets != { }) {
-        system.activationScripts.agenixMountSecrets.deps = mkIf (config.my.tmproot.persistence.dir != null) [
-          # The key used to decrypt is not going to exist!
-          "persist-files"
-        ];
-    })
     (mkIf config.my.build.isDevVM {
       age.identityPaths = [ cfg.vmKeyPath ];
     })

nixos/modules/vms.nix

@@ -1,6 +1,9 @@
 { lib, pkgs, config, ... }:
 let
-  inherit (lib) optional optionals optionalString flatten concatStringsSep mapAttrsToList mapAttrs' mkIf mkDefault;
+  inherit (builtins) filter any attrNames attrValues fetchGit;
+  inherit (lib)
+    unique optional optionals optionalString flatten concatStringsSep
+    concatMapStringsSep mapAttrsToList mapAttrs' mkIf mkDefault;
   inherit (lib.my) mkOpt' mkBoolOpt';
 
   flattenQEMUOpts = attrs:
@@ -38,6 +41,29 @@ let
           pass
     '';
 
+  # TODO: Upstream or something...
+  vfio-pci-bind = pkgs.stdenv.mkDerivation rec {
+    pname = "vfio-pci-bind";
+    version = "b41e4545b21de434fc51a34a9bf1d72e3ac66cc8";
+
+    src = fetchGit {
+      url = "https://github.com/andre-richter/vfio-pci-bind";
+      rev = version;
+    };
+
+    prePatch = ''
+      substituteInPlace vfio-pci-bind.sh \
+        --replace modprobe ${pkgs.kmod}/bin/modprobe
+      substituteInPlace 25-vfio-pci-bind.rules \
+        --replace vfio-pci-bind.sh "$out"/bin/vfio-pci-bind.sh
+    '';
+    installPhase = ''
+      mkdir -p "$out"/bin/ "$out"/lib/udev/rules.d
+      cp vfio-pci-bind.sh "$out"/bin/
+      cp 25-vfio-pci-bind.rules "$out"/lib/udev/rules.d/
+    '';
+  };
+
   cfg = config.my.vms;
 
   netOpts = with lib.types; { name, ... }: {
@@ -61,6 +87,12 @@ let
     };
   };
 
+  hostDevOpts = with lib.types; {
+    options = {
+      bindVFIO = mkBoolOpt' true "Whether to automatically bind the device to vfio-pci.";
+    };
+  };
+
   vmOpts = with lib.types; { name, ... }: {
     options = {
       qemuBin = mkOpt' path "${pkgs.qemu_kvm}/bin/qemu-kvm" "Path to QEMU executable.";
@@ -85,9 +117,16 @@ let
       spice.enable = mkBoolOpt' true "Whether to enable SPICE.";
       networks = mkOpt' (attrsOf (submodule netOpts)) { } "Networks to attach VM to.";
       drives = mkOpt' (attrsOf (submodule driveOpts)) { } "Drives to attach to VM.";
+      hostDevices = mkOpt' (attrsOf (submodule hostDevOpts)) { } "Host PCI devices to pass to the VM.";
     };
   };
 
+  allHostDevs =
+    flatten
+      (map
+        (i: mapAttrsToList (bdf: c: { inherit bdf; inherit (c) bindVFIO; }) i.hostDevices)
+        (attrValues cfg.instances));
+
   mkQemuCommand = n: i:
   let
     flags =
@@ -122,7 +161,8 @@ let
         "blockdev node-name=${dn}-backend,${c.backend}"
         "blockdev node-name=${dn}-format,${c.formatBackendProp}=${dn}-backend,${c.format}"
         ("device ${c.frontend},id=${dn},drive=${dn}-format" + (extraQEMUOpts c.frontendOpts))
-      ]) i.drives));
+      ]) i.drives)) ++
+      (map (bdf: "device vfio-pci,host=${bdf}") (attrNames i.hostDevices));
     args = map (v: "-${v}") flags;
   in
     concatStringsSep " " ([ i.qemuBin ] ++ args);
@@ -134,6 +174,30 @@ in
   };
 
   config = mkIf (cfg.instances != { }) {
+    assertions = [
+      {
+        assertion = let bdfs = map (d: d.bdf) allHostDevs; in (unique bdfs) == bdfs;
+        message = "VMs cannot share host devices!";
+      }
+    ];
+
+    services.udev = {
+      packages =
+        optionals
+          (any (d: d.bindVFIO) allHostDevs)
+          [
+            vfio-pci-bind
+            (pkgs.writeTextDir
+              "etc/udev/rules.d/20-vfio-tags.rules"
+              (concatMapStringsSep
+                "\n"
+                (d: ''ACTION=="add", SUBSYSTEM=="pci", KERNEL=="0000:${d.bdf}", TAG="vfio-pci-bind"'')
+                (filter (d: d.bindVFIO) allHostDevs)))
+          ];
+    };
+
+    my.tmproot.persistence.config.directories = [ "/var/lib/vms" ];
+
     # qemu-bridge-helper will fail otherwise
     environment.etc."qemu/bridge.conf".text = "allow all";
     systemd = {