From a92358f28b0e54faa3174e8770067d6a2b104cd8 Mon Sep 17 00:00:00 2001
From: Jack O'Sullivan <jackos1998@gmail.com>
Date: Thu, 30 Jun 2022 03:24:10 +0100
Subject: [PATCH] nixos: Add VM TAP FD, use real WAN (via macvtap) and fix deps

---
 nixos/boxes/colony/default.nix     |  8 +++++
 nixos/boxes/colony/vms/default.nix | 51 +++++++++++++++++++++++++++---
 nixos/modules/vms.nix              | 26 ++++++++++-----
 3 files changed, 73 insertions(+), 12 deletions(-)

diff --git a/nixos/boxes/colony/default.nix b/nixos/boxes/colony/default.nix
index 94fb177..2427510 100644
--- a/nixos/boxes/colony/default.nix
+++ b/nixos/boxes/colony/default.nix
@@ -128,6 +128,10 @@
                 Name = "base";
                 Kind = "bridge";
               };
+              "30-base-dummy".netdevConfig = {
+                Name = "base0";
+                Kind = "dummy";
+              };
 
               "25-vms".netdevConfig = {
                 Name = "vms";
@@ -141,6 +145,10 @@
 
             networks = {
               "80-base" = networkdAssignment "base" assignments.internal;
+              "80-base-dummy" = {
+                matchConfig.Name = "base0";
+                networkConfig.Bridge = "base";
+              };
               "80-base-ext" = {
                 matchConfig.Name = "base-ext";
                 networkConfig.Bridge = "base";
diff --git a/nixos/boxes/colony/vms/default.nix b/nixos/boxes/colony/vms/default.nix
index 56c0f2c..bcb1c43 100644
--- a/nixos/boxes/colony/vms/default.nix
+++ b/nixos/boxes/colony/vms/default.nix
@@ -39,6 +39,40 @@
     };
   in
   {
+    systemd = {
+      network = {
+        links = {
+          "10-wan10g" = {
+            matchConfig.Path = "pci-0000:2d:00.0";
+            linkConfig.Name = "wan10g";
+          };
+        };
+        netdevs = {
+          "25-vm-wan10g" = {
+            netdevConfig = {
+              Name = "vm-wan10g";
+              Kind = "macvtap";
+            };
+            # TODO: Upstream this missing section
+            extraConfig = ''
+              [MACVTAP]
+              Mode=passthru
+            '';
+          };
+        };
+        networks = {
+          "75-wan10g" = {
+            matchConfig.Name = "wan10g";
+            networkConfig.MACVTAP = "vm-wan10g";
+          };
+          "75-vm-wan10g" = {
+            matchConfig.Name = "vm-wan10g";
+            linkConfig.RequiredForOnline = "carrier";
+          };
+        };
+      };
+    };
+
     my = {
       vms = {
         instances = {
@@ -50,12 +84,21 @@
               threads = 2;
             };
             memory = 3072;
-            networks.base = {
-              waitOnline = "no-carrier";
-              mac = "52:54:00:15:1a:53";
+            networks = {
+              wan = {
+                ifname = "vm-wan10g";
+                bridge = null;
+                tapFD = 100;
+                # Real hardware MAC
+                mac = "00:02:c9:56:24:6e";
+              };
+              base = {
+                waitOnline = "carrier";
+                mac = "52:54:00:15:1a:53";
+              };
             };
             drives = [ ] ++ (optionals (!config.my.build.isDevVM) [
-              (vmLVM "estuary" "esp")
+              (mkMerge [ (vmLVM "estuary" "esp") { frontendOpts.bootindex = 0; } ])
               (vmLVM "estuary" "nix")
               (vmLVM "estuary" "persist")
             ]);
diff --git a/nixos/modules/vms.nix b/nixos/modules/vms.nix
index 96cfadd..e00feb6 100644
--- a/nixos/modules/vms.nix
+++ b/nixos/modules/vms.nix
@@ -69,10 +69,11 @@ let
 
   netOpts = with lib.types; { name, iName, ... }: {
     options = {
-      ifname = mkOpt' str "vm-${iName}" "TAP device to create ";
+      ifname = mkOpt' str "vm-${iName}" "TAP device to create / use.";
       bridge = mkOpt' (nullOr str) name "Network bridge to connect to (null to not attach to bridge).";
       waitOnline = mkOpt' (either bool str) true
-        "Whether to wait for networkd to consider the bridge online. Pass a string to set the OPERSTATE will wait for.";
+        "Whether to wait for networkd to consider the bridge / existing TAP device online. Pass a string to set the OPERSTATE will wait for.";
+      tapFD = mkOpt' (nullOr ints.unsigned) null "FD to use to pass existing TAP device.";
 
       model = mkOpt' str "virtio-net" "Device type for network interface.";
       mac = mkOpt' str null "Guest MAC address.";
@@ -171,7 +172,10 @@ let
       ]) ++
       (optional i.spice.enable "spice unix=on,addr=/run/vms/${n}/spice.sock,disable-ticketing=on") ++
       (flatten (mapAttrsToList (nn: c: [
-        "netdev tap,id=${nn},ifname=${c.ifname},script=no,downscript=no"
+        ("netdev tap,id=${nn}," + (
+          if (c.tapFD != null)
+            then "fd=${toString c.tapFD} ${toString c.tapFD}<>/dev/tap$(cat /sys/class/net/${c.ifname}/ifindex)"
+            else "ifname=${c.ifname},script=no,downscript=no"))
         ("device ${c.model},netdev=${nn},mac=${c.mac}" + (extraQEMUOpts c.extraOptions))
       ]) i.networks)) ++
       (flatten (map (d: [
@@ -244,16 +248,22 @@ in
               };
             })
             (filterAttrs (_: net: net.bridge != null) i.networks);
-        services."vm@${n}" = {
-          description = "Virtual machine '${n}'";
-          requires =
+        services."vm@${n}" =
+        let
+          dependencies =
             map
               (net:
               let
-                arg = if net.waitOnline == true then net.bridge else "${net.bridge}:${net.waitOnline}";
+                ifname = if net.bridge != null then net.bridge else net.ifname;
+                arg = if net.waitOnline == true then ifname else "${ifname}:${net.waitOnline}";
               in
               "systemd-networkd-wait-online@${arg}.service")
-              (filter (net: net.bridge != null && net.waitOnline != false) (attrValues i.networks));
+              (filter (net: (net.bridge != null || net.tapFD != null) && net.waitOnline != false) (attrValues i.networks));
+        in
+        {
+          description = "Virtual machine '${n}'";
+          requires = dependencies;
+          after = dependencies;
           serviceConfig = {
             ExecStop = mkIf i.cleanShutdown.enabled "${doCleanShutdown} /run/vms/${n}/monitor-qmp.sock";
             TimeoutStopSec = mkIf i.cleanShutdown.enabled i.cleanShutdown.timeout;