From 0a86a649a6b49a7e363d223a30f836172bd54889 Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Fri, 22 Dec 2023 01:34:28 +0000 Subject: [PATCH] nixos: Add SFH VM config --- nixos/boxes/home/palace/default.nix | 31 ++++++++- .../boxes/home/palace/vms/cellar/default.nix | 2 + nixos/boxes/home/palace/vms/default.nix | 66 ++++++++++++++----- nixos/modules/vms.nix | 17 +++-- 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/nixos/boxes/home/palace/default.nix b/nixos/boxes/home/palace/default.nix index 5885ae9..39aa1f2 100644 --- a/nixos/boxes/home/palace/default.nix +++ b/nixos/boxes/home/palace/default.nix @@ -94,7 +94,7 @@ in extraOptions = [ "-A /var/log/smartd/" "--interval=600" ]; }; udev.extraRules = '' - ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="2" + ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3" ''; }; @@ -110,7 +110,7 @@ in hwloc ]; - networking.domain = "h.${pubDomain}"; + networking = { inherit domain; }; systemd = { tmpfiles.rules = [ @@ -144,6 +144,13 @@ in netdevs = mkMerge [ (mkVLAN "lan-hi" vlans.hi) + (mkVLAN "lan-lo-phy" vlans.lo) + { + "25-lan-lo".netdevConfig = { + Name = "lan-lo"; + Kind = "bridge"; + }; + } ]; networks = { @@ -151,6 +158,7 @@ in (networkdAssignment "lan-core" assignments.core) { matchConfig.Name = "lan-core"; + vlan = [ "lan-lo-phy" ]; networkConfig.IPv6AcceptRA = mkForce false; } ]; @@ -173,9 +181,28 @@ in VirtualFunction=1 LinkState=yes MACAddress=52:54:00:8a:8a:f2 + + # sfh + [SR-IOV] + VirtualFunction=2 + VLANId=${toString vlans.hi} + LinkState=yes + MACAddress=52:54:00:ac:15:a9 ''; }; "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi; + + "50-lan-lo-phy" = { + matchConfig.Name = "lan-lo-phy"; + networkConfig = { + Bridge = "lan-lo"; + } // networkd.noL3; + }; + "60-lan-lo" = { + matchConfig.Name = "lan-lo"; + linkConfig.RequiredForOnline = "no"; + networkConfig = networkd.noL3; + }; }; }; }; diff --git a/nixos/boxes/home/palace/vms/cellar/default.nix b/nixos/boxes/home/palace/vms/cellar/default.nix index fee4f60..e1639e8 100644 --- a/nixos/boxes/home/palace/vms/cellar/default.nix +++ b/nixos/boxes/home/palace/vms/cellar/default.nix @@ -58,6 +58,8 @@ in }; }; + networking = { inherit domain; }; + environment.systemPackages = with pkgs; [ pciutils partclone diff --git a/nixos/boxes/home/palace/vms/default.nix b/nixos/boxes/home/palace/vms/default.nix index 1b83f4c..804fd43 100644 --- a/nixos/boxes/home/palace/vms/default.nix +++ b/nixos/boxes/home/palace/vms/default.nix @@ -55,21 +55,10 @@ }; }; - systemd.services = { - "vm@cellar" = { - serviceConfig = { - CPUAffinity = "numa"; - NUMAPolicy = "bind"; - NUMAMask = "1"; - }; - }; - "vm@river" = - let - vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device"; - in - { - requires = [ vtapUnit ]; - after = [ vtapUnit "vm@cellar.service" ]; + systemd.services = + let + awaitCellar = { + after = [ "vm@cellar.service" ]; bindsTo = [ "vm@cellar.service" ]; preStart = '' until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do @@ -77,6 +66,28 @@ done ''; }; + in + { + "vm@cellar" = { + serviceConfig = { + CPUAffinity = "numa"; + NUMAPolicy = "bind"; + NUMAMask = "1"; + }; + }; + + "vm@river" = + let + vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device"; + in + mkMerge [ + awaitCellar + { + requires = [ vtapUnit ]; + after = [ vtapUnit ]; + } + ]; + "vm@sfh" = awaitCellar; }; my = { @@ -128,7 +139,7 @@ threads = 2; }; memory = 4096; - cleanShutdown.timeout = 120; + cleanShutdown.timeout = 60; networks = { et1g0 = { ifname = "vm-et1g0"; @@ -150,6 +161,29 @@ }; }; }; + + sfh = { + uuid = "82ec149d-577c-421a-93e2-a9307c756cd8"; + cpu = "host,topoext"; + smp = { + cpus = 8; + threads = 2; + }; + memory = 32768; + cleanShutdown.timeout = 120; + networks.netboot = { + bridge = "lan-lo"; + waitOnline = "carrier"; + mac = "52:54:00:a5:7e:93"; + extraOptions.bootindex = 1; + }; + hostDevices = { + et100g0vf2 = { + index = 0; + hostBDF = "44:00.3"; + }; + }; + }; }; }; }; diff --git a/nixos/modules/vms.nix b/nixos/modules/vms.nix index 30038d5..ffcc571 100644 --- a/nixos/modules/vms.nix +++ b/nixos/modules/vms.nix @@ -116,7 +116,7 @@ let }); default = { }; }; - drives = mkOpt' (listOf (submodule driveOpts)) { } "Drives to attach to VM."; + drives = mkOpt' (listOf (submodule driveOpts)) [ ] "Drives to attach to VM."; hostDevices = mkOpt' (attrsOf (submodule hostDevOpts)) { } "Host PCI devices to pass to the VM."; }; }; @@ -126,8 +126,8 @@ let (map (i: mapAttrsToList (name: c: c // { inherit name; }) i.hostDevices) (attrValues cfg.instances)); - anyVfioDevs = any (d: d.bindVFIO) allHostDevs; - vfioHostDevs = filter (d: d.bindVFIO) allHostDevs; + anyVfioDevs = any (d: d.bindVFIO); + vfioHostDevs = filter (d: d.bindVFIO); mkQemuScript = n: i: let @@ -204,7 +204,7 @@ in services.udev = { packages = optionals - anyVfioDevs + (anyVfioDevs allHostDevs) [ pkgs.vfio-pci-bind (pkgs.writeTextDir @@ -212,7 +212,7 @@ in (concatMapStringsSep "\n" (d: ''ACTION=="add", SUBSYSTEM=="pci", KERNEL=="0000:${d.hostBDF}", TAG="vfio-pci-bind"'') - vfioHostDevs)) + (vfioHostDevs allHostDevs))) ]; }; @@ -261,12 +261,15 @@ in }; preStart = + let + hostDevs = attrValues i.hostDevices; + in '' if [ ! -e "$STATE_DIRECTORY"/ovmf_vars.bin ]; then cp "${cfg.ovmfPackage.fd}"/FV/OVMF_VARS.fd "$STATE_DIRECTORY"/ovmf_vars.bin fi - ${optionalString anyVfioDevs '' + ${optionalString (anyVfioDevs hostDevs) '' iommu_group() { g=/sys/bus/pci/devices/0000:$1/iommu_group until [ -e $g ]; do @@ -280,7 +283,7 @@ in done } - ${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") vfioHostDevs} + ${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") (vfioHostDevs hostDevs) } ''} ''; script = mkQemuScript n i;