dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos: Add SFH VM config
All checks were successful
CI / Check, build and cache Nix flake (push) Successful in 21m29s
Details

Browse Source
This commit is contained in:
Jack O'Sullivan 2023-12-22 01:34:28 +00:00
parent 85189e74f8
commit 0a86a649a6
4 changed files with 91 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
nixos/boxes/home/palace/default.nix
View File

@ -94,7 +94,7 @@ in
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
};
udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="2"
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="3"
'';
};
@ -110,7 +110,7 @@ in
hwloc
];
networking.domain = "h.${pubDomain}";
networking = { inherit domain; };
systemd = {
tmpfiles.rules = [
@ -144,6 +144,13 @@ in
netdevs = mkMerge [
(mkVLAN "lan-hi" vlans.hi)
(mkVLAN "lan-lo-phy" vlans.lo)
{
"25-lan-lo".netdevConfig = {
Name = "lan-lo";
Kind = "bridge";
};
}
];
networks = {
@ -151,6 +158,7 @@ in
(networkdAssignment "lan-core" assignments.core)
{
matchConfig.Name = "lan-core";
vlan = [ "lan-lo-phy" ];
networkConfig.IPv6AcceptRA = mkForce false;
}
];
@ -173,9 +181,28 @@ in
VirtualFunction=1
LinkState=yes
MACAddress=52:54:00:8a:8a:f2
# sfh
[SR-IOV]
VirtualFunction=2
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:ac:15:a9
'';
};
"60-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
"50-lan-lo-phy" = {
matchConfig.Name = "lan-lo-phy";
networkConfig = {
Bridge = "lan-lo";
} // networkd.noL3;
};
"60-lan-lo" = {
matchConfig.Name = "lan-lo";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
};
};
};

2
nixos/boxes/home/palace/vms/cellar/default.nix
View File

@ -58,6 +58,8 @@ in
};
};
networking = { inherit domain; };
environment.systemPackages = with pkgs; [
pciutils
partclone

66
nixos/boxes/home/palace/vms/default.nix
View File

@ -55,21 +55,10 @@
};
};
systemd.services = {
"vm@cellar" = {
serviceConfig = {
CPUAffinity = "numa";
NUMAPolicy = "bind";
NUMAMask = "1";
};
};
"vm@river" =
let
vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
in
{
requires = [ vtapUnit ];
after = [ vtapUnit "vm@cellar.service" ];
systemd.services =
let
awaitCellar = {
after = [ "vm@cellar.service" ];
bindsTo = [ "vm@cellar.service" ];
preStart = ''
until ${pkgs.netcat}/bin/nc -w1 -z ${allAssignments.cellar.hi.ipv4.address} 22; do
@ -77,6 +66,28 @@
done
'';
};
in
{
"vm@cellar" = {
serviceConfig = {
CPUAffinity = "numa";
NUMAPolicy = "bind";
NUMAMask = "1";
};
};
"vm@river" =
let
vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
in
mkMerge [
awaitCellar
{
requires = [ vtapUnit ];
after = [ vtapUnit ];
}
];
"vm@sfh" = awaitCellar;
};
my = {
@ -128,7 +139,7 @@
threads = 2;
};
memory = 4096;
cleanShutdown.timeout = 120;
cleanShutdown.timeout = 60;
networks = {
et1g0 = {
ifname = "vm-et1g0";
@ -150,6 +161,29 @@
};
};
};
sfh = {
uuid = "82ec149d-577c-421a-93e2-a9307c756cd8";
cpu = "host,topoext";
smp = {
cpus = 8;
threads = 2;
};
memory = 32768;
cleanShutdown.timeout = 120;
networks.netboot = {
bridge = "lan-lo";
waitOnline = "carrier";
mac = "52:54:00:a5:7e:93";
extraOptions.bootindex = 1;
};
hostDevices = {
et100g0vf2 = {
index = 0;
hostBDF = "44:00.3";
};
};
};
};
};
};

17
nixos/modules/vms.nix
View File

@ -116,7 +116,7 @@ let
});
default = { };
};
drives = mkOpt' (listOf (submodule driveOpts)) { } "Drives to attach to VM.";
drives = mkOpt' (listOf (submodule driveOpts)) [ ] "Drives to attach to VM.";
hostDevices = mkOpt' (attrsOf (submodule hostDevOpts)) { } "Host PCI devices to pass to the VM.";
};
};
@ -126,8 +126,8 @@ let
(map
(i: mapAttrsToList (name: c: c // { inherit name; }) i.hostDevices)
(attrValues cfg.instances));
anyVfioDevs = any (d: d.bindVFIO) allHostDevs;
vfioHostDevs = filter (d: d.bindVFIO) allHostDevs;
anyVfioDevs = any (d: d.bindVFIO);
vfioHostDevs = filter (d: d.bindVFIO);
mkQemuScript = n: i:
let
@ -204,7 +204,7 @@ in
services.udev = {
packages =
optionals
anyVfioDevs
(anyVfioDevs allHostDevs)
[
pkgs.vfio-pci-bind
(pkgs.writeTextDir
@ -212,7 +212,7 @@ in
(concatMapStringsSep
"\n"
(d: ''ACTION=="add", SUBSYSTEM=="pci", KERNEL=="0000:${d.hostBDF}", TAG="vfio-pci-bind"'')
vfioHostDevs))
(vfioHostDevs allHostDevs)))
];
};
@ -261,12 +261,15 @@ in
};
preStart =
let
hostDevs = attrValues i.hostDevices;
in
''
if [ ! -e "$STATE_DIRECTORY"/ovmf_vars.bin ]; then
cp "${cfg.ovmfPackage.fd}"/FV/OVMF_VARS.fd "$STATE_DIRECTORY"/ovmf_vars.bin
fi
${optionalString anyVfioDevs ''
${optionalString (anyVfioDevs hostDevs) ''
iommu_group() {
g=/sys/bus/pci/devices/0000:$1/iommu_group
until [ -e $g ]; do
@ -280,7 +283,7 @@ in
done
}
${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") vfioHostDevs}
${concatMapStringsSep "\n" (d: "wait_vfio ${d.hostBDF}") (vfioHostDevs hostDevs) }
''}
'';
script = mkQemuScript n i;