dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/portcullis: Tune NICs, IOMMU and EEE

Browse Source 
Router-sized 4096-entry rings on every port with GRO kept across
forwarding, the IOMMU in passthrough mode, and EEE pinned off on the
I226-V ports as one trigger for their link-drop erratum. The ring and
GRO settings are .link files, so they land on a device add event rather
than at switch time.

Also document the tuning deliberately not done -- coalescing and PCIe
ASPM -- and what measuring the NICs' ESP offload found, since the
esp4_offload modules are software batching and easy to mistake for it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Jack O'Sullivan
2026-08-23 10:54:22 +01:00
parent 81b7ca9d05
commit bf411e03e2
3 changed files with 104 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/boxes/colony/portcullis/default.nix
+24 -7
View File
@@ -32,6 +32,14 @@ in
inherit (lib) mkMerge;
inherit (lib.my) mkVLAN networkdAssignment;
inherit (lib.my.c) networkd;
# Router-sized rings rather than the driver defaults, and GRO kept across
# forwarding so UDP-encapsulated traffic stays batched.
nicTuning = {
RxBufferSize = 4096;
TxBufferSize = 4096;
GenericReceiveOffloadUDPForwarding = true;
};
in
{
hardware = {
@@ -43,7 +51,9 @@ in
boot = {
kernelModules = [ "kvm-intel" ];
kernelParams = [ "intel_iommu=on" ];
# Passthrough mode keeps the IOMMU available without paying DMA translation
# on the forwarding path.
kernelParams = [ "intel_iommu=on" "iommu=pt" ];
initrd = {
availableKernelModules = [ "xhci_pci" "nvme" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
kernelModules = [ "dm-snapshot" ];
@@ -69,6 +79,13 @@ in
networking = { inherit domain; };
# The I226-V link-drop erratum is driven by EEE as well as ASPM. The driver already
# leaves EEE off, so this pins a default rather than changing one; systemd.link has
# no knob for it.
services.udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", DRIVERS=="igc", RUN+="${pkgs.ethtool}/bin/ethtool --set-eee $name eee off"
'';
environment.systemPackages = with pkgs; [
pciutils
usbutils
@@ -87,28 +104,28 @@ in
links = {
"10-et2g5-0" = {
matchConfig.PermanentMACAddress = "00:d0:b4:05:ed:48";
linkConfig.Name = "et2g5-0";
linkConfig = nicTuning // { Name = "et2g5-0"; };
};
"10-et2g5-1" = {
matchConfig.PermanentMACAddress = "00:d0:b4:05:ed:49";
linkConfig.Name = "et2g5-1";
linkConfig = nicTuning // { Name = "et2g5-1"; };
};
"10-et2g5-2" = {
matchConfig.PermanentMACAddress = "00:d0:b4:05:ed:4a";
linkConfig.Name = "et2g5-2";
linkConfig = nicTuning // { Name = "et2g5-2"; };
};
"10-et2g5-3" = {
matchConfig.PermanentMACAddress = "00:d0:b4:05:ed:4b";
linkConfig.Name = "et2g5-3";
linkConfig = nicTuning // { Name = "et2g5-3"; };
};
"11-et10g-0" = {
matchConfig.PermanentMACAddress = "60:be:b4:2e:9b:a2";
linkConfig.Name = "et10g-0";
linkConfig = nicTuning // { Name = "et10g-0"; };
};
"11-et10g-1" = {
matchConfig.PermanentMACAddress = "60:be:b4:2e:9b:a3";
linkConfig.Name = "et10g-1";
linkConfig = nicTuning // { Name = "et10g-1"; };
};
};