dev/nixfiles
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

nixos/home: Separate router storage and uplinks

Browse Source 
Give `river` a dedicated RDMA VF and two RSTP trunk attachments.
Keep client services on the router VIPs and carry Digiweb VLANs
over both switch paths.

Set VF trust, multicast reception, bridge MAC and ARP behavior for
local switching. Wait for Kea interface readiness and apply RSTP
costs through `mstpd` despite port-registration and rename races.

Load `palace`'s RAID module in the initrd. Record RouterOS and
RouterBOOT upgrades and validate port, carrier and switch outages.

Document the deployed architecture, interface handoffs and recovery
procedures. Add physical socket maps, NIC capacities and cable
connections alongside the logical topology and failover checks.
This commit is contained in:
jackos1998
2026-11-15 16:23:06 +00:00
parent b02948d826
commit a04d11423a
26 changed files with 2037 additions and 243 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nixos/boxes/home/palace/default.nix
+24 -13
View File
@@ -57,6 +57,8 @@ in
kernelParams = [ "amd_iommu=on" ];
initrd = {
availableKernelModules = [ "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
# LVM autoactivation can discover the RAID LVs before stage-two module loading.
kernelModules = [ "dm-raid" ];
};
};
@@ -94,7 +96,7 @@ in
extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
};
udev.extraRules = ''
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="4"
ACTION=="add", SUBSYSTEM=="net", ENV{ID_NET_DRIVER}=="mlx5_core", ENV{ID_PATH}=="pci-0000:44:00.0", ATTR{device/sriov_numvfs}="5"
'';
};
@@ -119,24 +121,24 @@ in
network = {
links = {
"10-et1g0" = {
"10-river-jim-phy" = {
matchConfig = {
PermanentMACAddress = "e0:d5:5e:68:0c:6e";
Driver = "igb";
};
linkConfig.Name = "et1g0";
linkConfig.Name = "river-jim-phy";
};
"10-lan-core-phy" = {
"10-host-jim" = {
matchConfig.PermanentMACAddress = "e0:d5:5e:68:0c:70";
linkConfig.Name = "lan-core-phy";
linkConfig.Name = "host-jim";
};
"10-et100g" = {
"10-uplink-dave" = {
matchConfig = {
PermanentMACAddress = "24:8a:07:ac:59:c0";
Driver = "mlx5_core";
};
linkConfig = {
Name = "et100g";
Name = "uplink-dave";
MTUBytes = toString hiMTU;
};
};
@@ -169,8 +171,8 @@ in
}
];
"50-et100g" = {
matchConfig.Name = "et100g";
"50-uplink-dave" = {
matchConfig.Name = "uplink-dave";
vlan = [ "lan-hi" ];
networkConfig = networkd.noL3;
linkConfig.RequiredForOnline = "no";
@@ -182,9 +184,10 @@ in
LinkState=yes
MACAddress=52:54:00:cc:3e:70
# river
# river RDMA
[SR-IOV]
VirtualFunction=1
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:8a:8a:f2
@@ -195,12 +198,20 @@ in
LinkState=yes
MACAddress=52:54:00:ac:15:a9
# sfh bridge
# sfh containers
[SR-IOV]
VirtualFunction=3
VLANId=${toString vlans.hi}
LinkState=yes
MACAddress=52:54:00:90:34:95
# river LAN trunk
[SR-IOV]
VirtualFunction=4
LinkState=auto
MACAddress=52:54:00:8a:8a:f4
Trust=yes
MACSpoofCheck=no
'';
};
"60-lan-hi" = mkMerge [
@@ -208,8 +219,8 @@ in
{ networkConfig = vlanDns "hi"; }
];
"50-lan-core-phy" = {
matchConfig.Name = "lan-core-phy";
"50-host-jim" = {
matchConfig.Name = "host-jim";
networkConfig = {
Bridge = "lan-core";
} // networkd.noL3;
nixos/boxes/home/palace/vms/default.nix
+32 -17
View File
@@ -10,6 +10,7 @@
inherit (lib) mkMerge;
inherit (lib.my) vm;
inherit (lib.my.c) networkd;
inherit (lib.my.c.home) hiMTU;
installerDisk = {
name = "installer";
@@ -28,10 +29,11 @@
{
systemd.network = {
netdevs = {
"25-vm-et1g0" = {
"25-river-jim-tap" = {
netdevConfig = {
Name = "vm-et1g0";
Name = "river-jim-tap";
Kind = "macvtap";
MTUBytes = toString hiMTU;
};
# TODO: Upstream this missing section
extraConfig = ''
@@ -41,16 +43,24 @@
};
};
networks = {
"75-et1g0" = {
matchConfig.Name = "et1g0";
linkConfig.RequiredForOnline = "no";
"75-river-jim-phy" = {
matchConfig.Name = "river-jim-phy";
linkConfig = {
RequiredForOnline = "no";
MTUBytes = toString hiMTU;
};
networkConfig = {
MACVTAP = "vm-et1g0";
MACVTAP = "river-jim-tap";
} // networkd.noL3;
};
"75-vm-et1g0" = {
matchConfig.Name = "vm-et1g0";
linkConfig.RequiredForOnline = "no";
"75-river-jim-tap" = {
matchConfig.Name = "river-jim-tap";
linkConfig = {
RequiredForOnline = "no";
MTUBytes = toString hiMTU;
# Guest receive-filter changes are not synchronized to the host macvtap.
AllMulticast = true;
};
networkConfig = networkd.noL3;
};
};
@@ -79,7 +89,7 @@
"vm@river" =
let
vtapUnit = "sys-subsystem-net-devices-vm\\x2det1g0.device";
vtapUnit = "sys-subsystem-net-devices-river\\x2djim\\x2dtap.device";
in
mkMerge [
(awaitVM "cellar")
@@ -109,7 +119,7 @@
(vm.disk "cellar" "persist")
];
hostDevices = {
et100g0vf0 = {
cellar-hi = {
index = 0;
hostBDF = "44:00.1";
};
@@ -142,8 +152,8 @@
memory = 4096;
cleanShutdown.timeout = 60;
networks = {
et1g0 = {
ifname = "vm-et1g0";
lan-jim = {
ifname = "river-jim-tap";
bridge = null;
tapFD = 100;
# Real hardware MAC
@@ -156,10 +166,14 @@
(mkMerge [ (vm.disk "river" "esp") { frontendOpts.bootindex = 0; } ])
];
hostDevices = {
et100g0vf1 = {
river-rdma = {
index = 0;
hostBDF = "44:00.2";
};
river-dave = {
index = 1;
hostBDF = "44:00.5";
};
};
};
@@ -175,13 +189,14 @@
networks = {
netboot = {
bridge = "lan-lo";
ifname = "sfh-lo-tap";
waitOnline = "carrier";
mac = "52:54:00:a5:7e:93";
extraOptions.bootindex = 1;
};
core = {
bridge = "lan-core";
ifname = "vm-sfh-core";
ifname = "sfh-core-tap";
waitOnline = "carrier";
mac = "52:54:00:72:67:51";
};
@@ -190,11 +205,11 @@
(vm.lvmDisk' "hdds" "frigate")
];
hostDevices = {
et100g0vf2 = {
sfh-hi = {
index = 0;
hostBDF = "44:00.3";
};
et100g0vf3 = {
sfh-ctrs = {
index = 1;
hostBDF = "44:00.4";
};
nixos/boxes/home/palace/vms/river.nix
+90 -30
View File
@@ -1,3 +1,8 @@
{ lib, ... }:
let
inherit (lib.my) net;
inherit (lib.my.c.home) domain prefixes hiMTU;
in
{
imports = [ (import ../../routing-common 0) ];
@@ -6,6 +11,17 @@
nixpkgs = "mine";
home-manager = "mine";
assignments.rdma = {
name = "river-rdma";
inherit domain;
mtu = hiMTU;
ipv4 = {
address = net.cidr.host 3 prefixes.hi.v4;
mask = 32;
gateway = null;
};
};
configuration = { lib, modulesPath, pkgs, config, assignments, allAssignments, ... }:
let
inherit (builtins) elemAt;
@@ -14,11 +30,11 @@
inherit (lib.my.c) networkd;
inherit (lib.my.c.home) vlans domain prefixes roceBootModules routersPubV4;
# river reaches the ONT over its 100G `lan` uplink to the dave switch (which downlinks to
# brian, where the ONT lands). Digiweb delivers the ISP VLAN (pon-isp, 10) single-tagged at
# the ONT alongside the ONT's untagged management traffic. With a single ONT there's no VLAN
# collision, so the switches simply trunk the ISP's VLAN 10 straight through to river (PPPoE
# runs directly on it) and PVID the ONT's untagged management port onto wan-pon-ont (140).
# river reaches the ONT through its RSTP `lan` bridge, normally over `lan-dave`. Digiweb
# delivers the ISP VLAN (pon-isp, 10) single-tagged at the ONT alongside the ONT's untagged
# management traffic. With a single ONT there's no VLAN collision, so the switches simply
# trunk the ISP's VLAN 10 straight through to river (PPPoE runs directly on it) and PVID the
# ONT's untagged management port onto wan-pon-ont (140).
# river takes .100 in the ONT's /24 (matching stream's modem-mgmt .100) to reach its web
# UI at 192.168.100.1. (See docs/sites/home/switches.md for the switch side and the multi-ONT plan.)
ontV4 = net.cidr.host 100 prefixes.ont.v4;
@@ -35,24 +51,36 @@
boot = {
kernelModules = [ "kvm-amd" ];
kernelParams = [ "console=ttyS0,115200n8" ];
kernel.sysctl = {
# Keep storage and router addresses on their own interfaces on the shared L2 network.
"net.ipv4.conf.rdma-hi.arp_ignore" = 1;
"net.ipv4.conf.lan-hi.arp_ignore" = 1;
"net.ipv4.conf.rdma-hi.arp_announce" = 2;
"net.ipv4.conf.lan-hi.arp_announce" = 2;
};
initrd = {
availableKernelModules = [
"virtio_pci" "ahci" "sr_mod" "virtio_blk"
"8021q"
] ++ roceBootModules;
kernelModules = [ "dm-snapshot" ];
systemd = {
network = {
# Don't need to put the link config here, they're copied from main config
netdevs = mkVLAN "lan-hi" vlans.hi;
# NixOS copies the main system's .link units into the systemd initrd, so the
# rdma-hi rename is already available here. It does not copy .network units.
networks = {
"20-lan" = {
matchConfig.Name = "lan";
vlan = [ "lan-hi" ];
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
"30-lan-hi" = networkdAssignment "lan-hi" assignments.hi;
"20-rdma-hi" = mkMerge [
(networkdAssignment "rdma-hi" assignments.rdma)
{
networkConfig.IPv6AcceptRA = mkForce false;
routes = [
{
Destination = "${allAssignments.cellar.hi.ipv4.address}/32";
PreferredSource = assignments.rdma.ipv4.address;
Scope = "link";
}
];
}
];
};
};
};
@@ -86,9 +114,6 @@
};
fstrim.enable = true;
# TODO: re-enable once scheduling is tested
networkd-dispatcher.enable = mkForce false;
pppd = {
enable = true;
peers.digiweb = {
@@ -118,8 +143,8 @@
};
};
# PPPoE WAN (Digiweb): pppd owns the `wan` interface on top of wan-pon-isp (the switch's
# swap of the ISP's VLAN 10), and its ip-up/ip-down hooks toggle the shared
# PPPoE WAN (Digiweb): pppd owns the `wan` interface on top of wan-pon-isp (the ISP's
# VLAN 10, trunked untranslated), and its ip-up/ip-down hooks toggle the shared
# wan-online.target. Nothing else Wants the target, so it stays inactive until the link
# is actually up.
systemd.targets.wan-online.unitConfig.DefaultDependencies = false;
@@ -159,10 +184,18 @@
(mkVLAN "wan-pon-ont" vlans.wan-pon-ont)
# The ISP VLAN is trunked through untranslated, so this is the raw pon-isp (10)
(mkVLAN "wan-pon-isp" vlans.pon-isp)
{
# The adapter's local switching must recognize the router's destination MAC.
"25-lan".netdevConfig = {
Name = "lan";
Kind = "bridge";
MACAddress = "52:54:00:8a:8a:f4";
};
}
];
links = {
"10-wan-old" = {
"10-lan-jim" = {
matchConfig = {
# Matching against MAC address seems to break VLAN interfaces
# (since they share the same MAC address)
@@ -170,34 +203,61 @@
PermanentMACAddress = "e0:d5:5e:68:0c:6e";
};
linkConfig = {
Name = "wan-old";
Name = "lan-jim";
RxBufferSize = 4096;
TxBufferSize = 4096;
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
"10-lan" = {
"10-rdma-hi" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:8a:8a:f2";
};
linkConfig = {
Name = "lan";
Name = "rdma-hi";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
"10-lan-dave" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:8a:8a:f4";
};
linkConfig = {
Name = "lan-dave";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
};
networks = {
"30-rdma-hi" = mkMerge [
(networkdAssignment "rdma-hi" assignments.rdma)
{
networkConfig = {
IPv6AcceptRA = mkForce false;
# Keep the address and route established in the initrd: dropping either while
# stage 2 networkd takes over would interrupt the mounted NVMe-oF root.
KeepConfiguration = "static";
};
routes = [
{
Destination = "${allAssignments.cellar.hi.ipv4.address}/32";
PreferredSource = assignments.rdma.ipv4.address;
Scope = "link";
}
];
}
];
"55-lan" = {
# both WAN VLANs arrive single-tagged on the 100G uplink to dave: wan-pon-ont (140,
# the ONT's management, PVID-tagged at the brian edge) and the ISP's VLAN 10, trunked
# straight through
# Both WAN VLANs arrive single-tagged on the RSTP bridge: wan-pon-ont (140, the ONT's
# management, PVID-tagged at the brian edge) and the ISP's VLAN 10, trunked straight
# through.
vlan = [ "wan-pon-ont" "wan-pon-isp" ];
};
# So we don't drop the IP we use to connect to NVMe-oF!
"60-lan-hi".networkConfig.KeepConfiguration = "static";
# ONT management: the brian edge PVIDs the ONT's untagged port onto wan-pon-ont, so give
# ourselves an address in its /24 to reach the ONT's web UI at 192.168.100.1.
"70-wan-pon-ont" = {
nixos/boxes/home/palace/vms/sfh/default.nix
+15 -15
View File
@@ -105,29 +105,29 @@ in
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
"10-lan-hi-ctrs" = {
"10-ctrs-hi" = {
matchConfig = {
Driver = "mlx5_core";
PermanentMACAddress = "52:54:00:90:34:95";
};
linkConfig = {
Name = "lan-hi-ctrs";
Name = "ctrs-hi";
MTUBytes = toString lib.my.c.home.hiMTU;
};
};
"10-lan-core-ctrs" = {
"10-ctrs-core" = {
matchConfig = {
Driver = "virtio_net";
PermanentMACAddress = "52:54:00:72:67:51";
};
linkConfig.Name = "lan-core-ctrs";
linkConfig.Name = "ctrs-core";
};
"10-lan-lo-ctrs" = {
"10-ctrs-lo" = {
matchConfig = {
Driver = "virtio_net";
PermanentMACAddress = "52:54:00:a5:7e:93";
};
linkConfig.Name = "lan-lo-ctrs";
linkConfig.Name = "ctrs-lo";
};
};
@@ -139,18 +139,18 @@ in
networkConfig = vlanDns "hi" // { KeepConfiguration = "static"; };
}
];
"30-lan-hi-ctrs" = {
matchConfig.Name = "lan-hi-ctrs";
"30-ctrs-hi" = {
matchConfig.Name = "ctrs-hi";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
"30-lan-core-ctrs" = {
matchConfig.Name = "lan-core-ctrs";
"30-ctrs-core" = {
matchConfig.Name = "ctrs-core";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
"30-lan-lo-ctrs" = {
matchConfig.Name = "lan-lo-ctrs";
"30-ctrs-lo" = {
matchConfig.Name = "ctrs-lo";
linkConfig.RequiredForOnline = "no";
networkConfig = networkd.noL3;
};
@@ -160,12 +160,12 @@ in
systemd.nspawn = {
hass = {
networkConfig = {
MACVLAN = mkForce "lan-hi-ctrs:host0 lan-lo-ctrs:lan-lo";
MACVLAN = mkForce "ctrs-hi:host0 ctrs-lo:lan-lo";
};
};
unifi = {
networkConfig = {
MACVLAN = mkForce "lan-hi-ctrs:host0 lan-core-ctrs:lan-core";
MACVLAN = mkForce "ctrs-hi:host0 ctrs-core:lan-core";
};
};
};
@@ -217,7 +217,7 @@ in
mkMerge [
instances
(mapAttrs (n: i: {
networking.macVLAN = "lan-hi-ctrs";
networking.macVLAN = "ctrs-hi";
}) instances)
];
};
nixos/boxes/home/routing-common/default.nix
+20 -2
View File
@@ -3,7 +3,7 @@ let
inherit (builtins) elemAt;
inherit (lib.my) net mkVLAN;
inherit (lib.my.c) pubDomain;
inherit (lib.my.c.home) domain vlans prefixes vips routers routersPubV4;
inherit (lib.my.c.home) domain vlans prefixes vips routers routersPubV4 hiMTU;
name = elemAt routers index;
otherIndex = 1 - index;
@@ -106,7 +106,7 @@ in
./dns.nix
./radvd.nix
./kea.nix
];
] ++ [ ./mstpd.nix ];
# Per-box WAN-management specifics: the Virgin Media modem on stream lives on the `wan`
# interface itself, whereas river's ONT sits on its own interface. Declared as options the
@@ -203,6 +203,14 @@ in
netdevs = mkMerge [
{
"25-lan" = {
netdevConfig = {
Name = "lan";
Kind = "bridge";
MTUBytes = toString hiMTU;
};
bridgeConfig.STP = true;
};
"25-wan-ifb".netdevConfig = {
Name = "wan-ifb";
Kind = "ifb";
@@ -238,6 +246,16 @@ in
in
mkMerge [
{
"50-lan-dave" = {
matchConfig.Name = "lan-dave";
networkConfig.Bridge = "lan";
bridgeConfig.Cost = 10;
};
"50-lan-jim" = {
matchConfig.Name = "lan-jim";
networkConfig.Bridge = "lan";
bridgeConfig.Cost = 100;
};
"55-lan" = {
matchConfig.Name = "lan";
vlan = [ "lan-hi" "lan-lo" "lan-untrusted" "wan-tunnel" ];
nixos/boxes/home/routing-common/kea.nix
+12 -4
View File
@@ -26,10 +26,18 @@ in
};
systemd.services = {
kea-dhcp4-server.serviceConfig = {
# Sometimes interfaces might not be ready in time and Kea doesn't like that
Restart = "on-failure";
DynamicUser = mkForce false;
kea-dhcp4-server = {
unitConfig.StartLimitIntervalSec = 0;
preStart = ''
${config.systemd.package}/lib/systemd/systemd-networkd-wait-online \
--interface=lan-hi:routable --interface=lan-lo:routable \
--interface=lan-untrusted:routable --ipv4 --timeout=60
'';
serviceConfig = {
Restart = "on-failure";
RestartSec = 5;
DynamicUser = mkForce false;
};
};
kea-dhcp-ddns-server.serviceConfig.DynamicUser = mkForce false;
};
nixos/boxes/home/routing-common/mstpd.nix
+32 -2
View File
@@ -6,6 +6,34 @@ let
# Delete postInstall since it nukes the bridge-stp script we need
postInstall = "";
};
configureRSTP = pkgs.writeShellScript "configure-router-rstp" ''
set -eu
[ "$1" = lan ] || exit 0
event_port="''${2:-}"
${mstpd}/bin/mstpctl addbridge lan
${mstpd}/bin/mstpctl setforcevers lan rstp
for port in lan-dave lan-jim; do
if [ "$event_port" = "$port" ] || [ -e "/sys/class/net/lan/brif/$port" ]; then
case "$port" in
lan-dave) cost=10 ;;
lan-jim) cost=100 ;;
esac
# networkd can report configuration before mstpd has registered the port.
configured=false
for attempt in {1..50}; do
if ${mstpd}/bin/mstpctl setportpathcost lan "$port" "$cost"; then
configured=true
break
fi
${pkgs.coreutils}/bin/sleep 0.2
done
if [ "$configured" = false ]; then
echo "Failed to configure RSTP cost for $port" >&2
exit 1
fi
fi
done
'';
in
{
environment = {
@@ -17,6 +45,7 @@ in
"bridge-stp.conf".text = ''
MANAGE_MSTPD=n
MSTP_BRIDGES=lan
config_cmd=${configureRSTP}
'';
};
};
@@ -27,8 +56,9 @@ in
onState = [ "configured" ];
script = ''
#!${pkgs.runtimeShell}
if [ "$IFACE" = "lan" ]; then
${mstpd}/sbin/mstpctl setforcevers "$IFACE" rstp
# A queued event can retain the interface's name from before a rename.
if [ -d /sys/class/net/lan/bridge ]; then
${configureRSTP} lan "$IFACE"
fi
'';
};
nixos/boxes/home/stream.nix
-22
View File
@@ -18,8 +18,6 @@
modemV4 = net.cidr.host 100 prefixes.modem.v4;
in
{
imports = [ ./routing-common/mstpd.nix ];
config = {
boot = {
kernelModules = [ "kvm-intel" ];
@@ -109,15 +107,6 @@
};
systemd.network = {
netdevs = {
"25-lan" = {
netdevConfig = {
Name = "lan";
Kind = "bridge";
};
bridgeConfig.STP = true;
};
};
links = {
"10-wan" = {
matchConfig = {
@@ -169,17 +158,6 @@
};
};
networks = {
"50-lan-jim" = {
matchConfig.Name = "lan-jim";
networkConfig.Bridge = "lan";
bridgeConfig.Cost = 100;
};
"50-lan-dave" = {
matchConfig.Name = "lan-dave";
networkConfig.Bridge = "lan";
bridgeConfig.Cost = 10;
};
"50-wan-ifb" = {
matchConfig.Name = "wan-ifb";
networkConfig = networkd.noL3;