jackos1998
a04d11423a
Give `river` a dedicated RDMA VF and two RSTP trunk attachments. Keep client services on the router VIPs and carry Digiweb VLANs over both switch paths. Set VF trust, multicast reception, bridge MAC and ARP behavior for local switching. Wait for Kea interface readiness and apply RSTP costs through `mstpd` despite port-registration and rename races. Load `palace`'s RAID module in the initrd. Record RouterOS and RouterBOOT upgrades and validate port, carrier and switch outages. Document the deployed architecture, interface handoffs and recovery procedures. Add physical socket maps, NIC capacities and cable connections alongside the logical topology and failover checks.
408 lines
14 KiB
Nix
408 lines
14 KiB
Nix
index: { lib, allAssignments, ... }:
|
|
let
|
|
inherit (builtins) elemAt;
|
|
inherit (lib.my) net mkVLAN;
|
|
inherit (lib.my.c) pubDomain;
|
|
inherit (lib.my.c.home) domain vlans prefixes vips routers routersPubV4 hiMTU;
|
|
|
|
name = elemAt routers index;
|
|
otherIndex = 1 - index;
|
|
in
|
|
{
|
|
nixos.systems."${name}" = {
|
|
assignments = {
|
|
core = {
|
|
name = "${name}-core";
|
|
inherit domain;
|
|
mtu = 1500;
|
|
ipv4 = {
|
|
address = net.cidr.host (index + 1) prefixes.core.v4;
|
|
gateway = null;
|
|
};
|
|
};
|
|
hi = {
|
|
name = "${name}-hi";
|
|
inherit domain;
|
|
mtu = 9000;
|
|
ipv4 = {
|
|
address = net.cidr.host (index + 1) prefixes.hi.v4;
|
|
mask = 22;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host (index + 1) prefixes.hi.v6;
|
|
};
|
|
lo = {
|
|
name = "${name}-lo";
|
|
inherit domain;
|
|
mtu = 1500;
|
|
ipv4 = {
|
|
address = net.cidr.host (index + 1) prefixes.lo.v4;
|
|
mask = 21;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host (index + 1) prefixes.lo.v6;
|
|
};
|
|
untrusted = {
|
|
name = "${name}-ut";
|
|
inherit domain;
|
|
mtu = 1500;
|
|
ipv4 = {
|
|
address = net.cidr.host (index + 1) prefixes.untrusted.v4;
|
|
mask = 24;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host (index + 1) prefixes.untrusted.v6;
|
|
};
|
|
as211024 = {
|
|
ipv4 = {
|
|
address = net.cidr.host (index + 2) prefixes.as211024.v4;
|
|
gateway = null;
|
|
};
|
|
ipv6 = {
|
|
address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
|
|
gateway = net.cidr.host ((2*65536*65536*65536) + 1) prefixes.as211024.v6;
|
|
};
|
|
};
|
|
};
|
|
|
|
extraAssignments = {
|
|
router-hi.hi = {
|
|
name = "router-hi";
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = vips.hi.v4;
|
|
mask = 22;
|
|
};
|
|
ipv6.address = vips.hi.v6;
|
|
};
|
|
router-lo.lo = {
|
|
name = "router-lo";
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = vips.lo.v4;
|
|
mask = 21;
|
|
};
|
|
ipv6.address = vips.lo.v6;
|
|
};
|
|
router-ut.untrusted = {
|
|
name = "router-ut";
|
|
inherit domain;
|
|
ipv4.address = vips.untrusted.v4;
|
|
ipv6.address = vips.untrusted.v6;
|
|
};
|
|
};
|
|
|
|
configuration = { lib, pkgs, config, assignments, allAssignments, ... }:
|
|
let
|
|
inherit (lib) mkIf mkMerge mkForce optionalString concatStringsSep;
|
|
inherit (lib.my) mkOpt' networkdAssignment;
|
|
inherit (lib.my.c) networkd;
|
|
|
|
cfg = config.my.homeRouter;
|
|
in
|
|
{
|
|
imports = map (m: import m index) [
|
|
./keepalived.nix
|
|
./dns.nix
|
|
./radvd.nix
|
|
./kea.nix
|
|
] ++ [ ./mstpd.nix ];
|
|
|
|
# Per-box WAN-management specifics: the Virgin Media modem on stream lives on the `wan`
|
|
# interface itself, whereas river's ONT sits on its own interface. Declared as options the
|
|
# box sets so routing-common itself carries no modem/ONT knowledge.
|
|
options.my.homeRouter = with lib.types; {
|
|
dns.wanSkipBroadcasts = mkOpt' (listOf str) [ ] ''
|
|
Broadcast addresses to exclude when auto-selecting the router's own `wan` A record,
|
|
for extra static subnets that share the `wan` interface.
|
|
'';
|
|
firewall.untrustedRejectV4 = mkOpt' (listOf str) [ ] ''
|
|
IPv4 prefixes untrusted clients must be explicitly rejected from reaching. Only needed
|
|
for subnets sharing the `wan` interface, since `wan` egress is otherwise accepted.
|
|
'';
|
|
};
|
|
|
|
config = {
|
|
environment = {
|
|
systemPackages = with pkgs; [
|
|
ethtool
|
|
conntrack-tools
|
|
];
|
|
};
|
|
|
|
services = {
|
|
resolved.settings.Resolve = {
|
|
LLMNR = false;
|
|
MulticastDNS = false;
|
|
};
|
|
|
|
iperf3 = {
|
|
enable = true;
|
|
openFirewall = true;
|
|
};
|
|
|
|
networkd-dispatcher = {
|
|
enable = true;
|
|
rules = {
|
|
# tc filter hasn't been networkd-ified yet
|
|
setup-wan-mirror = {
|
|
onState = [ "configured" ];
|
|
script = ''
|
|
#!${pkgs.runtimeShell}
|
|
if [ "$IFACE" = "wan-ifb" ]; then
|
|
${pkgs.iproute2}/bin/tc filter add dev wan parent ffff: matchall action mirred egress redirect dev "$IFACE"
|
|
fi
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
nginx.enable = true;
|
|
};
|
|
|
|
networking = { inherit domain; };
|
|
|
|
# Uniform "WAN is up" gate. Consumers attach to this target (via wantedBy +
|
|
# partOf) rather than depending on it, so it is never pulled in / prematurely
|
|
# activated. Each box wires up how the target actually gets reached: stream
|
|
# gates it on networkd's wait-online, river drives it from the pppd hooks.
|
|
systemd.targets.wan-online.description = "WAN is online";
|
|
|
|
systemd.services = {
|
|
ipsec = {
|
|
after = [ "wan-online.target" ];
|
|
# strongswan/libreswan force wantedBy=multi-user.target; drop it so the
|
|
# target is a true gate rather than mere ordering. This matters most on
|
|
# river, where the target is hook-driven and not in the boot transaction,
|
|
# so plain ordering wouldn't hold ipsec back at all. partOf re-loads ipsec
|
|
# (re-orienting its connections) whenever the WAN drops and returns.
|
|
wantedBy = mkForce [ "wan-online.target" ];
|
|
partOf = [ "wan-online.target" ];
|
|
};
|
|
|
|
ipv6-clear-default-route = {
|
|
description = "Clear IPv6 RA default route";
|
|
after = [ "wan-online.target" ];
|
|
wantedBy = [ "wan-online.target" ];
|
|
partOf = [ "wan-online.target" ];
|
|
script = ''
|
|
# Seems like we can sometimes pick up a default route somehow...
|
|
${pkgs.iproute2}/bin/ip -6 route del default via fe80::1 || true
|
|
'';
|
|
serviceConfig.Type = "oneshot";
|
|
};
|
|
};
|
|
|
|
systemd.network = {
|
|
wait-online.enable = false;
|
|
config = {
|
|
networkConfig = {
|
|
ManageForeignRoutes = false;
|
|
};
|
|
};
|
|
|
|
netdevs = mkMerge [
|
|
{
|
|
"25-lan" = {
|
|
netdevConfig = {
|
|
Name = "lan";
|
|
Kind = "bridge";
|
|
MTUBytes = toString hiMTU;
|
|
};
|
|
bridgeConfig.STP = true;
|
|
};
|
|
"25-wan-ifb".netdevConfig = {
|
|
Name = "wan-ifb";
|
|
Kind = "ifb";
|
|
};
|
|
"30-lan-core".netdevConfig = {
|
|
Name = "lan-core";
|
|
Kind = "macvlan";
|
|
MTUBytes = "1500";
|
|
};
|
|
}
|
|
|
|
(mkVLAN "lan-hi" vlans.hi)
|
|
(mkVLAN "lan-lo" vlans.lo)
|
|
(mkVLAN "lan-untrusted" vlans.untrusted)
|
|
];
|
|
|
|
networks =
|
|
let
|
|
mkVLANConfig = name:
|
|
let
|
|
iface = "lan-${name}";
|
|
in
|
|
{
|
|
"60-${iface}" = mkMerge [
|
|
(networkdAssignment iface assignments."${name}")
|
|
{
|
|
dns = [ "127.0.0.1" "::1" ];
|
|
domains = [ config.networking.domain ];
|
|
networkConfig.IPv6AcceptRA = mkForce false;
|
|
}
|
|
];
|
|
};
|
|
in
|
|
mkMerge [
|
|
{
|
|
"50-lan-dave" = {
|
|
matchConfig.Name = "lan-dave";
|
|
networkConfig.Bridge = "lan";
|
|
bridgeConfig.Cost = 10;
|
|
};
|
|
"50-lan-jim" = {
|
|
matchConfig.Name = "lan-jim";
|
|
networkConfig.Bridge = "lan";
|
|
bridgeConfig.Cost = 100;
|
|
};
|
|
"55-lan" = {
|
|
matchConfig.Name = "lan";
|
|
vlan = [ "lan-hi" "lan-lo" "lan-untrusted" "wan-tunnel" ];
|
|
macvlan = [ "lan-core" ];
|
|
networkConfig = networkd.noL3;
|
|
};
|
|
"60-lan-core" = mkMerge [
|
|
(networkdAssignment "lan-core" assignments.core)
|
|
{
|
|
matchConfig.Name = "lan-core";
|
|
networkConfig.IPv6AcceptRA = mkForce false;
|
|
}
|
|
];
|
|
|
|
"90-l2mesh-as211024" = mkMerge [
|
|
(networkdAssignment "as211024" assignments.as211024)
|
|
{
|
|
matchConfig.Name = "as211024";
|
|
networkConfig.IPv6AcceptRA = mkForce false;
|
|
routes = [
|
|
{
|
|
Destination = lib.my.c.colony.prefixes.all.v4;
|
|
Gateway = allAssignments.estuary.as211024.ipv4.address;
|
|
}
|
|
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v4;
|
|
Gateway = allAssignments.britway.as211024.ipv4.address;
|
|
}
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v6;
|
|
Gateway = allAssignments.britway.as211024.ipv6.address;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
}
|
|
|
|
(mkVLANConfig "hi")
|
|
(mkVLANConfig "lo")
|
|
(mkVLANConfig "untrusted")
|
|
|
|
{
|
|
"60-lan-hi" = {
|
|
routes = [
|
|
{
|
|
Destination = elemAt routersPubV4 otherIndex;
|
|
Gateway = net.cidr.host (otherIndex + 1) prefixes.hi.v4;
|
|
}
|
|
];
|
|
};
|
|
}
|
|
];
|
|
};
|
|
|
|
my = {
|
|
secrets = {
|
|
files = {
|
|
"l2mesh/as211024.key" = {};
|
|
};
|
|
};
|
|
|
|
vpns = {
|
|
l2.pskFiles = {
|
|
as211024 = config.age.secrets."l2mesh/as211024.key".path;
|
|
};
|
|
};
|
|
firewall = {
|
|
trustedInterfaces = [ "lan-hi" "lan-lo" ];
|
|
udp.allowed = [ 5353 ];
|
|
tcp.allowed = [ 5353 ];
|
|
nat = {
|
|
enable = true;
|
|
externalInterface = "wan";
|
|
};
|
|
extraRules =
|
|
let
|
|
aa = allAssignments;
|
|
in
|
|
''
|
|
table inet filter {
|
|
chain input {
|
|
${lib.my.c.as211024.nftTrust}
|
|
iifname base meta l4proto { udp, tcp } th dport domain accept
|
|
iifname lan-core meta l4proto vrrp accept
|
|
}
|
|
|
|
chain routing-tcp {
|
|
ip daddr {
|
|
${aa.castle.hi.ipv4.address},
|
|
${aa.cellar.hi.ipv4.address},
|
|
${aa.palace.hi.ipv4.address}
|
|
} tcp dport ssh accept
|
|
ip6 daddr {
|
|
${aa.castle.hi.ipv6.address},
|
|
${aa.cellar.hi.ipv6.address},
|
|
${aa.palace.hi.ipv6.address}
|
|
} tcp dport ssh accept
|
|
|
|
return
|
|
}
|
|
chain routing-udp {
|
|
return
|
|
}
|
|
chain filter-routing {
|
|
tcp flags & (fin|syn|rst|ack) == syn ct state new jump routing-tcp
|
|
meta l4proto udp ct state new jump routing-udp
|
|
return
|
|
}
|
|
chain filter-untrusted {
|
|
${optionalString (cfg.firewall.untrustedRejectV4 != [ ]) "ip daddr { ${concatStringsSep ", " cfg.firewall.untrustedRejectV4} } reject"}
|
|
oifname wan accept
|
|
return
|
|
}
|
|
|
|
chain forward-early {
|
|
type filter hook forward priority -1; policy accept;
|
|
|
|
# MSS clamping to workaround IPv6 PMTUD being broken...
|
|
tcp flags syn tcp option maxseg size set rt mtu counter
|
|
|
|
# More Disney+ discrimination...
|
|
# TODO: This prefix could change (random AWS block)
|
|
ip6 daddr 2600:9000:2245::/48 drop
|
|
}
|
|
chain forward {
|
|
${lib.my.c.as211024.nftTrust}
|
|
iifname lan-untrusted jump filter-untrusted
|
|
iifname { wan, as211024, lan-untrusted } oifname { lan-hi, lan-lo } jump filter-routing
|
|
oifname as211024 accept
|
|
}
|
|
chain output { }
|
|
}
|
|
table inet nat {
|
|
chain prerouting {
|
|
ip daddr ${elemAt routersPubV4 index} meta l4proto { udp, tcp } th dport domain redirect to :5353
|
|
ip6 daddr ${assignments.as211024.ipv6.address} meta l4proto { udp, tcp } th dport domain redirect to :5353
|
|
}
|
|
chain postrouting {
|
|
oifname wan masquerade
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|