From 70f49c84386b1e5fa27d021bee9a556c7cbd6b82 Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Sat, 16 Dec 2023 15:59:33 +0000 Subject: [PATCH] nixos/home/routing-common: Working IPv6 router --- lib/constants.nix | 3 + lib/default.nix | 5 +- nixos/boxes/castle/default.nix | 8 +++ nixos/boxes/home/palace/default.nix | 22 +++---- .../boxes/home/palace/vms/cellar/default.nix | 22 +++---- nixos/boxes/home/routing-common/default.nix | 65 ++++++++++++------- nixos/boxes/home/routing-common/dns.nix | 6 +- .../boxes/home/routing-common/keepalived.nix | 20 ++++-- nixos/boxes/home/routing-common/radvd.nix | 28 ++++++++ nixos/default.nix | 1 + nixos/modules/spdk.nix | 4 +- 11 files changed, 123 insertions(+), 61 deletions(-) create mode 100644 nixos/boxes/home/routing-common/radvd.nix diff --git a/lib/constants.nix b/lib/constants.nix index 54d2b10..ee387af 100644 --- a/lib/constants.nix +++ b/lib/constants.nix @@ -180,14 +180,17 @@ rec { hi = { v4 = subnet 4 1 all.v4; v6 = subnet 4 1 all.v6; + mtu = hiMTU; }; lo = { v4 = subnet 3 1 all.v4; v6 = subnet 4 2 all.v6; + mtu = 1500; }; untrusted = { v4 = subnet 6 16 all.v4; v6 = subnet 4 3 all.v6; + mtu = 1500; }; inherit (colony.prefixes) as211024; }; diff --git a/lib/default.nix b/lib/default.nix index 4a8e7bd..ad177a4 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -4,7 +4,7 @@ let inherit (lib) genAttrs mapAttrsToList filterAttrsRecursive nameValuePair types mkOption mkOverride mkForce mkIf mergeEqualOption optional - showWarnings concatStringsSep flatten unique; + showWarnings concatStringsSep flatten unique optionalAttrs; inherit (lib.flake) defaultSystems; in rec { @@ -152,6 +152,9 @@ rec { LLDP = true; EmitLLDP = "customer-bridge"; }; + linkConfig = optionalAttrs (a.mtu != null) { + MTUBytes = toString a.mtu; + }; ipv6AcceptRAConfig = { Token = mkIf (a.ipv6.iid != null) "static:${a.ipv6.iid}"; UseDNS = true; diff --git a/nixos/boxes/castle/default.nix b/nixos/boxes/castle/default.nix index 70f3d06..0313c82 100644 --- a/nixos/boxes/castle/default.nix +++ b/nixos/boxes/castle/default.nix @@ -17,6 +17,10 @@ in mask = 22; gateway = null; }; + ipv6 = { + iid = "::3:1"; + address = net.cidr.host (65536*3+1) prefixes.hi.v6; + }; }; lo = { inherit domain; @@ -25,6 +29,10 @@ in mask = 21; gateway = null; }; + ipv6 = { + iid = "::3:1"; + address = net.cidr.host (65536*3+1) prefixes.lo.v6; + }; }; }; diff --git a/nixos/boxes/home/palace/default.nix b/nixos/boxes/home/palace/default.nix index cfab3c7..5885ae9 100644 --- a/nixos/boxes/home/palace/default.nix +++ b/nixos/boxes/home/palace/default.nix @@ -2,7 +2,7 @@ let inherit (lib.my) net mkVLAN; inherit (lib.my.c) pubDomain; - inherit (lib.my.c.home) domain vlans prefixes vips; + inherit (lib.my.c.home) domain vlans prefixes vips hiMTU; in { imports = [ ./vms ]; @@ -15,15 +15,21 @@ in assignments = { hi = { inherit domain; + mtu = hiMTU; ipv4 = { address = net.cidr.host 22 prefixes.hi.v4; mask = 22; gateway = vips.hi.v4; }; + ipv6 = { + iid = "::2:1"; + address = net.cidr.host (65536*2+1) prefixes.hi.v6; + }; }; core = { inherit domain; name = "palace-core"; + mtu = 1500; ipv4 = { address = net.cidr.host 20 prefixes.core.v4; gateway = null; @@ -131,7 +137,7 @@ in }; linkConfig = { Name = "et100g"; - MTUBytes = "9000"; + MTUBytes = toString hiMTU; }; }; }; @@ -169,17 +175,7 @@ in MACAddress=52:54:00:8a:8a:f2 ''; }; - "60-lan-hi" = mkMerge [ - (networkdAssignment "lan-hi" assignments.hi) - { - matchConfig.Name = "lan-hi"; - linkConfig.MTUBytes = "9000"; - networkConfig.DNS = [ - (allAssignments.stream.hi.ipv4.address) - # (allAssignments.river.hi.ipv4.address) - ]; - } - ]; + "60-lan-hi" = networkdAssignment "lan-hi" assignments.hi; }; }; }; diff --git a/nixos/boxes/home/palace/vms/cellar/default.nix b/nixos/boxes/home/palace/vms/cellar/default.nix index 5396c62..fee4f60 100644 --- a/nixos/boxes/home/palace/vms/cellar/default.nix +++ b/nixos/boxes/home/palace/vms/cellar/default.nix @@ -2,7 +2,7 @@ let inherit (lib.my) net; inherit (lib.my.c) pubDomain; - inherit (lib.my.c.home) domain prefixes vips; + inherit (lib.my.c.home) domain prefixes vips hiMTU; in { nixos.systems.cellar = { @@ -12,11 +12,16 @@ in assignments = { hi = { inherit domain; + mtu = hiMTU; ipv4 = { address = net.cidr.host 80 prefixes.hi.v4; mask = 22; gateway = vips.hi.v4; }; + ipv6 = { + iid = "::4:1"; + address = net.cidr.host (65536*4+1) prefixes.hi.v6; + }; }; }; @@ -66,23 +71,12 @@ in links = { "10-lan-hi" = { matchConfig.PermanentMACAddress = "52:54:00:cc:3e:70"; - linkConfig = { - Name = "lan-hi"; - MTUBytes = "9000"; - }; + linkConfig.Name = "lan-hi"; }; }; networks = { - "80-vms" = mkMerge [ - (networkdAssignment "lan-hi" assignments.hi) - { - networkConfig.DNS = [ - (allAssignments.stream.hi.ipv4.address) - (allAssignments.river.hi.ipv4.address) - ]; - } - ]; + "80-lan-hi" = networkdAssignment "lan-hi" assignments.hi; }; }; diff --git a/nixos/boxes/home/routing-common/default.nix b/nixos/boxes/home/routing-common/default.nix index 1c9e9c6..33ae252 100644 --- a/nixos/boxes/home/routing-common/default.nix +++ b/nixos/boxes/home/routing-common/default.nix @@ -3,7 +3,7 @@ let inherit (builtins) elemAt; inherit (lib.my) net mkVLAN; inherit (lib.my.c) pubDomain; - inherit (lib.my.c.home) domain vlans prefixes routers routersPubV4; + inherit (lib.my.c.home) domain vlans prefixes vips routers routersPubV4; name = elemAt routers index; otherIndex = 1 - index; @@ -20,14 +20,16 @@ in core = { name = "${name}-core"; inherit domain; + mtu = 1500; ipv4 = { address = net.cidr.host (index + 1) prefixes.core.v4; gateway = null; }; }; hi = { - inherit domain; name = "${name}-hi"; + inherit domain; + mtu = 9000; ipv4 = { address = net.cidr.host (index + 1) prefixes.hi.v4; mask = 22; @@ -38,6 +40,7 @@ in lo = { name = "${name}-lo"; inherit domain; + mtu = 1500; ipv4 = { address = net.cidr.host (index + 1) prefixes.lo.v4; mask = 21; @@ -48,6 +51,7 @@ in untrusted = { name = "${name}-ut"; inherit domain; + mtu = 1500; ipv4 = { address = net.cidr.host (index + 1) prefixes.untrusted.v4; mask = 24; @@ -67,6 +71,33 @@ in }; }; + extraAssignments = { + router-hi.hi = { + name = "router-hi"; + inherit domain; + ipv4 = { + address = vips.hi.v4; + mask = 22; + }; + ipv6.address = vips.hi.v6; + }; + router-lo.lo = { + name = "router-lo"; + inherit domain; + ipv4 = { + address = vips.lo.v4; + mask = 21; + }; + ipv6.address = vips.lo.v6; + }; + router-ut.untrusted = { + name = "router-ut"; + inherit domain; + ipv4.address = vips.untrusted.v4; + ipv6.address = vips.untrusted.v6; + }; + }; + configuration = { lib, pkgs, config, assignments, allAssignments, ... }: let inherit (lib) mkIf mkMerge mkForce; @@ -77,6 +108,7 @@ in imports = map (m: import m index) [ ./keepalived.nix ./dns.nix + ./radvd.nix ]; config = { @@ -158,7 +190,7 @@ in networks = let - mkVLANConfig = name: mtu: + mkVLANConfig = name: let iface = "lan-${name}"; in @@ -166,26 +198,9 @@ in "60-${iface}" = mkMerge [ (networkdAssignment iface assignments."${name}") { - linkConfig.MTUBytes = toString mtu; + dns = [ "127.0.0.1" "::1" ]; domains = [ config.networking.domain ]; - networkConfig = { - IPv6AcceptRA = mkForce false; - # IPv6SendRA = true; - }; - ipv6SendRAConfig = { - DNS = [ - (net.cidr.host 1 prefixes."${name}".v4) - (net.cidr.host 2 prefixes."${name}".v4) - (net.cidr.host 1 prefixes."${name}".v6) - (net.cidr.host 2 prefixes."${name}".v6) - ]; - Domains = [ config.networking.domain ]; - }; - ipv6Prefixes = [ - { - ipv6PrefixConfig.Prefix = prefixes."${name}".v6; - } - ]; + networkConfig.IPv6AcceptRA = mkForce false; } ]; }; @@ -256,9 +271,9 @@ in ]; } - (mkVLANConfig "hi" 9000) - (mkVLANConfig "lo" 1500) - (mkVLANConfig "untrusted" 1500) + (mkVLANConfig "hi") + (mkVLANConfig "lo") + (mkVLANConfig "untrusted") { "60-lan-hi" = { diff --git a/nixos/boxes/home/routing-common/dns.nix b/nixos/boxes/home/routing-common/dns.nix index a47e652..8d472b5 100644 --- a/nixos/boxes/home/routing-common/dns.nix +++ b/nixos/boxes/home/routing-common/dns.nix @@ -102,7 +102,7 @@ in bind.zones = let - names = [ "core" "hi" "lo" ]; + names = [ "core" "hi" "lo" "untrusted" ]; i = toString (index + 1); in { @@ -138,11 +138,15 @@ in jim-core IN A ${net.cidr.host 10 prefixes.core.v4} jim IN A ${net.cidr.host 10 prefixes.hi.v4} + jim IN AAAA ${net.cidr.host (65536+1) prefixes.hi.v6} jim-lo IN A ${net.cidr.host 10 prefixes.lo.v4} + jim-lo IN AAAA ${net.cidr.host (65536+1) prefixes.lo.v6} dave-core IN A ${net.cidr.host 11 prefixes.core.v4} dave IN A ${net.cidr.host 11 prefixes.hi.v4} + dave IN AAAA ${net.cidr.host (65536+2) prefixes.hi.v6} dave-lo IN A ${net.cidr.host 11 prefixes.lo.v4} + dave-lo IN AAAA ${net.cidr.host (65536+2) prefixes.lo.v6} ups IN A ${net.cidr.host 20 prefixes.lo.v4} palace-kvm IN A ${net.cidr.host 21 prefixes.lo.v4} diff --git a/nixos/boxes/home/routing-common/keepalived.nix b/nixos/boxes/home/routing-common/keepalived.nix index c8822d3..007f352 100644 --- a/nixos/boxes/home/routing-common/keepalived.nix +++ b/nixos/boxes/home/routing-common/keepalived.nix @@ -1,20 +1,30 @@ -index: { lib, pkgs, ... }: +index: { lib, pkgs, config, ... }: let - inherit (builtins) attrNames; + inherit (builtins) attrNames concatMap; + inherit (lib) optional; inherit (lib.my) net; inherit (lib.my.c.home) prefixes vips; vlanIface = vlan: if vlan == "as211024" then vlan else "lan-${vlan}"; - vrrpIPs = family: map (vlan: { - addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}"; + vrrpIPs = family: concatMap (vlan: [ + { + addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}"; + dev = vlanIface vlan; + } + ] ++ (optional (family == "v6") { + addr = "fe80::1/64"; dev = vlanIface vlan; - }) (attrNames vips); + })) (attrNames vips); mkVRRP = family: routerId: { state = if index == 0 then "MASTER" else "BACKUP"; interface = "lan-core"; priority = 255 - index; virtualRouterId = routerId; virtualIps = vrrpIPs family; + extraConfig = '' + notify_master "${config.systemd.package}/bin/systemctl start radvd.service" + notify_backup "${config.systemd.package}/bin/systemctl stop radvd.service" + ''; }; in { diff --git a/nixos/boxes/home/routing-common/radvd.nix b/nixos/boxes/home/routing-common/radvd.nix new file mode 100644 index 0000000..1288a9b --- /dev/null +++ b/nixos/boxes/home/routing-common/radvd.nix @@ -0,0 +1,28 @@ +index: { lib, pkgs, ... }: +let + inherit (lib) mkForce concatMapStringsSep; + inherit (lib.my) net; + inherit (lib.my.c.home) domain prefixes; + + mkInterface = name: '' + interface lan-${name} { + AdvSendAdvert on; + AdvRASrcAddress { fe80::1; }; + AdvLinkMTU ${toString prefixes."${name}".mtu}; + prefix ${prefixes."${name}".v6} {}; + RDNSS ${net.cidr.host 1 prefixes."${name}".v6} ${net.cidr.host 2 prefixes."${name}".v6} {}; + DNSSL ${domain} {}; + }; + ''; +in +{ + # To be started by keepalived + systemd.services.radvd.wantedBy = mkForce [ ]; + + services = { + radvd = { + enable = true; + config = concatMapStringsSep "\n" mkInterface [ "hi" "lo" "untrusted" ]; + }; + }; +} diff --git a/nixos/default.nix b/nixos/default.nix index 6a28cb5..763f345 100644 --- a/nixos/default.nix +++ b/nixos/default.nix @@ -100,6 +100,7 @@ let altNames = mkOpt' (listOf str) [ ] "Extra names to assign."; visible = mkBoolOpt' true "Whether or not this assignment should be visible."; domain = mkOpt' (nullOr str) null "Domain for this assignment."; + mtu = mkOpt' (nullOr ints.unsigned) null "Interface MTU."; ipv4 = { address = mkOpt' net.types.ipv4 null "IPv4 address."; mask = mkOpt' ints.u8 24 "Network mask."; diff --git a/nixos/modules/spdk.nix b/nixos/modules/spdk.nix index 6da6d7e..3e495f1 100644 --- a/nixos/modules/spdk.nix +++ b/nixos/modules/spdk.nix @@ -1,7 +1,7 @@ { lib, pkgs, config, ... }: let inherit (builtins) toJSON; - inherit (lib) optional mapAttrsToList mkIf withFeature; + inherit (lib) optional optionalAttrs mapAttrsToList mkIf withFeature; inherit (lib.my) mkOpt' mkBoolOpt'; rpcOpts = with lib.types; { @@ -17,7 +17,7 @@ let inherit subsystem; config = map (rpc: { inherit (rpc) method; - } // (if rpc.params != { } then { inherit (rpc) params; } else { })) c; + } // (optionalAttrs (rpc.params != { }) { inherit (rpc) params; })) c; }) cfg.config.subsystems; }; configJSON = pkgs.writeText "spdk-config.json" (toJSON config');