From 0cc35547f23802565ecc37724b41674fb4f4d358 Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Sun, 26 Nov 2023 01:29:44 +0000 Subject: [PATCH] nixos: Working l2mesh with IPsec --- lib/constants.nix | 19 +++ nixos/boxes/colony/vms/estuary/bgp.nix | 2 +- nixos/boxes/colony/vms/estuary/default.nix | 70 +++++------ nixos/boxes/home/routing-common/default.nix | 38 +++++- nixos/boxes/home/routing-common/dns.nix | 1 + .../boxes/home/routing-common/keepalived.nix | 3 +- nixos/default.nix | 3 + nixos/modules/l2mesh.nix | 113 +++++++++++++----- secrets/l2mesh/as211024.key.age | 15 +++ 9 files changed, 188 insertions(+), 76 deletions(-) create mode 100644 secrets/l2mesh/as211024.key.age diff --git a/lib/constants.nix b/lib/constants.nix index 32fe093..3c6560f 100644 --- a/lib/constants.nix +++ b/lib/constants.nix @@ -53,6 +53,7 @@ rec { pubDomain = "nul.ie"; colony = { domain = "ams1.int.${pubDomain}"; + pubV4 = "94.142.240.44"; prefixes = with lib.my.net.cidr; rec { all = { v4 = "10.100.0.0/16"; @@ -90,6 +91,12 @@ rec { vip1 = "94.142.241.224/30"; vip2 = "94.142.242.254/31"; + + as211024 = { + v4 = subnet 8 50 all.v4; + v6 = "2a0e:97c0:4df::/64"; + }; + home.v6 = "2a0e:97c0:4d0::/48"; }; fstrimConfig = { enable = true; @@ -97,6 +104,7 @@ rec { interval = "04:45"; }; }; + home = rec { domain = "h.${pubDomain}"; vlans = { @@ -110,6 +118,11 @@ rec { "river" "stream" ]; + routersPubV4 = [ + "109.255.252.123" # placeholder + "109.255.252.104" + ]; + prefixes = with lib.my.net.cidr; rec { modem = { v4 = "192.168.0.0/24"; @@ -133,6 +146,7 @@ rec { v4 = subnet 6 16 all.v4; v6 = subnet 4 3 all.v6; }; + inherit (colony.prefixes) as211024; }; vips = with lib.my.net.cidr; { hi = { @@ -147,8 +161,13 @@ rec { v4 = host 254 prefixes.untrusted.v4; v6 = host 65535 prefixes.untrusted.v6; }; + as211024 = { + v4 = host 4 prefixes.as211024.v4; + v6 = host ((1*65536*65536*65536) + 65535) prefixes.as211024.v6; + }; }; }; + kelder = { groups = { storage = 2000; diff --git a/nixos/boxes/colony/vms/estuary/bgp.nix b/nixos/boxes/colony/vms/estuary/bgp.nix index c67ee53..2cd5395 100644 --- a/nixos/boxes/colony/vms/estuary/bgp.nix +++ b/nixos/boxes/colony/vms/estuary/bgp.nix @@ -29,7 +29,7 @@ in define OWNNETSET6 = [ ${intnet6}, ${amsnet6}, ${homenet6} ]; #define TRANSSET6 = [ ::1/128 ]; - define DUB1IP6 = 2a0e:97c0:4df:0:2::1; + define DUB1IP6 = ${lib.my.c.home.vips.as211024.v6}; define PREFIXP = 110; define PREFPEER = 120; diff --git a/nixos/boxes/colony/vms/estuary/default.nix b/nixos/boxes/colony/vms/estuary/default.nix index e03606c..6c09908 100644 --- a/nixos/boxes/colony/vms/estuary/default.nix +++ b/nixos/boxes/colony/vms/estuary/default.nix @@ -1,9 +1,8 @@ { lib, ... }: let + inherit (builtins) elemAt; inherit (lib.my) net; - inherit (lib.my.c.colony) domain prefixes; - - pubV4 = "94.142.240.44"; + inherit (lib.my.c.colony) pubV4 domain prefixes; in { nixos = { @@ -11,9 +10,11 @@ in l2 = { as211024 = { vni = 211024; + security.enable = true; peers = { estuary.addr = pubV4; - home.addr = "188.141.75.2"; + # river.addr = elemAt lib.my.c.home.routersPubV4 0; + stream.addr = elemAt lib.my.c.home.routersPubV4 1; }; }; }; @@ -53,10 +54,10 @@ in }; as211024 = { ipv4 = { - address = "10.255.3.1"; + address = net.cidr.host 1 prefixes.as211024.v4; gateway = null; }; - ipv6.address = "2a0e:97c0:4df:0:3::1"; + ipv6.address = net.cidr.host 1 prefixes.as211024.v6; }; }; @@ -90,6 +91,7 @@ in environment = { systemPackages = with pkgs; [ ethtool + conntrack-tools wireguard-tools ]; }; @@ -114,34 +116,19 @@ in }; systemd = { - services = { - # Use this as a way to make sure the router always knows we're here (NDP seems kindy funky) - ipv6-neigh-keepalive = - let - waitOnline = "systemd-networkd-wait-online@wan.service"; - in - { - description = "Frequent ICMP6 neighbour solicitations"; - enable = false; - requires = [ waitOnline ]; - after = [ waitOnline ]; - script = '' - while true; do - ${pkgs.ndisc6}/bin/ndisc6 ${assignments.internal.ipv6.gateway} wan - sleep 10 - done - ''; - wantedBy = [ "multi-user.target" ]; - }; - - bird2 = - let - waitOnline = "systemd-networkd-wait-online@wan.service"; - in - { + services = + let + waitOnline = "systemd-networkd-wait-online@wan.service"; + in + { + bird2 = { after = [ waitOnline ]; # requires = [ waitOnline ]; }; + ipsec = { + after = [ waitOnline ]; + requires = [ waitOnline ]; + }; }; }; @@ -337,14 +324,13 @@ in } ]; - "90-l2mesh-as211024" = { - matchConfig.Name = "as211024"; - address = with assignments.as211024; [ - (with ipv4; "${address}/${toString mask}") - (with ipv6; "${address}/${toString mask}") - ]; - networkConfig.IPv6AcceptRA = false; - }; + "90-l2mesh-as211024" = mkMerge [ + (networkdAssignment "as211024" assignments.as211024) + { + matchConfig.Name = "as211024"; + networkConfig.IPv6AcceptRA = mkForce false; + } + ]; "95-kelder" = { matchConfig.Name = "kelder"; routes = [ @@ -366,10 +352,16 @@ in "estuary/kelder-wg.key" = { owner = "systemd-network"; }; + "l2mesh/as211024.key" = {}; }; }; server.enable = true; + vpns = { + l2.pskFiles = { + as211024 = config.age.secrets."l2mesh/as211024.key".path; + }; + }; firewall = { trustedInterfaces = [ "as211024" ]; udp.allowed = [ 5353 lib.my.c.kelder.vpn.port ]; diff --git a/nixos/boxes/home/routing-common/default.nix b/nixos/boxes/home/routing-common/default.nix index 2fd95e2..12f2738 100644 --- a/nixos/boxes/home/routing-common/default.nix +++ b/nixos/boxes/home/routing-common/default.nix @@ -1,4 +1,4 @@ -index: { lib, ... }: +index: { lib, allAssignments, ... }: let inherit (builtins) elemAt; inherit (lib.my) net; @@ -54,6 +54,13 @@ in }; ipv6.address = net.cidr.host (index + 1) prefixes.untrusted.v6; }; + as211024 = { + ipv4 = { + address = net.cidr.host (index + 2) prefixes.as211024.v4; + gateway = null; + }; + ipv6.address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6; + }; }; configuration = { lib, pkgs, config, assignments, allAssignments, ... }: @@ -72,6 +79,7 @@ in environment = { systemPackages = with pkgs; [ ethtool + conntrack-tools ]; }; @@ -107,6 +115,17 @@ in networking.domain = "h.${pubDomain}"; + systemd.services = { + ipsec = + let + waitOnline = "systemd-networkd-wait-online@wan.service"; + in + { + after = [ waitOnline ]; + requires = [ waitOnline ]; + }; + }; + systemd.network = { wait-online.enable = false; config = { @@ -277,6 +296,14 @@ in networkConfig.IPv6AcceptRA = mkForce false; } ]; + + "90-l2mesh-as211024" = mkMerge [ + (networkdAssignment "as211024" assignments.as211024) + { + matchConfig.Name = "as211024"; + networkConfig.IPv6AcceptRA = mkForce false; + } + ]; } (mkVLANConfig "hi" 9000) @@ -288,12 +315,15 @@ in my = { secrets = { files = { - # "estuary/kelder-wg.key" = { - # owner = "systemd-network"; - # }; + "l2mesh/as211024.key" = {}; }; }; + vpns = { + l2.pskFiles = { + as211024 = config.age.secrets."l2mesh/as211024.key".path; + }; + }; firewall = { trustedInterfaces = [ "lan-hi" "lan-lo" ]; udp.allowed = [ 5353 ]; diff --git a/nixos/boxes/home/routing-common/dns.nix b/nixos/boxes/home/routing-common/dns.nix index ad1d685..7805d41 100644 --- a/nixos/boxes/home/routing-common/dns.nix +++ b/nixos/boxes/home/routing-common/dns.nix @@ -49,6 +49,7 @@ in query-local-address = [ # TODO: IPv6 "0.0.0.0" + "::" # TODO: Dynamic IPv4 WAN address? # assignments.internal.ipv4.address # assignments.internal.ipv6.address diff --git a/nixos/boxes/home/routing-common/keepalived.nix b/nixos/boxes/home/routing-common/keepalived.nix index d2bfd5a..c8822d3 100644 --- a/nixos/boxes/home/routing-common/keepalived.nix +++ b/nixos/boxes/home/routing-common/keepalived.nix @@ -4,9 +4,10 @@ let inherit (lib.my) net; inherit (lib.my.c.home) prefixes vips; + vlanIface = vlan: if vlan == "as211024" then vlan else "lan-${vlan}"; vrrpIPs = family: map (vlan: { addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}"; - dev = "lan-${vlan}"; + dev = vlanIface vlan; }) (attrNames vips); mkVRRP = family: routerId: { state = if index == 0 then "MASTER" else "BACKUP"; diff --git a/nixos/default.nix b/nixos/default.nix index d55eaec..c480dd7 100644 --- a/nixos/default.nix +++ b/nixos/default.nix @@ -125,6 +125,9 @@ let l2MeshOpts = with lib.types; { name, ... }: { options = { interface = mkOpt' str name "Name of VXLAN interface."; + ipv6 = mkBoolOpt' false "Whether this mesh's underlay operates over IPv6."; + baseMTU = mkOpt' ints.unsigned 1500 "Base MTU to calculate VXLAN MTU with."; + l3Overhead = mkOpt' ints.unsigned 40 "Overhead of L3 header (to calculate MTU)."; firewall = mkBoolOpt' true "Whether to generate firewall rules."; vni = mkOpt' ints.unsigned 1 "VXLAN VNI."; peers = mkOpt' (attrsOf (submodule l2PeerOpts)) { } "Peers."; diff --git a/nixos/modules/l2mesh.nix b/nixos/modules/l2mesh.nix index aa6fe7c..6af107b 100644 --- a/nixos/modules/l2mesh.nix +++ b/nixos/modules/l2mesh.nix @@ -1,7 +1,8 @@ -{ lib, pkgs, config, vpns, ... }: +{ lib, config, vpns, ... }: let - inherit (lib) optionalString mapAttrsToList concatStringsSep filterAttrs mkIf mkMerge; - inherit (lib.my) isIPv6; + inherit (builtins) any attrValues; + inherit (lib) optionalString mapAttrsToList concatStringsSep concatMapStringsSep filterAttrs mkIf mkMerge; + inherit (lib.my) isIPv6 mkOpt'; vxlanPort = 4789; @@ -24,38 +25,34 @@ let Local = ownAddr; MacLearning = true; DestinationPort = vxlanPort; + PortRange = "${toString vxlanPort}-${toString (vxlanPort + 1)}"; Independent = true; }; }; - links."20-l2mesh-${name}" = { - matchConfig.Name = mesh.interface; - # TODO: ipv6? ipsec? - linkConfig.MTUBytes = "1450"; - }; networks."90-l2mesh-${name}" = { matchConfig.Name = mesh.interface; - extraConfig = concatStringsSep "\n" (mapAttrsToList (n: peer: '' - [BridgeFDB] - MACAddress=00:00:00:00:00:00 - Destination=${peer.addr} - '') otherPeers); + linkConfig.MTUBytes = + let + espOverhead = + if (!mesh.security.enable) then 0 + else + # SPI + seq + IV + pad / header + ICV + 4 + 4 + (if mesh.security.encrypt then 8 else 0) + 2 + 16; + # UDP + VXLAN + Ethernet + L3 (IPv4/IPv6) + overhead = espOverhead + 8 + 8 + 14 + mesh.l3Overhead; + in + toString (mesh.baseMTU - overhead); + + bridgeFDBs = mapAttrsToList (n: peer: { + bridgeFDBConfig = { + MACAddress = "00:00:00:00:00:00"; + Destination = peer.addr; + }; + }) otherPeers; }; }; - mkLibreswanConfig = name: mesh: with info mesh; { - enable = true; - # TODO: finish this... - connections."l2mesh-${name}" = '' - keyexchange=ike - type=transport - left=${ownAddr} - - auto=start - phase2=esp - ikev2=yes - ''; - }; - + vxlanAllow = vni: "udp dport ${toString vxlanPort} @th,96,24 ${toString vni} accept"; mkFirewallConfig = name: mesh: with info mesh; let netProto = if (isIPv6 ownAddr) then "ip6" else "ip"; @@ -63,8 +60,11 @@ let '' table inet filter { chain l2mesh-${name} { - ${optionalString mesh.security.enable "meta l4proto esp accept"} - udp dport ${toString vxlanPort} @th,96,24 ${toString mesh.vni} accept + ${optionalString mesh.security.enable '' + udp dport isakmp accept + meta l4proto esp accept + ''} + ${optionalString (!mesh.security.enable) (vxlanAllow mesh.vni)} return } chain input { @@ -72,12 +72,63 @@ let } } ''; + + mkLibreswanConfig = name: mesh: with info mesh; { + enable = true; + connections = mkMerge (mapAttrsToList + (pName: peer: { + "l2mesh-${name}-${pName}" = '' + keyexchange=ike + hostaddrfamily=ipv${if mesh.ipv6 then "6" else "4"} + type=transport + + left=${ownAddr} + leftprotoport=udp/${toString vxlanPort} + right=${peer.addr} + rightprotoport=udp/${toString vxlanPort} + rightupdown= + + auto=start + authby=secret + phase2=esp + esp=${if mesh.security.encrypt then "aes_gcm256" else "null-sha256"} + ikev2=yes + modecfgpull=no + ''; + }) + otherPeers); + }; + genSecrets = name: mesh: with info mesh; concatMapStringsSep "\n" (p: '' + echo "${ownAddr} ${p.addr} : PSK \"$(< "${config.my.vpns.l2.pskFiles.${name}}")\"" >> /run/l2mesh.secrets + '') (attrValues otherPeers); + anySecurity = any (c: c.security.enable) (attrValues memberMeshes); in { + options = { + my.vpns.l2 = with lib.types; { + pskFiles = mkOpt' (attrsOf str) { } "PSK files for secured meshes."; + }; + }; + config = { systemd.network = mkMerge (mapAttrsToList mkNetConfig memberMeshes); - # TODO: finish this... - #services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes)); + + environment.etc."ipsec.d/l2mesh.secrets" = mkIf anySecurity { + source = "/run/l2mesh.secrets"; + }; + systemd.services.ipsec = mkIf anySecurity { + preStart = '' + oldUmask="$(umask)" + umask 006 + + > /run/l2mesh.secrets + ${concatStringsSep "\n" (mapAttrsToList genSecrets memberMeshes)} + + umask "$oldUmask" + ''; + }; + + services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes)); my.firewall.extraRules = concatStringsSep "\n" (mapAttrsToList mkFirewallConfig (filterAttrs (_: c: c.firewall) memberMeshes)); }; } diff --git a/secrets/l2mesh/as211024.key.age b/secrets/l2mesh/as211024.key.age new file mode 100644 index 0000000..2bd6b8f --- /dev/null +++ b/secrets/l2mesh/as211024.key.age @@ -0,0 +1,15 @@ +-----BEGIN AGE ENCRYPTED FILE----- +YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IG44Q3BVdyBBY0ZU +RmFhZGFPdW1VTVJtYWRsUjJqdGlqVTd2Y3ovSFpQRGNJZ1pDODFvClNqQjlVK0hu +Uitvb2tydGo4OHZwRWZJMnhwYzNmdFVnaVdFWU9SYzBEYWMKLT4gc3NoLWVkMjU1 +MTkgcytxUmZnIDBSUW16OGpkalA3MFp1RlBCRnlMTVQ3WDZBLzFMMjd5djF0MW1Y +eVFjUWMKOUcyazgyeWNLWWJ3czFZM1hzK2htNWlLdnk5SmNEekZVRDFHNEZ0QlhY +SQotPiBYMjU1MTkgeWVXN0dqZlZwTWpCTTU3UXFBUkljWUFtcG9yRlNXSVJvR21X +QVQ3YnRSMApzb29oeUFQZmxZTVhSU2VmaUN5MEFzVWRrSGNLV0hRUmpnWGJTU1FC +V0ZVCi0+IEE1am8tZ3JlYXNlIGpmJktlK2pRCi9VcnQyQWFtd1pNL2xINGVMNTF0 +OTNkOVpIWThUcUxhdlVYTkw4NHZnWFgrZCt1SlhCcTdnOGMyazMwdWMzOXEKOVhr +NnQ3RzBCVzEvMUs1S0pkaGRXd3BBb1MwVEdKVXMKLS0tIEhMVnhDSFJRT01TS0Rp +Z2lubnRJd1cxN080YWJ0aTJidmcwR3BxdW5vUE0Kr9X2C1i5yj+gRZNFRek8b+2+ +7Ll+u7AxYLEdGeB/74ehp9v7oUVTTwhRnhXCLjmixYx9PRBivObFAVswk7fr6y8W +VGNFLmp6zpMzkbI= +-----END AGE ENCRYPTED FILE-----