nixos: Working l2mesh with IPsec

lib/constants.nix

@@ -53,6 +53,7 @@ rec {
   pubDomain = "nul.ie";
   colony = {
     domain = "ams1.int.${pubDomain}";
+    pubV4 = "94.142.240.44";
     prefixes = with lib.my.net.cidr; rec {
       all = {
         v4 = "10.100.0.0/16";
@@ -90,6 +91,12 @@ rec {
 
       vip1 = "94.142.241.224/30";
       vip2 = "94.142.242.254/31";
+
+      as211024 = {
+        v4 = subnet 8 50 all.v4;
+        v6 = "2a0e:97c0:4df::/64";
+      };
+      home.v6 = "2a0e:97c0:4d0::/48";
     };
     fstrimConfig = {
       enable = true;
@@ -97,6 +104,7 @@ rec {
       interval = "04:45";
     };
   };
+
   home = rec {
     domain = "h.${pubDomain}";
     vlans = {
@@ -110,6 +118,11 @@ rec {
       "river"
       "stream"
     ];
+    routersPubV4 = [
+      "109.255.252.123" # placeholder
+      "109.255.252.104"
+    ];
+
     prefixes = with lib.my.net.cidr; rec {
       modem = {
         v4 = "192.168.0.0/24";
@@ -133,6 +146,7 @@ rec {
         v4 = subnet 6 16 all.v4;
         v6 = subnet 4 3 all.v6;
       };
+      inherit (colony.prefixes) as211024;
     };
     vips = with lib.my.net.cidr; {
       hi = {
@@ -147,8 +161,13 @@ rec {
         v4 = host 254 prefixes.untrusted.v4;
         v6 = host 65535 prefixes.untrusted.v6;
       };
+      as211024 = {
+        v4 = host 4 prefixes.as211024.v4;
+        v6 = host ((1*65536*65536*65536) + 65535) prefixes.as211024.v6;
+      };
     };
   };
+
   kelder = {
     groups = {
       storage = 2000;

nixos/boxes/colony/vms/estuary/bgp.nix

@@ -29,7 +29,7 @@ in
           define OWNNETSET6 = [ ${intnet6}, ${amsnet6}, ${homenet6} ];
           #define TRANSSET6 = [ ::1/128 ];
 
-          define DUB1IP6 = 2a0e:97c0:4df:0:2::1;
+          define DUB1IP6 = ${lib.my.c.home.vips.as211024.v6};
 
           define PREFIXP = 110;
           define PREFPEER = 120;

nixos/boxes/colony/vms/estuary/default.nix

@@ -1,9 +1,8 @@
 { lib, ... }:
 let
+  inherit (builtins) elemAt;
   inherit (lib.my) net;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my.c.colony) pubV4 domain prefixes;
-
-  pubV4 = "94.142.240.44";
 in
 {
   nixos = {
@@ -11,9 +10,11 @@ in
       l2 = {
         as211024 = {
           vni = 211024;
+          security.enable = true;
           peers = {
             estuary.addr = pubV4;
-            home.addr = "188.141.75.2";
+            # river.addr = elemAt lib.my.c.home.routersPubV4 0;
+            stream.addr = elemAt lib.my.c.home.routersPubV4 1;
           };
         };
       };
@@ -53,10 +54,10 @@ in
       };
       as211024 = {
         ipv4 = {
-          address = "10.255.3.1";
+          address = net.cidr.host 1 prefixes.as211024.v4;
           gateway = null;
         };
-        ipv6.address = "2a0e:97c0:4df:0:3::1";
+        ipv6.address = net.cidr.host 1 prefixes.as211024.v6;
       };
     };
 
@@ -90,6 +91,7 @@ in
             environment = {
               systemPackages = with pkgs; [
                 ethtool
+                conntrack-tools
                 wireguard-tools
               ];
             };
@@ -114,34 +116,19 @@ in
             };
 
             systemd = {
-              services = {
+              services =
-                # Use this as a way to make sure the router always knows we're here (NDP seems kindy funky)
+              let
-                ipv6-neigh-keepalive =
+                waitOnline = "systemd-networkd-wait-online@wan.service";
-                let
+              in
-                  waitOnline = "systemd-networkd-wait-online@wan.service";
+              {
-                in
+                bird2 = {
-                {
-                  description = "Frequent ICMP6 neighbour solicitations";
-                  enable = false;
-                  requires = [ waitOnline ];
-                  after = [ waitOnline ];
-                  script = ''
-                    while true; do
-                      ${pkgs.ndisc6}/bin/ndisc6 ${assignments.internal.ipv6.gateway} wan
-                      sleep 10
-                    done
-                  '';
-                  wantedBy = [ "multi-user.target" ];
-                };
-
-                bird2 =
-                let
-                  waitOnline = "systemd-networkd-wait-online@wan.service";
-                in
-                {
                   after = [ waitOnline ];
                   # requires = [ waitOnline ];
                 };
+                ipsec = {
+                  after = [ waitOnline ];
+                  requires = [ waitOnline ];
+                };
               };
             };
 
@@ -337,14 +324,13 @@ in
                   }
                 ];
 
-                "90-l2mesh-as211024" = {
+                "90-l2mesh-as211024" = mkMerge [
-                  matchConfig.Name = "as211024";
+                  (networkdAssignment "as211024" assignments.as211024)
-                  address = with assignments.as211024; [
+                  {
-                    (with ipv4; "${address}/${toString mask}")
+                    matchConfig.Name = "as211024";
-                    (with ipv6; "${address}/${toString mask}")
+                    networkConfig.IPv6AcceptRA = mkForce false;
-                  ];
+                  }
-                  networkConfig.IPv6AcceptRA = false;
+                ];
-                };
                 "95-kelder" = {
                   matchConfig.Name = "kelder";
                   routes = [
@@ -366,10 +352,16 @@ in
                   "estuary/kelder-wg.key" = {
                     owner = "systemd-network";
                   };
+                  "l2mesh/as211024.key" = {};
                 };
               };
               server.enable = true;
 
+              vpns = {
+                l2.pskFiles = {
+                  as211024 = config.age.secrets."l2mesh/as211024.key".path;
+                };
+              };
               firewall = {
                 trustedInterfaces = [ "as211024" ];
                 udp.allowed = [ 5353 lib.my.c.kelder.vpn.port ];

nixos/boxes/home/routing-common/default.nix

@@ -1,4 +1,4 @@
-index: { lib, ... }:
+index: { lib, allAssignments, ... }:
 let
   inherit (builtins) elemAt;
   inherit (lib.my) net;
@@ -54,6 +54,13 @@ in
         };
         ipv6.address = net.cidr.host (index + 1) prefixes.untrusted.v6;
       };
+      as211024 = {
+        ipv4 = {
+          address = net.cidr.host (index + 2) prefixes.as211024.v4;
+          gateway = null;
+        };
+        ipv6.address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
+      };
     };
 
     configuration = { lib, pkgs, config, assignments, allAssignments, ... }:
@@ -64,6 +71,7 @@ in
       {
         imports = map (m: import m index) [
           ./mstpd.nix
+          ./keepalived.nix
           ./dns.nix
         ];
 
@@ -71,6 +79,7 @@ in
           environment = {
             systemPackages = with pkgs; [
               ethtool
+              conntrack-tools
             ];
           };
 
@@ -106,6 +115,17 @@ in
 
           networking.domain = "h.${pubDomain}";
 
+          systemd.services = {
+            ipsec =
+            let
+              waitOnline = "systemd-networkd-wait-online@wan.service";
+            in
+            {
+              after = [ waitOnline ];
+              requires = [ waitOnline ];
+            };
+          };
+
           systemd.network = {
             wait-online.enable = false;
             config = {
@@ -276,6 +296,14 @@ in
                     networkConfig.IPv6AcceptRA = mkForce false;
                   }
                 ];
+
+                "90-l2mesh-as211024" = mkMerge [
+                  (networkdAssignment "as211024" assignments.as211024)
+                  {
+                    matchConfig.Name = "as211024";
+                    networkConfig.IPv6AcceptRA = mkForce false;
+                  }
+                ];
               }
 
               (mkVLANConfig "hi" 9000)
@@ -287,12 +315,15 @@ in
           my = {
             secrets = {
               files = {
-                # "estuary/kelder-wg.key" = {
+                "l2mesh/as211024.key" = {};
-                #   owner = "systemd-network";
-                # };
               };
             };
 
+            vpns = {
+              l2.pskFiles = {
+                as211024 = config.age.secrets."l2mesh/as211024.key".path;
+              };
+            };
             firewall = {
               trustedInterfaces = [ "lan-hi" "lan-lo" ];
               udp.allowed = [ 5353 ];

nixos/boxes/home/routing-common/dns.nix

@@ -49,6 +49,7 @@ in
           query-local-address = [
             # TODO: IPv6
             "0.0.0.0"
+            "::"
             # TODO: Dynamic IPv4 WAN address?
             # assignments.internal.ipv4.address
             # assignments.internal.ipv6.address

nixos/boxes/home/routing-common/keepalived.nix

@@ -0,0 +1,42 @@
+index: { lib, pkgs, ... }:
+let
+  inherit (builtins) attrNames;
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) prefixes vips;
+
+  vlanIface = vlan: if vlan == "as211024" then vlan else "lan-${vlan}";
+  vrrpIPs = family: map (vlan: {
+    addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}";
+    dev = vlanIface vlan;
+  }) (attrNames vips);
+  mkVRRP = family: routerId: {
+    state = if index == 0 then "MASTER" else "BACKUP";
+    interface = "lan-core";
+    priority = 255 - index;
+    virtualRouterId = routerId;
+    virtualIps = vrrpIPs family;
+  };
+in
+{
+  services = {
+    keepalived = {
+      enable = true;
+      extraGlobalDefs = ''
+        vrrp_version 3
+        nftables keepalived
+      '';
+      vrrpInstances = {
+        v4 = mkVRRP "v4" 51;
+        v6 = mkVRRP "v6" 52;
+      };
+      extraConfig = ''
+        vrrp_sync_group main {
+          group {
+            v4
+            v6
+          }
+        }
+      '';
+    };
+  };
+}

nixos/default.nix

@@ -125,6 +125,9 @@ let
   l2MeshOpts = with lib.types; { name, ... }: {
     options = {
       interface = mkOpt' str name "Name of VXLAN interface.";
+      ipv6 = mkBoolOpt' false "Whether this mesh's underlay operates over IPv6.";
+      baseMTU = mkOpt' ints.unsigned 1500 "Base MTU to calculate VXLAN MTU with.";
+      l3Overhead = mkOpt' ints.unsigned 40 "Overhead of L3 header (to calculate MTU).";
       firewall = mkBoolOpt' true "Whether to generate firewall rules.";
       vni = mkOpt' ints.unsigned 1 "VXLAN VNI.";
       peers = mkOpt' (attrsOf (submodule l2PeerOpts)) { } "Peers.";

nixos/modules/l2mesh.nix

@@ -1,7 +1,8 @@
-{ lib, pkgs, config, vpns, ... }:
+{ lib, config, vpns, ... }:
 let
-  inherit (lib) optionalString mapAttrsToList concatStringsSep filterAttrs mkIf mkMerge;
+  inherit (builtins) any attrValues;
-  inherit (lib.my) isIPv6;
+  inherit (lib) optionalString mapAttrsToList concatStringsSep concatMapStringsSep filterAttrs mkIf mkMerge;
+  inherit (lib.my) isIPv6 mkOpt';
 
   vxlanPort = 4789;
 
@@ -24,38 +25,34 @@ let
         Local = ownAddr;
         MacLearning = true;
         DestinationPort = vxlanPort;
+        PortRange = "${toString vxlanPort}-${toString (vxlanPort + 1)}";
         Independent = true;
       };
     };
-    links."20-l2mesh-${name}" = {
-      matchConfig.Name = mesh.interface;
-      # TODO: ipv6? ipsec?
-      linkConfig.MTUBytes = "1450";
-    };
     networks."90-l2mesh-${name}" = {
       matchConfig.Name = mesh.interface;
-      extraConfig = concatStringsSep "\n" (mapAttrsToList (n: peer: ''
+      linkConfig.MTUBytes =
-        [BridgeFDB]
+      let
-        MACAddress=00:00:00:00:00:00
+        espOverhead =
-        Destination=${peer.addr}
+          if (!mesh.security.enable) then 0
-      '') otherPeers);
+          else
+            # SPI + seq + IV + pad / header + ICV
+            4 + 4 + (if mesh.security.encrypt then 8 else 0) + 2 + 16;
+        # UDP + VXLAN + Ethernet + L3 (IPv4/IPv6)
+        overhead = espOverhead + 8 + 8 + 14 + mesh.l3Overhead;
+      in
+      toString (mesh.baseMTU - overhead);
+
+      bridgeFDBs = mapAttrsToList (n: peer: {
+        bridgeFDBConfig = {
+          MACAddress = "00:00:00:00:00:00";
+          Destination = peer.addr;
+        };
+      }) otherPeers;
     };
   };
 
-  mkLibreswanConfig = name: mesh: with info mesh; {
+  vxlanAllow = vni: "udp dport ${toString vxlanPort} @th,96,24 ${toString vni} accept";
-    enable = true;
-    # TODO: finish this...
-    connections."l2mesh-${name}" = ''
-      keyexchange=ike
-      type=transport
-      left=${ownAddr}
-
-      auto=start
-      phase2=esp
-      ikev2=yes
-    '';
-  };
-
   mkFirewallConfig = name: mesh: with info mesh;
   let
     netProto = if (isIPv6 ownAddr) then "ip6" else "ip";
@@ -63,8 +60,11 @@ let
   ''
     table inet filter {
       chain l2mesh-${name} {
-        ${optionalString mesh.security.enable "meta l4proto esp accept"}
+        ${optionalString mesh.security.enable ''
-        udp dport ${toString vxlanPort} @th,96,24 ${toString mesh.vni} accept
+          udp dport isakmp accept
+          meta l4proto esp accept
+        ''}
+        ${optionalString (!mesh.security.enable) (vxlanAllow mesh.vni)}
         return
       }
       chain input {
@@ -72,12 +72,63 @@ let
       }
     }
   '';
+
+  mkLibreswanConfig = name: mesh: with info mesh; {
+    enable = true;
+    connections = mkMerge (mapAttrsToList
+      (pName: peer: {
+        "l2mesh-${name}-${pName}" = ''
+          keyexchange=ike
+          hostaddrfamily=ipv${if mesh.ipv6 then "6" else "4"}
+          type=transport
+
+          left=${ownAddr}
+          leftprotoport=udp/${toString vxlanPort}
+          right=${peer.addr}
+          rightprotoport=udp/${toString vxlanPort}
+          rightupdown=
+
+          auto=start
+          authby=secret
+          phase2=esp
+          esp=${if mesh.security.encrypt then "aes_gcm256" else "null-sha256"}
+          ikev2=yes
+          modecfgpull=no
+        '';
+      })
+    otherPeers);
+  };
+  genSecrets = name: mesh: with info mesh; concatMapStringsSep "\n" (p: ''
+    echo "${ownAddr} ${p.addr} : PSK \"$(< "${config.my.vpns.l2.pskFiles.${name}}")\"" >> /run/l2mesh.secrets
+  '') (attrValues otherPeers);
+  anySecurity = any (c: c.security.enable) (attrValues memberMeshes);
 in
 {
+  options = {
+    my.vpns.l2 = with lib.types; {
+      pskFiles = mkOpt' (attrsOf str) { } "PSK files for secured meshes.";
+    };
+  };
+
   config = {
     systemd.network = mkMerge (mapAttrsToList mkNetConfig memberMeshes);
-    # TODO: finish this...
+
-    #services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
+    environment.etc."ipsec.d/l2mesh.secrets" = mkIf anySecurity {
+      source = "/run/l2mesh.secrets";
+    };
+    systemd.services.ipsec = mkIf anySecurity {
+      preStart = ''
+        oldUmask="$(umask)"
+        umask 006
+
+        > /run/l2mesh.secrets
+        ${concatStringsSep "\n" (mapAttrsToList genSecrets memberMeshes)}
+
+        umask "$oldUmask"
+      '';
+    };
+
+    services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
     my.firewall.extraRules = concatStringsSep "\n" (mapAttrsToList mkFirewallConfig (filterAttrs (_: c: c.firewall) memberMeshes));
   };
 }

secrets/l2mesh/as211024.key.age

@@ -0,0 +1,15 @@
+-----BEGIN AGE ENCRYPTED FILE-----
+YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IG44Q3BVdyBBY0ZU
+RmFhZGFPdW1VTVJtYWRsUjJqdGlqVTd2Y3ovSFpQRGNJZ1pDODFvClNqQjlVK0hu
+Uitvb2tydGo4OHZwRWZJMnhwYzNmdFVnaVdFWU9SYzBEYWMKLT4gc3NoLWVkMjU1
+MTkgcytxUmZnIDBSUW16OGpkalA3MFp1RlBCRnlMTVQ3WDZBLzFMMjd5djF0MW1Y
+eVFjUWMKOUcyazgyeWNLWWJ3czFZM1hzK2htNWlLdnk5SmNEekZVRDFHNEZ0QlhY
+SQotPiBYMjU1MTkgeWVXN0dqZlZwTWpCTTU3UXFBUkljWUFtcG9yRlNXSVJvR21X
+QVQ3YnRSMApzb29oeUFQZmxZTVhSU2VmaUN5MEFzVWRrSGNLV0hRUmpnWGJTU1FC
+V0ZVCi0+IEE1am8tZ3JlYXNlIGpmJktlK2pRCi9VcnQyQWFtd1pNL2xINGVMNTF0
+OTNkOVpIWThUcUxhdlVYTkw4NHZnWFgrZCt1SlhCcTdnOGMyazMwdWMzOXEKOVhr
+NnQ3RzBCVzEvMUs1S0pkaGRXd3BBb1MwVEdKVXMKLS0tIEhMVnhDSFJRT01TS0Rp
+Z2lubnRJd1cxN080YWJ0aTJidmcwR3BxdW5vUE0Kr9X2C1i5yj+gRZNFRek8b+2+
+7Ll+u7AxYLEdGeB/74ehp9v7oUVTTwhRnhXCLjmixYx9PRBivObFAVswk7fr6y8W
+VGNFLmp6zpMzkbI=
+-----END AGE ENCRYPTED FILE-----