lib/constants.nix

@@ -53,7 +53,6 @@ rec {
   pubDomain = "nul.ie";
   colony = {
     domain = "ams1.int.${pubDomain}";
-    pubV4 = "94.142.240.44";
     prefixes = with lib.my.net.cidr; rec {
       all = {
         v4 = "10.100.0.0/16";
@@ -91,12 +90,6 @@ rec {
 
       vip1 = "94.142.241.224/30";
       vip2 = "94.142.242.254/31";
-
-      as211024 = {
-        v4 = subnet 8 50 all.v4;
-        v6 = "2a0e:97c0:4df::/64";
-      };
-      home.v6 = "2a0e:97c0:4d0::/48";
     };
     fstrimConfig = {
       enable = true;
@@ -104,7 +97,6 @@ rec {
       interval = "04:45";
     };
   };
-
   home = rec {
     domain = "h.${pubDomain}";
     vlans = {
@@ -118,11 +110,6 @@ rec {
       "river"
       "stream"
     ];
-    routersPubV4 = [
-      "109.255.252.123" # placeholder
-      "109.255.252.104"
-    ];
-
     prefixes = with lib.my.net.cidr; rec {
       modem = {
         v4 = "192.168.0.0/24";
@@ -146,7 +133,6 @@ rec {
         v4 = subnet 6 16 all.v4;
         v6 = subnet 4 3 all.v6;
       };
-      inherit (colony.prefixes) as211024;
     };
     vips = with lib.my.net.cidr; {
       hi = {
@@ -161,13 +147,8 @@ rec {
         v4 = host 254 prefixes.untrusted.v4;
         v6 = host 65535 prefixes.untrusted.v6;
       };
-      as211024 = {
-        v4 = host 4 prefixes.as211024.v4;
-        v6 = host ((1*65536*65536*65536) + 65535) prefixes.as211024.v6;
-      };
     };
   };
-
   kelder = {
     groups = {
       storage = 2000;

nixos/boxes/colony/vms/estuary/bgp.nix

@@ -29,7 +29,7 @@ in
           define OWNNETSET6 = [ ${intnet6}, ${amsnet6}, ${homenet6} ];
           #define TRANSSET6 = [ ::1/128 ];
 
-          define DUB1IP6 = ${lib.my.c.home.vips.as211024.v6};
+          define DUB1IP6 = 2a0e:97c0:4df:0:2::1;
 
           define PREFIXP = 110;
           define PREFPEER = 120;

nixos/boxes/colony/vms/estuary/default.nix

@@ -1,8 +1,9 @@
 { lib, ... }:
 let
-  inherit (builtins) elemAt;
   inherit (lib.my) net;
-  inherit (lib.my.c.colony) pubV4 domain prefixes;
+  inherit (lib.my.c.colony) domain prefixes;
+
+  pubV4 = "94.142.240.44";
 in
 {
   nixos = {
@@ -10,11 +11,9 @@ in
       l2 = {
         as211024 = {
           vni = 211024;
-          security.enable = true;
           peers = {
             estuary.addr = pubV4;
-            # river.addr = elemAt lib.my.c.home.routersPubV4 0;
+            home.addr = "188.141.75.2";
-            stream.addr = elemAt lib.my.c.home.routersPubV4 1;
           };
         };
       };
@@ -54,10 +53,10 @@ in
       };
       as211024 = {
         ipv4 = {
-          address = net.cidr.host 1 prefixes.as211024.v4;
+          address = "10.255.3.1";
           gateway = null;
         };
-        ipv6.address = net.cidr.host 1 prefixes.as211024.v6;
+        ipv6.address = "2a0e:97c0:4df:0:3::1";
       };
     };
 
@@ -91,7 +90,6 @@ in
             environment = {
               systemPackages = with pkgs; [
                 ethtool
-                conntrack-tools
                 wireguard-tools
               ];
             };
@@ -116,19 +114,34 @@ in
             };
 
             systemd = {
-              services =
+              services = {
-              let
+                # Use this as a way to make sure the router always knows we're here (NDP seems kindy funky)
-                waitOnline = "systemd-networkd-wait-online@wan.service";
+                ipv6-neigh-keepalive =
-              in
+                let
-              {
+                  waitOnline = "systemd-networkd-wait-online@wan.service";
-                bird2 = {
+                in
+                {
+                  description = "Frequent ICMP6 neighbour solicitations";
+                  enable = false;
+                  requires = [ waitOnline ];
+                  after = [ waitOnline ];
+                  script = ''
+                    while true; do
+                      ${pkgs.ndisc6}/bin/ndisc6 ${assignments.internal.ipv6.gateway} wan
+                      sleep 10
+                    done
+                  '';
+                  wantedBy = [ "multi-user.target" ];
+                };
+
+                bird2 =
+                let
+                  waitOnline = "systemd-networkd-wait-online@wan.service";
+                in
+                {
                   after = [ waitOnline ];
                   # requires = [ waitOnline ];
                 };
-                ipsec = {
-                  after = [ waitOnline ];
-                  requires = [ waitOnline ];
-                };
               };
             };
 
@@ -324,13 +337,14 @@ in
                   }
                 ];
 
-                "90-l2mesh-as211024" = mkMerge [
+                "90-l2mesh-as211024" = {
-                  (networkdAssignment "as211024" assignments.as211024)
+                  matchConfig.Name = "as211024";
-                  {
+                  address = with assignments.as211024; [
-                    matchConfig.Name = "as211024";
+                    (with ipv4; "${address}/${toString mask}")
-                    networkConfig.IPv6AcceptRA = mkForce false;
+                    (with ipv6; "${address}/${toString mask}")
-                  }
+                  ];
-                ];
+                  networkConfig.IPv6AcceptRA = false;
+                };
                 "95-kelder" = {
                   matchConfig.Name = "kelder";
                   routes = [
@@ -352,16 +366,10 @@ in
                   "estuary/kelder-wg.key" = {
                     owner = "systemd-network";
                   };
-                  "l2mesh/as211024.key" = {};
                 };
               };
               server.enable = true;
 
-              vpns = {
-                l2.pskFiles = {
-                  as211024 = config.age.secrets."l2mesh/as211024.key".path;
-                };
-              };
               firewall = {
                 trustedInterfaces = [ "as211024" ];
                 udp.allowed = [ 5353 lib.my.c.kelder.vpn.port ];

nixos/boxes/home/routing-common/default.nix

@@ -1,4 +1,4 @@
-index: { lib, allAssignments, ... }:
+index: { lib, ... }:
 let
   inherit (builtins) elemAt;
   inherit (lib.my) net;
@@ -54,13 +54,6 @@ in
         };
         ipv6.address = net.cidr.host (index + 1) prefixes.untrusted.v6;
       };
-      as211024 = {
-        ipv4 = {
-          address = net.cidr.host (index + 2) prefixes.as211024.v4;
-          gateway = null;
-        };
-        ipv6.address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
-      };
     };
 
     configuration = { lib, pkgs, config, assignments, allAssignments, ... }:
@@ -71,7 +64,6 @@ in
       {
         imports = map (m: import m index) [
           ./mstpd.nix
-          ./keepalived.nix
           ./dns.nix
         ];
 
@@ -79,7 +71,6 @@ in
           environment = {
             systemPackages = with pkgs; [
               ethtool
-              conntrack-tools
             ];
           };
 
@@ -115,17 +106,6 @@ in
 
           networking.domain = "h.${pubDomain}";
 
-          systemd.services = {
-            ipsec =
-            let
-              waitOnline = "systemd-networkd-wait-online@wan.service";
-            in
-            {
-              after = [ waitOnline ];
-              requires = [ waitOnline ];
-            };
-          };
-
           systemd.network = {
             wait-online.enable = false;
             config = {
@@ -296,14 +276,6 @@ in
                     networkConfig.IPv6AcceptRA = mkForce false;
                   }
                 ];
-
-                "90-l2mesh-as211024" = mkMerge [
-                  (networkdAssignment "as211024" assignments.as211024)
-                  {
-                    matchConfig.Name = "as211024";
-                    networkConfig.IPv6AcceptRA = mkForce false;
-                  }
-                ];
               }
 
               (mkVLANConfig "hi" 9000)
@@ -315,15 +287,12 @@ in
           my = {
             secrets = {
               files = {
-                "l2mesh/as211024.key" = {};
+                # "estuary/kelder-wg.key" = {
+                #   owner = "systemd-network";
+                # };
               };
             };
 
-            vpns = {
-              l2.pskFiles = {
-                as211024 = config.age.secrets."l2mesh/as211024.key".path;
-              };
-            };
             firewall = {
               trustedInterfaces = [ "lan-hi" "lan-lo" ];
               udp.allowed = [ 5353 ];

nixos/boxes/home/routing-common/dns.nix

@@ -49,7 +49,6 @@ in
           query-local-address = [
             # TODO: IPv6
             "0.0.0.0"
-            "::"
             # TODO: Dynamic IPv4 WAN address?
             # assignments.internal.ipv4.address
             # assignments.internal.ipv6.address

nixos/boxes/home/routing-common/keepalived.nix

@@ -1,42 +0,0 @@
-index: { lib, pkgs, ... }:
-let
-  inherit (builtins) attrNames;
-  inherit (lib.my) net;
-  inherit (lib.my.c.home) prefixes vips;
-
-  vlanIface = vlan: if vlan == "as211024" then vlan else "lan-${vlan}";
-  vrrpIPs = family: map (vlan: {
-    addr = "${vips.${vlan}.${family}}/${toString (net.cidr.length prefixes.${vlan}.${family})}";
-    dev = vlanIface vlan;
-  }) (attrNames vips);
-  mkVRRP = family: routerId: {
-    state = if index == 0 then "MASTER" else "BACKUP";
-    interface = "lan-core";
-    priority = 255 - index;
-    virtualRouterId = routerId;
-    virtualIps = vrrpIPs family;
-  };
-in
-{
-  services = {
-    keepalived = {
-      enable = true;
-      extraGlobalDefs = ''
-        vrrp_version 3
-        nftables keepalived
-      '';
-      vrrpInstances = {
-        v4 = mkVRRP "v4" 51;
-        v6 = mkVRRP "v6" 52;
-      };
-      extraConfig = ''
-        vrrp_sync_group main {
-          group {
-            v4
-            v6
-          }
-        }
-      '';
-    };
-  };
-}

nixos/default.nix

@@ -125,9 +125,6 @@ let
   l2MeshOpts = with lib.types; { name, ... }: {
     options = {
       interface = mkOpt' str name "Name of VXLAN interface.";
-      ipv6 = mkBoolOpt' false "Whether this mesh's underlay operates over IPv6.";
-      baseMTU = mkOpt' ints.unsigned 1500 "Base MTU to calculate VXLAN MTU with.";
-      l3Overhead = mkOpt' ints.unsigned 40 "Overhead of L3 header (to calculate MTU).";
       firewall = mkBoolOpt' true "Whether to generate firewall rules.";
       vni = mkOpt' ints.unsigned 1 "VXLAN VNI.";
       peers = mkOpt' (attrsOf (submodule l2PeerOpts)) { } "Peers.";

nixos/modules/l2mesh.nix

@@ -1,8 +1,7 @@
-{ lib, config, vpns, ... }:
+{ lib, pkgs, config, vpns, ... }:
 let
-  inherit (builtins) any attrValues;
+  inherit (lib) optionalString mapAttrsToList concatStringsSep filterAttrs mkIf mkMerge;
-  inherit (lib) optionalString mapAttrsToList concatStringsSep concatMapStringsSep filterAttrs mkIf mkMerge;
+  inherit (lib.my) isIPv6;
-  inherit (lib.my) isIPv6 mkOpt';
 
   vxlanPort = 4789;
 
@@ -25,34 +24,38 @@ let
         Local = ownAddr;
         MacLearning = true;
         DestinationPort = vxlanPort;
-        PortRange = "${toString vxlanPort}-${toString (vxlanPort + 1)}";
         Independent = true;
       };
     };
+    links."20-l2mesh-${name}" = {
+      matchConfig.Name = mesh.interface;
+      # TODO: ipv6? ipsec?
+      linkConfig.MTUBytes = "1450";
+    };
     networks."90-l2mesh-${name}" = {
       matchConfig.Name = mesh.interface;
-      linkConfig.MTUBytes =
+      extraConfig = concatStringsSep "\n" (mapAttrsToList (n: peer: ''
-      let
+        [BridgeFDB]
-        espOverhead =
+        MACAddress=00:00:00:00:00:00
-          if (!mesh.security.enable) then 0
+        Destination=${peer.addr}
-          else
+      '') otherPeers);
-            # SPI + seq + IV + pad / header + ICV
-            4 + 4 + (if mesh.security.encrypt then 8 else 0) + 2 + 16;
-        # UDP + VXLAN + Ethernet + L3 (IPv4/IPv6)
-        overhead = espOverhead + 8 + 8 + 14 + mesh.l3Overhead;
-      in
-      toString (mesh.baseMTU - overhead);
-
-      bridgeFDBs = mapAttrsToList (n: peer: {
-        bridgeFDBConfig = {
-          MACAddress = "00:00:00:00:00:00";
-          Destination = peer.addr;
-        };
-      }) otherPeers;
     };
   };
 
-  vxlanAllow = vni: "udp dport ${toString vxlanPort} @th,96,24 ${toString vni} accept";
+  mkLibreswanConfig = name: mesh: with info mesh; {
+    enable = true;
+    # TODO: finish this...
+    connections."l2mesh-${name}" = ''
+      keyexchange=ike
+      type=transport
+      left=${ownAddr}
+
+      auto=start
+      phase2=esp
+      ikev2=yes
+    '';
+  };
+
   mkFirewallConfig = name: mesh: with info mesh;
   let
     netProto = if (isIPv6 ownAddr) then "ip6" else "ip";
@@ -60,11 +63,8 @@ let
   ''
     table inet filter {
       chain l2mesh-${name} {
-        ${optionalString mesh.security.enable ''
+        ${optionalString mesh.security.enable "meta l4proto esp accept"}
-          udp dport isakmp accept
+        udp dport ${toString vxlanPort} @th,96,24 ${toString mesh.vni} accept
-          meta l4proto esp accept
-        ''}
-        ${optionalString (!mesh.security.enable) (vxlanAllow mesh.vni)}
         return
       }
       chain input {
@@ -72,63 +72,12 @@ let
       }
     }
   '';
-
-  mkLibreswanConfig = name: mesh: with info mesh; {
-    enable = true;
-    connections = mkMerge (mapAttrsToList
-      (pName: peer: {
-        "l2mesh-${name}-${pName}" = ''
-          keyexchange=ike
-          hostaddrfamily=ipv${if mesh.ipv6 then "6" else "4"}
-          type=transport
-
-          left=${ownAddr}
-          leftprotoport=udp/${toString vxlanPort}
-          right=${peer.addr}
-          rightprotoport=udp/${toString vxlanPort}
-          rightupdown=
-
-          auto=start
-          authby=secret
-          phase2=esp
-          esp=${if mesh.security.encrypt then "aes_gcm256" else "null-sha256"}
-          ikev2=yes
-          modecfgpull=no
-        '';
-      })
-    otherPeers);
-  };
-  genSecrets = name: mesh: with info mesh; concatMapStringsSep "\n" (p: ''
-    echo "${ownAddr} ${p.addr} : PSK \"$(< "${config.my.vpns.l2.pskFiles.${name}}")\"" >> /run/l2mesh.secrets
-  '') (attrValues otherPeers);
-  anySecurity = any (c: c.security.enable) (attrValues memberMeshes);
 in
 {
-  options = {
-    my.vpns.l2 = with lib.types; {
-      pskFiles = mkOpt' (attrsOf str) { } "PSK files for secured meshes.";
-    };
-  };
-
   config = {
     systemd.network = mkMerge (mapAttrsToList mkNetConfig memberMeshes);
-
+    # TODO: finish this...
-    environment.etc."ipsec.d/l2mesh.secrets" = mkIf anySecurity {
+    #services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
-      source = "/run/l2mesh.secrets";
-    };
-    systemd.services.ipsec = mkIf anySecurity {
-      preStart = ''
-        oldUmask="$(umask)"
-        umask 006
-
-        > /run/l2mesh.secrets
-        ${concatStringsSep "\n" (mapAttrsToList genSecrets memberMeshes)}
-
-        umask "$oldUmask"
-      '';
-    };
-
-    services.libreswan = mkMerge (mapAttrsToList mkLibreswanConfig (filterAttrs (_: c: c.security.enable) memberMeshes));
     my.firewall.extraRules = concatStringsSep "\n" (mapAttrsToList mkFirewallConfig (filterAttrs (_: c: c.firewall) memberMeshes));
   };
 }

secrets/l2mesh/as211024.key.age

@@ -1,15 +0,0 @@
------BEGIN AGE ENCRYPTED FILE-----
-YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IHNzaC1lZDI1NTE5IG44Q3BVdyBBY0ZU
-RmFhZGFPdW1VTVJtYWRsUjJqdGlqVTd2Y3ovSFpQRGNJZ1pDODFvClNqQjlVK0hu
-Uitvb2tydGo4OHZwRWZJMnhwYzNmdFVnaVdFWU9SYzBEYWMKLT4gc3NoLWVkMjU1
-MTkgcytxUmZnIDBSUW16OGpkalA3MFp1RlBCRnlMTVQ3WDZBLzFMMjd5djF0MW1Y
-eVFjUWMKOUcyazgyeWNLWWJ3czFZM1hzK2htNWlLdnk5SmNEekZVRDFHNEZ0QlhY
-SQotPiBYMjU1MTkgeWVXN0dqZlZwTWpCTTU3UXFBUkljWUFtcG9yRlNXSVJvR21X
-QVQ3YnRSMApzb29oeUFQZmxZTVhSU2VmaUN5MEFzVWRrSGNLV0hRUmpnWGJTU1FC
-V0ZVCi0+IEE1am8tZ3JlYXNlIGpmJktlK2pRCi9VcnQyQWFtd1pNL2xINGVMNTF0
-OTNkOVpIWThUcUxhdlVYTkw4NHZnWFgrZCt1SlhCcTdnOGMyazMwdWMzOXEKOVhr
-NnQ3RzBCVzEvMUs1S0pkaGRXd3BBb1MwVEdKVXMKLS0tIEhMVnhDSFJRT01TS0Rp
-Z2lubnRJd1cxN080YWJ0aTJidmcwR3BxdW5vUE0Kr9X2C1i5yj+gRZNFRek8b+2+
-7Ll+u7AxYLEdGeB/74ehp9v7oUVTTwhRnhXCLjmixYx9PRBivObFAVswk7fr6y8W
-VGNFLmp6zpMzkbI=
------END AGE ENCRYPTED FILE-----