{ lib, ... }: let inherit (builtins) elemAt; inherit (lib.my) net mkVLAN; inherit (lib.my.c.colony) pubV4 domain prefixes firewallForwards; in { nixos = { vpns = { l2 = { as211024 = { vni = 211024; security.enable = true; peers = { estuary.addr = pubV4; river.addr = elemAt lib.my.c.home.routersPubV4 0; stream.addr = elemAt lib.my.c.home.routersPubV4 1; britway.addr = lib.my.c.britway.pubV4; }; }; }; }; }; nixos.systems.estuary = { system = "x86_64-linux"; nixpkgs = "mine"; home-manager = "mine"; assignments = { internal = { name = "estuary-vm"; altNames = [ "fw" ]; inherit domain; ipv4 = { address = pubV4; mask = 24; gateway = "94.142.240.254"; genPTR = false; }; ipv6 = { address = "2a02:898:0:20::329:1"; mask = 64; gateway = "2a02:898:0:20::1"; genPTR = false; }; }; base = { name = "estuary-vm-base"; inherit domain; ipv4 = { address = net.cidr.host 1 prefixes.base.v4; gateway = null; }; ipv6.address = net.cidr.host 1 prefixes.base.v6; }; as211024 = { ipv4 = { address = net.cidr.host 1 prefixes.as211024.v4; gateway = null; }; ipv6.address = net.cidr.host 1 prefixes.as211024.v6; }; }; configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }: let inherit (lib) flatten mkIf mkMerge mkForce; inherit (lib.my) networkdAssignment; inherit (lib.my.c) networkd; in { imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./dns.nix ./bgp.nix ]; config = mkMerge [ { boot.kernelParams = [ "console=ttyS0,115200n8" ]; fileSystems = { "/boot" = { device = "/dev/disk/by-label/ESP"; fsType = "vfat"; }; "/nix" = { device = "/dev/disk/by-label/nix"; fsType = "ext4"; }; "/persist" = { device = "/dev/disk/by-label/persist"; fsType = "ext4"; neededForBoot = true; }; }; environment = { systemPackages = with pkgs; [ ethtool conntrack-tools wireguard-tools ]; }; services = { fstrim = lib.my.c.colony.fstrimConfig; lvm = { dmeventd.enable = true; }; resolved = { llmnr = "false"; extraConfig = '' MulticastDNS=false ''; }; netdata.enable = true; iperf3 = { enable = true; openFirewall = true; }; }; systemd = { services = let waitOnline = "systemd-networkd-wait-online@wan.service"; in { bird2 = { after = [ waitOnline ]; # requires = [ waitOnline ]; }; ipsec = { after = [ waitOnline ]; requires = [ waitOnline ]; }; }; }; #systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug"; systemd.network = { wait-online.enable = false; config = { networkConfig = { ManageForeignRoutes = false; }; }; netdevs = mkMerge [ (mkVLAN "ifog" 409) (mkVLAN "frys-ix" 701) (mkVLAN "nl-ix" 1845) (mkVLAN "fogixp" 1147) (mkVLAN "ifog-transit" 702) { "30-kelder" = { netdevConfig = { Name = "kelder"; Kind = "wireguard"; }; wireguardConfig = { PrivateKeyFile = config.age.secrets."estuary/kelder-wg.key".path; ListenPort = lib.my.c.kelder.vpn.port; }; wireguardPeers = [ { wireguardPeerConfig = { PublicKey = "7N9YdQaCMWWIwAnW37vrthm9ZpbnG4Lx3gheHeRYz2E="; AllowedIPs = [ allAssignments.kelder.estuary.ipv4.address ]; PersistentKeepalive = 25; }; } ]; }; } ]; links = { "10-wan" = { matchConfig = { Driver = "igb"; Path = "pci-0000:01:00.0"; # Matching against MAC address seems to break VLAN interfaces (since they share the same MAC address) #MACAddress = "d0:50:99:fa:a7:99"; }; linkConfig = { Name = "wan"; RxBufferSize = 4096; TxBufferSize = 4096; MTUBytes = "9000"; }; }; # Mellanox ConnectX-2 #"10-wan" = { # matchConfig.MACAddress = "00:02:c9:56:24:6e"; # linkConfig.Name = "wan"; #}; "10-base" = { matchConfig.MACAddress = "52:54:00:15:1a:53"; linkConfig.Name = "base"; }; }; networks = let mkIXPConfig = name: ipv4: ipv6: { "85-${name}" = { matchConfig.Name = name; address = [ ipv4 ipv6 ]; linkConfig.MTUBytes = "1500"; networkConfig = { DHCP = "no"; LLDP = false; EmitLLDP = false; IPv6AcceptRA = false; }; }; }; in mkMerge [ (mkIXPConfig "frys-ix" "185.1.203.196/24" "2001:7f8:10f::3:3850:196/64") (mkIXPConfig "nl-ix" "193.239.116.145/22" "2001:7f8:13::a521:1024:1/64") (mkIXPConfig "fogixp" "185.1.147.159/24" "2001:7f8:ca:1::159/64") { "80-wan" = { matchConfig.Name = "wan"; vlan = [ "ifog" ]; DHCP = "no"; address = with assignments.internal; [ (with ipv4; "${address}/${toString mask}") (with ipv6; "${address}/${toString mask}") ]; gateway = with assignments.internal; [ ipv4.gateway ipv6.gateway ]; networkConfig = { # We're using an explicit gateway and Linux uses link local address for neighbour discovery, so we # get lost to the router... (this was true in 23M Frankfurt) #LinkLocalAddressing = "no"; IPv6AcceptRA = false; }; }; "85-ifog" = { matchConfig = { Name = "ifog"; Kind = "vlan"; }; vlan = [ "frys-ix" "nl-ix" "fogixp" "ifog-transit" ]; networkConfig = networkd.noL3; }; "85-ifog-transit" = { matchConfig.Name = "ifog-transit"; address = [ "2a0c:9a40:100f:370::2/64" ]; linkConfig.MTUBytes = "1500"; networkConfig = { DHCP = "no"; LLDP = false; EmitLLDP = false; IPv6AcceptRA = false; }; }; "80-base" = mkMerge [ (networkdAssignment "base" assignments.base) { dns = [ "127.0.0.1" "::1" ]; domains = [ config.networking.domain ]; networkConfig = { IPv6AcceptRA = mkForce false; IPv6SendRA = true; }; ipv6SendRAConfig = { DNS = [ assignments.base.ipv6.address ]; Domains = [ config.networking.domain ]; }; ipv6Prefixes = [ { ipv6PrefixConfig.Prefix = prefixes.base.v6; } ]; routes = map (r: { routeConfig = r; }) (flatten ([ { Destination = prefixes.vip1; Gateway = allAssignments.colony.routing.ipv4.address; } { Destination = prefixes.vip3; Gateway = allAssignments.colony.routing.ipv4.address; } { Destination = prefixes.darts.v4; Gateway = allAssignments.colony.routing.ipv4.address; } { Destination = prefixes.cust.v6; Gateway = allAssignments.colony.internal.ipv6.address; } { Destination = lib.my.c.tailscale.prefix.v4; Gateway = allAssignments.colony.routing.ipv4.address; } { Destination = lib.my.c.tailscale.prefix.v6; Gateway = allAssignments.colony.internal.ipv6.address; } ] ++ (map (pName: [ { Gateway = allAssignments.colony.routing.ipv4.address; Destination = prefixes."${pName}".v4; } { Destination = prefixes."${pName}".v6; Gateway = allAssignments.colony.internal.ipv6.address; } ]) [ "vms" "ctrs" "oci" ]))); } ]; "90-l2mesh-as211024" = mkMerge [ (networkdAssignment "as211024" assignments.as211024) { matchConfig.Name = "as211024"; networkConfig.IPv6AcceptRA = mkForce false; routes = map (r: { routeConfig = r; }) [ { Destination = lib.my.c.home.prefixes.all.v4; Gateway = lib.my.c.home.vips.as211024.v4; } ]; } ]; "95-kelder" = { matchConfig.Name = "kelder"; routes = [ { routeConfig = { Destination = allAssignments.kelder.estuary.ipv4.address; Scope = "link"; }; } ]; }; } ]; }; my = { secrets = { key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9up7pXu6M/OWCKufTOfSiGcxMUk4VqUe7fLuatNFFA"; files = { "estuary/kelder-wg.key" = { owner = "systemd-network"; }; "l2mesh/as211024.key" = {}; }; }; server.enable = true; vpns = { l2.pskFiles = { as211024 = config.age.secrets."l2mesh/as211024.key".path; }; }; firewall = { udp.allowed = [ 5353 lib.my.c.kelder.vpn.port ]; tcp.allowed = [ 5353 "bgp" ]; nat = { enable = true; externalInterface = "wan"; forwardPorts."${assignments.internal.ipv4.address}" = firewallForwards allAssignments; }; extraRules = let aa = allAssignments; matchInet = rule: sys: '' ip daddr ${aa."${sys}".internal.ipv4.address} ${rule} ip6 daddr ${aa."${sys}".internal.ipv6.address} ${rule} ''; in '' define ixps = { frys-ix, nl-ix, fogixp, ifog-transit } table inet filter { chain input { iifname base meta l4proto { udp, tcp } th dport domain accept } chain routing-tcp { # Safe enough to allow all SSH tcp dport ssh accept ip6 daddr ${aa.middleman.internal.ipv6.address} tcp dport { http, https, 8448 } accept ${matchInet "tcp dport { http, https } accept" "git"} ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} tcp dport { 25565, 25575 } accept ip6 daddr ${aa.simpcraft-staging-oci.internal.ipv6.address} tcp dport 25565 accept return } chain routing-udp { ip6 daddr ${aa.valheim-oci.internal.ipv6.address} udp dport { 2456-2457 } accept ip6 daddr ${aa.waffletail.internal.ipv6.address} udp dport 41641 accept ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} udp dport 25565 accept return } chain filter-routing { ip daddr { ${prefixes.mail.v4}, ${prefixes.darts.v4} } accept ip6 daddr ${prefixes.cust.v6} accept tcp flags & (fin|syn|rst|ack) == syn ct state new jump routing-tcp meta l4proto udp ct state new jump routing-udp return } chain ixp { ether type != { ip, ip6, arp, vlan } reject return } chain forward { ${lib.my.c.as211024.nftTrust} iifname { wan, as211024, $ixps } oifname base jump filter-routing oifname $ixps jump ixp iifname base oifname { base, wan, $ixps } accept oifname { as211024, kelder } accept } chain output { oifname ifog ether type != vlan reject oifname $ixps jump ixp } } table inet nat { chain prerouting { ${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" "estuary"} } chain postrouting { ip saddr ${prefixes.all.v4} oifname != as211024 snat to ${assignments.internal.ipv4.address} } } ''; }; }; } (mkIf config.my.build.isDevVM { systemd.network = { netdevs."05-dummy-base".netdevConfig = { Name = "base"; Kind = "dummy"; }; }; }) ]; }; }; }