Attach `clue-ix` VLAN 121 directly to `wan` with the assigned IPv6 address. Add sessions to both AS218511 route servers using `rs_client` roles and mandatory GTSM, following the existing IXP policy. Include the interface in the direct protocol and IXP firewall set, and document the VLAN and peers. Omit IPv4 peering because no address was assigned.
542 lines
20 KiB
Nix
542 lines
20 KiB
Nix
{ lib, ... }:
|
|
let
|
|
inherit (builtins) elemAt;
|
|
inherit (lib.my) net mkVLAN;
|
|
inherit (lib.my.c.colony) pubV4 domain prefixes firewallForwards;
|
|
in
|
|
{
|
|
nixos = {
|
|
vpns = {
|
|
l2 = {
|
|
as211024 = {
|
|
udpEncapsulation = true;
|
|
vni = 211024;
|
|
security.enable = true;
|
|
peers = {
|
|
estuary.addr = pubV4;
|
|
river.addr = elemAt lib.my.c.home.routersPubV4 0;
|
|
stream.addr = elemAt lib.my.c.home.routersPubV4 1;
|
|
britway.addr = lib.my.c.britway.pubV4;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
};
|
|
nixos.systems.estuary = {
|
|
system = "x86_64-linux";
|
|
nixpkgs = "mine";
|
|
home-manager = "mine";
|
|
|
|
assignments = {
|
|
internal = {
|
|
name = "estuary-vm";
|
|
altNames = [ "fw" ];
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = pubV4;
|
|
mask = 24;
|
|
gateway = "94.142.240.254";
|
|
genPTR = false;
|
|
};
|
|
ipv6 = {
|
|
address = "2a02:898:0:20::329:1";
|
|
mask = 64;
|
|
gateway = "2a02:898:0:20::1";
|
|
genPTR = false;
|
|
};
|
|
};
|
|
base = {
|
|
name = "estuary-vm-base";
|
|
inherit domain;
|
|
ipv4 = {
|
|
address = net.cidr.host 1 prefixes.base.v4;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host 1 prefixes.base.v6;
|
|
};
|
|
as211024 = {
|
|
ipv4 = {
|
|
address = net.cidr.host 1 prefixes.as211024.v4;
|
|
gateway = null;
|
|
};
|
|
ipv6.address = net.cidr.host 1 prefixes.as211024.v6;
|
|
};
|
|
};
|
|
|
|
configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
|
|
let
|
|
inherit (lib) flatten mkIf mkMerge mkForce optional;
|
|
inherit (lib.my) networkdAssignment;
|
|
inherit (lib.my.c) networkd;
|
|
in
|
|
{
|
|
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./dns.nix ./bgp.nix ];
|
|
|
|
config = mkMerge [
|
|
{
|
|
boot.kernelParams = [ "console=ttyS0,115200n8" ];
|
|
fileSystems = {
|
|
"/boot" = {
|
|
device = "/dev/disk/by-label/ESP";
|
|
fsType = "vfat";
|
|
};
|
|
"/nix" = {
|
|
device = "/dev/disk/by-label/nix";
|
|
fsType = "ext4";
|
|
};
|
|
"/persist" = {
|
|
device = "/dev/disk/by-label/persist";
|
|
fsType = "ext4";
|
|
neededForBoot = true;
|
|
};
|
|
};
|
|
|
|
environment = {
|
|
systemPackages = with pkgs; [
|
|
ethtool
|
|
conntrack-tools
|
|
wireguard-tools
|
|
];
|
|
};
|
|
|
|
services = {
|
|
fstrim = lib.my.c.colony.fstrimConfig;
|
|
lvm = {
|
|
dmeventd.enable = true;
|
|
};
|
|
resolved.settings.Resolve = {
|
|
LLMNR = false;
|
|
MulticastDNS = false;
|
|
};
|
|
netdata.enable = true;
|
|
|
|
iperf3 = {
|
|
enable = true;
|
|
openFirewall = true;
|
|
};
|
|
};
|
|
|
|
systemd = {
|
|
services =
|
|
let
|
|
waitOnline = "systemd-networkd-wait-online@wan.service";
|
|
in
|
|
{
|
|
bird2 = {
|
|
after = [ waitOnline ];
|
|
# requires = [ waitOnline ];
|
|
};
|
|
ipsec = {
|
|
after = [ waitOnline ];
|
|
requires = [ waitOnline ];
|
|
};
|
|
};
|
|
};
|
|
|
|
#systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
|
|
systemd.network = {
|
|
wait-online.enable = false;
|
|
config = {
|
|
networkConfig = {
|
|
ManageForeignRoutes = false;
|
|
};
|
|
};
|
|
|
|
netdevs = mkMerge [
|
|
(mkVLAN "ifog" 409)
|
|
(mkVLAN "clue-ix" 121)
|
|
|
|
(mkVLAN "frys-ix" 701)
|
|
(mkVLAN "nl-ix" 1845)
|
|
(mkVLAN "fogixp" 1147)
|
|
(mkVLAN "ifog-transit" 702)
|
|
|
|
{
|
|
"30-kelder" = {
|
|
netdevConfig = {
|
|
Name = "kelder";
|
|
Kind = "wireguard";
|
|
};
|
|
wireguardConfig = {
|
|
PrivateKeyFile = config.age.secrets."estuary/kelder-wg.key".path;
|
|
ListenPort = lib.my.c.kelder.vpn.port;
|
|
};
|
|
wireguardPeers = [
|
|
{
|
|
PublicKey = "7N9YdQaCMWWIwAnW37vrthm9ZpbnG4Lx3gheHeRYz2E=";
|
|
AllowedIPs = [ allAssignments.kelder.estuary.ipv4.address ];
|
|
PersistentKeepalive = 25;
|
|
}
|
|
];
|
|
};
|
|
}
|
|
{
|
|
"30-hillcrest" = {
|
|
netdevConfig = {
|
|
Name = "hillcrest";
|
|
Kind = "wireguard";
|
|
};
|
|
wireguardConfig = {
|
|
PrivateKeyFile = config.age.secrets."estuary/hillcrest-wg.key".path;
|
|
ListenPort = lib.my.c.hillcrest.vpn.port;
|
|
};
|
|
wireguardPeers = [
|
|
{
|
|
PublicKey = "+67Ks+ZRk1ssNCfg5BFKmIE9NtLasAxRE6XMqufx5GY=";
|
|
AllowedIPs = [ (net.cidr.host 2 prefixes.hillcrest.v4) ];
|
|
PersistentKeepalive = 25;
|
|
}
|
|
];
|
|
};
|
|
}
|
|
{
|
|
"30-john-valorant" = {
|
|
netdevConfig = {
|
|
Name = "john-valorant";
|
|
Kind = "wireguard";
|
|
};
|
|
wireguardConfig = {
|
|
PrivateKeyFile = config.age.secrets."estuary/john-valorant-wg.key".path;
|
|
ListenPort = lib.my.c.john-valorant.vpn.port;
|
|
};
|
|
wireguardPeers = [
|
|
{
|
|
PublicKey = "xyqKF0yOAv1bObN1paL2vATFh77pdFfvN+JmuAxaTCk=";
|
|
AllowedIPs = [ (net.cidr.host 2 prefixes.john-valorant.v4) ];
|
|
PersistentKeepalive = 25;
|
|
}
|
|
];
|
|
};
|
|
}
|
|
];
|
|
|
|
links = {
|
|
"10-wan" = {
|
|
matchConfig = {
|
|
Driver = "igb";
|
|
Path = "pci-0000:01:00.0";
|
|
# Matching against MAC address seems to break VLAN interfaces (since they share the same MAC address)
|
|
#MACAddress = "d0:50:99:fa:a7:99";
|
|
};
|
|
linkConfig = {
|
|
Name = "wan";
|
|
RxBufferSize = 4096;
|
|
TxBufferSize = 4096;
|
|
MTUBytes = "9000";
|
|
};
|
|
};
|
|
# Mellanox ConnectX-2
|
|
#"10-wan" = {
|
|
# matchConfig.MACAddress = "00:02:c9:56:24:6e";
|
|
# linkConfig.Name = "wan";
|
|
#};
|
|
|
|
"10-base" = {
|
|
matchConfig.MACAddress = "52:54:00:15:1a:53";
|
|
linkConfig.Name = "base";
|
|
};
|
|
};
|
|
|
|
networks =
|
|
let
|
|
mkIXPConfig = name: ipv4: ipv6: {
|
|
"85-${name}" = {
|
|
matchConfig.Name = name;
|
|
address = optional (ipv4 != null) ipv4 ++ [ ipv6 ];
|
|
linkConfig.MTUBytes = "1500";
|
|
networkConfig = {
|
|
DHCP = "no";
|
|
LLDP = false;
|
|
EmitLLDP = false;
|
|
IPv6AcceptRA = false;
|
|
};
|
|
};
|
|
};
|
|
in
|
|
mkMerge
|
|
[
|
|
(mkIXPConfig "clue-ix" null "2001:7f8:154::21:1024:1/64")
|
|
(mkIXPConfig "frys-ix" "185.1.160.196/23" "2001:7f8:10f::3:3850:196/64")
|
|
(mkIXPConfig "nl-ix" "193.239.116.145/22" "2001:7f8:13::a521:1024:1/64")
|
|
(mkIXPConfig "fogixp" "185.1.147.159/24" "2001:7f8:ca:1::159/64")
|
|
{
|
|
"80-wan" = {
|
|
matchConfig.Name = "wan";
|
|
vlan = [ "ifog" "clue-ix" ];
|
|
DHCP = "no";
|
|
address = with assignments.internal; [
|
|
(with ipv4; "${address}/${toString mask}")
|
|
(with ipv6; "${address}/${toString mask}")
|
|
];
|
|
gateway = with assignments.internal; [
|
|
ipv4.gateway
|
|
ipv6.gateway
|
|
];
|
|
networkConfig = {
|
|
# We're using an explicit gateway and Linux uses link local address for neighbour discovery, so we
|
|
# get lost to the router... (this was true in 23M Frankfurt)
|
|
#LinkLocalAddressing = "no";
|
|
IPv6AcceptRA = false;
|
|
};
|
|
};
|
|
"85-ifog" = {
|
|
matchConfig = {
|
|
Name = "ifog";
|
|
Kind = "vlan";
|
|
};
|
|
vlan = [ "frys-ix" "nl-ix" "fogixp" "ifog-transit" ];
|
|
networkConfig = networkd.noL3;
|
|
};
|
|
"85-ifog-transit" = {
|
|
matchConfig.Name = "ifog-transit";
|
|
address = [ "2a0c:9a40:100f:370::2/64" ];
|
|
linkConfig.MTUBytes = "1500";
|
|
networkConfig = {
|
|
DHCP = "no";
|
|
LLDP = false;
|
|
EmitLLDP = false;
|
|
IPv6AcceptRA = false;
|
|
};
|
|
};
|
|
|
|
"80-base" = mkMerge [
|
|
(networkdAssignment "base" assignments.base)
|
|
{
|
|
dns = [ "127.0.0.1" "::1" ];
|
|
domains = [ config.networking.domain ];
|
|
networkConfig = {
|
|
IPv6AcceptRA = mkForce false;
|
|
IPv6SendRA = true;
|
|
};
|
|
ipv6SendRAConfig = {
|
|
DNS = [ assignments.base.ipv6.address ];
|
|
Domains = [ config.networking.domain ];
|
|
};
|
|
ipv6Prefixes = [
|
|
{
|
|
Prefix = prefixes.base.v6;
|
|
}
|
|
];
|
|
routes = flatten ([
|
|
{
|
|
Destination = prefixes.vip1;
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = prefixes.vip3;
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = prefixes.darts.v4;
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = prefixes.cust.v6;
|
|
Gateway = allAssignments.colony.internal.ipv6.address;
|
|
}
|
|
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v4;
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
}
|
|
{
|
|
Destination = lib.my.c.tailscale.prefix.v6;
|
|
Gateway = allAssignments.colony.internal.ipv6.address;
|
|
}
|
|
|
|
{
|
|
Destination = prefixes.qclk.v4;
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
}
|
|
] ++
|
|
(map (pName: [
|
|
{
|
|
Gateway = allAssignments.colony.routing.ipv4.address;
|
|
Destination = prefixes."${pName}".v4;
|
|
}
|
|
{
|
|
Destination = prefixes."${pName}".v6;
|
|
Gateway = allAssignments.colony.internal.ipv6.address;
|
|
}
|
|
]) [ "vms" "ctrs" "oci" ]));
|
|
}
|
|
];
|
|
|
|
"90-l2mesh-as211024" = mkMerge [
|
|
(networkdAssignment "as211024" assignments.as211024)
|
|
{
|
|
matchConfig.Name = "as211024";
|
|
networkConfig.IPv6AcceptRA = mkForce false;
|
|
routes = [
|
|
{
|
|
Destination = lib.my.c.home.prefixes.all.v4;
|
|
Gateway = lib.my.c.home.vips.as211024.v4;
|
|
}
|
|
];
|
|
}
|
|
];
|
|
"95-kelder" = {
|
|
matchConfig.Name = "kelder";
|
|
routes = [
|
|
{
|
|
Destination = allAssignments.kelder.estuary.ipv4.address;
|
|
Scope = "link";
|
|
}
|
|
];
|
|
};
|
|
"95-hillcrest" = {
|
|
matchConfig.Name = "hillcrest";
|
|
address = [ "${net.cidr.host 1 prefixes.hillcrest.v4}/32" ];
|
|
routes = [
|
|
{
|
|
Destination = net.cidr.host 2 prefixes.hillcrest.v4;
|
|
Scope = "link";
|
|
}
|
|
];
|
|
};
|
|
"95-john-valorant" = {
|
|
matchConfig.Name = "john-valorant";
|
|
address = [ "${net.cidr.host 1 prefixes.john-valorant.v4}/32" ];
|
|
routes = [
|
|
{
|
|
Destination = net.cidr.host 2 prefixes.john-valorant.v4;
|
|
Scope = "link";
|
|
}
|
|
];
|
|
};
|
|
} ];
|
|
};
|
|
|
|
my = {
|
|
secrets = {
|
|
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9up7pXu6M/OWCKufTOfSiGcxMUk4VqUe7fLuatNFFA";
|
|
files = {
|
|
"estuary/kelder-wg.key" = {
|
|
owner = "systemd-network";
|
|
};
|
|
"estuary/hillcrest-wg.key" = {
|
|
owner = "systemd-network";
|
|
};
|
|
"estuary/john-valorant-wg.key" = {
|
|
owner = "systemd-network";
|
|
};
|
|
"l2mesh/as211024.key" = {};
|
|
};
|
|
};
|
|
server.enable = true;
|
|
|
|
vpns = {
|
|
l2.pskFiles = {
|
|
as211024 = config.age.secrets."l2mesh/as211024.key".path;
|
|
};
|
|
};
|
|
firewall = {
|
|
udp.allowed = [
|
|
5353
|
|
|
|
lib.my.c.kelder.vpn.port
|
|
lib.my.c.hillcrest.vpn.port
|
|
lib.my.c.john-valorant.vpn.port
|
|
];
|
|
tcp.allowed = [ 5353 "bgp" ];
|
|
nat = {
|
|
enable = true;
|
|
externalInterface = "wan";
|
|
forwardPorts."${assignments.internal.ipv4.address}" = firewallForwards allAssignments;
|
|
};
|
|
extraRules =
|
|
let
|
|
aa = allAssignments;
|
|
matchInet = rule: sys: ''
|
|
ip daddr ${aa."${sys}".internal.ipv4.address} ${rule}
|
|
ip6 daddr ${aa."${sys}".internal.ipv6.address} ${rule}
|
|
'';
|
|
in
|
|
''
|
|
define ixps = { clue-ix, frys-ix, nl-ix, fogixp, ifog-transit }
|
|
|
|
table inet filter {
|
|
chain input {
|
|
iifname base meta l4proto { udp, tcp } th dport domain accept
|
|
}
|
|
|
|
chain routing-tcp {
|
|
# Safe enough to allow all SSH
|
|
tcp dport ssh accept
|
|
|
|
# jam-ctr forwards
|
|
ip daddr ${aa.shill.internal.ipv4.address} tcp dport 60022 accept
|
|
|
|
ip6 daddr ${aa.middleman.internal.ipv6.address} tcp dport { http, https, 8448 } accept
|
|
${matchInet "tcp dport { http, https } accept" "git"}
|
|
ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} tcp dport 25565 accept
|
|
ip6 daddr ${aa.simpcraft-staging-oci.internal.ipv6.address} tcp dport 25565 accept
|
|
ip6 daddr ${aa.kevcraft-oci.internal.ipv6.address} tcp dport 25567 accept
|
|
ip6 daddr ${aa.kinkcraft-oci.internal.ipv6.address} tcp dport 25568 accept
|
|
ip6 daddr ${aa.graeme-oci.internal.ipv6.address} tcp dport 25569 accept
|
|
ip6 daddr ${aa.gam.internal.ipv6.address} tcp dport 7777 accept
|
|
return
|
|
}
|
|
chain routing-udp {
|
|
ip6 daddr ${aa.valheim-oci.internal.ipv6.address} udp dport { 2456-2457 } accept
|
|
ip6 daddr ${aa.waffletail.internal.ipv6.address} udp dport 41641 accept
|
|
ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} udp dport 25565 accept
|
|
ip6 daddr ${aa.enshrouded-oci.internal.ipv6.address} udp dport { 15636-15637 } accept
|
|
ip6 daddr ${aa.kevcraft-oci.internal.ipv6.address} udp dport 25567 accept
|
|
ip6 daddr ${aa.kinkcraft-oci.internal.ipv6.address} udp dport 25568 accept
|
|
ip6 daddr ${aa.graeme-oci.internal.ipv6.address} udp dport 25569 accept
|
|
ip6 daddr ${aa.gam.internal.ipv6.address} udp dport 7777 accept
|
|
return
|
|
}
|
|
chain filter-routing {
|
|
ip daddr { ${prefixes.mail.v4}, ${prefixes.darts.v4} } accept
|
|
ip6 daddr ${prefixes.cust.v6} accept
|
|
|
|
tcp flags & (fin|syn|rst|ack) == syn ct state new jump routing-tcp
|
|
meta l4proto udp ct state new jump routing-udp
|
|
return
|
|
}
|
|
chain ixp {
|
|
ether type != { ip, ip6, arp, vlan } reject
|
|
return
|
|
}
|
|
|
|
chain forward {
|
|
${lib.my.c.as211024.nftTrust}
|
|
iifname { wan, as211024, $ixps } oifname base jump filter-routing
|
|
oifname $ixps jump ixp
|
|
iifname base oifname { base, wan, $ixps } accept
|
|
oifname { as211024, kelder, hillcrest, john-valorant } accept
|
|
}
|
|
chain output {
|
|
oifname ifog ether type != vlan reject
|
|
oifname $ixps jump ixp
|
|
}
|
|
}
|
|
table inet nat {
|
|
chain prerouting {
|
|
${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" "estuary"}
|
|
}
|
|
chain postrouting {
|
|
oifname hillcrest snat ip to ${net.cidr.host 1 prefixes.hillcrest.v4}
|
|
oifname john-valorant snat ip to ${net.cidr.host 1 prefixes.john-valorant.v4}
|
|
ip saddr ${prefixes.all.v4} oifname != as211024 snat to ${assignments.internal.ipv4.address}
|
|
}
|
|
}
|
|
'';
|
|
};
|
|
};
|
|
}
|
|
(mkIf config.my.build.isDevVM {
|
|
systemd.network = {
|
|
netdevs."05-dummy-base".netdevConfig = {
|
|
Name = "base";
|
|
Kind = "dummy";
|
|
};
|
|
};
|
|
})
|
|
];
|
|
};
|
|
};
|
|
}
|