Files
nixfiles/nixos/boxes/colony/vms/estuary/default.nix
T
jackos1998 c1242b12ed nixos/estuary: Add Clue-IX IPv6 peering
Attach `clue-ix` VLAN 121 directly to `wan` with the assigned IPv6
address. Add sessions to both AS218511 route servers using `rs_client`
roles and mandatory GTSM, following the existing IXP policy.

Include the interface in the direct protocol and IXP firewall set,
and document the VLAN and peers. Omit IPv4 peering because no address
was assigned.
2026-11-17 12:47:15 +00:00

542 lines
20 KiB
Nix

{ lib, ... }:
let
inherit (builtins) elemAt;
inherit (lib.my) net mkVLAN;
inherit (lib.my.c.colony) pubV4 domain prefixes firewallForwards;
in
{
nixos = {
vpns = {
l2 = {
as211024 = {
udpEncapsulation = true;
vni = 211024;
security.enable = true;
peers = {
estuary.addr = pubV4;
river.addr = elemAt lib.my.c.home.routersPubV4 0;
stream.addr = elemAt lib.my.c.home.routersPubV4 1;
britway.addr = lib.my.c.britway.pubV4;
};
};
};
};
};
nixos.systems.estuary = {
system = "x86_64-linux";
nixpkgs = "mine";
home-manager = "mine";
assignments = {
internal = {
name = "estuary-vm";
altNames = [ "fw" ];
inherit domain;
ipv4 = {
address = pubV4;
mask = 24;
gateway = "94.142.240.254";
genPTR = false;
};
ipv6 = {
address = "2a02:898:0:20::329:1";
mask = 64;
gateway = "2a02:898:0:20::1";
genPTR = false;
};
};
base = {
name = "estuary-vm-base";
inherit domain;
ipv4 = {
address = net.cidr.host 1 prefixes.base.v4;
gateway = null;
};
ipv6.address = net.cidr.host 1 prefixes.base.v6;
};
as211024 = {
ipv4 = {
address = net.cidr.host 1 prefixes.as211024.v4;
gateway = null;
};
ipv6.address = net.cidr.host 1 prefixes.as211024.v6;
};
};
configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
let
inherit (lib) flatten mkIf mkMerge mkForce optional;
inherit (lib.my) networkdAssignment;
inherit (lib.my.c) networkd;
in
{
imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./dns.nix ./bgp.nix ];
config = mkMerge [
{
boot.kernelParams = [ "console=ttyS0,115200n8" ];
fileSystems = {
"/boot" = {
device = "/dev/disk/by-label/ESP";
fsType = "vfat";
};
"/nix" = {
device = "/dev/disk/by-label/nix";
fsType = "ext4";
};
"/persist" = {
device = "/dev/disk/by-label/persist";
fsType = "ext4";
neededForBoot = true;
};
};
environment = {
systemPackages = with pkgs; [
ethtool
conntrack-tools
wireguard-tools
];
};
services = {
fstrim = lib.my.c.colony.fstrimConfig;
lvm = {
dmeventd.enable = true;
};
resolved.settings.Resolve = {
LLMNR = false;
MulticastDNS = false;
};
netdata.enable = true;
iperf3 = {
enable = true;
openFirewall = true;
};
};
systemd = {
services =
let
waitOnline = "systemd-networkd-wait-online@wan.service";
in
{
bird2 = {
after = [ waitOnline ];
# requires = [ waitOnline ];
};
ipsec = {
after = [ waitOnline ];
requires = [ waitOnline ];
};
};
};
#systemd.services.systemd-networkd.environment.SYSTEMD_LOG_LEVEL = "debug";
systemd.network = {
wait-online.enable = false;
config = {
networkConfig = {
ManageForeignRoutes = false;
};
};
netdevs = mkMerge [
(mkVLAN "ifog" 409)
(mkVLAN "clue-ix" 121)
(mkVLAN "frys-ix" 701)
(mkVLAN "nl-ix" 1845)
(mkVLAN "fogixp" 1147)
(mkVLAN "ifog-transit" 702)
{
"30-kelder" = {
netdevConfig = {
Name = "kelder";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets."estuary/kelder-wg.key".path;
ListenPort = lib.my.c.kelder.vpn.port;
};
wireguardPeers = [
{
PublicKey = "7N9YdQaCMWWIwAnW37vrthm9ZpbnG4Lx3gheHeRYz2E=";
AllowedIPs = [ allAssignments.kelder.estuary.ipv4.address ];
PersistentKeepalive = 25;
}
];
};
}
{
"30-hillcrest" = {
netdevConfig = {
Name = "hillcrest";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets."estuary/hillcrest-wg.key".path;
ListenPort = lib.my.c.hillcrest.vpn.port;
};
wireguardPeers = [
{
PublicKey = "+67Ks+ZRk1ssNCfg5BFKmIE9NtLasAxRE6XMqufx5GY=";
AllowedIPs = [ (net.cidr.host 2 prefixes.hillcrest.v4) ];
PersistentKeepalive = 25;
}
];
};
}
{
"30-john-valorant" = {
netdevConfig = {
Name = "john-valorant";
Kind = "wireguard";
};
wireguardConfig = {
PrivateKeyFile = config.age.secrets."estuary/john-valorant-wg.key".path;
ListenPort = lib.my.c.john-valorant.vpn.port;
};
wireguardPeers = [
{
PublicKey = "xyqKF0yOAv1bObN1paL2vATFh77pdFfvN+JmuAxaTCk=";
AllowedIPs = [ (net.cidr.host 2 prefixes.john-valorant.v4) ];
PersistentKeepalive = 25;
}
];
};
}
];
links = {
"10-wan" = {
matchConfig = {
Driver = "igb";
Path = "pci-0000:01:00.0";
# Matching against MAC address seems to break VLAN interfaces (since they share the same MAC address)
#MACAddress = "d0:50:99:fa:a7:99";
};
linkConfig = {
Name = "wan";
RxBufferSize = 4096;
TxBufferSize = 4096;
MTUBytes = "9000";
};
};
# Mellanox ConnectX-2
#"10-wan" = {
# matchConfig.MACAddress = "00:02:c9:56:24:6e";
# linkConfig.Name = "wan";
#};
"10-base" = {
matchConfig.MACAddress = "52:54:00:15:1a:53";
linkConfig.Name = "base";
};
};
networks =
let
mkIXPConfig = name: ipv4: ipv6: {
"85-${name}" = {
matchConfig.Name = name;
address = optional (ipv4 != null) ipv4 ++ [ ipv6 ];
linkConfig.MTUBytes = "1500";
networkConfig = {
DHCP = "no";
LLDP = false;
EmitLLDP = false;
IPv6AcceptRA = false;
};
};
};
in
mkMerge
[
(mkIXPConfig "clue-ix" null "2001:7f8:154::21:1024:1/64")
(mkIXPConfig "frys-ix" "185.1.160.196/23" "2001:7f8:10f::3:3850:196/64")
(mkIXPConfig "nl-ix" "193.239.116.145/22" "2001:7f8:13::a521:1024:1/64")
(mkIXPConfig "fogixp" "185.1.147.159/24" "2001:7f8:ca:1::159/64")
{
"80-wan" = {
matchConfig.Name = "wan";
vlan = [ "ifog" "clue-ix" ];
DHCP = "no";
address = with assignments.internal; [
(with ipv4; "${address}/${toString mask}")
(with ipv6; "${address}/${toString mask}")
];
gateway = with assignments.internal; [
ipv4.gateway
ipv6.gateway
];
networkConfig = {
# We're using an explicit gateway and Linux uses link local address for neighbour discovery, so we
# get lost to the router... (this was true in 23M Frankfurt)
#LinkLocalAddressing = "no";
IPv6AcceptRA = false;
};
};
"85-ifog" = {
matchConfig = {
Name = "ifog";
Kind = "vlan";
};
vlan = [ "frys-ix" "nl-ix" "fogixp" "ifog-transit" ];
networkConfig = networkd.noL3;
};
"85-ifog-transit" = {
matchConfig.Name = "ifog-transit";
address = [ "2a0c:9a40:100f:370::2/64" ];
linkConfig.MTUBytes = "1500";
networkConfig = {
DHCP = "no";
LLDP = false;
EmitLLDP = false;
IPv6AcceptRA = false;
};
};
"80-base" = mkMerge [
(networkdAssignment "base" assignments.base)
{
dns = [ "127.0.0.1" "::1" ];
domains = [ config.networking.domain ];
networkConfig = {
IPv6AcceptRA = mkForce false;
IPv6SendRA = true;
};
ipv6SendRAConfig = {
DNS = [ assignments.base.ipv6.address ];
Domains = [ config.networking.domain ];
};
ipv6Prefixes = [
{
Prefix = prefixes.base.v6;
}
];
routes = flatten ([
{
Destination = prefixes.vip1;
Gateway = allAssignments.colony.routing.ipv4.address;
}
{
Destination = prefixes.vip3;
Gateway = allAssignments.colony.routing.ipv4.address;
}
{
Destination = prefixes.darts.v4;
Gateway = allAssignments.colony.routing.ipv4.address;
}
{
Destination = prefixes.cust.v6;
Gateway = allAssignments.colony.internal.ipv6.address;
}
{
Destination = lib.my.c.tailscale.prefix.v4;
Gateway = allAssignments.colony.routing.ipv4.address;
}
{
Destination = lib.my.c.tailscale.prefix.v6;
Gateway = allAssignments.colony.internal.ipv6.address;
}
{
Destination = prefixes.qclk.v4;
Gateway = allAssignments.colony.routing.ipv4.address;
}
] ++
(map (pName: [
{
Gateway = allAssignments.colony.routing.ipv4.address;
Destination = prefixes."${pName}".v4;
}
{
Destination = prefixes."${pName}".v6;
Gateway = allAssignments.colony.internal.ipv6.address;
}
]) [ "vms" "ctrs" "oci" ]));
}
];
"90-l2mesh-as211024" = mkMerge [
(networkdAssignment "as211024" assignments.as211024)
{
matchConfig.Name = "as211024";
networkConfig.IPv6AcceptRA = mkForce false;
routes = [
{
Destination = lib.my.c.home.prefixes.all.v4;
Gateway = lib.my.c.home.vips.as211024.v4;
}
];
}
];
"95-kelder" = {
matchConfig.Name = "kelder";
routes = [
{
Destination = allAssignments.kelder.estuary.ipv4.address;
Scope = "link";
}
];
};
"95-hillcrest" = {
matchConfig.Name = "hillcrest";
address = [ "${net.cidr.host 1 prefixes.hillcrest.v4}/32" ];
routes = [
{
Destination = net.cidr.host 2 prefixes.hillcrest.v4;
Scope = "link";
}
];
};
"95-john-valorant" = {
matchConfig.Name = "john-valorant";
address = [ "${net.cidr.host 1 prefixes.john-valorant.v4}/32" ];
routes = [
{
Destination = net.cidr.host 2 prefixes.john-valorant.v4;
Scope = "link";
}
];
};
} ];
};
my = {
secrets = {
key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIF9up7pXu6M/OWCKufTOfSiGcxMUk4VqUe7fLuatNFFA";
files = {
"estuary/kelder-wg.key" = {
owner = "systemd-network";
};
"estuary/hillcrest-wg.key" = {
owner = "systemd-network";
};
"estuary/john-valorant-wg.key" = {
owner = "systemd-network";
};
"l2mesh/as211024.key" = {};
};
};
server.enable = true;
vpns = {
l2.pskFiles = {
as211024 = config.age.secrets."l2mesh/as211024.key".path;
};
};
firewall = {
udp.allowed = [
5353
lib.my.c.kelder.vpn.port
lib.my.c.hillcrest.vpn.port
lib.my.c.john-valorant.vpn.port
];
tcp.allowed = [ 5353 "bgp" ];
nat = {
enable = true;
externalInterface = "wan";
forwardPorts."${assignments.internal.ipv4.address}" = firewallForwards allAssignments;
};
extraRules =
let
aa = allAssignments;
matchInet = rule: sys: ''
ip daddr ${aa."${sys}".internal.ipv4.address} ${rule}
ip6 daddr ${aa."${sys}".internal.ipv6.address} ${rule}
'';
in
''
define ixps = { clue-ix, frys-ix, nl-ix, fogixp, ifog-transit }
table inet filter {
chain input {
iifname base meta l4proto { udp, tcp } th dport domain accept
}
chain routing-tcp {
# Safe enough to allow all SSH
tcp dport ssh accept
# jam-ctr forwards
ip daddr ${aa.shill.internal.ipv4.address} tcp dport 60022 accept
ip6 daddr ${aa.middleman.internal.ipv6.address} tcp dport { http, https, 8448 } accept
${matchInet "tcp dport { http, https } accept" "git"}
ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} tcp dport 25565 accept
ip6 daddr ${aa.simpcraft-staging-oci.internal.ipv6.address} tcp dport 25565 accept
ip6 daddr ${aa.kevcraft-oci.internal.ipv6.address} tcp dport 25567 accept
ip6 daddr ${aa.kinkcraft-oci.internal.ipv6.address} tcp dport 25568 accept
ip6 daddr ${aa.graeme-oci.internal.ipv6.address} tcp dport 25569 accept
ip6 daddr ${aa.gam.internal.ipv6.address} tcp dport 7777 accept
return
}
chain routing-udp {
ip6 daddr ${aa.valheim-oci.internal.ipv6.address} udp dport { 2456-2457 } accept
ip6 daddr ${aa.waffletail.internal.ipv6.address} udp dport 41641 accept
ip6 daddr ${aa.simpcraft-oci.internal.ipv6.address} udp dport 25565 accept
ip6 daddr ${aa.enshrouded-oci.internal.ipv6.address} udp dport { 15636-15637 } accept
ip6 daddr ${aa.kevcraft-oci.internal.ipv6.address} udp dport 25567 accept
ip6 daddr ${aa.kinkcraft-oci.internal.ipv6.address} udp dport 25568 accept
ip6 daddr ${aa.graeme-oci.internal.ipv6.address} udp dport 25569 accept
ip6 daddr ${aa.gam.internal.ipv6.address} udp dport 7777 accept
return
}
chain filter-routing {
ip daddr { ${prefixes.mail.v4}, ${prefixes.darts.v4} } accept
ip6 daddr ${prefixes.cust.v6} accept
tcp flags & (fin|syn|rst|ack) == syn ct state new jump routing-tcp
meta l4proto udp ct state new jump routing-udp
return
}
chain ixp {
ether type != { ip, ip6, arp, vlan } reject
return
}
chain forward {
${lib.my.c.as211024.nftTrust}
iifname { wan, as211024, $ixps } oifname base jump filter-routing
oifname $ixps jump ixp
iifname base oifname { base, wan, $ixps } accept
oifname { as211024, kelder, hillcrest, john-valorant } accept
}
chain output {
oifname ifog ether type != vlan reject
oifname $ixps jump ixp
}
}
table inet nat {
chain prerouting {
${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" "estuary"}
}
chain postrouting {
oifname hillcrest snat ip to ${net.cidr.host 1 prefixes.hillcrest.v4}
oifname john-valorant snat ip to ${net.cidr.host 1 prefixes.john-valorant.v4}
ip saddr ${prefixes.all.v4} oifname != as211024 snat to ${assignments.internal.ipv4.address}
}
}
'';
};
};
}
(mkIf config.my.build.isDevVM {
systemd.network = {
netdevs."05-dummy-base".netdevConfig = {
Name = "base";
Kind = "dummy";
};
};
})
];
};
};
}