nixos/shill: Replicate port forwards for internal routing

lib/constants.nix

@@ -114,6 +114,33 @@ rec {
       };
       home.v6 = "2a0e:97c0:4d0::/48";
     };
+
+    firewallForwards = aa: [
+      {
+        port = "http";
+        dst = aa.middleman.internal.ipv4.address;
+      }
+      {
+        port = "https";
+        dst = aa.middleman.internal.ipv4.address;
+      }
+      {
+        port = 8448;
+        dst = aa.middleman.internal.ipv4.address;
+      }
+
+      {
+        port = 2456;
+        dst = aa.valheim-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+      {
+        port = 2457;
+        dst = aa.valheim-oci.internal.ipv4.address;
+        proto = "udp";
+      }
+    ];
+
     fstrimConfig = {
       enable = true;
       # backup happens at 05:00

lib/default.nix

@@ -1,6 +1,6 @@
 { lib }:
 let
-  inherit (builtins) length match elemAt filter;
+  inherit (builtins) length match elemAt filter replaceStrings;
   inherit (lib)
     genAttrs mapAttrsToList filterAttrsRecursive nameValuePair types
     mkOption mkOverride mkForce mkIf mergeEqualOption optional
@@ -123,6 +123,12 @@ rec {
     home-manager = mkOpt' (enum [ "unstable" "stable" "mine" "mine-stable" ]) "unstable" "Branch of home-manager to use.";
   };
 
+  nft = rec {
+    ipEscape = replaceStrings ["." ":"] ["-" "-"];
+    natFilterChain = ip: "filter-fwd-${ipEscape ip}";
+    dnatChain = ip: "fwd-${ipEscape ip}";
+  };
+
   mkVLAN = name: vid: {
     "25-${name}" = {
       netdevConfig = {

nixos/boxes/colony/default.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 let
   inherit (lib.my) net;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my.c.colony) domain prefixes firewallForwards;
 in
 {
   imports = [ ./vms ];
@@ -351,6 +351,7 @@ in
 
           firewall = {
             trustedInterfaces = [ "vms" ];
+            nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
             extraRules = ''
               define cust = { vm-mail, vm-darts }
               table inet filter {

nixos/boxes/colony/vms/estuary/default.nix

@@ -2,7 +2,7 @@
 let
   inherit (builtins) elemAt;
   inherit (lib.my) net mkVLAN;
-  inherit (lib.my.c.colony) pubV4 domain prefixes;
+  inherit (lib.my.c.colony) pubV4 domain prefixes firewallForwards;
 in
 {
   nixos = {
@@ -356,32 +356,7 @@ in
                 nat = {
                   enable = true;
                   externalInterface = "wan";
-                  externalIP = assignments.internal.ipv4.address;
-                  forwardPorts = [
-                    {
-                      port = "http";
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-                    {
-                      port = "https";
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-                    {
-                      port = 8448;
-                      dst = allAssignments.middleman.internal.ipv4.address;
-                    }
-
-                    {
-                      port = 2456;
-                      dst = allAssignments.valheim-oci.internal.ipv4.address;
-                      proto = "udp";
-                    }
-                    {
-                      port = 2457;
-                      dst = allAssignments.valheim-oci.internal.ipv4.address;
-                      proto = "udp";
-                    }
-                  ];
+                  forwardPorts."${assignments.internal.ipv4.address}" = firewallForwards allAssignments;
                 };
                 extraRules =
                 let

nixos/boxes/colony/vms/shill/default.nix

@@ -1,7 +1,7 @@
 { lib, ... }:
 let
-  inherit (lib.my) net;
-  inherit (lib.my.c.colony) domain prefixes;
+  inherit (lib.my) net nft;
+  inherit (lib.my.c.colony) domain prefixes firewallForwards;
 in
 {
   imports = [ ./containers ];
@@ -151,6 +151,7 @@ in
               firewall = {
                 tcp.allowed = [ 19999 ];
                 trustedInterfaces = [ "ctrs" ];
+                nat.forwardPorts."${allAssignments.estuary.internal.ipv4.address}" = firewallForwards allAssignments;
                 extraRules = ''
                   table inet filter {
                     chain forward {
@@ -158,6 +159,17 @@ in
                       iifname vms oifname ctrs accept
                     }
                   }
+                  table inet nat {
+                    # Hack to fix our NAT situation with internal routing
+                    # We need to snat to our public IP, otherwise on the return path from e.g. middleman it will
+                    # try to forward packet directly with its own IP, bypassing our carefully crafted DNAT...
+                    chain ${nft.dnatChain allAssignments.estuary.internal.ipv4.address} {
+                      ct mark set 0x1337
+                    }
+                    chain postrouting {
+                      ct mark 0x1337 snat ip to ${assignments.internal.ipv4.address}
+                    }
+                  }
                 '';
               };
 

nixos/modules/firewall.nix

@@ -1,6 +1,9 @@
 { lib, options, config, ... }:
 let
-  inherit (lib) optionalString concatStringsSep concatMapStringsSep optionalAttrs mkIf mkDefault mkMerge mkOverride;
+  inherit (builtins) typeOf attrNames;
+  inherit (lib)
+    optionalString concatStringsSep concatMapStringsSep mapAttrsToList optionalAttrs mkIf
+    mkDefault mkMerge mkOverride;
   inherit (lib.my) isIPv6 mkOpt' mkBoolOpt';
 
   allowICMP = ''
@@ -63,8 +66,8 @@ in
 
     nat = with options.networking.nat; {
       enable = mkBoolOpt' true "Whether to enable IP forwarding and NAT.";
-      inherit externalInterface externalIP;
-      forwardPorts = mkOpt' (listOf (submodule forwardOpts)) [ ] "List of port forwards.";
+      inherit externalInterface;
+      forwardPorts = mkOpt' (either (listOf (submodule forwardOpts)) (attrsOf (listOf (submodule forwardOpts)))) [ ] "IPv4 port forwards";
     };
   };
 
@@ -137,6 +140,9 @@ in
                 chain postrouting {
                   type nat hook postrouting priority srcnat;
                 }
+                chain input {
+                  type nat hook input priority srcnat;
+                }
               }
 
               ${cfg.extraRules}
@@ -144,11 +150,16 @@ in
         };
       };
     }
-    (mkIf cfg.nat.enable {
+    (mkIf cfg.nat.enable (
+    let
+      iifForward = typeOf cfg.nat.forwardPorts == "list" && cfg.nat.forwardPorts != [ ];
+      dipForward = typeOf cfg.nat.forwardPorts == "set" && cfg.nat.forwardPorts != { };
+    in
+    {
       assertions = [
         {
-          assertion = with cfg.nat; (forwardPorts != [ ]) -> (externalInterface != null);
-          message = "my.firewall.nat.forwardPorts requires my.firewall.nat.external{Interface,IP}";
+          assertion = with cfg.nat; iifForward -> (externalInterface != null);
+          message = "my.firewall.nat.forwardPorts as list requires my.firewall.nat.externalInterface";
         }
       ];
 
@@ -171,43 +182,75 @@ in
 
       my.firewall.extraRules =
         let
+          inherit (lib.my.nft) natFilterChain dnatChain;
+          ipK = ip: "ip${optionalString (isIPv6 ip) "6"}";
+
           makeFilter = f:
-          let
-            v6 = isIPv6 f.dst;
-          in
-            "ip${optionalString v6 "6"} daddr ${f.dst} ${f.proto} dport ${toString f.dstPort} accept";
+            "${ipK f.dst} daddr ${f.dst} ${f.proto} dport ${toString f.dstPort} accept";
           makeForward = f:
-            let
-              v6 = isIPv6 f.dst;
-            in
-              "${f.proto} dport ${toString f.port} dnat ip${optionalString v6 "6"} to ${f.dst}:${toString f.dstPort}";
+            "${f.proto} dport ${toString f.port} dnat ${ipK f.dst} to ${f.dst}:${toString f.dstPort}";
+
+          dnatJumps = ''
+            ${optionalString
+              iifForward
+              "iifname ${cfg.nat.externalInterface} jump iif-port-forward"}
+            ${optionalString
+              dipForward
+              (concatMapStringsSep "\n    " (ip: "${ipK ip} daddr ${ip} jump ${dnatChain ip}") (attrNames cfg.nat.forwardPorts))}
+          '';
         in
         ''
           table inet filter {
-            chain filter-port-forwards {
+            ${optionalString iifForward ''
+              chain filter-iif-port-forwards {
                 ${concatMapStringsSep "\n    " makeFilter cfg.nat.forwardPorts}
                 return
               }
+            ''}
+            ${optionalString
+              dipForward
+              (concatStringsSep "\n" (mapAttrsToList (ip: fs: ''
+                chain ${natFilterChain ip} {
+                  ${concatMapStringsSep "\n    " makeFilter fs}
+                  return
+                }
+              '') cfg.nat.forwardPorts))}
+
             chain forward {
               ${optionalString
-                (cfg.nat.externalInterface != null)
-                "iifname ${cfg.nat.externalInterface} jump filter-port-forwards"}
+                iifForward
+                "iifname ${cfg.nat.externalInterface} jump filter-iif-port-forwards"}
+              ${optionalString
+                dipForward
+                (concatMapStringsSep "\n    " (ip: "${ipK ip} daddr ${ip} jump ${natFilterChain ip}") (attrNames cfg.nat.forwardPorts))}
             }
           }
 
           table inet nat {
-            chain port-forward {
+            ${optionalString iifForward ''
+              chain iif-port-forward {
                 ${concatMapStringsSep "\n    " makeForward cfg.nat.forwardPorts}
                 return
               }
-            chain prerouting {
+            ''}
             ${optionalString
-                (cfg.nat.externalInterface != null)
-                "${if (cfg.nat.externalIP != null) then "ip daddr ${cfg.nat.externalIP}" else "iifname ${cfg.nat.externalInterface}"} jump port-forward"}
+              dipForward
+              (concatStringsSep "\n" (mapAttrsToList (ip: fs: ''
+                chain ${dnatChain ip} {
+                  ${concatMapStringsSep "\n    " makeForward fs}
+                  return
+                }
+              '') cfg.nat.forwardPorts))}
+
+            chain prerouting {
+              ${dnatJumps}
+            }
+            chain output {
+              ${dnatJumps}
             }
           }
         '';
-    })
+    }))
   ]);
 
   meta.buildDocsInSandbox = false;