nixos: Add initial palace

2023-12-03 22:58:28 +00:00
parent a1778e0f1e
commit c4e6896a0c
58 changed files with 1066 additions and 837 deletions
--- a/nixos/boxes/castle/default.nix
+++ b/nixos/boxes/castle/default.nix
@@ -1,12 +1,37 @@
-{ lib, ... }: {
+{ lib, ... }:
+let
+  inherit (lib.my) net;
+  inherit (lib.my.c.home) domain vlans prefixes;
+in
+{
  nixos.systems.castle = {
    system = "x86_64-linux";
    nixpkgs = "mine";
    home-manager = "mine";

+    assignments = {
+      hi = {
+        inherit domain;
+        ipv4 = {
+          address = net.cidr.host 40 prefixes.hi.v4;
+          mask = 22;
+          gateway = null;
+        };
+      };
+      lo = {
+        inherit domain;
+        ipv4 = {
+          address = net.cidr.host 40 prefixes.lo.v4;
+          mask = 21;
+          gateway = null;
+        };
+      };
+    };
+
    configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
      let
        inherit (lib) mkIf mkMerge mkForce;
+        inherit (lib.my) mkVLAN networkdAssignment;
      in
      {
        hardware = {
@@ -117,6 +142,10 @@
        systemd = {
          network = {
            wait-online.enable = false;
+            netdevs = mkMerge [
+              (mkVLAN "lan-hi" vlans.hi)
+              (mkVLAN "lan-lo" vlans.lo)
+            ];
            links = {
              "10-et2.5g" = {
                matchConfig.MACAddress = "c8:7f:54:6e:17:0f";
@@ -127,8 +156,11 @@
                linkConfig.Name = "et10g";
              };
              "12-et100g" = {
-                matchConfig.MACAddress = "24:8a:07:a8:fe:3a";
-                linkConfig.Name = "et100g";
+                matchConfig.PermanentMACAddress = "24:8a:07:a8:fe:3a";
+                linkConfig = {
+                  Name = "et100g";
+                  MTUBytes = "9000";
+                };
              };
            };
            networks = {
@@ -136,6 +168,26 @@
                matchConfig.Name = "et2.5g";
                DHCP = "yes";
              };
+
+              "50-et100g" = {
+                matchConfig.Name = "et100g";
+                vlan = [ "lan-hi" "lan-lo" ];
+                networkConfig.IPv6AcceptRA = false;
+              };
+              "60-lan-hi" = mkMerge [
+                (networkdAssignment "lan-hi" assignments.hi)
+                {
+                  matchConfig.Name = "lan-hi";
+                  linkConfig.MTUBytes = "9000";
+                }
+              ];
+              "60-lan-lo" = mkMerge [
+                (networkdAssignment "lan-lo" assignments.lo)
+                {
+                  matchConfig.Name = "lan-lo";
+                  linkConfig.MTUBytes = "1500";
+                }
+              ];
            };
          };
        };
--- a/nixos/boxes/colony/vms/estuary/default.nix
+++ b/nixos/boxes/colony/vms/estuary/default.nix
@@ -1,7 +1,7 @@
 { lib, ... }:
 let
  inherit (builtins) elemAt;
-  inherit (lib.my) net;
+  inherit (lib.my) net mkVLAN;
  inherit (lib.my.c.colony) pubV4 domain prefixes;
 in
 {
@@ -141,19 +141,7 @@ in
                };
              };

-              netdevs =
-              let
-                mkVLAN = name: vid: {
-                  "25-${name}" = {
-                    netdevConfig = {
-                      Name = name;
-                      Kind = "vlan";
-                    };
-                    vlanConfig.Id = vid;
-                  };
-                };
-              in
-              mkMerge [
+              netdevs = mkMerge [
                (mkVLAN "ifog" 409)

                (mkVLAN "frys-ix" 701)
--- a/nixos/boxes/home/palace/default.nix
+++ b/nixos/boxes/home/palace/default.nix
@@ -0,0 +1,179 @@
+{ lib, ... }:
+let
+  inherit (lib.my) net mkVLAN;
+  inherit (lib.my.c) pubDomain;
+  inherit (lib.my.c.home) domain vlans prefixes vips;
+in
+{
+  nixos.systems.palace = {
+    system = "x86_64-linux";
+    nixpkgs = "mine-stable";
+    home-manager = "mine-stable";
+
+    assignments = {
+      hi = {
+        inherit domain;
+        ipv4 = {
+          address = net.cidr.host 22 prefixes.hi.v4;
+          mask = 22;
+          gateway = vips.hi.v4;
+        };
+      };
+      core = {
+        inherit domain;
+        name = "palace-core";
+        ipv4 = {
+          address = net.cidr.host 20 prefixes.core.v4;
+          gateway = null;
+        };
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, systems, assignments, allAssignments, ... }:
+      let
+        inherit (lib) mkForce mkMerge;
+        inherit (lib.my) networkdAssignment;
+      in
+      {
+        boot = {
+          kernelModules = [ "kvm-amd" ];
+          kernelParams = [ "amd_iommu=on" ];
+          initrd = {
+            availableKernelModules = [ "nvme" "xhci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" "sr_mod" ];
+          };
+        };
+
+        hardware = {
+          enableRedistributableFirmware = true;
+          cpu = {
+            amd.updateMicrocode = true;
+          };
+        };
+
+        fileSystems = {
+          "/boot" = {
+            device = "/dev/disk/by-partuuid/c06a8d24-2af9-4416-bf5e-cfe6defdbd47";
+            fsType = "vfat";
+          };
+          "/nix" = {
+            device = "/dev/disk/by-partuuid/71695225-b306-41e6-83f8-c5cde57c06f7";
+            fsType = "ext4";
+          };
+          "/persist" = {
+            device = "/dev/disk/by-partuuid/9991aec3-c062-41d1-971e-e056b63370f0";
+            fsType = "ext4";
+            neededForBoot = true;
+          };
+        };
+
+        services = {
+          smartd = {
+            enable = true;
+            autodetect = true;
+            extraOptions = [ "-A /var/log/smartd/" "--interval=600" ];
+          };
+        };
+
+        environment.systemPackages = with pkgs; [
+          pciutils
+          usbutils
+          partclone
+          lm_sensors
+          linuxPackages.cpupower
+          smartmontools
+          mstflint
+          ethtool
+        ];
+
+        networking.domain = "h.${pubDomain}";
+
+        systemd = {
+          tmpfiles.rules = [
+            "d /var/log/smartd 0755 root root"
+          ];
+
+          services = {
+            setup-mlx5-vfs = {
+              description = "Enable 100G NIC VFs";
+              serviceConfig = {
+                Type = "oneshot";
+              };
+              script = ''
+                vfsFile=/sys/class/infiniband/mlx5_0/device/sriov_numvfs
+                until [ -f "$vfsFile" ]; do
+                  sleep 0.2
+                done
+
+                echo 3 > "$vfsFile"
+              '';
+              wantedBy = [ "multi-user.target" ];
+              before = [ "network-pre.target" ];
+            };
+          };
+
+          network = {
+            links = {
+              "10-et1g0" = {
+                matchConfig.MACAddress = "e0:d5:5e:68:0c:6e";
+                linkConfig.Name = "et1g0";
+              };
+              "10-lan-core" = {
+                matchConfig.MACAddress = "e0:d5:5e:68:0c:70";
+                linkConfig.Name = "lan-core";
+              };
+              "10-et100g" = {
+                matchConfig = {
+                  PermanentMACAddress = "24:8a:07:ac:59:c0";
+                  Driver = "mlx5_core";
+                };
+                linkConfig = {
+                  Name = "et100g";
+                  MTUBytes = "9000";
+                };
+              };
+            };
+
+            netdevs = mkMerge [
+              (mkVLAN "lan-hi" vlans.hi)
+            ];
+
+            networks = {
+              "50-lan-core" = mkMerge [
+                (networkdAssignment "lan-core" assignments.core)
+                {
+                  matchConfig.Name = "lan-core";
+                  networkConfig.IPv6AcceptRA = mkForce false;
+                }
+              ];
+
+              "50-et100g" = {
+                matchConfig.Name = "et100g";
+                vlan = [ "lan-hi" ];
+                networkConfig.IPv6AcceptRA = false;
+              };
+              "60-lan-hi" = mkMerge [
+                (networkdAssignment "lan-hi" assignments.hi)
+                {
+                  matchConfig.Name = "lan-hi";
+                  linkConfig.MTUBytes = "9000";
+                  networkConfig.DNS = [
+                    (allAssignments.stream.hi.ipv4.address)
+                    # (allAssignments.river.hi.ipv4.address)
+                  ];
+                }
+              ];
+            };
+          };
+        };
+
+        my = {
+          secrets = {
+            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHzVJpoDz/AAYLJGzU8t6DgZ2sY3oehRqrlSO7C+GWiK";
+          };
+
+          server.enable = true;
+          deploy.node.hostname = "192.168.68.22";
+        };
+      };
+  };
+}
--- a/nixos/boxes/home/routing-common/default.nix
+++ b/nixos/boxes/home/routing-common/default.nix
@@ -1,9 +1,9 @@
 index: { lib, allAssignments, ... }:
 let
  inherit (builtins) elemAt;
-  inherit (lib.my) net;
+  inherit (lib.my) net mkVLAN;
  inherit (lib.my.c) pubDomain;
-  inherit (lib.my.c.home) domain vlans prefixes routers;
+  inherit (lib.my.c.home) domain vlans prefixes routers routersPubV4;

  name = elemAt routers index;
 in
@@ -59,7 +59,10 @@ in
          address = net.cidr.host (index + 2) prefixes.as211024.v4;
          gateway = null;
        };
-        ipv6.address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
+        ipv6 = {
+          address = net.cidr.host ((1*65536*65536*65536) + index + 1) prefixes.as211024.v6;
+          gateway = net.cidr.host 1 prefixes.as211024.v6;
+        };
      };
    };

@@ -70,7 +73,6 @@ in
      in
      {
        imports = map (m: import m index) [
-          ./mstpd.nix
          ./keepalived.nix
          ./dns.nix
        ];
@@ -134,19 +136,7 @@ in
              };
            };

-            netdevs =
-            let
-              mkVLAN = name: vid: {
-                "25-${name}" = {
-                  netdevConfig = {
-                    Name = name;
-                    Kind = "vlan";
-                  };
-                  vlanConfig.Id = vid;
-                };
-              };
-            in
-            mkMerge [
+            netdevs = mkMerge [
              {
                "25-wan-phy-ifb".netdevConfig = {
                  Name = "wan-phy-ifb";
@@ -156,16 +146,6 @@ in
                  Name = "wan";
                  Kind = "bridge";
                };
-                "25-lan" = {
-                  netdevConfig = {
-                    Name = "lan";
-                    Kind = "bridge";
-                  };
-                  extraConfig = ''
-                    [Bridge]
-                    STP=true
-                  '';
-                };
                "30-lan-core".netdevConfig = {
                  Name = "lan-core";
                  Kind = "macvlan";
@@ -270,14 +250,6 @@ in
                  }
                ];

-                "50-lan-jim" = {
-                  matchConfig.Name = "lan-jim";
-                  networkConfig.Bridge = "lan";
-                };
-                "50-lan-dave" = {
-                  matchConfig.Name = "lan-dave";
-                  networkConfig.Bridge = "lan";
-                };
                "55-lan" = {
                  matchConfig.Name = "lan";
                  vlan = [ "lan-hi" "lan-lo" "lan-untrusted" "wan-tunnel" ];
@@ -339,15 +311,7 @@ in
                  # }
                ];
              };
-              extraRules =
-              let
-                aa = allAssignments;
-                matchInet = rule: sys: ''
-                  ip daddr ${aa."${sys}".hi.ipv4.address} ${rule}
-                  ip6 daddr ${aa."${sys}".hi.ipv6.address} ${rule}
-                '';
-              in
-              ''
+              extraRules = ''
                table inet filter {
                  chain input {
                    iifname base meta l4proto { udp, tcp } th dport domain accept
@@ -381,7 +345,8 @@ in
                }
                table inet nat {
                  chain prerouting {
-                    ${matchInet "meta l4proto { udp, tcp } th dport domain redirect to :5353" name}
+                    ip daddr ${elemAt routersPubV4 index} meta l4proto { udp, tcp } th dport domain redirect to :5353
+                    ip6 daddr ${assignments.as211024.ipv6.address} meta l4proto { udp, tcp } th dport domain redirect to :5353
                  }
                  chain postrouting {
                    oifname wan masquerade
--- a/nixos/boxes/home/routing-common/dns.nix
+++ b/nixos/boxes/home/routing-common/dns.nix
@@ -145,6 +145,7 @@ in
            dave-lo IN A ${net.cidr.host 11 prefixes.lo.v4}

            ups IN A ${net.cidr.host 20 prefixes.lo.v4}
+            palace-kvm IN A ${net.cidr.host 21 prefixes.lo.v4}

            ${lib.my.dns.fwdRecords {
              inherit allAssignments names;
--- a/nixos/boxes/home/routing-common/mstpd.nix
+++ b/nixos/boxes/home/routing-common/mstpd.nix
@@ -1,4 +1,4 @@
-index: { lib, pkgs, ... }:
+{ lib, pkgs, ... }:
 let
  # TODO: Move into nixpkgs
  mstpd = pkgs.mstpd.overrideAttrs {
--- a/nixos/boxes/home/stream.nix
+++ b/nixos/boxes/home/stream.nix
@@ -11,6 +11,8 @@
      inherit (lib);
    in
    {
+      imports = [ ./routing-common/mstpd.nix ];
+
      config = {
        boot = {
          kernelModules = [ "kvm-intel" ];
@@ -42,6 +44,18 @@
        };

        systemd.network = {
+          netdevs = {
+            "25-lan" = {
+              netdevConfig = {
+                Name = "lan";
+                Kind = "bridge";
+              };
+              extraConfig = ''
+                [Bridge]
+                STP=true
+              '';
+            };
+          };
          links = {
            "10-wan-phy" = {
              matchConfig = {
@@ -92,6 +106,16 @@
              linkConfig.Name = "et5";
            };
          };
+          networks = {
+            "50-lan-jim" = {
+              matchConfig.Name = "lan-jim";
+              networkConfig.Bridge = "lan";
+            };
+            "50-lan-dave" = {
+              matchConfig.Name = "lan-dave";
+              networkConfig.Bridge = "lan";
+            };
+          };
        };

        my = {
@@ -99,7 +123,7 @@
            key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPYTB4zeAqotrEJ8M+AiGm/s9PFsWlAodz3hYSROGuDb";
          };
          server.enable = true;
-          deploy.node.hostname = "192.168.72.2";
+          deploy.node.hostname = "192.168.68.2";
        };
      };
    };