Add OCI container VM

2022-08-01 17:44:08 +01:00
parent 12aef1e47b
commit 9750bc5052
25 changed files with 267 additions and 55 deletions
--- a/lib/default.nix
+++ b/lib/default.nix
@@ -217,6 +217,10 @@ rec {
        v4 = "${start.all.v4}2.";
        v6 = "${start.all.v6}2::";
      };
+      oci = {
+        v4 = "${start.all.v4}3.";
+        v6 = "${start.all.v6}3::";
+      };
    };
    prefixes = {
      all = {
@@ -232,6 +236,10 @@ rec {
        v4 = "${start.ctrs.v4}0/24";
        v6 = "${start.ctrs.v6}/64";
      };
+      oci = {
+        v4 = "${start.oci.v4}0/24";
+        v6 = "${start.oci.v6}/64";
+      };
    };
  };
  sshKeyFiles = {
--- a/nixos/boxes/colony/default.nix
+++ b/nixos/boxes/colony/default.nix
@@ -100,6 +100,7 @@
          lm_sensors
          linuxPackages.cpupower
          smartmontools
+          xfsprogs
        ];

        systemd = {
@@ -179,6 +180,15 @@
                      Gateway = allAssignments.shill.internal.ipv6.address;
                      Destination = lib.my.colony.prefixes.ctrs.v6;
                    }
+
+                    {
+                      Gateway = allAssignments.whale2.internal.ipv4.address;
+                      Destination = lib.my.colony.prefixes.oci.v4;
+                    }
+                    {
+                      Gateway = allAssignments.whale2.internal.ipv6.address;
+                      Destination = lib.my.colony.prefixes.oci.v6;
+                    }
                  ];
                }
              ];
--- a/nixos/boxes/colony/vms/default.nix
+++ b/nixos/boxes/colony/vms/default.nix
@@ -2,6 +2,7 @@
  imports = [
    ./estuary
    ./shill
+    ./whale2
  ];

  nixos.systems.colony.configuration = { lib, pkgs, config, systems, ... }:
@@ -28,7 +29,8 @@
      name = "installer";
      backend = {
        driver = "file";
-        filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos-installer-devplayer0.iso";
+        #filename = "${systems.installer.configuration.config.my.buildAs.iso}/iso/nixos-installer-devplayer0.iso";
+        filename = "/persist/home/dev/nixos-installer-devplayer0.iso";
        read-only = "on";
      };
      format.driver = "raw";
@@ -164,6 +166,36 @@
              }
            ]);
          };
+
+          whale2 = {
+            uuid = "6d31b672-1f32-4e2b-a39f-78a5b5e949a0";
+            cpu = "host,topoext";
+            smp = {
+              cpus = 8;
+              threads = 2;
+            };
+            memory = 16384;
+            networks.vms.mac = "52:54:00:d5:d9:c6";
+            cleanShutdown.timeout = 120;
+            drives = [ ] ++ (optionals (!config.my.build.isDevVM) [
+              (mkMerge [ (vmLVM "whale2" "esp") { frontendOpts.bootindex = 0; } ])
+              (vmLVM "whale2" "nix")
+              (vmLVM "whale2" "persist")
+              {
+                name = "oci";
+                backend = {
+                  driver = "host_device";
+                  filename = "/dev/ssds/oci";
+                  discard = "unmap";
+                };
+                format = {
+                  driver = "raw";
+                  discard = "unmap";
+                };
+                frontend = "virtio-blk";
+              }
+            ]);
+          };
        };
      };
    };
--- a/nixos/boxes/colony/vms/estuary/default.nix
+++ b/nixos/boxes/colony/vms/estuary/default.nix
@@ -35,7 +35,7 @@

    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
      let
-        inherit (lib) mkIf mkMerge mkForce;
+        inherit (lib) flatten mkIf mkMerge mkForce;
        inherit (lib.my) networkdAssignment;
      in
      {
@@ -150,25 +150,18 @@
                        ipv6PrefixConfig.Prefix = lib.my.colony.prefixes.base.v6;
                      }
                    ];
-                    routes = map (r: { routeConfig = r; }) [
-                      {
-                        Gateway = allAssignments.colony.internal.ipv4.address;
-                        Destination = lib.my.colony.prefixes.vms.v4;
-                      }
-                      {
-                        Gateway = allAssignments.colony.internal.ipv6.address;
-                        Destination = lib.my.colony.prefixes.vms.v6;
-                      }
-
-                      {
-                        Gateway = allAssignments.colony.internal.ipv4.address;
-                        Destination = lib.my.colony.prefixes.ctrs.v4;
-                      }
-                      {
-                        Gateway = allAssignments.colony.internal.ipv6.address;
-                        Destination = lib.my.colony.prefixes.ctrs.v6;
-                      }
-                    ];
+                    routes = map (r: { routeConfig = r; }) (flatten
+                      ([  ] ++
+                      (map (pName: [
+                        {
+                          Gateway = allAssignments.colony.internal.ipv4.address;
+                          Destination = lib.my.colony.prefixes."${pName}".v4;
+                        }
+                        {
+                          Gateway = allAssignments.colony.internal.ipv6.address;
+                          Destination = lib.my.colony.prefixes."${pName}".v6;
+                        }
+                      ]) [ "vms" "ctrs" "oci" ])));
                  }
                ];
              };
--- a/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix
+++ b/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix
@@ -89,7 +89,7 @@ in
      let
        hosts = [
          "vm"
-          "fw" "ctr"
+          "fw" "ctr" "oci"
          "http" "jackflix-ctr" "chatterbox-ctr" "colony-psql-ctr"
        ];
        matchHosts = concatStringsSep "|" hosts;
--- a/nixos/boxes/colony/vms/whale2/default.nix
+++ b/nixos/boxes/colony/vms/whale2/default.nix
@@ -0,0 +1,154 @@
+{ lib, ... }: {
+  nixos.systems.whale2 = {
+    system = "x86_64-linux";
+    nixpkgs = "mine";
+
+    assignments = {
+      internal = {
+        name = "whale-vm";
+        altNames = [ "oci" ];
+        domain = lib.my.colony.domain;
+        ipv4.address = "${lib.my.colony.start.vms.v4}3";
+        ipv6 = {
+          iid = "::3";
+          address = "${lib.my.colony.start.vms.v6}3";
+        };
+      };
+      oci = {
+        name = "whale-vm-oci";
+        domain = lib.my.colony.domain;
+        ipv4 = {
+          address = "${lib.my.colony.start.oci.v4}1";
+          gateway = null;
+        };
+        ipv6.address = "${lib.my.colony.start.oci.v6}1";
+      };
+    };
+
+    configuration = { lib, pkgs, modulesPath, config, assignments, allAssignments, ... }:
+      let
+        inherit (builtins) mapAttrs toJSON;
+        inherit (lib) mkIf mkMerge mkForce;
+        inherit (lib.my) networkdAssignment;
+      in
+      {
+        imports = [
+          "${modulesPath}/profiles/qemu-guest.nix"
+
+
+        ];
+
+        config = mkMerge [
+          {
+            boot = {
+              kernelParams = [ "console=ttyS0,115200n8" ];
+            };
+
+            fileSystems = {
+              "/boot" = {
+                device = "/dev/disk/by-label/ESP";
+                fsType = "vfat";
+              };
+              "/nix" = {
+                device = "/dev/disk/by-label/nix";
+                fsType = "ext4";
+              };
+              "/persist" = {
+                device = "/dev/disk/by-label/persist";
+                fsType = "ext4";
+                neededForBoot = true;
+              };
+
+              "/var/lib/containers" = {
+                device = "/dev/disk/by-label/oci";
+                fsType = "xfs";
+                options = [ "pquota" ];
+              };
+            };
+
+            services = {
+              fstrim.enable = true;
+              netdata.enable = true;
+            };
+
+            virtualisation = {
+              podman = {
+                enable = true;
+              };
+            };
+
+            environment = {
+              etc = {
+                "cni/net.d/90-colony.conflist".text = toJSON {
+                  cniVersion = "0.4.0";
+                  name = "colony";
+                  plugins = [
+                    {
+                      type = "bridge";
+                      bridge = "oci";
+                      isGateway = true;
+                      ipMasq = false;
+                      hairpinMode = true;
+                      ipam = {
+                        type = "host-local";
+                        routes = [
+                          { dst = "0.0.0.0/0"; }
+                          { dst = "::/0"; }
+                        ];
+                        ranges = [
+                          [
+                            {
+                              subnet = lib.my.colony.prefixes.oci.v4;
+                              gateway = lib.my.colony.start.oci.v4 + "1";
+                            }
+                          ]
+                          [
+                            {
+                              subnet = lib.my.colony.prefixes.oci.v6;
+                              gateway = lib.my.colony.start.oci.v6 + "1";
+                            }
+                          ]
+                        ];
+                      };
+                      capabilities.ips = true;
+                    }
+                  ];
+                };
+              };
+            };
+
+            systemd.network = {
+              links = {
+                "10-vms" = {
+                  matchConfig.MACAddress = "52:54:00:d5:d9:c6";
+                  linkConfig.Name = "vms";
+                };
+              };
+
+              networks = {
+                "80-vms" = networkdAssignment "vms" assignments.internal;
+              };
+            };
+
+            my = {
+              secrets.key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDBTIj1jVdknXLNNroMJfgy7S2cSUC/qgFdnaUopEUzZ";
+              server.enable = true;
+
+              firewall = {
+                tcp.allowed = [ 19999 ];
+                trustedInterfaces = [ "oci" ];
+                extraRules = ''
+                  table inet filter {
+                    chain forward {
+                      # Trust that the outer firewall has done the filtering!
+                      iifname vms oifname oci accept
+                    }
+                  }
+                '';
+              };
+            };
+          }
+        ];
+      };
+  };
+}
--- a/nixos/modules/tmproot.nix
+++ b/nixos/modules/tmproot.nix
@@ -129,6 +129,8 @@ in
          "/root/.nix-defexpr"

          "/var/lib/logrotate.status"
+
+          "/etc/cni/net.d/cni.lock"
        ];
        persistence.config = {
          # In impermanence the key in `environment.persistence.*` (aka name passed the attrsOf submodule) sets the
@@ -326,6 +328,15 @@ in
          }
        ];
      })
+      (mkIf config.virtualisation.podman.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = "/var/cache/containers";
+            mode = "750";
+          }
+          "/var/lib/cni"
+        ];
+      })
      (mkIf config.my.build.isDevVM {
        fileSystems = mkVMOverride {
          # Hijack the "root" device for persistence in the VM
--- a/secrets/chatterbox/nul.ie.signing.key.age
+++ b/secrets/chatterbox/nul.ie.signing.key.age
@@ -1,9 +1,10 @@
 age-encryption.org/v1
-> ssh-ed25519 ZB3e6Q sQJFhvr8FRUhNhBMue77730wcbg28fTFnsszgerwEBo
-7VzmwSkllK2wbSyFSCClvjY4X6sT6vLLPBAcXSbmnRU
-> X25519 DufjAOGVQtGU2oiDCymV7rv9bdw5Llk3KjbOj5wJxxs
-9sOvYKIfp+fUKcW6zbhAU3kwaUrF9PCBlu56qmGhOss
-> m-grease s$ A ,2 =sKpm
-lLRsEhRI4PsWw9K6uygWxFznKZSJUXesteKQ7hZ/wWJXkRHq
--- XYl7iGPy1+YfKOWNoZoiYvfFjctfqhWWzR4hMCWmXYU
-<EFBFBD>5<EFBFBD>0K<EFBFBD><EFBFBD><EFBFBD><EFBFBD><07>2Rp)w<>.(rh<72>U~w|%j͂<6A><CD82><EFBFBD><EFBFBD><0C><> 9<><39><EFBFBD>V<EFBFBD><11><><EFBFBD> 0d<30>{<7B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>E<EFBFBD>\Vag<1B>~"T-<2D><>3\<5C>)N<><4E><EFBFBD>S"I
+-> ssh-ed25519 ZB3e6Q iCLxItNihRG7KUDgcUm4vrtWQblN5hdYwvAegw0m5DQ
+nQSrxGdOaWjtjYssejOg1DoNRnIYNznRzDJUEcWCUgA
+-> X25519 eE1k40fJ67VXFqUJ8pB2Ll8/s1K0kD3YkfMQnOqKiTw
+nH9+nHG8pAVLn5krLSNGc18FEMcp6o5NKkf/ciuFPY8
+-> U|8z(Y7-grease n 6
+DNyQQUnKJ9kGTrZY0pj67eeuEMpyn69awH4v0+RZiS9GaVRNPz9dv6VfzI178NDv
+wb2gQLYc/5QFlvKo1pYx12AxxF3LvrwhNm8w9nvVjXUzFqn7SvoFxszxtw
+--- bQBm6Njo6zu9+Xwao1BlMfBUXYL8TbytByW27Hde/Tg
+<EFBFBD><EFBFBD>v<EFBFBD><EFBFBD><EFBFBD>\<5C>'<27><>_<05><><EFBFBD><EFBFBD>H<EFBFBD><48><EFBFBD><EFBFBD><EFBFBD>(=a<><61>Jf<4A><66><16>+<2B><><EFBFBD>R<EFBFBD><1F><><EFBFBD><EFBFBD>,(<28><>+<2B><>W<EFBFBD>{?<02>Zn<5A>Q<EFBFBD><51>~<7E>a<EFBFBD>><3E><>)<29><><EFBFBD>gF<67><46>X<EFBFBD>rM4<4D><34>y<EFBFBD><79>
--- a/secrets/chatterbox/synapse.yaml.age
+++ b/secrets/chatterbox/synapse.yaml.age
--- a/secrets/dhparams.pem.age
+++ b/secrets/dhparams.pem.age
--- a/secrets/estuary/netdata/powerdns.conf.age
+++ b/secrets/estuary/netdata/powerdns.conf.age
--- a/secrets/estuary/netdata/powerdns_recursor.conf.age
+++ b/secrets/estuary/netdata/powerdns_recursor.conf.age
--- a/secrets/estuary/pdns/auth.conf.age
+++ b/secrets/estuary/pdns/auth.conf.age
@@ -1,9 +1,10 @@
 age-encryption.org/v1
-> ssh-ed25519 n8CpUw ACIVtxzORRq2ptG0/MNlBt83MQZJu3Pc3R/5QRpgi2A
-NSO8o2fL/EDLXegZ/kkzMW4/Za79q/6QfMQ1t0Sk9BM
-> X25519 nHYed6I+w6lIxgQNPUdeO35HlHmd0tKATpvnbtB5WzU
-IWRKvT2csHQplib3ms1akiqdzGS37xQ2ev45yGW5d+w
-> %YW{-grease
-4/tMk8Gzztby5x5ojQXj3853G0V8t7AoZA
--- 6vzp2wJk0Eh0O33xXCLrQiNbqeV7oMgvvqrgyRMK9Mg
-2<EFBFBD><EFBFBD><EFBFBD>ȬGh<EFBFBD><12><>l<EFBFBD><6C>)N˺<4E><CBBA><17><>[N<>
+-> ssh-ed25519 n8CpUw gSOLNKBwaCiP9TqcaIBrRF7HnQrXziYl13GzjVS1ryk
+kgXnpg8IMVfNnb9meGPbAYGbgkeiWF5USDd7KlJGJmA
+-> X25519 oL6s/UbRmFIcZ62H7766Q0Bu4KoFwzICgGPB/ogTvj0
+FTWqAvm3Eq2AzhC+5xAUGMuZYbVtrPt+c1QBtXMdv/A
+-> 54{PX{A-grease CyetKe> >}$Pn iQ)-0sK r
+68Ze/tRYRoVy0x619dD1ibTGYaAGoljMxE2Ll5Sx+V9jRzi/DHtq/xyQTgvJfv3z
+JM7E+KJZetXLLlvpOGKw3GBm
+--- TWJdBHQyXz0rCxKloRqmXut0GODBw32Lwjnj9gFJAFI
+<EFBFBD><EFBFBD>!=<0B><><19><12><>I0r<><72><EFBFBD>J<EFBFBD>v<EFBFBD><76>#(<28>2<EFBFBD><32>R<EFBFBD>8	[-<0F>VI<56>}p<>,}v<>j<EFBFBD>H<>#<23>qJ?<3F><><EFBFBD>!<21><10>v<EFBFBD>~P<><50>
--- a/secrets/estuary/pdns/recursor.conf.age
+++ b/secrets/estuary/pdns/recursor.conf.age
--- a/secrets/hercules/aws-credentials.ini.age
+++ b/secrets/hercules/aws-credentials.ini.age
--- a/secrets/hercules/binary-caches.json.age
+++ b/secrets/hercules/binary-caches.json.age
--- a/secrets/hercules/cluster-join-token.key.age
+++ b/secrets/hercules/cluster-join-token.key.age
--- a/secrets/jackflix/mullvad-privkey.age
+++ b/secrets/jackflix/mullvad-privkey.age
--- a/secrets/middleman/cloudflare-credentials.conf.age
+++ b/secrets/middleman/cloudflare-credentials.conf.age
--- a/secrets/middleman/nginx-sso.yaml.age
+++ b/secrets/middleman/nginx-sso.yaml.age
--- a/secrets/minio.env.age
+++ b/secrets/minio.env.age
@@ -1,10 +1,12 @@
 age-encryption.org/v1
-> ssh-ed25519 hkbtvg IrwZ+4sEJvFpB/zrFR/8Lu4GgpPppm84IYOAP7QWH0o
-Px7RM+aKmjRQKdr0Ta/v+s9M+rRvRTNs9YYaZnNeORk
-> X25519 lSIE40xvHLkKFMCgsKjVhbxYfv7ddDJ3xyMlcDdxgxU
-sc+2ibxqyLtlcpFUPCab+x4imPjuedQadA4b1Qg63a4
-> [J"78S~E-grease S||B(wq} suB8~I ~?E@d}
-/3IplD0a0o3phrEIX85CAVkFRvLcCh3ncK/0Reur0bvKsqOjg37KH+Az5dDh2h9D
-63kpJpGxwNKlRntnWQWxeYN2PN3cZrggH25/EJuJT3td2Q
--- a+cb3+9Z7WWk6vGGaiXz11G2fKUqLbYuUPyzturVFXY
-<EFBFBD><EFBFBD><EFBFBD><08>CEB<45>(c#<1A><>Y$J<><4A>4g*t<>~<7E><><EFBFBD>)<29><><EFBFBD>h<>:<1C>H<EFBFBD>`<60><>ݷ<EFBFBD>a'<27>
+-> ssh-ed25519 hkbtvg G/+xT6RqgxbeZc6fafYkqFs7FyWL58+PhUIrN4g7lVA
+h/lSiNjqSnoBv+nuSyRuQegzIrpyDJ/JmH2z0+WjxJc
+-> X25519 S/BUrpWmbVbEzRWzLLtLctqR+aiir7slufy+o2Wq+Vs
+Hf3NrG88+kISvWbRGTjkNRTNLnpjRY/W/Ukg4N133lg
+-> 9KLY0A-grease b%;W R dR$
+ijMZxH1fad+vLWdei7kZsMYO9u92jjVlx7lPgMbIMFqkFy3xqoGL8jpi96Oz7+nS
+BPbCv6bJQyfo+fUgg6U8Indc3XdbCbcqVPNzguCohQoYxUAC+j+DRQVz3ePadXKY
+fZo
+--- vWyTykBiq5nFO1UxCC8r1eXZiRxdRzVaj02zi3iCLKc
+G<EFBFBD>O<EFBFBD><EFBFBD><12><>t<EFBFBD><74>P<EFBFBD><16><><EFBFBD>)<29>E7<45><37><EFBFBD><EFBFBD>R<EFBFBD>;<3B><><EFBFBD>ݻ<EFBFBD><0B><><EFBFBD>J<4A>s<EFBFBD>5i=lS<6C><53>N<05><13>(.b@<40>]<5D>VƓ<56><C693><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD>x<EFBFBD><EFBFBD>&\<5C>XFF<46>ۖCD~ <20>#<23>h
--- a/secrets/nix-cache-gc.ini.age
+++ b/secrets/nix-cache-gc.ini.age
--- a/secrets/pdns-file-records.key.age
+++ b/secrets/pdns-file-records.key.age
--- a/secrets/user-passwd.txt.age
+++ b/secrets/user-passwd.txt.age
--- a/secrets/vaultwarden.env.age
+++ b/secrets/vaultwarden.env.age
@@ -1,9 +1,9 @@
 age-encryption.org/v1
-> ssh-ed25519 HJ/J7A SyRXLRxv1tu111Xne8u3JUIz0qy8C6HYD7X4uD4pGUI
-QuTY1ZZqLjE8gC3Df7pBT1lWcRPL3EIxHA7dGjp+NEg
-> X25519 N5sWlzsAMGknj5na/nbY00pT5CWedf2S0j1iCKCTFSU
-vM70N7ymKgC3J9OrOAj6CFU/smAz2lLGX4inh1usTzc
-> 'cQcC'-grease Cq7J w# .Tc
-2ScwMdP22ccCa7g3Xbhw2kT4qjW9Cg
--- wwRUIFVC5mOE9w8NRr2Ld7GjeK0sFlsDnvEFke9Rzc0
-<EFBFBD><EFBFBD>(<28>ǹ|u<>fK<66><4B>
+-> ssh-ed25519 HJ/J7A A4ybdNG0bDSIBDnjktzi1DpmGrkvNt0SE+YqCHNokEg
+gwL+6yhXPM3oFkq3S/4PlWzi1h43yBRW1atvYbg2Ax4
+-> X25519 R8AIKLRKCLCUmJB3A/z+9iQOfwbqNRm7GgZQX1PgHXM
+nP+UagGakkcI4c59CHSldzGvJLzDXJE16u+LggSLUcM
+-> iS[]-grease
+NLqKdqlhdrhVyfNihGFsQC+jvA9wu60
+--- KDffMrsRX2L2uqdu0ReWQnIcqkYjWfNh4s7KgXTYpDA
+<05>-<2D><>)<29><>h<EFBFBD><68>i<EFBFBD>@X"Āe<C480><65><EFBFBD>Ʃ<EFBFBD>q}J<>a&rJ	<09>!I<><49><EFBFBD>:<3A>7;~<7E>v<EFBFBD><76><EFBFBD><EFBFBD>-<2D><><EFBFBD><EFBFBD><EFBFBD>*=<3D>e<EFBFBD>N<EFBFBD><4E><EFBFBD>Kb<4B>Wp#<23><>B<EFBFBD><42>m<EFBFBD>ux<75><78>q<EFBFBD><71><EFBFBD>Xn<7F>+<2B><>B<EFBFBD>G<EFBFBD>aLȂ<4C>Der<65><04>O1<4F><31>^<5E>t]c"<22>dI<><18>RR<52><52><EFBFBD><EFBFBD>G|Q