nixos: Add Hercules CI and Nix cache

lib/default.nix

@@ -174,6 +174,26 @@ rec {
     filterOpts = filterAttrsRecursive (_: v: v != null);
   };
 
+  nginx = {
+    proxyHeaders = ''
+      # Setting any proxy_header in a child (e.g. location) will nuke the parents...
+      proxy_set_header X-Origin-URI $request_uri;
+      proxy_set_header Host $host;
+      proxy_set_header X-Host $http_host;
+      proxy_set_header X-Forwarded-Host $http_host;
+      proxy_set_header X-Forwarded-Server $host;
+      proxy_set_header X-Real-IP $remote_addr;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto $scheme;
+      proxy_set_header X-Forwarded-Protocol $scheme;
+      proxy_set_header X-Scheme $scheme;
+    '';
+  };
+
+  nix = {
+    cacheKey = "nix-cache.nul.ie-1:XofkqdHQSGFoPjB6aRohQbCU2ILKFqhNjWfoOdQgF5Y=";
+  };
+
   pubDomain = "nul.ie";
   colony = rec {
     domain = "fra1.int.${pubDomain}";

nixos/boxes/colony/vms/shill/containers/middleman/default.nix

@@ -203,7 +203,9 @@
               # Based on recommended*Settings, but probably better to be explicit about these
               appendHttpConfig = ''
                 # NixOS provides a logrotate config that auto-compresses :)
-                access_log /var/log/nginx/access.log combined;
+                log_format main
+                  '$remote_addr - $remote_user [$time_local] $scheme "$host" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"';
+                access_log /var/log/nginx/access.log main;
 
                 # optimisation
                 sendfile on;
@@ -236,19 +238,16 @@
                 proxy_send_timeout 60s;
                 proxy_http_version 1.1;
 
-                # proxy headers
+                ${lib.my.nginx.proxyHeaders}
-                proxy_set_header X-Origin-URI $request_uri;
-                proxy_set_header Host $host;
-                proxy_set_header X-Host $http_host;
-                proxy_set_header X-Forwarded-Host $http_host;
-                proxy_set_header X-Forwarded-Server $host;
-                proxy_set_header X-Real-IP $remote_addr;
-                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-                proxy_set_header X-Forwarded-Proto $scheme;
-                proxy_set_header X-Forwarded-Protocol $scheme;
-                proxy_set_header X-Scheme $scheme;
 
                 vhost_traffic_status_zone;
+
+                map $upstream_status $nix_cache_control {
+                  "~20(0|6)" "public, max-age=315360000, immutable";
+                }
+                map $upstream_status $nix_expires {
+                  "~20(0|6)" "Thu, 31 Dec 2037 23:55:55 GMT";
+                }
               '';
             };
           };

nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix

@@ -104,6 +104,7 @@ in
                 proxyPass = "http://$behost.${config.networking.domain}:19999/$ndpath$is_args$args";
                 extraConfig = ''
                   proxy_pass_request_headers on;
+                  ${lib.my.nginx.proxyHeaders}
                   proxy_set_header Connection "keep-alive";
                   proxy_store off;
 
@@ -130,6 +131,7 @@ in
           "/notifications/hub" = {
             proxyPass = upstream;
             proxyWebsockets = true;
+            extraConfig = lib.my.nginx.proxyHeaders;
           };
           "/notifications/hub/negotiate".proxyPass = upstream;
         };
@@ -235,6 +237,7 @@ in
             {
               proxyPass = "http://jackflix-ctr.${config.networking.domain}:7878";
               proxyWebsockets = true;
+              extraConfig = lib.my.nginx.proxyHeaders;
             }
             (ssoLoc "generic")
           ];
@@ -248,6 +251,7 @@ in
             {
               proxyPass = "http://jackflix-ctr.${config.networking.domain}:8989";
               proxyWebsockets = true;
+              extraConfig = lib.my.nginx.proxyHeaders;
             }
             (ssoLoc "generic")
           ];
@@ -275,39 +279,80 @@ in
           "/socket" = {
             proxyPass = upstream;
             proxyWebsockets = true;
+            extraConfig = lib.my.nginx.proxyHeaders;
           };
         };
         useACMEHost = lib.my.pubDomain;
       };
+    };
 
-      "minio.${lib.my.pubDomain}" = {
+    minio =
+    let
+      host = "object-ctr.${config.networking.domain}";
+      s3Upstream = "http://${host}:9000";
       extraConfig = ''
         chunked_transfer_encoding off;
+        ignore_invalid_headers off;
       '';
+
+      nixCacheableRegex = ''^\/(\S+\.narinfo|nar\/\S+\.nar\.\S+)$'';
+      nixCacheHeaders = ''
+        proxy_hide_header "X-Amz-Request-Id";
+        add_header Cache-Control $nix_cache_control;
+        add_header Expires $nix_expires;
+      '';
+    in
+    {
+      "minio.${lib.my.pubDomain}" = {
+        inherit extraConfig;
         locations = {
-          "/".proxyPass = "http://object-ctr.${config.networking.domain}:9001";
+          "/" = {
+            proxyPass = "http://${host}:9001";
+          };
+          "/ws" = {
+            proxyPass = "http://${host}:9001";
+            proxyWebsockets = true;
+            extraConfig = lib.my.nginx.proxyHeaders;
+          };
         };
         useACMEHost = lib.my.pubDomain;
       };
       "s3.${lib.my.pubDomain}" = {
         serverAliases = [ "*.s3.${lib.my.pubDomain}" ];
-        extraConfig = ''
+        inherit extraConfig;
-          chunked_transfer_encoding off;
+        locations."/".proxyPass = s3Upstream;
-        '';
-        locations = {
-          "/".proxyPass = "http://object-ctr.${config.networking.domain}:9000";
-        };
         useACMEHost = lib.my.pubDomain;
       };
+
+      "nix-cache.${lib.my.pubDomain}" = {
+        extraConfig = ''
+          ${extraConfig}
+          proxy_set_header Host "nix-cache.s3.nul.ie";
+        '';
+        locations = {
+          "/".proxyPass = s3Upstream;
+          "~ ${nixCacheableRegex}" = {
+            proxyPass = s3Upstream;
+            extraConfig = nixCacheHeaders;
           };
-  in
+        };
-  mkMerge [
+        useACMEHost = lib.my.pubDomain;
-    hosts
+        onlySSL = false;
-    (mapAttrs (n: _: {
+      };
+    };
+
+    defaultsFor = mapAttrs (n: _: {
       onlySSL = mkDefault true;
       useACMEHost = mkDefault "${config.networking.domain}";
       kTLS = mkDefault true;
       http2 = mkDefault true;
-    }) hosts)
+    });
+  in
+  mkMerge [
+    hosts
+    (defaultsFor hosts)
+
+    minio
+    (defaultsFor minio)
   ];
 }

nixos/boxes/colony/vms/shill/containers/object.nix

@@ -57,6 +57,7 @@
               region = "eu-central-1";
               browser = true;
               rootCredentialsFile = config.age.secrets."minio.env".path;
+              dataDir = [ "/mnt/minio" ];
             };
           };
         }

nixos/boxes/colony/vms/shill/default.nix

@@ -34,7 +34,7 @@
         inherit (lib.my) networkdAssignment;
       in
       {
-        imports = [ "${modulesPath}/profiles/qemu-guest.nix" ];
+        imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./hercules.nix ];
 
         config = mkMerge [
           {

nixos/boxes/colony/vms/shill/hercules.nix

@@ -0,0 +1,60 @@
+{ lib, pkgs, config, ... }: {
+  config = {
+    system = {
+      activationScripts.herculesAWSCredsRoot.text = ''
+        mkdir -p /root/.aws
+        ln -sf "${config.age.secrets."hercules/aws-credentials.ini".path}" /root/.aws/credentials
+      '';
+    };
+
+    systemd = {
+      services = {
+        hercules-ci-agent-pre =
+        let
+          deps = [ "hercules-ci-agent.service" ];
+          awsCredsPath = "${config.services.hercules-ci-agent.settings.baseDirectory}/.aws/credentials";
+        in
+        {
+          before = deps;
+          requiredBy = deps;
+          serviceConfig = {
+            Type = "oneshot";
+            User = "hercules-ci-agent";
+          };
+          script = ''
+            mkdir -p "$(dirname "${awsCredsPath}")"
+            ln -sf "${config.age.secrets."hercules/aws-credentials.ini".path}" "${awsCredsPath}"
+          '';
+        };
+      };
+    };
+
+    services = {
+      hercules-ci-agent = {
+        enable = true;
+        settings = {
+          concurrentTasks = 20;
+          clusterJoinTokenPath = config.age.secrets."hercules/cluster-join-token.key".path;
+          binaryCachesPath = config.age.secrets."hercules/binary-caches.json".path;
+        };
+      };
+    };
+
+    my = {
+      secrets = {
+        files =
+        let
+          ownedByAgent = {
+            owner = "hercules-ci-agent";
+            group = "hercules-ci-agent";
+          };
+        in
+        {
+          "hercules/cluster-join-token.key" = ownedByAgent;
+          "hercules/binary-caches.json" = ownedByAgent;
+          "hercules/aws-credentials.ini" = ownedByAgent;
+        };
+      };
+    };
+  };
+}

nixos/modules/tmproot.nix

@@ -294,6 +294,16 @@ in
           }
         ];
       })
+      (mkIf config.services.hercules-ci-agent.enable {
+        my.tmproot.persistence.config.directories = [
+          {
+            directory = config.services.hercules-ci-agent.settings.baseDirectory;
+            mode = "0750";
+            user = "hercules-ci-agent";
+            group = "hercules-ci-agent";
+          }
+        ];
+      })
       (persistSimpleSvc "transmission")
       (persistSimpleSvc "jackett")
       (persistSimpleSvc "radarr")