From 0ca615a3899af016d5c8523118bd256a23a36697 Mon Sep 17 00:00:00 2001 From: Jack O'Sullivan Date: Sat, 16 Jul 2022 21:01:18 +0100 Subject: [PATCH] nixos: Add Hercules CI and Nix cache --- lib/default.nix | 20 +++++ .../shill/containers/middleman/default.nix | 23 +++--- .../vms/shill/containers/middleman/vhosts.nix | 75 ++++++++++++++---- .../colony/vms/shill/containers/object.nix | 1 + nixos/boxes/colony/vms/shill/default.nix | 2 +- nixos/boxes/colony/vms/shill/hercules.nix | 60 ++++++++++++++ nixos/modules/tmproot.nix | 10 +++ secrets/hercules/aws-credentials.ini.age | Bin 0 -> 470 bytes secrets/hercules/binary-caches.json.age | Bin 0 -> 779 bytes .../cluster-join-token.key.age} | Bin 10 files changed, 163 insertions(+), 28 deletions(-) create mode 100644 nixos/boxes/colony/vms/shill/hercules.nix create mode 100644 secrets/hercules/aws-credentials.ini.age create mode 100644 secrets/hercules/binary-caches.json.age rename secrets/{hercules-cluster-join-token.key.age => hercules/cluster-join-token.key.age} (100%) diff --git a/lib/default.nix b/lib/default.nix index 852254b..b350015 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -174,6 +174,26 @@ rec { filterOpts = filterAttrsRecursive (_: v: v != null); }; + nginx = { + proxyHeaders = '' + # Setting any proxy_header in a child (e.g. location) will nuke the parents... + proxy_set_header X-Origin-URI $request_uri; + proxy_set_header Host $host; + proxy_set_header X-Host $http_host; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Protocol $scheme; + proxy_set_header X-Scheme $scheme; + ''; + }; + + nix = { + cacheKey = "nix-cache.nul.ie-1:XofkqdHQSGFoPjB6aRohQbCU2ILKFqhNjWfoOdQgF5Y="; + }; + pubDomain = "nul.ie"; colony = rec { domain = "fra1.int.${pubDomain}"; diff --git a/nixos/boxes/colony/vms/shill/containers/middleman/default.nix b/nixos/boxes/colony/vms/shill/containers/middleman/default.nix index f19c4e2..2bda636 100644 --- a/nixos/boxes/colony/vms/shill/containers/middleman/default.nix +++ b/nixos/boxes/colony/vms/shill/containers/middleman/default.nix @@ -203,7 +203,9 @@ # Based on recommended*Settings, but probably better to be explicit about these appendHttpConfig = '' # NixOS provides a logrotate config that auto-compresses :) - access_log /var/log/nginx/access.log combined; + log_format main + '$remote_addr - $remote_user [$time_local] $scheme "$host" "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'; + access_log /var/log/nginx/access.log main; # optimisation sendfile on; @@ -236,19 +238,16 @@ proxy_send_timeout 60s; proxy_http_version 1.1; - # proxy headers - proxy_set_header X-Origin-URI $request_uri; - proxy_set_header Host $host; - proxy_set_header X-Host $http_host; - proxy_set_header X-Forwarded-Host $http_host; - proxy_set_header X-Forwarded-Server $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-Protocol $scheme; - proxy_set_header X-Scheme $scheme; + ${lib.my.nginx.proxyHeaders} vhost_traffic_status_zone; + + map $upstream_status $nix_cache_control { + "~20(0|6)" "public, max-age=315360000, immutable"; + } + map $upstream_status $nix_expires { + "~20(0|6)" "Thu, 31 Dec 2037 23:55:55 GMT"; + } ''; }; }; diff --git a/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix b/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix index a237279..1c99ea5 100644 --- a/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix +++ b/nixos/boxes/colony/vms/shill/containers/middleman/vhosts.nix @@ -104,6 +104,7 @@ in proxyPass = "http://$behost.${config.networking.domain}:19999/$ndpath$is_args$args"; extraConfig = '' proxy_pass_request_headers on; + ${lib.my.nginx.proxyHeaders} proxy_set_header Connection "keep-alive"; proxy_store off; @@ -130,6 +131,7 @@ in "/notifications/hub" = { proxyPass = upstream; proxyWebsockets = true; + extraConfig = lib.my.nginx.proxyHeaders; }; "/notifications/hub/negotiate".proxyPass = upstream; }; @@ -235,6 +237,7 @@ in { proxyPass = "http://jackflix-ctr.${config.networking.domain}:7878"; proxyWebsockets = true; + extraConfig = lib.my.nginx.proxyHeaders; } (ssoLoc "generic") ]; @@ -248,6 +251,7 @@ in { proxyPass = "http://jackflix-ctr.${config.networking.domain}:8989"; proxyWebsockets = true; + extraConfig = lib.my.nginx.proxyHeaders; } (ssoLoc "generic") ]; @@ -275,39 +279,80 @@ in "/socket" = { proxyPass = upstream; proxyWebsockets = true; + extraConfig = lib.my.nginx.proxyHeaders; }; }; useACMEHost = lib.my.pubDomain; }; + }; + minio = + let + host = "object-ctr.${config.networking.domain}"; + s3Upstream = "http://${host}:9000"; + extraConfig = '' + chunked_transfer_encoding off; + ignore_invalid_headers off; + ''; + + nixCacheableRegex = ''^\/(\S+\.narinfo|nar\/\S+\.nar\.\S+)$''; + nixCacheHeaders = '' + proxy_hide_header "X-Amz-Request-Id"; + add_header Cache-Control $nix_cache_control; + add_header Expires $nix_expires; + ''; + in + { "minio.${lib.my.pubDomain}" = { - extraConfig = '' - chunked_transfer_encoding off; - ''; + inherit extraConfig; locations = { - "/".proxyPass = "http://object-ctr.${config.networking.domain}:9001"; + "/" = { + proxyPass = "http://${host}:9001"; + }; + "/ws" = { + proxyPass = "http://${host}:9001"; + proxyWebsockets = true; + extraConfig = lib.my.nginx.proxyHeaders; + }; }; useACMEHost = lib.my.pubDomain; }; "s3.${lib.my.pubDomain}" = { serverAliases = [ "*.s3.${lib.my.pubDomain}" ]; - extraConfig = '' - chunked_transfer_encoding off; - ''; - locations = { - "/".proxyPass = "http://object-ctr.${config.networking.domain}:9000"; - }; + inherit extraConfig; + locations."/".proxyPass = s3Upstream; useACMEHost = lib.my.pubDomain; }; + + "nix-cache.${lib.my.pubDomain}" = { + extraConfig = '' + ${extraConfig} + proxy_set_header Host "nix-cache.s3.nul.ie"; + ''; + locations = { + "/".proxyPass = s3Upstream; + "~ ${nixCacheableRegex}" = { + proxyPass = s3Upstream; + extraConfig = nixCacheHeaders; + }; + }; + useACMEHost = lib.my.pubDomain; + onlySSL = false; + }; }; - in - mkMerge [ - hosts - (mapAttrs (n: _: { + + defaultsFor = mapAttrs (n: _: { onlySSL = mkDefault true; useACMEHost = mkDefault "${config.networking.domain}"; kTLS = mkDefault true; http2 = mkDefault true; - }) hosts) + }); + in + mkMerge [ + hosts + (defaultsFor hosts) + + minio + (defaultsFor minio) ]; } diff --git a/nixos/boxes/colony/vms/shill/containers/object.nix b/nixos/boxes/colony/vms/shill/containers/object.nix index 7a01db7..c3c7d5d 100644 --- a/nixos/boxes/colony/vms/shill/containers/object.nix +++ b/nixos/boxes/colony/vms/shill/containers/object.nix @@ -57,6 +57,7 @@ region = "eu-central-1"; browser = true; rootCredentialsFile = config.age.secrets."minio.env".path; + dataDir = [ "/mnt/minio" ]; }; }; } diff --git a/nixos/boxes/colony/vms/shill/default.nix b/nixos/boxes/colony/vms/shill/default.nix index 070441c..73ac394 100644 --- a/nixos/boxes/colony/vms/shill/default.nix +++ b/nixos/boxes/colony/vms/shill/default.nix @@ -34,7 +34,7 @@ inherit (lib.my) networkdAssignment; in { - imports = [ "${modulesPath}/profiles/qemu-guest.nix" ]; + imports = [ "${modulesPath}/profiles/qemu-guest.nix" ./hercules.nix ]; config = mkMerge [ { diff --git a/nixos/boxes/colony/vms/shill/hercules.nix b/nixos/boxes/colony/vms/shill/hercules.nix new file mode 100644 index 0000000..af3daf7 --- /dev/null +++ b/nixos/boxes/colony/vms/shill/hercules.nix @@ -0,0 +1,60 @@ +{ lib, pkgs, config, ... }: { + config = { + system = { + activationScripts.herculesAWSCredsRoot.text = '' + mkdir -p /root/.aws + ln -sf "${config.age.secrets."hercules/aws-credentials.ini".path}" /root/.aws/credentials + ''; + }; + + systemd = { + services = { + hercules-ci-agent-pre = + let + deps = [ "hercules-ci-agent.service" ]; + awsCredsPath = "${config.services.hercules-ci-agent.settings.baseDirectory}/.aws/credentials"; + in + { + before = deps; + requiredBy = deps; + serviceConfig = { + Type = "oneshot"; + User = "hercules-ci-agent"; + }; + script = '' + mkdir -p "$(dirname "${awsCredsPath}")" + ln -sf "${config.age.secrets."hercules/aws-credentials.ini".path}" "${awsCredsPath}" + ''; + }; + }; + }; + + services = { + hercules-ci-agent = { + enable = true; + settings = { + concurrentTasks = 20; + clusterJoinTokenPath = config.age.secrets."hercules/cluster-join-token.key".path; + binaryCachesPath = config.age.secrets."hercules/binary-caches.json".path; + }; + }; + }; + + my = { + secrets = { + files = + let + ownedByAgent = { + owner = "hercules-ci-agent"; + group = "hercules-ci-agent"; + }; + in + { + "hercules/cluster-join-token.key" = ownedByAgent; + "hercules/binary-caches.json" = ownedByAgent; + "hercules/aws-credentials.ini" = ownedByAgent; + }; + }; + }; + }; +} diff --git a/nixos/modules/tmproot.nix b/nixos/modules/tmproot.nix index 5c499f3..acd182a 100644 --- a/nixos/modules/tmproot.nix +++ b/nixos/modules/tmproot.nix @@ -294,6 +294,16 @@ in } ]; }) + (mkIf config.services.hercules-ci-agent.enable { + my.tmproot.persistence.config.directories = [ + { + directory = config.services.hercules-ci-agent.settings.baseDirectory; + mode = "0750"; + user = "hercules-ci-agent"; + group = "hercules-ci-agent"; + } + ]; + }) (persistSimpleSvc "transmission") (persistSimpleSvc "jackett") (persistSimpleSvc "radarr") diff --git a/secrets/hercules/aws-credentials.ini.age b/secrets/hercules/aws-credentials.ini.age new file mode 100644 index 0000000000000000000000000000000000000000..ac5208dd72cec9172186ca4b5ba09c1720e2819a GIT binary patch literal 470 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCT4N({^iN>|AA^~-ZN zcPdG*O43g?GRsag2{AX%bMX$S$_eufHV-u|&oFk0sxmi9Pv-KEDDrUj)z2<2Dlkb; zGYyFhFstxPH7YX=3@a&g$qdOZ2{9}R39vMdas=5H0kO!X*elP&&)M9>#H=(cKeZ$! z)zw@(Ehoq=Jufpj*ux^PGS8@1(OtmHHgV<+=4 zA4kWWg0P5W(^PL4H%A}OaKHQvql{v%aBqzQM{VAvG3DucN9Ui>ci7FT=(ak%>3B7_ zdD`TC+c{(I*0nGlZI@r56R#$|KO|S4->)KMabB~^_5R*1K1*9&luj`fd1OpdO7n%6f~oN-^^`8p?t`TAcOP6{5&k&=7d F4*;aVph^G$ literal 0 HcmV?d00001 diff --git a/secrets/hercules/binary-caches.json.age b/secrets/hercules/binary-caches.json.age new file mode 100644 index 0000000000000000000000000000000000000000..752fe02b0c12faa4691bca9b1f1134f737ff92e3 GIT binary patch literal 779 zcmV+m1N8i1XJsvAZewzJaCB*JZZ2 zFHux_D>hrLM`l%SNKH^iFG&h5J|I{!H8n9gAWv6Mby!F^ zYjGXh>*9Zb1qyEiE8rM{adU zV|GDdWm#%XXKOPrX<;%&MleJ)Z7*m{XklS%Vs=PtL3LJFL1zjiv236}<88-DP?zJ0 zHC+H_ZPK_vl0a=HBlg*yCJ%j^#*>?emp-#Xr?)X(zxH>ncUzFY(Yd>jLDrH_Dbm7m7@7xQhh>EVv8CR7>rnq}}#Jdc-a zjwj+Gh#0F`l1UI~>A@&=9?ADz>IJ1I=%RRr7|d(bHrBsGv&G{avMEUHA^xs2Qu!MY z7q+l;lKIKpVW5uI^5|lstz4dE3xkX%ap?qi^xt7apg`k)VtVY$#Ui%`rp12`J`2uz zo;yw$HWS5S2fBtdhk$pQ^Wf^bMT1g3wq)u-ORnkJuW82EA@@&r Ju0brxN