nixos/weblate: init module and test

Co-authored-by: Taeer Bar-Yam <Radvendii@users.noreply.github.com>

nixos/modules/module-list.nix

@@ -1484,6 +1484,7 @@
   ./services/web-apps/trilium.nix
   ./services/web-apps/tt-rss.nix
   ./services/web-apps/vikunja.nix
+  ./services/web-apps/weblate.nix
   ./services/web-apps/whitebophir.nix
   ./services/web-apps/wiki-js.nix
   ./services/web-apps/windmill.nix

nixos/modules/services/web-apps/weblate.nix

@@ -0,0 +1,388 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.weblate;
+
+  dataDir = "/var/lib/weblate";
+  settingsDir = "${dataDir}/settings";
+
+  finalPackage = cfg.package.overridePythonAttrs (old: {
+    # We only support the PostgreSQL backend in this module
+    dependencies = old.dependencies ++ cfg.package.optional-dependencies.postgres;
+    # Use a settings module in dataDir, to avoid having to rebuild the package
+    # when user changes settings.
+    makeWrapperArgs = (old.makeWrapperArgs or [ ]) ++ [
+      "--set PYTHONPATH  \"${settingsDir}\""
+      "--set DJANGO_SETTINGS_MODULE \"settings\""
+    ];
+  });
+  inherit (finalPackage) python;
+
+  pythonEnv = python.buildEnv.override {
+    extraLibs = with python.pkgs; [
+      (toPythonModule finalPackage)
+      celery
+    ];
+  };
+
+  # This extends and overrides the weblate/settings_example.py code found in upstream.
+  weblateConfig =
+    ''
+      # This was autogenerated by the NixOS module.
+
+      SITE_TITLE = "Weblate"
+      SITE_DOMAIN = "${cfg.localDomain}"
+      # TLS terminates at the reverse proxy, but this setting controls how links to weblate are generated.
+      ENABLE_HTTPS = True
+      SESSION_COOKIE_SECURE = ENABLE_HTTPS
+      DATA_DIR = "${dataDir}"
+      CACHE_DIR = f"{DATA_DIR}/cache"
+      STATIC_ROOT = "${finalPackage.static}/static"
+      MEDIA_ROOT = "/var/lib/weblate/media"
+      COMPRESS_ROOT = "${finalPackage.static}/compressor-cache"
+      DEBUG = False
+
+      DATABASES = {
+        "default": {
+          "ENGINE": "django.db.backends.postgresql",
+          "HOST": "/run/postgresql",
+          "NAME": "weblate",
+          "USER": "weblate",
+        }
+      }
+
+      with open("${cfg.djangoSecretKeyFile}") as f:
+        SECRET_KEY = f.read().rstrip("\n")
+
+      CACHES = {
+        "default": {
+          "BACKEND": "django_redis.cache.RedisCache",
+          "LOCATION": "unix://${config.services.redis.servers.weblate.unixSocket}",
+          "OPTIONS": {
+              "CLIENT_CLASS": "django_redis.client.DefaultClient",
+              "PASSWORD": None,
+              "CONNECTION_POOL_KWARGS": {},
+          },
+          "KEY_PREFIX": "weblate",
+          "TIMEOUT": 3600,
+        },
+        "avatar": {
+          "BACKEND": "django.core.cache.backends.filebased.FileBasedCache",
+          "LOCATION": "/var/lib/weblate/avatar-cache",
+          "TIMEOUT": 86400,
+          "OPTIONS": {"MAX_ENTRIES": 1000},
+        }
+      }
+
+
+      CELERY_TASK_ALWAYS_EAGER = False
+      CELERY_BROKER_URL = "redis+socket://${config.services.redis.servers.weblate.unixSocket}"
+      CELERY_RESULT_BACKEND = CELERY_BROKER_URL
+
+      VCS_BACKENDS = ("weblate.vcs.git.GitRepository",)
+
+    ''
+    + lib.optionalString cfg.smtp.enable ''
+      ADMINS = (("Weblate Admin", "${cfg.smtp.user}"),)
+
+      EMAIL_HOST = "${cfg.smtp.host}"
+      EMAIL_USE_TLS = True
+      EMAIL_HOST_USER = "${cfg.smtp.user}"
+      SERVER_EMAIL = "${cfg.smtp.user}"
+      DEFAULT_FROM_EMAIL = "${cfg.smtp.user}"
+      EMAIL_PORT = 587
+      with open("${cfg.smtp.passwordFile}") as f:
+        EMAIL_HOST_PASSWORD = f.read().rstrip("\n")
+
+    ''
+    + cfg.extraConfig;
+  settings_py =
+    pkgs.runCommand "weblate_settings.py"
+      {
+        inherit weblateConfig;
+        passAsFile = [ "weblateConfig" ];
+      }
+      ''
+        mkdir -p $out
+        cat \
+          ${finalPackage}/${python.sitePackages}/weblate/settings_example.py \
+          $weblateConfigPath \
+          > $out/settings.py
+      '';
+
+  environment = {
+    PYTHONPATH = "${settingsDir}:${pythonEnv}/${python.sitePackages}/";
+    DJANGO_SETTINGS_MODULE = "settings";
+    # We run Weblate through gunicorn, so we can't utilise the env var set in the wrapper.
+    inherit (finalPackage) GI_TYPELIB_PATH;
+  };
+
+  weblatePath = with pkgs; [
+    gitSVN
+
+    #optional
+    git-review
+    tesseract
+    licensee
+    mercurial
+  ];
+in
+{
+
+  options = {
+    services.weblate = {
+      enable = lib.mkEnableOption "Weblate service";
+
+      package = lib.mkPackageOption pkgs "weblate" { };
+
+      localDomain = lib.mkOption {
+        description = "The domain name serving your Weblate instance.";
+        example = "weblate.example.org";
+        type = lib.types.str;
+      };
+
+      djangoSecretKeyFile = lib.mkOption {
+        description = ''
+          Location of the Django secret key.
+
+          This should be a path pointing to a file with secure permissions (not /nix/store).
+
+          Can be generated with `weblate-generate-secret-key` which is available as the `weblate` user.
+        '';
+        type = lib.types.path;
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = ''
+          Text to append to `settings.py` Weblate configuration file.
+        '';
+      };
+
+      smtp = {
+        enable = lib.mkEnableOption "Weblate SMTP support";
+        user = lib.mkOption {
+          description = "SMTP login name.";
+          example = "weblate@example.org";
+          type = lib.types.str;
+        };
+
+        host = lib.mkOption {
+          description = "SMTP host used when sending emails to users.";
+          type = lib.types.str;
+          example = "127.0.0.1";
+        };
+
+        passwordFile = lib.mkOption {
+          description = ''
+            Location of a file containing the SMTP password.
+
+            This should be a path pointing to a file with secure permissions (not /nix/store).
+          '';
+          type = lib.types.path;
+        };
+      };
+
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.tmpfiles.rules = [ "L+ ${settingsDir} - - - - ${settings_py}" ];
+
+    services.nginx = {
+      enable = true;
+      virtualHosts."${cfg.localDomain}" = {
+
+        forceSSL = true;
+        enableACME = true;
+
+        locations = {
+          "= /favicon.ico".alias = "${finalPackage}/${python.sitePackages}/weblate/static/favicon.ico";
+          "/static/".alias = "${finalPackage.static}/static/";
+          "/static/CACHE/".alias = "${finalPackage.static}/compressor-cache/CACHE/";
+          "/media/".alias = "/var/lib/weblate/media/";
+          "/".proxyPass = "http://unix:///run/weblate.socket";
+        };
+
+      };
+    };
+
+    systemd.services.weblate-postgresql-setup = {
+      description = "Weblate PostgreSQL setup";
+      after = [ "postgresql.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        User = "postgres";
+        Group = "postgres";
+        ExecStart = ''
+          ${config.services.postgresql.package}/bin/psql weblate -c "CREATE EXTENSION IF NOT EXISTS pg_trgm"
+        '';
+      };
+    };
+
+    systemd.services.weblate-migrate = {
+      description = "Weblate migration";
+      after = [ "weblate-postgresql-setup.service" ];
+      requires = [ "weblate-postgresql-setup.service" ];
+      # We want this to be active on boot, not just on socket activation
+      wantedBy = [ "multi-user.target" ];
+      inherit environment;
+      path = weblatePath;
+      serviceConfig = {
+        Type = "oneshot";
+        StateDirectory = "weblate";
+        User = "weblate";
+        Group = "weblate";
+        ExecStart = "${finalPackage}/bin/weblate migrate --noinput";
+      };
+    };
+
+    systemd.services.weblate-celery = {
+      description = "Weblate Celery";
+      after = [
+        "network.target"
+        "redis.service"
+        "postgresql.service"
+      ];
+      # We want this to be active on boot, not just on socket activation
+      wantedBy = [ "multi-user.target" ];
+      environment = environment // {
+        CELERY_WORKER_RUNNING = "1";
+      };
+      path = weblatePath;
+      # Recommendations from:
+      # https://github.com/WeblateOrg/weblate/blob/main/weblate/examples/celery-weblate.service
+      serviceConfig =
+        let
+          # We have to push %n through systemd's replacement, therefore %%n.
+          pidFile = "/run/celery/weblate-%%n.pid";
+          nodes = "celery notify memory backup translate";
+          cmd = verb: ''
+            ${pythonEnv}/bin/celery multi ${verb} \
+              ${nodes} \
+              -A "weblate.utils" \
+              --pidfile=${pidFile} \
+              --logfile=/var/log/celery/weblate-%%n%%I.log \
+              --loglevel=DEBUG \
+              --beat:celery \
+              --queues:celery=celery \
+              --prefetch-multiplier:celery=4 \
+              --queues:notify=notify \
+              --prefetch-multiplier:notify=10 \
+              --queues:memory=memory \
+              --prefetch-multiplier:memory=10 \
+              --queues:translate=translate \
+              --prefetch-multiplier:translate=4 \
+              --concurrency:backup=1 \
+              --queues:backup=backup \
+              --prefetch-multiplier:backup=2
+          '';
+        in
+        {
+          Type = "forking";
+          User = "weblate";
+          Group = "weblate";
+          WorkingDirectory = "${finalPackage}/${python.sitePackages}/weblate/";
+          RuntimeDirectory = "celery";
+          RuntimeDirectoryPreserve = "restart";
+          LogsDirectory = "celery";
+          ExecStart = cmd "start";
+          ExecReload = cmd "restart";
+          ExecStop = ''
+            ${pythonEnv}/bin/celery multi stopwait \
+              ${nodes} \
+              --pidfile=${pidFile}
+          '';
+          Restart = "always";
+        };
+    };
+
+    systemd.services.weblate = {
+      description = "Weblate Gunicorn app";
+      after = [
+        "network.target"
+        "weblate-migrate.service"
+        "weblate-celery.service"
+      ];
+      requires = [
+        "weblate-migrate.service"
+        "weblate-celery.service"
+        "weblate.socket"
+      ];
+      inherit environment;
+      path = weblatePath;
+      serviceConfig = {
+        Type = "notify";
+        NotifyAccess = "all";
+        ExecStart =
+          let
+            gunicorn = python.pkgs.gunicorn.overridePythonAttrs (old: {
+              # Allows Gunicorn to set a meaningful process name
+              dependencies = (old.dependencies or [ ]) ++ old.optional-dependencies.setproctitle;
+            });
+          in
+          ''
+            ${gunicorn}/bin/gunicorn \
+              --name=weblate \
+              --bind='unix:///run/weblate.socket' \
+              weblate.wsgi
+          '';
+        ExecReload = "kill -s HUP $MAINPID";
+        KillMode = "mixed";
+        PrivateTmp = true;
+        WorkingDirectory = dataDir;
+        StateDirectory = "weblate";
+        RuntimeDirectory = "weblate";
+        User = "weblate";
+        Group = "weblate";
+      };
+    };
+
+    systemd.sockets.weblate = {
+      before = [ "nginx.service" ];
+      wantedBy = [ "sockets.target" ];
+      socketConfig = {
+        ListenStream = "/run/weblate.socket";
+        SocketUser = "weblate";
+        SocketGroup = "weblate";
+        SocketMode = "770";
+      };
+    };
+
+    services.redis.servers.weblate = {
+      enable = true;
+      user = "weblate";
+      unixSocket = "/run/redis-weblate/redis.sock";
+      unixSocketPerm = 770;
+    };
+
+    services.postgresql = {
+      enable = true;
+      ensureUsers = [
+        {
+          name = "weblate";
+          ensureDBOwnership = true;
+        }
+      ];
+      ensureDatabases = [ "weblate" ];
+    };
+
+    users.users.weblate = {
+      isSystemUser = true;
+      group = "weblate";
+      packages = [ finalPackage ] ++ weblatePath;
+    };
+
+    users.groups.weblate.members = [ config.services.nginx.user ];
+  };
+
+  meta.maintainers = with lib.maintainers; [ erictapen ];
+
+}

nixos/tests/all-tests.nix

@@ -1073,6 +1073,7 @@ in {
   wastebin = handleTest ./wastebin.nix {};
   watchdogd = handleTest ./watchdogd.nix {};
   webhook = runTest ./webhook.nix;
+  weblate = handleTest ./web-apps/weblate.nix {};
   wiki-js = handleTest ./wiki-js.nix {};
   wine = handleTest ./wine.nix {};
   wireguard = handleTest ./wireguard {};

nixos/tests/web-apps/weblate.nix

@@ -0,0 +1,104 @@
+import ../make-test-python.nix (
+  { pkgs, ... }:
+
+  let
+    certs = import ../common/acme/server/snakeoil-certs.nix;
+
+    serverDomain = certs.domain;
+
+    admin = {
+      username = "admin";
+      password = "snakeoilpass";
+    };
+    # An API token that we manually insert into the db as a valid one.
+    apiToken = "OVJh65sXaAfQMZ4NTcIGbFZIyBZbEZqWTi7azdDf";
+  in
+  {
+    name = "weblate";
+    meta.maintainers = with pkgs.lib.maintainers; [ erictapen ];
+
+    nodes.server =
+      { pkgs, lib, ... }:
+      {
+        virtualisation.memorySize = 2048;
+
+        services.weblate = {
+          enable = true;
+          localDomain = "${serverDomain}";
+          djangoSecretKeyFile = pkgs.writeText "weblate-django-secret" "thisissnakeoilsecretwithmorethan50characterscorrecthorsebatterystaple";
+          extraConfig = ''
+            # Weblate tries to fetch Avatars from the network
+            ENABLE_AVATARS = False
+          '';
+        };
+
+        services.nginx.virtualHosts."${serverDomain}" = {
+          enableACME = lib.mkForce false;
+          sslCertificate = certs."${serverDomain}".cert;
+          sslCertificateKey = certs."${serverDomain}".key;
+        };
+
+        security.pki.certificateFiles = [ certs.ca.cert ];
+
+        networking.hosts."::1" = [ "${serverDomain}" ];
+        networking.firewall.allowedTCPPorts = [
+          80
+          443
+        ];
+
+        users.users.weblate.shell = pkgs.bashInteractive;
+      };
+
+    nodes.client =
+      { pkgs, nodes, ... }:
+      {
+        environment.systemPackages = [ pkgs.wlc ];
+
+        environment.etc."xdg/weblate".text = ''
+          [weblate]
+          url = https://${serverDomain}/api/
+          key = ${apiToken}
+        '';
+
+        networking.hosts."${nodes.server.networking.primaryIPAddress}" = [ "${serverDomain}" ];
+
+        security.pki.certificateFiles = [ certs.ca.cert ];
+      };
+
+    testScript = ''
+      import json
+
+      start_all()
+      server.wait_for_unit("weblate.socket")
+      server.wait_until_succeeds("curl -f https://${serverDomain}/")
+      server.succeed("sudo -iu weblate -- weblate createadmin --username ${admin.username} --password ${admin.password} --email weblate@example.org")
+
+      # It's easier to replace the generated API token with a predefined one than
+      # to extract it at runtime.
+      server.succeed("sudo -iu weblate -- psql -d weblate -c \"UPDATE authtoken_token SET key = '${apiToken}' WHERE user_id = (SELECT id FROM weblate_auth_user WHERE username = 'admin');\"")
+
+      client.wait_for_unit("multi-user.target")
+
+      # Test the official Weblate client wlc.
+      client.wait_until_succeeds("REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt wlc --debug list-projects")
+
+      def call_wl_api(arg):
+          (rv, result) = client.execute("curl -H \"Content-Type: application/json\" -H \"Authorization: Token ${apiToken}\" https://${serverDomain}/api/{}".format(arg))
+          assert rv == 0
+          print(result)
+
+      call_wl_api("users/ --data '{}'".format(
+        json.dumps(
+          {"username": "test1",
+            "full_name": "test1",
+            "email": "test1@example.org"
+          })))
+
+      # TODO: Check sending and receiving email.
+      # server.wait_for_unit("postfix.service")
+
+      # TODO: The goal is for this to succeed, but there are still some checks failing.
+      # server.succeed("sudo -iu weblate -- weblate check --deploy")
+    '';
+  }
+)