gitlab: Build frontend assets from source

Instead of extracting prebuilt assets from the debian build, build
them from the source. This should give faster package updates and
reduces the amount of data needed to be downloaded by more than 500MB.

pkgs/applications/version-management/gitlab/default.nix

@@ -1,9 +1,21 @@
 { stdenv, lib, fetchurl, fetchFromGitLab, bundlerEnv
-, ruby, tzdata, git, nettools, nixosTests
-, gitlabEnterprise ? false
+, ruby, tzdata, git, nettools, nixosTests, nodejs
+, gitlabEnterprise ? false, callPackage, yarn
+, yarn2nix-moretea
 }:
 
 let
+  flavour = if gitlabEnterprise then "ee" else "ce";
+  data = (builtins.fromJSON (builtins.readFile ./data.json)).${flavour};
+
+  version = data.version;
+  src = fetchFromGitLab {
+    owner = data.owner;
+    repo = data.repo;
+    rev = data.rev;
+    sha256 = data.repo_hash;
+  };
+
   rubyEnv = bundlerEnv rec {
     name = "gitlab-env-${version}";
     inherit ruby;
@@ -26,28 +38,64 @@ let
     ignoreCollisions = true;
   };
 
-  flavour = if gitlabEnterprise then "ee" else "ce";
-  data = (builtins.fromJSON (builtins.readFile ./data.json)).${flavour};
+  yarnOfflineCache = (callPackage ./yarnPkgs.nix {}).offline_cache;
 
-  version = data.version;
-  sources = {
-    gitlab = fetchFromGitLab {
-      owner = data.owner;
-      repo = data.repo;
-      rev = data.rev;
-      sha256 = data.repo_hash;
-    };
-    gitlabDeb = fetchurl {
-      url = data.deb_url;
-      sha256 = data.deb_hash;
-    };
+  assets = stdenv.mkDerivation {
+    pname = "gitlab-assets";
+    inherit version src;
+
+    nativeBuildInputs = [ rubyEnv.wrappedRuby rubyEnv.bundler nodejs yarn ];
+
+    configurePhase = ''
+      runHook preConfigure
+
+      # Some rake tasks try to run yarn automatically, which won't work
+      rm lib/tasks/yarn.rake
+
+      # The rake tasks won't run without a basic configuration in place
+      mv config/database.yml.env config/database.yml
+      mv config/gitlab.yml.example config/gitlab.yml
+
+      # Yarn and bundler wants a real home directory to write cache, config, etc to
+      export HOME=$NIX_BUILD_TOP/fake_home
+
+      # Make yarn install packages from our offline cache, not the registry
+      yarn config --offline set yarn-offline-mirror ${yarnOfflineCache}
+
+      # Fixup "resolved"-entries in yarn.lock to match our offline cache
+      ${yarn2nix-moretea.fixup_yarn_lock}/bin/fixup_yarn_lock yarn.lock
+
+      yarn install --offline --frozen-lockfile --ignore-scripts --no-progress --non-interactive
+
+      patchShebangs node_modules/
+
+      runHook postConfigure
+    '';
+
+    buildPhase = ''
+      runHook preBuild
+
+      bundle exec rake gettext:po_to_json RAILS_ENV=production NODE_ENV=production
+      bundle exec rake rake:assets:precompile RAILS_ENV=production NODE_ENV=production
+      bundle exec rake webpack:compile RAILS_ENV=production NODE_ENV=production NODE_OPTIONS="--max_old_space_size=4096"
+      bundle exec rake gitlab:assets:fix_urls RAILS_ENV=production NODE_ENV=production
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      mv public/assets $out
+
+      runHook postInstall
+    '';
   };
 in
-
 stdenv.mkDerivation {
   name = "gitlab${if gitlabEnterprise then "-ee" else ""}-${version}";
 
-  src = sources.gitlab;
+  inherit src;
 
   buildInputs = [
     rubyEnv rubyEnv.wrappedRuby rubyEnv.bundler tzdata git nettools
@@ -59,7 +107,7 @@ stdenv.mkDerivation {
     # For reasons I don't understand "bundle exec" ignores the
     # RAILS_ENV causing tests to be executed that fail because we're
     # not installing development and test gems above. Deleting the
-    # tests works though.:
+    # tests works though.
     rm lib/tasks/test.rake
 
     rm config/initializers/gitlab_shell_secret_token.rb
@@ -69,27 +117,15 @@ stdenv.mkDerivation {
   '';
 
   buildPhase = ''
-    mv config/gitlab.yml.example config/gitlab.yml
-
-    # Building this requires yarn, node &c, so we just get it from the deb
-    ar p ${sources.gitlabDeb} data.tar.gz | gunzip > gitlab-deb-data.tar
-    # Work around unpacking deb containing binary with suid bit
-    tar -f gitlab-deb-data.tar --delete ./opt/gitlab/embedded/bin/ksu
-    tar -xf gitlab-deb-data.tar
-    rm gitlab-deb-data.tar
-
-    mv -v opt/gitlab/embedded/service/gitlab-rails/public/assets public
-    rm -rf opt # only directory in data.tar.gz
-
-    mv config/gitlab.yml config/gitlab.yml.example
     rm -f config/secrets.yml
     mv config config.dist
+    rm -r tmp
   '';
 
   installPhase = ''
-    rm -r tmp
     mkdir -p $out/share
     cp -r . $out/share/gitlab
+    ln -sf ${assets} $out/share/gitlab/public/assets
     rm -rf $out/share/gitlab/log
     ln -sf /run/gitlab/log $out/share/gitlab/log
     ln -sf /run/gitlab/uploads $out/share/gitlab/public/uploads

pkgs/applications/version-management/gitlab/update.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env nix-shell
-#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix
+#! nix-shell -i python3 -p bundix common-updater-scripts nix nix-prefetch-git python3 python3Packages.requests python3Packages.lxml python3Packages.click python3Packages.click-log vgo2nix yarn2nix-moretea.yarn2nix
 
 import click
 import click_log
@@ -46,24 +46,6 @@ class GitLabRepo:
         j = json.loads(out)
         return j['sha256']
 
-    def get_deb_url(self, flavour: str, version: str, arch: str = 'amd64') -> str:
-        """
-        gitlab builds debian packages, which we currently need as we don't build the frontend on our own
-        this returns the url of a given flavour, version and arch
-        :param flavour: 'ce' or 'ee'
-        :param version: a version, without 'v' prefix and '-ee' suffix
-        :param arch: amd64
-        :return: url of the debian package
-        """
-        if flavour not in ['ce', 'ee']:
-            raise Exception(f"don't know how to get deb_url for flavour {flavour}")
-        return f"https://packages.gitlab.com/gitlab/gitlab-{flavour}/packages" + \
-               f"/debian/stretch/gitlab-{flavour}_{version}-{flavour}.0_{arch}.deb/download.deb"
-
-    def get_deb_hash(self, flavour: str, version: str) -> str:
-        out = subprocess.check_output(['nix-prefetch-url', self.get_deb_url(flavour, version)])
-        return out.decode('utf-8').strip()
-
     @staticmethod
     def rev2version(tag: str) -> str:
         """
@@ -93,8 +75,6 @@ class GitLabRepo:
                                                                'GITLAB_SHELL_VERSION', 'GITLAB_WORKHORSE_VERSION']}
         return dict(version=self.rev2version(rev),
                     repo_hash=self.get_git_hash(rev),
-                    deb_hash=self.get_deb_hash(flavour, version),
-                    deb_url=self.get_deb_url(flavour, version),
                     owner=self.owner,
                     repo=self.repo,
                     rev=rev,
@@ -191,6 +171,27 @@ def update_rubyenv(flavour):
     subprocess.check_output(['bundix'], cwd=rubyenv_dir)
 
 
+@cli.command('update-yarnpkgs')
+def update_yarnpkgs():
+    """Update yarnPkgs"""
+
+    # yarn.lock is identical between the repos
+    repo = _flavour2gitlabrepo('ee')
+    yarnpkgs_dir = pathlib.Path(__file__).parent
+
+    # load rev from data.json
+    data = _get_data_json()
+    rev = data['ee']['rev']
+
+    with open(yarnpkgs_dir / 'yarn.lock', 'w') as f:
+        f.write(repo.get_file('yarn.lock', rev))
+
+    with open(yarnpkgs_dir / 'yarnPkgs.nix', 'w') as f:
+        subprocess.run(['yarn2nix'], cwd=yarnpkgs_dir, check=True, stdout=f)
+
+    os.unlink(yarnpkgs_dir / 'yarn.lock')
+
+
 @cli.command('update-gitaly')
 def update_gitaly():
     """Update gitaly"""
@@ -267,6 +268,7 @@ def update_all(ctx):
     for flavour in ['ce', 'ee']:
         ctx.invoke(update_data, rev='latest', flavour=flavour)
         ctx.invoke(update_rubyenv, flavour=flavour)
+    ctx.invoke(update_yarnpkgs)
     ctx.invoke(update_gitaly)
     ctx.invoke(update_gitlab_shell)
     ctx.invoke(update_gitlab_workhorse)