texlive: generate "fixed-hashes.nix" from Nix expression

pkgs/tools/typesetting/tex/texlive/UPGRADING.md

@@ -45,24 +45,39 @@ in Nixpkgs matches the one that generated from `texlive.tlpdb.xz`.
 
 ### Build packages locally and generate fix hashes
 
-To save disk space and prevent unnecessary rebuilds, texlive packages are built
-as fixed-output derivations whose hashes are contained in `fixedHashes.nix`.
-
-Updating the list of fixed hashes requires a local build of *all* packages,
-which is a resource-intensive process:
+To prevent unnecessary rebuilds, texlive packages are built as fixed-output
+derivations whose hashes are contained in `fixed-hashes.nix`.
 
+Updating the list of fixed hashes requires a local build of all new packages,
+which is a resource-intensive process. First build the hashes for the new
+packages. Consider tweaking the `-j` option to maximise core usage.
 
 ```bash
-# move fixedHashes away, otherwise build will fail on updated packages
-mv fixedHashes.nix fixedHashes-old.nix
-# start with empty fixedHashes
-echo '{}' > fixedHashes.nix
-
-nix-build ../../../../.. -Q --no-out-link -A texlive.scheme-full.pkgs | ./fixHashes.awk > ./fixedHashes-new.nix
-
-mv fixedHashes-new.nix fixedHashes.nix
+nix-build generate-fixed-hashes.nix -A newHashes -j 8
 ```
 
+Then build the Nix expression containing all the hashes, old and new. This step
+cannot be parallelized because it relies on 'import from derivation'.
+
+```bash
+nix-build generate-fixed-hashes.nix -A fixedHashesNix
+```
+
+Finally, copy the result to `fixed-hashes.nix`.
+
+**Warning.** The expression `fixedHashesNix` reuses the *previous* fixed hashes
+when possible. This is based on two assumptions: that `.tar.xz` archives with
+the same names remain identical in time (which is the intended behaviour of
+CTAN and the various mirrors) and that the build recipe continues to produce
+the same output. Should those assumptions not hold, remove the previous fixed
+hashes for the relevant package, or for all packages.
+
 ### Commit changes
 
-Commit the updated `tlpdb.nix` and `fixedHashes.nix` to the repository.
+Commit the updated `tlpdb.nix` and `fixed-hashes.nix` to the repository with
+a message like
+
+> texlive: 2022-final -> 2023.20230401
+
+Please make sure to follow the [CONTRIBUTING](https://github.com/NixOS/nixpkgs/blob/master/CONTRIBUTING.md)
+guidelines.

pkgs/tools/typesetting/tex/texlive/default.nix

@@ -18,9 +18,6 @@ let
     };
   };
 
-  # map: name -> fixed-output hash
-  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixedHashes.nix);
-
   # function for creating a working environment from a set of TL packages
   combine = import ./combine.nix {
     inherit bin combinePkgs buildEnv lib makeWrapper writeText runCommand
@@ -59,7 +56,8 @@ let
       };
 
       texdoc = orig.texdoc // {
-        version = orig.texdoc.version + "-tlpdb-" + (toString tlpdbVersion.revision);
+        extraRevision = ".tlpdb${toString tlpdbVersion.revision}";
+        extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";
 
         # build Data.tlpdb.lua (part of the 'tlType == "run"' package)
         postUnpack = ''
@@ -155,13 +153,24 @@ let
     xzcat "$tlpdbxz" | sed -rn -f "$tl2nix" | uniq > "$out"
   '';
 
+  # map: name -> fixed-output hash
+  fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixed-hashes.nix);
+
+  # NOTE: the fixed naming scheme must match generated-fixed-hashes.nix
+  # name for the URL
+  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
+  # name + revision for the fixed output hashes
+  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
+  # name + version for the derivation
+  mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
+
   # create a derivation that contains an unpacked upstream TL package
-  mkPkg = { pname, tlType, revision, version, sha512, postUnpack ? "", stripPrefix ? 1, ... }@args:
+  mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, ... }@args:
     let
       # the basename used by upstream (without ".tar.xz" suffix)
-      urlName = pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
-      tlName = urlName + lib.optionalString (tlType == "tlpkg") ".tlpkg" + "-${version}";
-      fixedHash = fixedHashes.${tlName} or null; # be graceful about missing hashes
+      urlName = mkURLName args;
+      tlName = mkTLName args;
+      fixedHash = fixedHashes.${mkFixedName args} or null; # be graceful about missing hashes
 
       urls = args.urls or (if args ? url then [ args.url ] else
         map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
@@ -172,7 +181,7 @@ let
           inherit stripPrefix tlType;
           # metadata for texlive.combine
           passthru = {
-            inherit pname tlType version;
+            inherit pname tlType revision version extraRevision;
           } // lib.optionalAttrs (tlType == "run" && args ? deps) {
             tlDeps = map (n: tl.${n}) args.deps;
           } // lib.optionalAttrs (tlType == "run") {

pkgs/tools/typesetting/tex/texlive/fixHashes.awk

@@ -1,24 +0,0 @@
-#!/usr/bin/env nix-shell
-#! nix-shell -i "gawk -f" -p gawk
-
-BEGIN {
-  print "{"
-}
-
-/-texlive-/ && !/\.bin/ {
-  if (match($0, /-texlive-([^\/]*)/, m) == 0) {
-    print "No match for \""$0"\"" > "/dev/stderr"
-    exit 1
-  }
-  cmd="nix-hash --type sha256 --base32 "$0
-  if (( cmd | getline hash ) <= 0) {
-    print "Error executing nix-hash" > "/dev/stderr"
-    exit 1
-  }
-  close(cmd)
-  printf("\"%s\"=\"%s\";\n", m[1], hash)
-}
-
-END {
-  print "}"
-}

pkgs/tools/typesetting/tex/texlive/generate-fixed-hashes.nix

@@ -0,0 +1,41 @@
+with import ../../../../.. { };
+
+with lib; let
+  # NOTE: the fixed naming scheme must match default.nix
+  # name for the URL
+  mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
+  # name + revision for the fixed output hashes
+  mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
+
+  uniqueByName = fods: catAttrs "fod" (genericClosure {
+    startSet = map (fod: { key = fod.name; inherit fod; }) fods;
+    operator = _: [ ];
+  });
+
+  # ugly hack to extract combine from collection-latexextra, since it is masked by texlive.combine
+  combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = []; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;
+  all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs;
+
+  # fixed hashes only for run, doc, source, tlpkg types
+  fods = sort (a: b: a.name < b.name) (uniqueByName (filter (p: isDerivation p && p.tlType != "bin") all));
+
+  computeHash = fod: runCommand "${fod.name}-fixed-hash"
+    { buildInputs = [ nix ]; inherit fod; }
+    ''echo -n "$(nix-hash --base32 --type sha256 "$fod")" >"$out"'';
+
+  hash = fod: fod.outputHash or (builtins.readFile (computeHash fod));
+  hashLine = fod: ''
+    "${mkFixedName fod}"="${hash fod}";
+  '';
+in
+{
+  # fixedHashesNix uses 'import from derivation' which does not parallelize well
+  # you should build newHashes first, before evaluating (and building) fixedHashesNix
+  newHashes = map computeHash (filter (fod: ! fod ? outputHash) fods);
+
+  fixedHashesNix = writeText "fixed-hashes.nix"
+  ''
+    {
+    ${lib.concatMapStrings hashLine fods}}
+  '';
+}