dev/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #247550 from xworld21/build-texlive-package

Browse Source 
texlive: refactor package builder in separate expression
This commit is contained in:
Dmitry Kalinkin 2023-08-20 15:18:03 -04:00 committed by GitHub
commit fdbb20045d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 4427 additions and 10056 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

140
pkgs/tools/typesetting/tex/texlive/build-texlive-package.nix Normal file
View File

@ -0,0 +1,140 @@
{ lib
, fetchurl
, runCommand
# script interpreters
, bash
, jdk
, perl
, python3
, ruby
, snobol4
, tk
# TeX Live prerequisites
, texliveBinaries
}:
{ pname
, revision
, version ? toString revision
, sha512
, mirrors
, extraVersion ? ""
, fixedHashes ? { }
, postUnpack ? ""
, stripPrefix ? 1
, license ? [ ]
, hasHyphens ? false
, hasManpages ? false
, hasRunfiles ? false
, hasTlpkg ? false
, ...
}@args:
let
meta = { license = map (x: lib.licenses.${x}) license; };
commonPassthru = {
inherit pname revision version;
} // lib.optionalAttrs (args ? extraRevision) {
inherit (args) extraRevision;
};
# build run, doc, source, tlpkg containers
mkContainer = tlType: passthru: sha512:
let
# NOTE: the fixed naming scheme must match generated-fixed-hashes.nix
# the basename used by upstream (without ".tar.xz" suffix)
urlName = pname + (lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}");
# name + version for the derivation
tlName = urlName + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
fixedHash = fixedHashes.${tlType} or null; # be graceful about missing hashes
urls = args.urls or (if args ? url then [ args.url ] else
map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") mirrors);
in
runCommand "texlive-${tlName}"
({
src = fetchurl { inherit urls sha512; };
inherit meta passthru stripPrefix tlType;
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
})
(''
mkdir "$out"
if [[ "$tlType" == "tlpkg" ]]; then
tar -xf "$src" \
--strip-components=1 \
-C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \
tlpkg
else
tar -xf "$src" \
--strip-components="$stripPrefix" \
-C "$out" --anchored --exclude=tlpkg --keep-old-files
fi
'' + postUnpack);
tex = [
(
let passthru = commonPassthru
// lib.optionalAttrs (args ? deps) { tlDeps = args.deps; }
// lib.optionalAttrs (args ? formats) { inherit (args) formats; }
// lib.optionalAttrs hasHyphens { inherit hasHyphens; }; in
if hasRunfiles then mkContainer "run" passthru sha512.run
else (passthru // { tlType = "run"; })
)
];
doc = let passthru = commonPassthru
// lib.optionalAttrs hasManpages { inherit hasManpages; }; in
lib.optional (sha512 ? doc) (mkContainer "doc" passthru sha512.doc);
source = lib.optional (sha512 ? source) (mkContainer "source" commonPassthru sha512.source);
tlpkg = let passthru = commonPassthru
// lib.optionalAttrs (args ? postactionScript) { postactionScript = args.postactionScript; }; in
lib.optional hasTlpkg (mkContainer "tlpkg" passthru sha512.run);
bin = lib.optional (args ? binfiles && args.binfiles != [ ]) (
let
# find interpreters for the script extensions found in tlpdb
extToInput = {
jar = jdk;
lua = texliveBinaries.luatex;
py = python3;
rb = ruby;
sno = snobol4;
tcl = tk;
texlua = texliveBinaries.luatex;
tlu = texliveBinaries.luatex;
};
run = lib.head tex;
in
runCommand "texlive-${pname}.bin-${version}"
{
passthru = commonPassthru // { tlType = "bin"; };
inherit meta;
# shebang interpreters
buildInputs = (args.extraBuildInputs or [ ]) ++ [ bash perl ] ++ (lib.attrVals (args.scriptExts or [ ]) extToInput);
# absolute scripts folder
scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + args.scriptsFolder or pname);
# binaries info
inherit (args) binfiles;
binlinks = builtins.attrNames (args.binlinks or { });
bintargets = builtins.attrValues (args.binlinks or { });
binfolders = [ (lib.getBin texliveBinaries.core) ] ++
lib.optional (texliveBinaries ? ${pname}) (lib.getBin texliveBinaries.${pname});
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;
}
''
. "$makeBinContainers"
${args.postFixup or ""}
''
);
in
{ pkgs = tex ++ doc ++ source ++ tlpkg ++ bin; }

157
pkgs/tools/typesetting/tex/texlive/default.nix
View File

@ -33,7 +33,7 @@ let
tlpdbVersion = tlpdb."00texlive.config";
# the set of TeX Live packages, collections, and schemes; using upstream naming
tl = let
overriddenTlpdb = let
# most format -> engine links are generated by texlinks according to fmtutil.cnf at combine time
# so we remove them from binfiles, and add back the ones texlinks purposefully ignore (e.g. mptopdf)
removeFormatLinks = lib.mapAttrs (_: attrs:
@ -43,9 +43,9 @@ let
in if binNotFormats != [] then attrs // { binfiles = binNotFormats; } else removeAttrs attrs [ "binfiles" ]
else attrs);
orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]);
orig = removeFormatLinks (removeAttrs tlpdb [ "00texlive.config" ]); in
overridden = lib.recursiveUpdate orig rec {
lib.recursiveUpdate orig rec {
#### overrides of texlive.tlpdb
#### nonstandard script folders
@ -355,8 +355,8 @@ let
# hardcode revision numbers (since texlive.infra, tlshell are not in either system or user texlive.tlpdb)
tlshell.postFixup = ''
substituteInPlace "$out"/bin/tlshell \
--replace '[dict get $::pkgs texlive.infra localrev]' '${toString overridden."texlive.infra".revision}' \
--replace '[dict get $::pkgs tlshell localrev]' '${toString overridden.tlshell.revision}'
--replace '[dict get $::pkgs texlive.infra localrev]' '${toString orig."texlive.infra".revision}' \
--replace '[dict get $::pkgs tlshell localrev]' '${toString orig.tlshell.revision}'
'';
#### dependency changes
@ -380,7 +380,7 @@ let
npp-for-context.license = [ "gpl3Only" ];
texdoc = {
extraRevision = ".tlpdb${toString tlpdbVersion.revision}";
extraRevision = "-tlpdb${toString tlpdbVersion.revision}";
extraVersion = "-tlpdb-${toString tlpdbVersion.revision}";
# build Data.tlpdb.lua (part of the 'tlType == "run"' package)
@ -428,39 +428,6 @@ let
};
}; # overrides
in lib.mapAttrs mkTLPkg overridden;
# create a TeX package: an attribute set { pkgs = [ ... ]; ... } where pkgs is a list of derivations
mkTLPkg = pname: attrs:
let
version = attrs.version or (toString attrs.revision);
mkPkgV = tlType: let
pkg = attrs // {
sha512 = attrs.sha512.${if tlType == "tlpkg" then "run" else tlType};
inherit pname tlType version;
} // lib.optionalAttrs (attrs.hasManpages or false) {
hasManpages = true;
};
in mkPkg pkg;
run = if (attrs.hasRunfiles or false) then mkPkgV "run"
# the fake derivations are used for filtering of hyphenation patterns and formats
else ({
inherit pname version;
tlType = "run";
hasHyphens = attrs.hasHyphens or false;
tlDeps = map (n: tl.${n}) (attrs.deps or []);
} // lib.optionalAttrs (attrs ? formats) { inherit (attrs) formats; });
in {
# TL pkg contains lists of packages: runtime files, docs, sources, tlpkg, binaries
pkgs =
# tarball of a collection/scheme itself only contains a tlobj file
[ run ]
++ lib.optional (attrs.sha512 ? doc) (mkPkgV "doc")
++ lib.optional (attrs.sha512 ? source) (mkPkgV "source")
++ lib.optional (attrs.hasTlpkg or false) (mkPkgV "tlpkg")
++ lib.optional (attrs ? binfiles && attrs.binfiles != []) (mkPkgBin pname version run attrs);
};
version = {
# day of the snapshot being taken
year = "2023";
@ -477,7 +444,7 @@ let
# need to be used instead. Ideally, for the release branches of NixOS we
# should be switching to the tlnet-final versions
# (https://tug.org/historic/).
urlPrefixes = with version; lib.optionals final [
mirrors = with version; lib.optionals final [
# tlnet-final snapshot; used when texlive.tlpdb is frozen
# the TeX Live yearly freeze typically happens in mid-March
"http://ftp.math.utah.edu/pub/tex/historic/systems/texlive/${toString texliveYear}/tlnet-final"
@ -491,7 +458,7 @@ let
];
tlpdbxz = fetchurl {
urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") urlPrefixes;
urls = map (up: "${up}/tlpkg/texlive.tlpdb.xz") mirrors;
hash = "sha256-vm7DmkH/h183pN+qt1p1wZ6peT2TcMk/ae0nCXsCoMw=";
};
@ -506,102 +473,17 @@ let
# map: name -> fixed-output hash
fixedHashes = lib.optionalAttrs useFixedHashes (import ./fixed-hashes.nix);
# NOTE: the fixed naming scheme must match generated-fixed-hashes.nix
# name for the URL
mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
# name + revision for the fixed output hashes
mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
# name + version for the derivation
mkTLName = { tlType, version, extraVersion ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + "-${version}${extraVersion}";
buildTeXLivePackage = import ./build-texlive-package.nix {
inherit lib fetchurl runCommand bash jdk perl python3 ruby snobol4 tk;
texliveBinaries = bin;
};
# build tlType == "bin" containers based on `binfiles` in TLPDB
# see UPGRADING.md for how to keep the list of shebangs up to date
mkPkgBin = let extToInput = {
jar = jdk;
lua = bin.luatex;
py = python3;
rb = ruby;
sno = snobol4;
tcl = tk;
texlua = bin.luatex;
tlu = bin.luatex;
}; in pname: version: run:
{ binfiles, scriptsFolder ? pname, postFixup ? "", scriptExts ? [], extraBuildInputs ? [], binlinks ? {}, ... }@args:
runCommand "texlive-${pname}.bin-${version}"
{
# metadata for texlive.combine
passthru = {
inherit pname version;
tlType = "bin";
};
# shebang interpreters
buildInputs = extraBuildInputs ++ [ bash perl ] ++ (lib.attrVals scriptExts extToInput);
# absolute scripts folder
scriptsFolder = lib.optionalString (run ? outPath) (run.outPath + "/scripts/" + scriptsFolder);
# binaries info
inherit binfiles;
binlinks = builtins.attrNames binlinks;
bintargets = builtins.attrValues binlinks;
binfolders = [ (lib.getBin bin.core) ] ++ lib.optional (bin ? ${pname}) (lib.getBin bin.${pname});
# build scripts
patchScripts = ./patch-scripts.sed;
makeBinContainers = ./make-bin-containers.sh;
}
''
. "$makeBinContainers"
${postFixup}
'';
# create a derivation that contains an unpacked upstream TL package
mkPkg = { pname, tlType, revision, version, sha512, extraRevision ? "", postUnpack ? "", stripPrefix ? 1, hasManpages ? false, ... }@args:
let
# the basename used by upstream (without ".tar.xz" suffix)
urlName = mkURLName args;
tlName = mkTLName args;
fixedHash = fixedHashes.${mkFixedName args} or null; # be graceful about missing hashes
urls = args.urls or (if args ? url then [ args.url ] else
map (up: "${up}/archive/${urlName}.r${toString revision}.tar.xz") (args.urlPrefixes or urlPrefixes));
in runCommand "texlive-${tlName}"
( {
src = fetchurl { inherit urls sha512; };
meta = {
license = map (x: lib.licenses.${x}) (args.license or []);
};
inherit stripPrefix tlType;
# metadata for texlive.combine
passthru = {
inherit pname tlType revision version extraRevision;
} // lib.optionalAttrs (tlType == "run" && args ? deps) {
tlDeps = map (n: tl.${n}) args.deps;
} // lib.optionalAttrs (tlType == "run") {
hasHyphens = args.hasHyphens or false;
} // lib.optionalAttrs (tlType == "tlpkg" && args ? postactionScript) {
postactionScript = args.postactionScript;
}
// lib.optionalAttrs (tlType == "run" && args ? formats) { inherit (args) formats; }
// lib.optionalAttrs (tlType == "doc" && hasManpages) { hasManpages = true; };
} // lib.optionalAttrs (fixedHash != null) {
outputHash = fixedHash;
outputHashAlgo = "sha256";
outputHashMode = "recursive";
}
)
( ''
mkdir "$out"
if [[ "$tlType" == "tlpkg" ]]; then
tar -xf "$src" \
--strip-components=1 \
-C "$out" --anchored --exclude=tlpkg/tlpobj --keep-old-files \
tlpkg
else
tar -xf "$src" \
--strip-components="$stripPrefix" \
-C "$out" --anchored --exclude=tlpkg --keep-old-files
fi
'' + postUnpack
);
tl = lib.mapAttrs (pname: { revision, extraRevision ? "", ... }@args:
buildTeXLivePackage (args
# NOTE: the fixed naming scheme must match generate-fixed-hashes.nix
// { inherit mirrors pname; fixedHashes = fixedHashes."${pname}-${toString revision}${extraRevision}" or { }; }
// lib.optionalAttrs (args ? deps) { deps = map (n: tl.${n}) (args.deps or [ ]); })
) overriddenTlpdb;
# combine a set of TL packages into a single TL meta-package
combinePkgs = pkgList: lib.catAttrs "pkg" (
@ -640,6 +522,7 @@ in
# for backward compatibility
latexindent = lib.findFirst (p: p.tlType == "bin") tl.latexindent.pkgs;
};
combine = assert assertions; combine;
# Pre-defined combined packages for TeX Live schemes,
@ -652,7 +535,7 @@ in
scheme-basic = [ free gfl gpl1Only gpl2 gpl2Plus knuth lgpl21 lppl1 lppl13c mit ofl publicDomain ];
scheme-context = [ bsd2 bsd3 cc-by-sa-40 free gfl gfsl gpl1Only gpl2 gpl2Plus gpl3 gpl3Plus knuth lgpl2 lgpl21
lppl1 lppl13c mit ofl publicDomain x11 ];
scheme-full = [ artistic1-cl8 asl20 bsd2 bsd3 bsdOriginal cc-by-10 cc-by-40 cc-by-sa-10 cc-by-sa-20
scheme-full = [ artistic1-cl8 artistic2 asl20 bsd2 bsd3 bsdOriginal cc-by-10 cc-by-40 cc-by-sa-10 cc-by-sa-20
cc-by-sa-30 cc-by-sa-40 cc0 fdl13Only free gfl gfsl gpl1Only gpl2 gpl2Plus gpl3 gpl3Plus isc knuth
lgpl2 lgpl21 lgpl3 lppl1 lppl12 lppl13a lppl13c mit ofl publicDomain x11 ];
scheme-gust = [ artistic1-cl8 asl20 bsd2 bsd3 cc-by-40 cc-by-sa-40 cc0 fdl13Only free gfl gfsl gpl1Only gpl2

14139
pkgs/tools/typesetting/tex/texlive/fixed-hashes.nix
View File

File diff suppressed because it is too large Load Diff

47
pkgs/tools/typesetting/tex/texlive/generate-fixed-hashes.nix
View File

@ -1,32 +1,33 @@
with import ../../../../.. { };
with lib; let
# NOTE: the fixed naming scheme must match default.nix
# name for the URL
mkURLName = { pname, tlType, ... }: pname + lib.optionalString (tlType != "run" && tlType != "tlpkg") ".${tlType}";
# name + revision for the fixed output hashes
mkFixedName = { tlType, revision, extraRevision ? "", ... }@attrs: mkURLName attrs + (lib.optionalString (tlType == "tlpkg") ".tlpkg") + ".r${toString revision}${extraRevision}";
uniqueByName = fods: catAttrs "fod" (genericClosure {
startSet = map (fod: { key = fod.name; inherit fod; }) fods;
operator = _: [ ];
});
isFod = p: p.tlType != "bin" && isDerivation p;
# ugly hack to extract combine from collection-latexextra, since it is masked by texlive.combine
combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = []; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;
all = concatLists (map (p: p.pkgs or []) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ]))) ++ combine.pkgs;
combine = lib.findFirst (p: (lib.head p.pkgs).pname == "combine") { pkgs = [ ]; } (lib.head texlive.collection-latexextra.pkgs).tlDeps;
all = filter (p: p ? pkgs) (attrValues (removeAttrs texlive [ "bin" "combine" "combined" "tlpdb" ])) ++ [ combine ];
sorted = sort (a: b: (head a.pkgs).pname < (head b.pkgs).pname) all;
fods = filter isFod (concatMap (p: p.pkgs or [ ]) all);
# fixed hashes only for run, doc, source, tlpkg types
fods = sort (a: b: a.name < b.name) (uniqueByName (filter (p: isDerivation p && p.tlType != "bin") all));
computeHash = fod: runCommand "${fod.name}-fixed-hash"
computeHash = fod: runCommand "${fod.pname}-${fod.tlType}-fixed-hash"
{ buildInputs = [ nix ]; inherit fod; }
''echo -n "$(nix-hash --base32 --type sha256 "$fod")" >"$out"'';
hash = fod: fod.outputHash or (builtins.readFile (computeHash fod));
hashLine = fod: ''
"${mkFixedName fod}"="${hash fod}";
'';
hashes = { pkgs }:
concatMapStrings ({ tlType, ... }@p: lib.optionalString (isFod p) (''${tlType}="${hash p}";'')) pkgs;
hashLine = { pkgs }@pkg:
let
fods = lib.filter isFod pkgs;
first = lib.head fods;
# NOTE: the fixed naming scheme must match default.nix
fixedName = with first; "${pname}-${toString revision}${first.extraRevision or ""}";
in
lib.optionalString (fods != [ ]) ''
${strings.escapeNixIdentifier fixedName}={${hashes pkg}};
'';
in
{
# fixedHashesNix uses 'import from derivation' which does not parallelize well
@ -34,8 +35,8 @@ in
newHashes = map computeHash (filter (fod: ! fod ? outputHash) fods);
fixedHashesNix = writeText "fixed-hashes.nix"
''
{
${lib.concatMapStrings hashLine fods}}
'';
''
{
${lib.concatMapStrings hashLine sorted}}
'';
}