Python 3.x: do not regenerate _sysconfigdata
This commit fixes several issues: - as reported in https://github.com/NixOS/nixpkgs/issues/24924 it was possible that the file _sysconfigdata.pyc was generated after the actual build of the CPython interpreter. We forgot to regenerate that file during the build. This is now fixed - the expression of the 3.3 interpreter now also includes some of the determinism patches even though the output isn't yet reproducible. The reason for adding them is that this makes the expressions of the different interpreters more similar. - references to -dev packages are now also removed in the 3.6 package, thereby reducing its closure size
This commit is contained in:
Frederik Rietdijk
parent
f7a4f146c9
commit
5c8ffe0311
@ -50,7 +50,17 @@ in stdenv.mkDerivation {
|
||||
|
||||
NIX_LDFLAGS = stdenv.lib.optionalString stdenv.isLinux "-lgcc_s";
|
||||
|
||||
postPatch = optionalString (x11Support && (tix != null)) ''
|
||||
# Determinism: The interpreter is patched to write null timestamps when compiling python files.
|
||||
# This way python doesn't try to update them when we freeze timestamps in nix store.
|
||||
DETERMINISTIC_BUILD=1;
|
||||
# Determinism: We fix the hashes of str, bytes and datetime objects.
|
||||
PYTHONHASHSEED=0;
|
||||
|
||||
postPatch = ''
|
||||
# Determinism
|
||||
substituteInPlace "Lib/py_compile.py" --replace "source_stats['mtime']" "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"
|
||||
# # We do not patch `Lib/importlib/_bootstrap_external.py` because it does not exist.
|
||||
'' + optionalString (x11Support && (tix != null)) ''
|
||||
substituteInPlace "Lib/tkinter/tix.py" --replace "os.environ.get('TIX_LIBRARY')" "os.environ.get('TIX_LIBRARY') or '${tix}/lib'"
|
||||
'';
|
||||
|
||||
@ -80,12 +90,17 @@ in stdenv.mkDerivation {
|
||||
fi
|
||||
done
|
||||
touch $out/lib/python${majorVersion}/test/__init__.py
|
||||
|
||||
ln -s "$out/include/python${majorVersion}m" "$out/include/python${majorVersion}"
|
||||
paxmark E $out/bin/python${majorVersion}
|
||||
|
||||
# Python on Nix is not manylinux1 compatible. https://github.com/NixOS/nixpkgs/issues/18484
|
||||
echo "manylinux1_compatible=False" >> $out/lib/${libPrefix}/_manylinux.py
|
||||
|
||||
# Determinism: Windows installers were not deterministic.
|
||||
# We're also not interested in building Windows installers.
|
||||
find "$out" -name 'wininst*.exe' | xargs -r rm -f
|
||||
|
||||
# Use Python3 as default python
|
||||
ln -s "$out/bin/idle3" "$out/bin/idle"
|
||||
ln -s "$out/bin/pip3" "$out/bin/pip"
|
||||
@ -93,18 +108,20 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3" "$out/bin/python"
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# FIXME: should regenerate this.
|
||||
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
passthru = let
|
||||
|
||||
@ -132,6 +132,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
@ -140,18 +147,6 @@ in stdenv.mkDerivation {
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# FIXME: should regenerate this.
|
||||
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
|
||||
|
||||
'';
|
||||
|
||||
passthru = let
|
||||
pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
|
||||
in rec {
|
||||
|
||||
@ -134,6 +134,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
@ -142,17 +149,6 @@ in stdenv.mkDerivation {
|
||||
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata.py $out/lib/python${majorVersion}/config-${majorVersion}m/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# FIXME: should regenerate this.
|
||||
rm $out/lib/python${majorVersion}/__pycache__/_sysconfigdata.cpython*
|
||||
'';
|
||||
|
||||
passthru = let
|
||||
pythonPackages = callPackage ../../../../../top-level/python-packages.nix {python=self; overrides=packageOverrides;};
|
||||
in rec {
|
||||
|
||||
@ -134,6 +134,13 @@ in stdenv.mkDerivation {
|
||||
ln -s "$out/bin/python3-config" "$out/bin/python-config"
|
||||
ln -s "$out/lib/pkgconfig/python3.pc" "$out/lib/pkgconfig/python.pc"
|
||||
|
||||
# Get rid of retained dependencies on -dev packages, and remove
|
||||
# some $TMPDIR references to improve binary reproducibility.
|
||||
# Note that the .pyc file of _sysconfigdata.py should be regenerated!
|
||||
for i in $out/lib/python${majorVersion}/_sysconfigdata*.py $out/lib/python${majorVersion}/config-${majorVersion}m*/Makefile; do
|
||||
sed -i $i -e "s|-I/nix/store/[^ ']*||g" -e "s|-L/nix/store/[^ ']*||g" -e "s|$TMPDIR|/no-such-path|g"
|
||||
done
|
||||
|
||||
# Determinism: rebuild all bytecode
|
||||
# We exclude lib2to3 because that's Python 2 code which fails
|
||||
# We rebuild three times, once for each optimization level
|
||||
|
||||
Loading…
Reference in New Issue
Block a user