7d40fbbc04
nix-prefetch-git is either run as part of a build, usually sandboxed, or outside a build, unsandboxed, to prefetch something that will later be used in a build. It's important that the latter use produces hashes that can be reproduced by the former. One way that they can differ is if the user's git config does something that changes the result of git clone. I ran into this, because my global git config automatically enables git-lfs, whereas nix-prefetch-git otherwise only uses git-lfs if specifically requested. This led to very confusing hash mismatches.
473 lines
13 KiB
Bash
Executable File
473 lines
13 KiB
Bash
Executable File
#! /usr/bin/env bash
|
|
|
|
set -e -o pipefail
|
|
|
|
url=
|
|
rev=
|
|
expHash=
|
|
hashType=$NIX_HASH_ALGO
|
|
deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
|
|
leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
|
|
fetchSubmodules=
|
|
fetchLFS=
|
|
builder=
|
|
branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
|
|
|
|
# ENV params
|
|
out=${out:-}
|
|
http_proxy=${http_proxy:-}
|
|
|
|
# allow overwriting cacert's ca-bundle.crt with a custom one
|
|
# this can be done by setting NIX_GIT_SSL_CAINFO and NIX_SSL_CERT_FILE environment variables for the nix-daemon
|
|
GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$GIT_SSL_CAINFO}
|
|
|
|
# populated by clone_user_rev()
|
|
fullRev=
|
|
humanReadableRev=
|
|
commitDate=
|
|
commitDateStrict8601=
|
|
|
|
if test -n "$deepClone"; then
|
|
deepClone=true
|
|
else
|
|
deepClone=
|
|
fi
|
|
|
|
if test "$leaveDotGit" != 1; then
|
|
leaveDotGit=
|
|
else
|
|
leaveDotGit=true
|
|
fi
|
|
|
|
usage(){
|
|
echo >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
|
|
|
|
Options:
|
|
--out path Path where the output would be stored.
|
|
--url url Any url understood by 'git clone'.
|
|
--rev ref Any sha1 or references (such as refs/heads/master)
|
|
--hash h Expected hash.
|
|
--branch-name Branch name to check out into
|
|
--sparse-checkout Only fetch and checkout part of the repository.
|
|
--non-cone-mode Use non-cone mode for sparse checkouts.
|
|
--deepClone Clone the entire repository.
|
|
--no-deepClone Make a shallow clone of just the required ref.
|
|
--leave-dotGit Keep the .git directories.
|
|
--fetch-lfs Fetch git Large File Storage (LFS) files.
|
|
--fetch-submodules Fetch submodules.
|
|
--builder Clone as fetchgit does, but url, rev, and out option are mandatory.
|
|
--quiet Only print the final json summary.
|
|
"
|
|
exit 1
|
|
}
|
|
|
|
# some git commands print to stdout, which would contaminate our JSON output
|
|
clean_git(){
|
|
git "$@" >&2
|
|
}
|
|
|
|
argi=0
|
|
argfun=""
|
|
for arg; do
|
|
if test -z "$argfun"; then
|
|
case $arg in
|
|
--out) argfun=set_out;;
|
|
--url) argfun=set_url;;
|
|
--rev) argfun=set_rev;;
|
|
--hash) argfun=set_hashType;;
|
|
--branch-name) argfun=set_branchName;;
|
|
--deepClone) deepClone=true;;
|
|
--sparse-checkout) argfun=set_sparseCheckout;;
|
|
--non-cone-mode) nonConeMode=true;;
|
|
--quiet) QUIET=true;;
|
|
--no-deepClone) deepClone=;;
|
|
--leave-dotGit) leaveDotGit=true;;
|
|
--fetch-lfs) fetchLFS=true;;
|
|
--fetch-submodules) fetchSubmodules=true;;
|
|
--builder) builder=true;;
|
|
-h|--help) usage; exit;;
|
|
*)
|
|
: $((++argi))
|
|
case $argi in
|
|
1) url=$arg;;
|
|
2) rev=$arg;;
|
|
3) expHash=$arg;;
|
|
*) exit 1;;
|
|
esac
|
|
;;
|
|
esac
|
|
else
|
|
case $argfun in
|
|
set_*)
|
|
var=${argfun#set_}
|
|
eval "$var=$(printf %q "$arg")"
|
|
;;
|
|
esac
|
|
argfun=""
|
|
fi
|
|
done
|
|
|
|
if test -z "$url"; then
|
|
usage
|
|
fi
|
|
|
|
|
|
init_remote(){
|
|
local url=$1
|
|
clean_git init --initial-branch=master
|
|
clean_git remote add origin "$url"
|
|
if [ -n "$sparseCheckout" ]; then
|
|
git config remote.origin.partialclonefilter "blob:none"
|
|
echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
|
|
fi
|
|
( [ -n "$http_proxy" ] && clean_git config http.proxy "$http_proxy" ) || true
|
|
}
|
|
|
|
# Return the reference of an hash if it exists on the remote repository.
|
|
ref_from_hash(){
|
|
local hash=$1
|
|
git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
|
|
}
|
|
|
|
# Return the hash of a reference if it exists on the remote repository.
|
|
hash_from_ref(){
|
|
local ref=$1
|
|
git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
|
|
}
|
|
|
|
# Returns a name based on the url and reference
|
|
#
|
|
# This function needs to be in sync with nix's fetchgit implementation
|
|
# of urlToName() to re-use the same nix store paths.
|
|
url_to_name(){
|
|
local url=$1
|
|
local ref=$2
|
|
local base
|
|
base=$(basename "$url" .git | cut -d: -f2)
|
|
|
|
if [[ $ref =~ ^[a-z0-9]+$ ]]; then
|
|
echo "$base-${ref:0:7}"
|
|
else
|
|
echo "$base"
|
|
fi
|
|
}
|
|
|
|
# Fetch and checkout the right sha1
|
|
checkout_hash(){
|
|
local hash="$1"
|
|
local ref="$2"
|
|
|
|
if test -z "$hash"; then
|
|
hash=$(hash_from_ref "$ref")
|
|
fi
|
|
|
|
[[ -z "$deepClone" ]] && \
|
|
clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
|
|
clean_git fetch -t ${builder:+--progress} origin || return 1
|
|
|
|
local object_type=$(git cat-file -t "$hash")
|
|
if [[ "$object_type" == "commit" ]]; then
|
|
clean_git checkout -b "$branchName" "$hash" || return 1
|
|
elif [[ "$object_type" == "tree" ]]; then
|
|
clean_git config user.email "nix-prefetch-git@localhost"
|
|
clean_git config user.name "nix-prefetch-git"
|
|
local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
|
|
clean_git checkout -b "$branchName" "$commit_id" || return 1
|
|
else
|
|
echo "Unrecognized git object type: $object_type"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Fetch only a branch/tag and checkout it.
|
|
checkout_ref(){
|
|
local hash="$1"
|
|
local ref="$2"
|
|
|
|
if [[ -n "$deepClone" ]]; then
|
|
# The caller explicitly asked for a deep clone. Deep clones
|
|
# allow "git describe" and similar tools to work. See
|
|
# https://marc.info/?l=nix-dev&m=139641582514772
|
|
# for a discussion.
|
|
return 1
|
|
fi
|
|
|
|
if test -z "$ref"; then
|
|
ref=$(ref_from_hash "$hash")
|
|
fi
|
|
|
|
if test -n "$ref"; then
|
|
# --depth option is ignored on http repository.
|
|
clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
|
|
clean_git checkout -b "$branchName" FETCH_HEAD || return 1
|
|
else
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Update submodules
|
|
init_submodules(){
|
|
clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1}
|
|
}
|
|
|
|
clone(){
|
|
local top=$PWD
|
|
local dir="$1"
|
|
local url="$2"
|
|
local hash="$3"
|
|
local ref="$4"
|
|
|
|
cd "$dir"
|
|
|
|
# Initialize the repository.
|
|
init_remote "$url"
|
|
|
|
# Download data from the repository.
|
|
checkout_ref "$hash" "$ref" ||
|
|
checkout_hash "$hash" "$ref" || (
|
|
echo 1>&2 "Unable to checkout $hash$ref from $url."
|
|
exit 1
|
|
)
|
|
|
|
# Checkout linked sources.
|
|
if test -n "$fetchSubmodules"; then
|
|
init_submodules
|
|
fi
|
|
|
|
if [ -z "$builder" ] && [ -f .topdeps ]; then
|
|
if tg help &>/dev/null; then
|
|
echo "populating TopGit branches..."
|
|
tg remote --populate origin
|
|
else
|
|
echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
|
|
echo "WARNING: install TopGit to fix the problem" >&2
|
|
fi
|
|
fi
|
|
|
|
cd "$top"
|
|
}
|
|
|
|
# Remove all remote branches, remove tags not reachable from HEAD, do a full
|
|
# repack and then garbage collect unreferenced objects.
|
|
make_deterministic_repo(){
|
|
local repo="$1"
|
|
|
|
# run in sub-shell to not touch current working directory
|
|
(
|
|
cd "$repo"
|
|
# Remove files that contain timestamps or otherwise have non-deterministic
|
|
# properties.
|
|
rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
|
|
.git/refs/remotes/origin/HEAD .git/config
|
|
|
|
# Remove all remote branches.
|
|
git branch -r | while read -r branch; do
|
|
clean_git branch -rD "$branch"
|
|
done
|
|
|
|
# Remove tags not reachable from HEAD. If we're exactly on a tag, don't
|
|
# delete it.
|
|
maybe_tag=$(git tag --points-at HEAD)
|
|
git tag --contains HEAD | while read -r tag; do
|
|
if [ "$tag" != "$maybe_tag" ]; then
|
|
clean_git tag -d "$tag"
|
|
fi
|
|
done
|
|
|
|
# Do a full repack. Must run single-threaded, or else we lose determinism.
|
|
clean_git config pack.threads 1
|
|
clean_git repack -A -d -f
|
|
rm -f .git/config
|
|
|
|
# Garbage collect unreferenced objects.
|
|
# Note: --keep-largest-pack prevents non-deterministic ordering of packs
|
|
# listed in .git/objects/info/packs by only using a single pack
|
|
clean_git gc --prune=all --keep-largest-pack
|
|
)
|
|
}
|
|
|
|
|
|
clone_user_rev() {
|
|
local dir="$1"
|
|
local url="$2"
|
|
local rev="${3:-HEAD}"
|
|
|
|
if [ -n "$fetchLFS" ]; then
|
|
clean_git lfs install
|
|
fi
|
|
|
|
# Perform the checkout.
|
|
case "$rev" in
|
|
HEAD|refs/*)
|
|
clone "$dir" "$url" "" "$rev" 1>&2;;
|
|
*)
|
|
if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
|
|
clone "$dir" "$url" "$rev" "" 1>&2
|
|
else
|
|
# if revision is not hexadecimal it might be a tag
|
|
clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
|
|
fi;;
|
|
esac
|
|
|
|
pushd "$dir" >/dev/null
|
|
fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
|
|
humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
|
|
commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
|
|
commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
|
|
popd >/dev/null
|
|
|
|
# Allow doing additional processing before .git removal
|
|
eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
|
|
if test -z "$leaveDotGit"; then
|
|
echo "removing \`.git'..." >&2
|
|
find "$dir" -name .git -print0 | xargs -0 rm -rf
|
|
else
|
|
find "$dir" -name .git | while read -r gitdir; do
|
|
make_deterministic_repo "$(readlink -f "$gitdir/..")"
|
|
done
|
|
fi
|
|
}
|
|
|
|
exit_handlers=()
|
|
|
|
run_exit_handlers() {
|
|
exit_status=$?
|
|
for handler in "${exit_handlers[@]}"; do
|
|
eval "$handler $exit_status"
|
|
done
|
|
}
|
|
|
|
trap run_exit_handlers EXIT
|
|
|
|
quiet_exit_handler() {
|
|
exec 2>&3 3>&-
|
|
if [ $1 -ne 0 ]; then
|
|
cat "$errfile" >&2
|
|
fi
|
|
rm -f "$errfile"
|
|
}
|
|
|
|
quiet_mode() {
|
|
errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
|
|
exit_handlers+=(quiet_exit_handler)
|
|
exec 3>&2 2>"$errfile"
|
|
}
|
|
|
|
json_escape() {
|
|
local s="$1"
|
|
s="${s//\\/\\\\}" # \
|
|
s="${s//\"/\\\"}" # "
|
|
s="${s//^H/\\\b}" # \b (backspace)
|
|
s="${s//^L/\\\f}" # \f (form feed)
|
|
s="${s//
|
|
/\\\n}" # \n (newline)
|
|
s="${s//^M/\\\r}" # \r (carriage return)
|
|
s="${s// /\\t}" # \t (tab)
|
|
echo "$s"
|
|
}
|
|
|
|
print_results() {
|
|
hash="$1"
|
|
if ! test -n "$QUIET"; then
|
|
echo "" >&2
|
|
echo "git revision is $fullRev" >&2
|
|
if test -n "$finalPath"; then
|
|
echo "path is $finalPath" >&2
|
|
fi
|
|
echo "git human-readable version is $humanReadableRev" >&2
|
|
echo "Commit date is $commitDate" >&2
|
|
if test -n "$hash"; then
|
|
echo "hash is $hash" >&2
|
|
fi
|
|
fi
|
|
if test -n "$hash"; then
|
|
cat <<EOF
|
|
{
|
|
"url": "$(json_escape "$url")",
|
|
"rev": "$(json_escape "$fullRev")",
|
|
"date": "$(json_escape "$commitDateStrict8601")",
|
|
"path": "$(json_escape "$finalPath")",
|
|
"$(json_escape "$hashType")": "$(json_escape "$hash")",
|
|
"hash": "$(nix-hash --to-sri --type $hashType $hash)",
|
|
"fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
|
|
"fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
|
|
"deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
|
|
"leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false)
|
|
}
|
|
EOF
|
|
fi
|
|
}
|
|
|
|
remove_tmpPath() {
|
|
rm -rf "$tmpPath"
|
|
}
|
|
|
|
remove_tmpHomePath() {
|
|
rm -rf "$tmpHomePath"
|
|
}
|
|
|
|
if test -n "$QUIET"; then
|
|
quiet_mode
|
|
fi
|
|
|
|
if test -z "$branchName"; then
|
|
branchName=fetchgit
|
|
fi
|
|
|
|
tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
|
|
exit_handlers+=(remove_tmpHomePath)
|
|
HOME="$tmpHomePath"
|
|
unset XDG_CONFIG_HOME
|
|
export GIT_CONFIG_NOSYSTEM=1
|
|
|
|
if test -n "$builder"; then
|
|
test -n "$out" -a -n "$url" -a -n "$rev" || usage
|
|
mkdir -p "$out"
|
|
clone_user_rev "$out" "$url" "$rev"
|
|
else
|
|
if test -z "$hashType"; then
|
|
hashType=sha256
|
|
fi
|
|
|
|
# If the hash was given, a file with that hash may already be in the
|
|
# store.
|
|
if test -n "$expHash"; then
|
|
finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
|
|
if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
|
|
finalPath=
|
|
fi
|
|
hash=$expHash
|
|
fi
|
|
|
|
# If we don't know the hash or a path with that hash doesn't exist,
|
|
# download the file and add it to the store.
|
|
if test -z "$finalPath"; then
|
|
|
|
tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
|
|
exit_handlers+=(remove_tmpPath)
|
|
|
|
tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
|
|
mkdir -p "$tmpFile"
|
|
|
|
# Perform the checkout.
|
|
clone_user_rev "$tmpFile" "$url" "$rev"
|
|
|
|
# Compute the hash.
|
|
hash=$(nix-hash --type $hashType --base32 "$tmpFile")
|
|
|
|
# Add the downloaded file to the Nix store.
|
|
finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
|
|
|
|
if test -n "$expHash" -a "$expHash" != "$hash"; then
|
|
echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
print_results "$hash"
|
|
|
|
if test -n "$PRINT_PATH"; then
|
|
echo "$finalPath"
|
|
fi
|
|
fi
|