push sheeet

pkgs/build-support/fetchgit/builder.sh

@@ -0,0 +1,27 @@
+# tested so far with:
+# - no revision specified and remote has a HEAD which is used
+# - revision specified and remote has a HEAD
+# - revision specified and remote without HEAD
+#
+
+echo "exporting $url (rev $rev) into $out"
+
+runHook preFetch
+
+if [ -n "$gitConfigFile" ]; then
+  echo "using GIT_CONFIG_GLOBAL=$gitConfigFile"
+  export GIT_CONFIG_GLOBAL="$gitConfigFile"
+fi
+
+$SHELL $fetcher --builder --url "$url" --out "$out" --rev "$rev" --name "$name" \
+  ${leaveDotGit:+--leave-dotGit} \
+  ${fetchLFS:+--fetch-lfs} \
+  ${deepClone:+--deepClone} \
+  ${fetchSubmodules:+--fetch-submodules} \
+  ${fetchTags:+--fetch-tags} \
+  ${sparseCheckout:+--sparse-checkout "$sparseCheckout"} \
+  ${nonConeMode:+--non-cone-mode} \
+  ${branchName:+--branch-name "$branchName"} \
+  ${rootDir:+--root-dir "$rootDir"}
+
+runHook postFetch

pkgs/build-support/fetchgit/default.nix

@@ -0,0 +1,201 @@
+{
+  config,
+  lib,
+  stdenvNoCC,
+  writeText,
+  git,
+  git-lfs,
+  cacert,
+}:
+
+let
+  urlToName =
+    {
+      url,
+      rev,
+      append,
+    }:
+    let
+      shortRev = lib.sources.shortRev rev;
+      appendShort = lib.optionalString ((builtins.match "[a-f0-9]*" rev) != null) "-${shortRev}";
+    in
+    "${lib.sources.urlToName url}${if append == "" then appendShort else append}";
+in
+
+lib.makeOverridable (
+  lib.fetchers.withNormalizedHash { } (
+    # NOTE Please document parameter additions or changes in
+    #   ../../../doc/build-helpers/fetchers.chapter.md
+    {
+      url,
+      tag ? null,
+      rev ? null,
+      name ? urlToName {
+        inherit url;
+        rev = lib.revOrTag rev tag;
+        # when rootDir is specified, avoid invalidating the result when rev changes
+        append = if rootDir != "" then "-${lib.strings.sanitizeDerivationName rootDir}" else "";
+      },
+      leaveDotGit ? deepClone || fetchTags,
+      outputHash ? lib.fakeHash,
+      outputHashAlgo ? null,
+      fetchSubmodules ? true,
+      deepClone ? false,
+      branchName ? null,
+      sparseCheckout ? lib.optional (rootDir != "") rootDir,
+      nonConeMode ? rootDir != "",
+      nativeBuildInputs ? [ ],
+      # Shell code executed before the file has been fetched.  This, in
+      # particular, can do things like set NIX_PREFETCH_GIT_CHECKOUT_HOOK to
+      # run operations between the checkout completing and deleting the .git
+      # directory.
+      preFetch ? "",
+      # Shell code executed after the file has been fetched
+      # successfully. This can do things like check or transform the file.
+      postFetch ? "",
+      preferLocalBuild ? true,
+      fetchLFS ? false,
+      # Shell code to build a netrc file for BASIC auth
+      netrcPhase ? null,
+      # Impure env vars (https://nixos.org/nix/manual/#sec-advanced-attributes)
+      # needed for netrcPhase
+      netrcImpureEnvVars ? [ ],
+      meta ? { },
+      allowedRequisites ? null,
+      # fetch all tags after tree (useful for git describe)
+      fetchTags ? false,
+      # make this subdirectory the root of the result
+      rootDir ? "",
+      # GIT_CONFIG_GLOBAL (as a file)
+      gitConfigFile ? config.gitConfigFile,
+    }:
+
+    /*
+      NOTE:
+      fetchgit has one problem: git fetch only works for refs.
+      This is because fetching arbitrary (maybe dangling) commits creates garbage collection risks
+      and checking whether a commit belongs to a ref is expensive. This may
+      change in the future when some caching is added to git (?)
+      Usually refs are either tags (refs/tags/*) or branches (refs/heads/*)
+      Cloning branches will make the hash check fail when there is an update.
+      But not all patches we want can be accessed by tags.
+
+      The workaround is getting the last n commits so that it's likely that they
+      still contain the hash we want.
+
+      for now : increase depth iteratively (TODO)
+
+      real fix: ask git folks to add a
+      git fetch $HASH contained in $BRANCH
+      facility because checking that $HASH is contained in $BRANCH is less
+      expensive than fetching --depth $N.
+      Even if git folks implemented this feature soon it may take years until
+      server admins start using the new version?
+    */
+
+    assert nonConeMode -> (sparseCheckout != [ ]);
+    assert fetchTags -> leaveDotGit;
+    assert rootDir != "" -> !leaveDotGit;
+
+    let
+      revWithTag =
+        let
+          warningMsg = "fetchgit requires one of either `rev` or `tag` to be provided (not both).";
+          otherIsNull = other: lib.assertMsg (other == null) warningMsg;
+        in
+        if tag != null then
+          assert (otherIsNull rev);
+          "refs/tags/${tag}"
+        else if rev != null then
+          assert (otherIsNull tag);
+          rev
+        else
+          # FIXME fetching HEAD if no rev or tag is provided is problematic at best
+          "HEAD";
+    in
+
+    if builtins.isString sparseCheckout then
+      # Changed to throw on 2023-06-04
+      throw
+        "Please provide directories/patterns for sparse checkout as a list of strings. Passing a (multi-line) string is not supported any more."
+    else
+      stdenvNoCC.mkDerivation {
+        inherit name;
+
+        builder = ./builder.sh;
+        fetcher = ./nix-prefetch-git;
+
+        nativeBuildInputs = [
+          git
+          cacert
+        ]
+        ++ lib.optionals fetchLFS [ git-lfs ]
+        ++ nativeBuildInputs;
+
+        inherit outputHash outputHashAlgo;
+        outputHashMode = "recursive";
+
+        # git-sparse-checkout(1) says:
+        # > When the --stdin option is provided, the directories or patterns are read
+        # > from standard in as a newline-delimited list instead of from the arguments.
+        sparseCheckout = builtins.concatStringsSep "\n" sparseCheckout;
+
+        inherit
+          url
+          leaveDotGit
+          fetchLFS
+          fetchSubmodules
+          deepClone
+          branchName
+          nonConeMode
+          preFetch
+          postFetch
+          fetchTags
+          rootDir
+          gitConfigFile
+          ;
+        rev = revWithTag;
+
+        postHook =
+          if netrcPhase == null then
+            null
+          else
+            ''
+              ${netrcPhase}
+              # required that git uses the netrc file
+              mv {,.}netrc
+              export NETRC=$PWD/.netrc
+              export HOME=$PWD
+            '';
+
+        impureEnvVars =
+          lib.fetchers.proxyImpureEnvVars
+          ++ netrcImpureEnvVars
+          ++ [
+            "GIT_PROXY_COMMAND"
+            "NIX_GIT_SSL_CAINFO"
+            "SOCKS_SERVER"
+
+            # This is a parameter intended to be set by setup hooks or preFetch
+            # scripts that want per-URL control over HTTP proxies used by Git
+            # (if per-URL control isn't needed, `http_proxy` etc. will
+            # suffice). It must be a whitespace-separated (with backslash as an
+            # escape character) list of pairs like this:
+            #
+            #   http://domain1/path1 proxy1 https://domain2/path2 proxy2
+            #
+            # where the URLs are as documented in the `git-config` manual page
+            # under `http.<url>.*`, and the proxies are as documented on the
+            # same page under `http.proxy`.
+            "FETCHGIT_HTTP_PROXIES"
+          ];
+
+        inherit preferLocalBuild meta allowedRequisites;
+
+        passthru = {
+          gitRepoUrl = url;
+          inherit tag;
+        };
+      }
+  )
+)

pkgs/build-support/fetchgit/deterministic-git

@@ -0,0 +1,45 @@
+#!/bin/sh
+
+# some git commands print to stdout, which would contaminate our JSON output
+clean_git(){
+    git "$@" >&2
+}
+
+# Remove all remote branches, remove tags not reachable from HEAD, do a full
+# repack and then garbage collect unreferenced objects.
+make_deterministic_repo(){
+    local repo="$1"
+
+    # run in sub-shell to not touch current working directory
+    (
+    cd "$repo"
+    # Remove files that contain timestamps or otherwise have non-deterministic
+    # properties.
+    rm -rf .git/logs/ .git/hooks/ .git/index .git/FETCH_HEAD .git/ORIG_HEAD \
+        .git/refs/remotes/origin/HEAD .git/config
+
+    # Remove all remote branches.
+    git branch -r | while read -r branch; do
+        clean_git branch -rD "$branch"
+    done
+
+    # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
+    # delete it.
+    maybe_tag=$(git tag --points-at HEAD)
+    git tag --contains HEAD | while read -r tag; do
+        if [ "$tag" != "$maybe_tag" ]; then
+            clean_git tag -d "$tag"
+        fi
+    done
+
+    # Do a full repack. Must run single-threaded, or else we lose determinism.
+    clean_git config pack.threads 1
+    clean_git repack -A -d -f
+    rm -f .git/config
+
+    # Garbage collect unreferenced objects.
+    # Note: --keep-largest-pack prevents non-deterministic ordering of packs
+    #   listed in .git/objects/info/packs by only using a single pack
+    clean_git gc --prune=all --keep-largest-pack
+    )
+}

pkgs/build-support/fetchgit/nix-prefetch-git

@@ -0,0 +1,544 @@
+#! /usr/bin/env bash
+
+set -e -o pipefail
+
+url=
+rev=
+expHash=
+hashType=$NIX_HASH_ALGO
+deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
+leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
+fetchSubmodules=
+fetchLFS=
+builder=
+fetchTags=
+branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
+
+# ENV params
+out=${out:-}
+http_proxy=${http_proxy:-}
+
+# NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred
+# as of PR#303307
+GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
+
+# populated by clone_user_rev()
+fullRev=
+humanReadableRev=
+commitDate=
+commitDateStrict8601=
+
+if test -n "$deepClone"; then
+    deepClone=true
+else
+    deepClone=
+fi
+
+if test "$leaveDotGit" != 1; then
+    leaveDotGit=
+else
+    leaveDotGit=true
+fi
+
+usage(){
+    echo  >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
+
+Options:
+      --out path      Path where the output would be stored.
+      --url url       Any url understood by 'git clone'.
+      --rev ref       Any sha1 or references (such as refs/heads/master)
+      --hash h        Expected hash.
+      --name n        Symbolic store path name to use for the result (default: based on URL)
+      --branch-name   Branch name to check out into
+      --sparse-checkout Only fetch and checkout part of the repository.
+      --non-cone-mode Use non-cone mode for sparse checkouts.
+      --deepClone     Clone the entire repository.
+      --no-deepClone  Make a shallow clone of just the required ref.
+      --leave-dotGit  Keep the .git directories.
+      --fetch-lfs     Fetch git Large File Storage (LFS) files.
+      --fetch-submodules Fetch submodules.
+      --fetch-tags    Fetch all tags (useful for git describe).
+      --builder       Clone as fetchgit does, but url, rev, and out option are mandatory.
+      --no-add-path   Do not actually add the contents of the git repo to the store.
+      --root-dir dir  Directory in the repository that will be copied to the output instead of the full repository.
+      --quiet         Only print the final json summary.
+"
+    exit 1
+}
+
+# some git commands print to stdout, which would contaminate our JSON output
+clean_git(){
+    git "$@" >&2
+}
+
+argi=0
+argfun=""
+for arg; do
+    if test -z "$argfun"; then
+        case $arg in
+            --out) argfun=set_out;;
+            --url) argfun=set_url;;
+            --rev) argfun=set_rev;;
+            --hash) argfun=set_hashType;;
+            --name) argfun=set_symbolicName;;
+            --branch-name) argfun=set_branchName;;
+            --deepClone) deepClone=true;;
+            --sparse-checkout) argfun=set_sparseCheckout;;
+            --non-cone-mode) nonConeMode=true;;
+            --quiet) QUIET=true;;
+            --no-deepClone) deepClone=;;
+            --leave-dotGit) leaveDotGit=true;;
+            --fetch-lfs) fetchLFS=true;;
+            --fetch-submodules) fetchSubmodules=true;;
+            --fetch-tags) fetchTags=true;;
+            --builder) builder=true;;
+            --no-add-path) noAddPath=true;;
+            --root-dir) argfun=set_rootDir;;
+            -h|--help) usage; exit;;
+            *)
+                : $((++argi))
+                case $argi in
+                    1) url=$arg;;
+                    2) rev=$arg;;
+                    3) expHash=$arg;;
+                    *) exit 1;;
+                esac
+                ;;
+        esac
+    else
+        case $argfun in
+            set_*)
+                var=${argfun#set_}
+                eval "$var=$(printf %q "$arg")"
+                ;;
+        esac
+        argfun=""
+    fi
+done
+
+if test -z "$url"; then
+    usage
+fi
+
+
+init_remote(){
+    local url=$1
+    clean_git init --initial-branch=master
+    clean_git remote add origin "$url"
+    if [ -n "$sparseCheckout" ]; then
+        git config remote.origin.partialclonefilter "blob:none"
+        echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
+    fi
+    ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
+    local proxy_pairs i
+    read -a proxy_pairs <<< "${FETCHGIT_HTTP_PROXIES:-}"
+    for ((i = 1; i < ${#proxy_pairs[@]}; i += 2)); do
+        clean_git config --global "http.${proxy_pairs[$i - 1]}.proxy" "${proxy_pairs[$i]}"
+    done
+}
+
+# Return the reference of an hash if it exists on the remote repository.
+ref_from_hash(){
+    local hash=$1
+    git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
+}
+
+# Return the hash of a reference if it exists on the remote repository.
+hash_from_ref(){
+    local ref=$1
+    git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
+}
+
+# Returns a name based on the url and reference
+#
+# This function needs to be in sync with nix's fetchgit implementation
+# of urlToName() to re-use the same nix store paths.
+url_to_name(){
+    local url=$1
+    local ref=$2
+    local base
+    base=$(basename "$url" .git | cut -d: -f2)
+
+    if test -n "$rootDir"; then
+        # Sanitize by removing leading dots and replacing all invalid character sequences with dashes.
+        # See sanitizeDerivationName in ../../../lib/strings.nix for reference.
+        echo "$base-$(sed -E 's/^\.+//;s/[^[:alnum:]+._?=-]+/-/g' <<< $rootDir)"
+    else
+        if [[ $ref =~ ^[a-z0-9]+$ ]]; then
+            echo "$base-${ref:0:7}"
+        else
+            echo "$base"
+        fi
+    fi
+}
+
+# Fetch and checkout the right sha1
+checkout_hash(){
+    local hash="$1"
+    local ref="$2"
+
+    if test -z "$hash"; then
+        hash=$(hash_from_ref "$ref")
+    fi
+
+    [[ -z "$deepClone" ]] && \
+    clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
+    clean_git fetch -t ${builder:+--progress} origin || return 1
+
+    local object_type=$(git cat-file -t "$hash")
+    if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
+        clean_git checkout -b "$branchName" "$hash" || return 1
+    elif [[ "$object_type" == "tree" ]]; then
+        clean_git config user.email "nix-prefetch-git@localhost"
+        clean_git config user.name "nix-prefetch-git"
+        local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
+        clean_git checkout -b "$branchName" "$commit_id" || return 1
+    else
+        echo "Unrecognized git object type: $object_type"
+        return 1
+    fi
+}
+
+# Fetch only a branch/tag and checkout it.
+checkout_ref(){
+    local hash="$1"
+    local ref="$2"
+
+    if [[ -n "$deepClone" ]]; then
+        # The caller explicitly asked for a deep clone.  Deep clones
+        # allow "git describe" and similar tools to work.  See
+        # https://marc.info/?l=nix-dev&m=139641582514772
+        # for a discussion.
+        return 1
+    fi
+
+    if test -z "$ref"; then
+        ref=$(ref_from_hash "$hash")
+    fi
+
+    if test -n "$ref"; then
+        # --depth option is ignored on http repository.
+        clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
+        clean_git checkout -b "$branchName" FETCH_HEAD || return 1
+    else
+        return 1
+    fi
+}
+
+# Update submodules
+init_submodules(){
+    # shallow with leaveDotGit will change hashes
+    [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
+    clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
+    clean_git submodule update --init --recursive --checkout -j ${NIX_BUILD_CORES:-1} --progress
+}
+
+clone(){
+    local top=$PWD
+    local dir="$1"
+    local url="$2"
+    local hash="$3"
+    local ref="$4"
+
+    cd "$dir"
+
+    # Initialize the repository.
+    init_remote "$url"
+
+    # Download data from the repository.
+    checkout_ref "$hash" "$ref" ||
+    checkout_hash "$hash" "$ref" || (
+        echo 1>&2 "Unable to checkout $hash$ref from $url."
+        exit 1
+    )
+
+    # Fetch all tags if requested
+    if test -n "$fetchTags"; then
+        echo "fetching all tags..." >&2
+        clean_git fetch origin 'refs/tags/*:refs/tags/*' || echo "warning: failed to fetch some tags" >&2
+    fi
+
+    # Checkout linked sources.
+    if test -n "$fetchSubmodules"; then
+        init_submodules
+    fi
+
+    if [ -z "$builder" ] && [ -f .topdeps ]; then
+        if tg help &>/dev/null; then
+            echo "populating TopGit branches..."
+            tg remote --populate origin
+        else
+            echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
+            echo "WARNING: install TopGit to fix the problem" >&2
+        fi
+    fi
+
+    cd "$top"
+}
+
+# Remove all remote branches, remove tags not reachable from HEAD, do a full
+# repack and then garbage collect unreferenced objects.
+make_deterministic_repo(){
+    local repo="$1"
+
+    # run in sub-shell to not touch current working directory
+    (
+    cd "$repo"
+    # Remove files that contain timestamps or otherwise have non-deterministic
+    # properties.
+    if [ -f .git ]; then
+        local dotgit_content=$(<.git)
+        local dotgit_dir="${dotgit_content#gitdir: }"
+    else
+        local dotgit_dir=".git"
+    fi
+    pushd "$dotgit_dir" >/dev/null
+        rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
+    popd >/dev/null
+    # Remove all remote branches.
+    git branch -r | while read -r branch; do
+        clean_git branch -rD "$branch"
+    done
+
+    # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
+    # delete it.
+    maybe_tag=$(git tag --points-at HEAD)
+    git tag --contains HEAD | while read -r tag; do
+        if [ "$tag" != "$maybe_tag" ]; then
+            clean_git tag -d "$tag"
+        fi
+    done
+
+    # Do a full repack. Must run single-threaded, or else we lose determinism.
+    clean_git config pack.threads 1
+    clean_git repack -A -d -f
+    rm -f "$dotgit_dir/config"
+
+    # Garbage collect unreferenced objects.
+    # Note: --keep-largest-pack prevents non-deterministic ordering of packs
+    #   listed in .git/objects/info/packs by only using a single pack
+    clean_git gc --prune=all --keep-largest-pack
+    )
+}
+
+
+clone_user_rev() {
+    local dir="$1"
+    local url="$2"
+    local rev="${3:-HEAD}"
+
+    if [ -n "$fetchLFS" ]; then
+        clean_git lfs install
+    fi
+
+    # Perform the checkout.
+    case "$rev" in
+        HEAD|refs/*)
+            clone "$dir" "$url" "" "$rev" 1>&2;;
+        *)
+            if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
+                clone "$dir" "$url" "$rev" "" 1>&2
+            else
+                # if revision is not hexadecimal it might be a tag
+                clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
+            fi;;
+    esac
+
+    pushd "$dir" >/dev/null
+    fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
+    humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
+    commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
+    commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
+    popd >/dev/null
+
+    # Allow doing additional processing before .git removal
+    eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
+    if test -z "$leaveDotGit"; then
+        echo "removing \`.git'..." >&2
+        find "$dir" -name .git -print0 | xargs -0 rm -rf
+    else
+        find "$dir" -name .git | while read -r gitdir; do
+            make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
+        done
+    fi
+}
+
+clone_user_rev_to_tmpfile(){
+    local url="$1"
+    local rev="${2:-HEAD}"
+
+    # nix>=2.20 rejects adding symlinked paths to the store, so use realpath
+    # to resolve to a physical path. https://github.com/NixOS/nix/issues/11941
+    tmpPath="$(realpath "$(mktemp -d --tmpdir git-checkout-tmp-XXXXXXXX)")"
+    exit_handlers+=(remove_tmpPath)
+
+    tmpOut="$tmpPath/out/$storePathName"
+    tmpClone="$tmpPath/clone"
+    mkdir -p "$tmpPath/out" "$tmpClone"
+
+    # Perform the checkout.
+    clone_user_rev "$tmpClone" "$url" "$rev"
+}
+
+exit_handlers=()
+
+run_exit_handlers() {
+    exit_status=$?
+    for handler in "${exit_handlers[@]}"; do
+        eval "$handler $exit_status"
+    done
+}
+
+trap run_exit_handlers EXIT
+
+quiet_exit_handler() {
+    exec 2>&3 3>&-
+    if [ $1 -ne 0 ]; then
+        cat "$errfile" >&2
+    fi
+    rm -f "$errfile"
+}
+
+quiet_mode() {
+    errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
+    exit_handlers+=(quiet_exit_handler)
+    exec 3>&2 2>"$errfile"
+}
+
+json_escape() {
+    local s="$1"
+    s="${s//\\/\\\\}" # \
+    s="${s//\"/\\\"}" # "
+    s="${s//^H/\\\b}" # \b (backspace)
+    s="${s//^L/\\\f}" # \f (form feed)
+    s="${s//
+/\\\n}" # \n (newline)
+    s="${s//^M/\\\r}" # \r (carriage return)
+    s="${s//   /\\t}" # \t (tab)
+    echo "$s"
+}
+
+print_results() {
+    hash="$1"
+    if ! test -n "$QUIET"; then
+        echo "" >&2
+        echo "git revision is $fullRev" >&2
+        if test -n "$finalPath"; then
+            echo "path is $finalPath" >&2
+        fi
+        echo "git human-readable version is $humanReadableRev" >&2
+        echo "Commit date is $commitDate" >&2
+        if test -n "$hash"; then
+            echo "hash is $hash" >&2
+        fi
+    fi
+    if test -n "$hash"; then
+        cat <<EOF
+{
+  "url": "$(json_escape "$url")",
+  "rev": "$(json_escape "$fullRev")",
+  "date": "$(json_escape "$commitDateStrict8601")",
+  "path": "$(json_escape "$finalPath")",
+  "$(json_escape "$hashType")": "$(json_escape "$hash")",
+  "hash": "$(nix-hash --to-sri --type $hashType $hash)",
+  "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
+  "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
+  "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
+  "fetchTags": $([[ -n "$fetchTags" ]] && echo true || echo false),
+  "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false),
+  "rootDir": "$(json_escape "$rootDir")"
+}
+EOF
+    fi
+}
+
+remove_tmpPath() {
+    rm -rf "$tmpPath"
+}
+
+remove_tmpHomePath() {
+    chmod -R u+w "$tmpHomePath"
+    rm -rf "$tmpHomePath"
+}
+
+if test -n "$QUIET"; then
+    quiet_mode
+fi
+
+if test -z "$branchName"; then
+    branchName=fetchgit
+fi
+
+if [ -v symbolicName ]; then
+    storePathName="$symbolicName"
+else
+    storePathName="$(url_to_name "$url" "$rev")"
+fi
+
+tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
+exit_handlers+=(remove_tmpHomePath)
+ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
+HOME="$tmpHomePath"
+unset XDG_CONFIG_HOME
+export GIT_CONFIG_NOSYSTEM=1
+
+if test -n "$builder"; then
+    test -n "$out" -a -n "$url" -a -n "$rev" || usage
+    if test -n "$rootDir"; then
+        clone_user_rev_to_tmpfile "$url" "$rev"
+        mv "$tmpClone/$rootDir" "$out"
+    else
+        mkdir -p "$out"
+        clone_user_rev "$out" "$url" "$rev"
+    fi
+else
+    if test -z "$hashType"; then
+        hashType=sha256
+    fi
+
+    # If the hash was given, a file with that hash may already be in the
+    # store.
+    if test -n "$expHash"; then
+        finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$storePathName")
+        if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
+            finalPath=
+        fi
+        hash=$expHash
+    fi
+
+    # If we don't know the hash or a path with that hash doesn't exist,
+    # download the file and add it to the store.
+    if test -z "$finalPath"; then
+        clone_user_rev_to_tmpfile "$url" "$rev"
+
+        if test -z "$rootDir"; then
+            mv "$tmpClone" "$tmpOut"
+        else
+            mv "$tmpClone/$rootDir" "$tmpOut"
+        fi
+
+        # Compute the hash.
+        hash=$(nix-hash --type $hashType --base32 "$tmpOut")
+
+        # Add the downloaded file to the Nix store.
+
+        if test -z "$noAddPath"; then
+            finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpOut") \
+                || { printf "maybe try again with \`nix-prefetch-git --no-add-path <...>' ?\n" >&2; exit 1; }
+        else
+            printf "the path for \`%s' has NOT been added to the store\n" "$url" >&2
+            finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$hash" "$storePathName")
+        fi
+
+        if test -n "$expHash" -a "$expHash" != "$hash"; then
+            echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
+            exit 1
+        fi
+    fi
+
+    print_results "$hash"
+
+    if test -n "$PRINT_PATH"; then
+        echo "$finalPath"
+    fi
+fi

pkgs/build-support/fetchgit/tests.nix

@@ -0,0 +1,212 @@
+{
+  runCommand,
+  testers,
+  fetchgit,
+  nix-prefetch-git,
+  jq,
+  cacert,
+  nix,
+  closureInfo,
+  ...
+}:
+{
+  simple = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "simple-nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
+  };
+
+  sparseCheckout = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "sparse-checkout-nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    sparseCheckout = [
+      "src"
+      "tests"
+    ];
+    sha256 = "sha256-g1PHGTWgAcd/+sXHo1o6AjVWCvC6HiocOfMbMh873LQ=";
+  };
+
+  sparseCheckoutNonConeMode = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "sparse-checkout-non-cone-nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    sparseCheckout = [
+      "src"
+      "tests"
+    ];
+    nonConeMode = true;
+    sha256 = "sha256-FknO6C/PSnMPfhUqObD4vsW4PhkwdmPa9blNzcNvJQ4=";
+  };
+
+  leave-git = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "leave-git-nix-source";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    sha256 = "sha256-VmQ38+lr+rNPaTnjjV41uC2XSN4fkfZAfytE2uKyLfo=";
+    leaveDotGit = true;
+  };
+
+  submodule-simple = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "submodule-simple-source";
+    url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";
+    rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a";
+    sha256 = "sha256-rmP8PQT0wJBopdtr/hsB7Y/L1G+ZPdHC2r9LB05Qrj4=";
+    fetchSubmodules = true;
+  };
+
+  submodule-leave-git = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "submodule-leave-git-source";
+    url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";
+    rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a";
+    sha256 = "sha256-EC2PMEEtA7f5OFdsluHn7pi4QXhCZuFML8tib4pV7Ek=";
+    leaveDotGit = true;
+    fetchSubmodules = true;
+  };
+
+  submodule-deep = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "submodule-deep-source";
+    url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";
+    rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a";
+    sha256 = "sha256-3zWogs6EZBnzUfz6gBnigETTKGYl9KFKFgsy6Bl4DME=";
+    deepClone = true;
+    fetchSubmodules = true;
+    # deepClone implies leaveDotGit, so delete the .git directory after
+    # fetching to distinguish from the submodule-leave-git-deep test.
+    postFetch = "rm -r $out/.git";
+  };
+
+  submodule-leave-git-deep = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "submodule-leave-git-deep-source";
+    url = "https://github.com/pineapplehunter/nix-test-repo-with-submodule";
+    rev = "26473335b84ead88ee0a3b649b1c7fa4a91cfd4a";
+    sha256 = "sha256-ieYn9I/0RgeSwQkSqwKaU3RgjKFlRqMg7zw0Nvu3azA=";
+    deepClone = true;
+    leaveDotGit = true;
+    fetchSubmodules = true;
+  };
+
+  dumb-http-signed-tag = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "dumb-http-signed-tag-source";
+    url = "https://git.scottworley.com/pub/git/pinch";
+    rev = "v3.0.14";
+    sha256 = "sha256-bd0Lx75Gd1pcBJtwz5WGki7XoYSpqhinCT3a77wpY2c=";
+  };
+
+  fetchTags = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "fetchgit-fetch-tags-test";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    fetchTags = true;
+    leaveDotGit = true;
+    sha256 = "sha256-y7l+46lVP2pzJwGON5qEV0EoxWofRoWAym5q9VXvpc8=";
+    postFetch = ''
+      cd $out && git describe --tags --always > describe-output.txt 2>&1 || echo "git describe failed" > describe-output.txt
+      # See https://github.com/NixOS/nixpkgs/issues/412967#issuecomment-2927452118
+      rm -rf .git
+    '';
+  };
+
+  rootDir = testers.invalidateFetcherByDrvHash fetchgit {
+    name = "fetchgit-with-rootdir";
+    url = "https://github.com/NixOS/nix";
+    rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+    rootDir = "misc/systemd";
+    sha256 = "sha256-UhxHk4SrXYq7ZDMtXLig5SigpbITrVgkpFTmryuvpcM=";
+  };
+
+  # Make sure that if an expected hash is given and the corresponding store path exists already, no fetch is done
+  cached-prefetch-avoids-fetch =
+    let
+      name = "cached-prefetch-avoids-fetch";
+      url = "https://github.com/NixOS/nix";
+      rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+      sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
+      fetched = fetchgit {
+        inherit
+          name
+          url
+          rev
+          sha256
+          ;
+      };
+    in
+    runCommand "cached-prefetch-avoids-fetch"
+      {
+        nativeBuildInputs = [
+          nix-prefetch-git
+          nix
+        ];
+      }
+      ''
+        export NIX_REMOTE=local?root=$(mktemp -d)
+        nix-store --load-db < ${closureInfo { rootPaths = fetched; }}/registration
+        nix-prefetch-git --name "${name}" "${url}" "${rev}" "${sha256}" > $out
+      '';
+
+  prefetch-git-no-add-path =
+    testers.invalidateFetcherByDrvHash
+      (
+        {
+          name,
+          url,
+          rev,
+          hash,
+          ...
+        }:
+        runCommand name
+          {
+            buildInputs = [
+              nix-prefetch-git
+              nix
+              cacert
+              jq
+            ];
+            outputHashMode = "recursive";
+            outputHashAlgo = null;
+            outputHash = hash;
+            inherit url rev;
+          }
+          ''
+            store_root="$(mktemp -d)"
+            prefetch() { NIX_REMOTE="local?root=$store_root" nix-prefetch-git $@ "$url" --rev "$rev" | jq -r .path; }
+            path="$(prefetch --no-add-path)"
+            if test -e "$store_root/$path"; then
+              echo "$path exists in $NIX_REMOTE when it shouldn't" >&2
+              exit 1
+            fi
+            path_added="$(prefetch)"
+            if ! test -e "$store_root/$path"; then
+              echo "$path_added doesn't exist in NIX_REMOTE when it should" >&2
+              exit 1
+            fi
+            if test "$path" != "$path_added"; then
+              echo "Paths are different with and without --no-add-path: $path != $path_added" >&2
+              exit 1
+            fi
+            cp -r "$store_root/$path_added" "$out"
+          ''
+      )
+      {
+        name = "nix-prefetch-git-no-add-path";
+        url = "https://github.com/NixOS/nix";
+        rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+        hash = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
+      };
+
+  withGitConfig =
+    let
+      pkgs = import ../../.. {
+        config.gitConfig = {
+          url."https://github.com".insteadOf = "https://doesntexist.forsure";
+        };
+      };
+    in
+    pkgs.testers.invalidateFetcherByDrvHash pkgs.fetchgit {
+      name = "fetchgit-with-config";
+      url = "https://doesntexist.forsure/NixOS/nix";
+      rev = "9d9dbe6ed05854e03811c361a3380e09183f4f4a";
+      sha256 = "sha256-7DszvbCNTjpzGRmpIVAWXk20P0/XTrWZ79KSOGLrUWY=";
+    };
+}