push sheeet

pkgs/build-support/node/build-npm-package/default.nix

@@ -0,0 +1,110 @@
+{
+  lib,
+  stdenv,
+  fetchNpmDeps,
+  buildPackages,
+  nodejs,
+  cctools,
+}@topLevelArgs:
+
+lib.extendMkDerivation {
+  constructDrv = stdenv.mkDerivation;
+
+  extendDrvArgs =
+    finalAttrs:
+    {
+      name ? "${args.pname}-${args.version}",
+      src ? null,
+      srcs ? null,
+      sourceRoot ? null,
+      prePatch ? "",
+      patches ? [ ],
+      postPatch ? "",
+      patchFlags ? [ ],
+      nativeBuildInputs ? [ ],
+      buildInputs ? [ ],
+      # The output hash of the dependencies for this project.
+      # Can be calculated in advance with prefetch-npm-deps.
+      npmDepsHash ? "",
+      # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile.
+      # Use with care.
+      forceGitDeps ? false,
+      # Whether to force allow an empty dependency cache.
+      # This can be enabled if there are truly no remote dependencies, but generally an empty cache indicates something is wrong.
+      forceEmptyCache ? false,
+      # Whether to make the cache writable prior to installing dependencies.
+      # Don't set this unless npm tries to write to the cache directory, as it can slow down the build.
+      makeCacheWritable ? false,
+      # The script to run to build the project.
+      npmBuildScript ? "build",
+      # Flags to pass to all npm commands.
+      npmFlags ? [ ],
+      # Flags to pass to `npm ci`.
+      npmInstallFlags ? [ ],
+      # Flags to pass to `npm rebuild`.
+      npmRebuildFlags ? [ ],
+      # Flags to pass to `npm run ${npmBuildScript}`.
+      npmBuildFlags ? [ ],
+      # Flags to pass to `npm pack`.
+      npmPackFlags ? [ ],
+      # Flags to pass to `npm prune`.
+      npmPruneFlags ? npmInstallFlags,
+      # Value for npm `--workspace` flag and directory in which the files to be installed are found.
+      npmWorkspace ? null,
+      nodejs ? topLevelArgs.nodejs,
+      npmDeps ? fetchNpmDeps {
+        inherit
+          forceGitDeps
+          forceEmptyCache
+          src
+          srcs
+          sourceRoot
+          prePatch
+          patches
+          postPatch
+          patchFlags
+          ;
+        name = "${name}-npm-deps";
+        hash = npmDepsHash;
+      },
+      # Custom npmConfigHook
+      npmConfigHook ? null,
+      # Custom npmBuildHook
+      npmBuildHook ? null,
+      # Custom npmInstallHook
+      npmInstallHook ? null,
+      ...
+    }@args:
+
+    let
+      # .override {} negates splicing, so we need to use buildPackages explicitly
+      npmHooks = buildPackages.npmHooks.override {
+        inherit nodejs;
+      };
+    in
+    {
+      inherit npmDeps npmBuildScript;
+
+      nativeBuildInputs =
+        nativeBuildInputs
+        ++ [
+          nodejs
+          # Prefer passed hooks
+          (if npmConfigHook != null then npmConfigHook else npmHooks.npmConfigHook)
+          (if npmBuildHook != null then npmBuildHook else npmHooks.npmBuildHook)
+          (if npmInstallHook != null then npmInstallHook else npmHooks.npmInstallHook)
+          nodejs.python
+        ]
+        ++ lib.optionals stdenv.hostPlatform.isDarwin [ cctools ];
+      buildInputs = buildInputs ++ [ nodejs ];
+
+      strictDeps = true;
+
+      # Stripping takes way too long with the amount of files required by a typical Node.js project.
+      dontStrip = args.dontStrip or true;
+
+      meta = (args.meta or { }) // {
+        platforms = args.meta.platforms or nodejs.meta.platforms;
+      };
+    };
+}

pkgs/build-support/node/build-npm-package/hooks/default.nix

@@ -0,0 +1,54 @@
+{
+  lib,
+  srcOnly,
+  stdenv,
+  makeSetupHook,
+  makeWrapper,
+  nodejs,
+  jq,
+  prefetch-npm-deps,
+  diffutils,
+  installShellFiles,
+  nodejsInstallManuals,
+  nodejsInstallExecutables,
+}:
+
+{
+  npmConfigHook = makeSetupHook {
+    name = "npm-config-hook";
+    substitutions = {
+      nodeSrc = srcOnly nodejs;
+      nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+      npmArch = stdenv.targetPlatform.node.arch;
+      npmPlatform = stdenv.targetPlatform.node.platform;
+
+      # Specify `diff`, `jq`, and `prefetch-npm-deps` by abspath to ensure that the user's build
+      # inputs do not cause us to find the wrong binaries.
+      diff = "${diffutils}/bin/diff";
+      jq = "${jq}/bin/jq";
+      prefetchNpmDeps = "${prefetch-npm-deps}/bin/prefetch-npm-deps";
+
+      nodeVersion = nodejs.version;
+      nodeVersionMajor = lib.versions.major nodejs.version;
+    };
+  } ./npm-config-hook.sh;
+
+  npmBuildHook = makeSetupHook {
+    name = "npm-build-hook";
+  } ./npm-build-hook.sh;
+
+  npmInstallHook = makeSetupHook {
+    name = "npm-install-hook";
+    propagatedBuildInputs = [
+      installShellFiles
+      makeWrapper
+      nodejsInstallManuals
+      (nodejsInstallExecutables.override {
+        inherit nodejs;
+      })
+    ];
+    substitutions = {
+      jq = "${jq}/bin/jq";
+    };
+  } ./npm-install-hook.sh;
+}

pkgs/build-support/node/build-npm-package/hooks/npm-build-hook.sh

@@ -0,0 +1,38 @@
+# shellcheck shell=bash
+
+npmBuildHook() {
+    echo "Executing npmBuildHook"
+
+    runHook preBuild
+
+    if [ -z "${npmBuildScript-}" ]; then
+        echo
+        echo "ERROR: no build script was specified"
+        echo 'Hint: set `npmBuildScript`, override `buildPhase`, or set `dontNpmBuild = true`.'
+        echo
+
+        exit 1
+    fi
+
+    if ! npm run ${npmWorkspace+--workspace=$npmWorkspace} "$npmBuildScript" $npmBuildFlags "${npmBuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo 'ERROR: `npm build` failed'
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo "1. Make sure your build script ($npmBuildScript) exists"
+        echo '  If there is none, set `dontNpmBuild = true`.'
+        echo '2. If the error being thrown is something similar to "error:0308010C:digital envelope routines::unsupported", add `NODE_OPTIONS = "--openssl-legacy-provider"` to your derivation'
+        echo "  See https://github.com/webpack/webpack/issues/14532 for more information."
+        echo
+
+        exit 1
+    fi
+
+    runHook postBuild
+
+    echo "Finished npmBuildHook"
+}
+
+if [ -z "${dontNpmBuild-}" ] && [ -z "${buildPhase-}" ]; then
+    buildPhase=npmBuildHook
+fi

pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh

@@ -0,0 +1,123 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    # Use npm patches in the nodejs package
+    export NIX_NODEJS_BUILDNPMPACKAGE=1
+    export prefetchNpmDeps="@prefetchNpmDeps@"
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    echo "Configuring npm"
+
+    export HOME="$TMPDIR"
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+    export npm_config_arch="@npmArch@"
+    export npm_config_platform="@npmPlatform@"
+
+    if [ -z "${npmDeps-}" ]; then
+        echo
+        echo "ERROR: no dependencies were specified"
+        echo 'Hint: set `npmDeps` if using these hooks individually. If this is happening with `buildNpmPackage`, please open an issue.'
+        echo
+
+        exit 1
+    fi
+
+    local -r cacheLockfile="$npmDeps/package-lock.json"
+    local -r srcLockfile="$PWD/package-lock.json"
+
+    echo "Validating consistency between $srcLockfile and $cacheLockfile"
+
+    if ! @diff@ "$srcLockfile" "$cacheLockfile"; then
+      # If the diff failed, first double-check that the file exists, so we can
+      # give a friendlier error msg.
+      if ! [ -e "$srcLockfile" ]; then
+        echo
+        echo "ERROR: Missing package-lock.json from src. Expected to find it at: $srcLockfile"
+        echo "Hint: You can copy a vendored package-lock.json file via postPatch."
+        echo
+
+        exit 1
+      fi
+
+      if ! [ -e "$cacheLockfile" ]; then
+        echo
+        echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile"
+        echo
+
+        exit 1
+      fi
+
+      echo
+      echo "ERROR: npmDepsHash is out of date"
+      echo
+      echo "The package-lock.json in src is not the same as the in $npmDeps."
+      echo
+      echo "To fix the issue:"
+      echo '1. Use `lib.fakeHash` as the npmDepsHash value'
+      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
+      echo "3. Copy the 'got: sha256-' value back into the npmDepsHash field"
+      echo
+
+      exit 1
+    fi
+
+    export CACHE_MAP_PATH="$TMP/MEOW"
+    @prefetchNpmDeps@ --map-cache
+
+    @prefetchNpmDeps@ --fixup-lockfile "$srcLockfile"
+
+    local cachePath
+
+    if [ -z "${makeCacheWritable-}" ]; then
+        cachePath="$npmDeps"
+    else
+        echo "Making cache writable"
+        cp -r "$npmDeps" "$TMPDIR/cache"
+        chmod -R 700 "$TMPDIR/cache"
+        cachePath="$TMPDIR/cache"
+    fi
+
+    echo "Setting npm_config_cache to $cachePath"
+    # do not use npm config to avoid modifying .npmrc
+    export npm_config_cache="$cachePath"
+    export npm_config_offline="true"
+    export npm_config_progress="false"
+
+    echo "Installing dependencies"
+
+    if ! npm ci --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `makeCacheWritable = true`'
+        echo "  Note that this won't help if npm is complaining about not being able to write to the logs directory -- look above that for the actual error."
+        echo '2. Set `npmFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
+
+    patchShebangs node_modules
+
+    rm "$CACHE_MAP_PATH"
+    unset CACHE_MAP_PATH
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postPatchHooks+=(npmConfigHook)

pkgs/build-support/node/build-npm-package/hooks/npm-install-hook.sh

@@ -0,0 +1,49 @@
+# shellcheck shell=bash
+
+npmInstallHook() {
+    echo "Executing npmInstallHook"
+
+    runHook preInstall
+
+    local -r packageOut="$out/lib/node_modules/$(@jq@ --raw-output '.name' package.json)"
+
+    # `npm pack` writes to cache so temporarily override it
+    while IFS= read -r file; do
+        local dest="$packageOut/$(dirname "$file")"
+        mkdir -p "$dest"
+        cp "${npmWorkspace-.}/$file" "$dest"
+    done < <(@jq@ --raw-output '.[0].files | map(.path | select(. | startswith("node_modules/") | not)) | join("\n")' <<< "$(npm_config_cache="$HOME/.npm" npm pack --json --dry-run --loglevel=warn --no-foreground-scripts ${npmWorkspace+--workspace=$npmWorkspace} $npmPackFlags "${npmPackFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}")")
+
+    nodejsInstallExecutables "${npmWorkspace-.}/package.json"
+
+    nodejsInstallManuals "${npmWorkspace-.}/package.json"
+
+    local -r nodeModulesPath="$packageOut/node_modules"
+
+    if [ ! -d "$nodeModulesPath" ]; then
+        if [ -z "${dontNpmPrune-}" ]; then
+            if ! npm prune --omit=dev --no-save ${npmWorkspace+--workspace=$npmWorkspace} $npmPruneFlags "${npmPruneFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+              echo
+              echo
+              echo "ERROR: npm prune step failed"
+              echo
+              echo 'If npm tried to download additional dependencies above, try setting `dontNpmPrune = true`.'
+              echo
+
+              exit 1
+            fi
+        fi
+
+        find node_modules -maxdepth 1 -type d -empty -delete
+
+        cp -r node_modules "$nodeModulesPath"
+    fi
+
+    runHook postInstall
+
+    echo "Finished npmInstallHook"
+}
+
+if [ -z "${dontNpmInstall-}" ] && [ -z "${installPhase-}" ]; then
+    installPhase=npmInstallHook
+fi

pkgs/build-support/node/fetch-npm-deps/.gitignore

@@ -0,0 +1 @@
+/target

pkgs/build-support/node/fetch-npm-deps/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "prefetch-npm-deps"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.82"
+backoff = "0.4.0"
+data-encoding = "2.5.0"
+digest = "0.10.7"
+env_logger = "0.11.3"
+isahc = { version = "1.7.2", default_features = false }
+log = "0.4.21"
+nix-nar = "0.3.0"
+rayon = "1.10.0"
+serde = { version = "1.0.198", features = ["derive"] }
+serde_json = "1.0.116"
+sha1 = "0.10.6"
+sha2 = "0.10.8"
+tempfile = "3.10.1"
+url = { version = "2.5.0", features = ["serde"] }
+walkdir = "2.5.0"

pkgs/build-support/node/fetch-npm-deps/default.nix

@@ -0,0 +1,265 @@
+{
+  lib,
+  stdenvNoCC,
+  rustPlatform,
+  makeWrapper,
+  pkg-config,
+  curl,
+  gnutar,
+  gzip,
+  testers,
+  fetchurl,
+  cacert,
+  prefetch-npm-deps,
+  fetchNpmDeps,
+}:
+
+{
+  prefetch-npm-deps = rustPlatform.buildRustPackage {
+    pname = "prefetch-npm-deps";
+    version = (lib.importTOML ./Cargo.toml).package.version;
+
+    src = lib.cleanSourceWith {
+      src = ./.;
+      filter =
+        name: type:
+        let
+          name' = baseNameOf name;
+        in
+        name' != "default.nix" && name' != "target";
+    };
+
+    cargoLock.lockFile = ./Cargo.lock;
+
+    nativeBuildInputs = [
+      makeWrapper
+      pkg-config
+    ];
+    buildInputs = [ curl ];
+
+    postInstall = ''
+      wrapProgram "$out/bin/prefetch-npm-deps" --prefix PATH : ${
+        lib.makeBinPath [
+          gnutar
+          gzip
+        ]
+      }
+    '';
+
+    passthru.tests =
+      let
+        makeTestSrc =
+          { name, src }:
+          stdenvNoCC.mkDerivation {
+            name = "${name}-src";
+
+            inherit src;
+
+            buildCommand = ''
+              mkdir -p $out
+              cp $src $out/package-lock.json
+            '';
+          };
+
+        makeTest =
+          {
+            name,
+            src,
+            hash,
+            forceGitDeps ? false,
+            forceEmptyCache ? false,
+          }:
+          testers.invalidateFetcherByDrvHash fetchNpmDeps {
+            inherit
+              name
+              hash
+              forceGitDeps
+              forceEmptyCache
+              ;
+
+            src = makeTestSrc { inherit name src; };
+          };
+      in
+      {
+        lockfileV1 = makeTest {
+          name = "lockfile-v1";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/jellyfin/jellyfin-web/v10.8.4/package-lock.json";
+            hash = "sha256-uQmc+S+V1co1Rfc4d82PpeXjmd1UqdsG492ADQFcZGA=";
+          };
+
+          hash = "sha256-wca1QvxUw3OrLStfYN9Co6oVBR1LbfcNUKlDqvObps4=";
+        };
+
+        lockfileV2 = makeTest {
+          name = "lockfile-v2";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/jesec/flood/v4.7.0/package-lock.json";
+            hash = "sha256-qS29tq5QPnGxV+PU40VgMAtdwVLtLyyhG2z9GMeYtC4=";
+          };
+
+          hash = "sha256-tuEfyePwlOy2/mOPdXbqJskO6IowvAP4DWg8xSZwbJw=";
+        };
+
+        hashPrecedence = makeTest {
+          name = "hash-precedence";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/matrix-org/matrix-appservice-irc/0.34.0/package-lock.json";
+            hash = "sha256-1+0AQw9EmbHiMPA/H8OP8XenhrkhLRYBRhmd1cNPFjk=";
+          };
+
+          hash = "sha256-oItUls7AXcCECuyA+crQO6B0kv4toIr8pBubNwB7kAM=";
+        };
+
+        hostedGitDeps = makeTest {
+          name = "hosted-git-deps";
+
+          src = fetchurl {
+            url = "https://cyberchaos.dev/yuka/trainsearch/-/raw/e3cba6427e8ecfd843d0f697251ddaf5e53c2327/package-lock.json";
+            hash = "sha256-X9mCwPqV5yP0S2GonNvpYnLSLJMd/SUIked+hMRxDpA=";
+          };
+
+          hash = "sha256-tEdElWJ+KBTxBobzXBpPopQSwK2usGW/it1+yfbVzBw=";
+        };
+
+        linkDependencies = makeTest {
+          name = "link-dependencies";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/evcc-io/evcc/0.106.3/package-lock.json";
+            hash = "sha256-6ZTBMyuyPP/63gpQugggHhKVup6OB4hZ2rmSvPJ0yEs=";
+          };
+
+          hash = "sha256-VzQhArHoznYSXUT7l9HkJV4yoSOmoP8eYTLel1QwmB4=";
+        };
+
+        # This package has no resolved deps whatsoever, which will not actually work but does test the forceEmptyCache option.
+        emptyCache = makeTest {
+          name = "empty-cache";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/bufbuild/protobuf-es/v1.2.1/package-lock.json";
+            hash = "sha256-UdBUEb4YRHsbvyjymIyjemJEiaI9KQRirqt+SFSK0wA=";
+          };
+
+          hash = "sha256-Cdv40lQjRszzJtJydZt25uYfcJVeJGwH54A+agdH9wI=";
+
+          forceEmptyCache = true;
+        };
+
+        # This package contains both hosted Git shorthand, and a bundled dependency that happens to override an existing one.
+        etherpadLite1818 = makeTest {
+          name = "etherpad-lite-1.8.18";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/ether/etherpad-lite/1.8.18/src/package-lock.json";
+            hash = "sha256-1fGNxYJi1I4cXK/jinNG+Y6tPEOhP3QAqWOBEQttS9E=";
+          };
+
+          hash = "sha256-+KA8/orSBJ4EhuSyQO8IKSxsN/FAsYU3lOzq+awuxNQ=";
+
+          forceGitDeps = true;
+        };
+
+        # This package has a lockfile v1 git dependency with no `dependencies` attribute, since it semantically has no dependencies.
+        jitsiMeet9111 = makeTest {
+          name = "jitsi-meet-9111";
+
+          src = fetchurl {
+            url = "https://raw.githubusercontent.com/jitsi/jitsi-meet/stable/jitsi-meet_9111/package-lock.json";
+            hash = "sha256-NU+eQD4WZ4BMur8uX79uk8wUPsZvIT02KhPWHTmaihk=";
+          };
+
+          hash = "sha256-FhxlJ0HdJMPiWe7+n1HaGLWOr/2HJEPwiS65uqXZM8Y=";
+        };
+      };
+
+    meta = with lib; {
+      description = "Prefetch dependencies from npm (for use with `fetchNpmDeps`)";
+      mainProgram = "prefetch-npm-deps";
+      maintainers = with maintainers; [ winter ];
+      license = licenses.mit;
+    };
+  };
+
+  fetchNpmDeps =
+    {
+      name ? "npm-deps",
+      hash ? "",
+      forceGitDeps ? false,
+      forceEmptyCache ? false,
+      nativeBuildInputs ? [ ],
+      ...
+    }@args:
+    let
+      hash_ =
+        if hash != "" then
+          {
+            outputHash = hash;
+          }
+        else
+          {
+            outputHash = "";
+            outputHashAlgo = "sha256";
+          };
+
+      forceGitDeps_ = lib.optionalAttrs forceGitDeps { FORCE_GIT_DEPS = true; };
+      forceEmptyCache_ = lib.optionalAttrs forceEmptyCache { FORCE_EMPTY_CACHE = true; };
+    in
+    stdenvNoCC.mkDerivation (
+      args
+      // {
+        inherit name;
+
+        nativeBuildInputs = nativeBuildInputs ++ [ prefetch-npm-deps ];
+
+        buildPhase = ''
+          runHook preBuild
+
+          if [[ ! -e package-lock.json ]]; then
+            echo
+            echo "ERROR: The package-lock.json file does not exist!"
+            echo
+            echo "package-lock.json is required to make sure that npmDepsHash doesn't change"
+            echo "when packages are updated on npm."
+            echo
+            echo "Hint: You can copy a vendored package-lock.json file via postPatch."
+            echo
+
+            exit 1
+          fi
+
+          prefetch-npm-deps package-lock.json $out
+
+          runHook postBuild
+        '';
+
+        dontInstall = true;
+
+        # NIX_NPM_TOKENS environment variable should be a JSON mapping in the shape of:
+        # `{ "registry.example.com": "example-registry-bearer-token", ... }`
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ "NIX_NPM_TOKENS" ];
+
+        SSL_CERT_FILE =
+          if
+            (
+              hash_.outputHash == ""
+              || hash_.outputHash == lib.fakeSha256
+              || hash_.outputHash == lib.fakeSha512
+              || hash_.outputHash == lib.fakeHash
+            )
+          then
+            "${cacert}/etc/ssl/certs/ca-bundle.crt"
+          else
+            "/no-cert-file.crt";
+
+        outputHashMode = "recursive";
+      }
+      // hash_
+      // forceGitDeps_
+      // forceEmptyCache_
+    );
+}

pkgs/build-support/node/fetch-npm-deps/src/cacache.rs

@@ -0,0 +1,128 @@
+use data_encoding::BASE64;
+use digest::{Digest, Update};
+use serde::{Deserialize, Serialize};
+use sha1::Sha1;
+use sha2::{Sha256, Sha512};
+use std::{
+    fmt::Write as FmtWrite,
+    fs::{self, File},
+    io::Write,
+    path::PathBuf,
+};
+use url::Url;
+
+#[allow(clippy::struct_field_names)]
+#[derive(Serialize, Deserialize)]
+pub(super) struct Key {
+    pub(super) key: String,
+    pub(super) integrity: String,
+    pub(super) time: u8,
+    pub(super) size: usize,
+    pub(super) metadata: Metadata,
+}
+
+#[derive(Serialize, Deserialize)]
+pub(super) struct Metadata {
+    pub(super) url: Url,
+    pub(super) options: Options,
+}
+
+#[derive(Serialize, Deserialize)]
+pub(super) struct Options {
+    pub(super) compress: bool,
+}
+
+pub struct Cache(PathBuf);
+
+fn push_hash_segments(path: &mut PathBuf, hash: &str) {
+    path.push(&hash[0..2]);
+    path.push(&hash[2..4]);
+    path.push(&hash[4..]);
+}
+
+impl Cache {
+    pub fn new(path: PathBuf) -> Cache {
+        Cache(path)
+    }
+
+    pub fn init(&self) -> anyhow::Result<()> {
+        fs::create_dir_all(self.0.join("content-v2"))?;
+        fs::create_dir_all(self.0.join("index-v5"))?;
+
+        Ok(())
+    }
+
+    pub fn put(
+        &self,
+        key: String,
+        url: Url,
+        data: &[u8],
+        integrity: Option<String>,
+    ) -> anyhow::Result<()> {
+        let (algo, hash, integrity) = if let Some(integrity) = integrity {
+            let (algo, hash) = integrity
+                .split_once('-')
+                .expect("hash should be SRI format");
+
+            (algo.to_string(), BASE64.decode(hash.as_bytes())?, integrity)
+        } else {
+            let hash = Sha512::new().chain(data).finalize();
+
+            (
+                String::from("sha512"),
+                hash.to_vec(),
+                format!("sha512-{}", BASE64.encode(&hash)),
+            )
+        };
+
+        let content_path = {
+            let mut p = self.0.join("content-v2");
+
+            p.push(algo);
+
+            push_hash_segments(
+                &mut p,
+                &hash.into_iter().fold(String::new(), |mut out, n| {
+                    let _ = write!(out, "{n:02x}");
+                    out
+                }),
+            );
+
+            p
+        };
+
+        fs::create_dir_all(content_path.parent().unwrap())?;
+
+        fs::write(content_path, data)?;
+
+        let index_path = {
+            let mut p = self.0.join("index-v5");
+
+            push_hash_segments(
+                &mut p,
+                &format!("{:x}", Sha256::new().chain(&key).finalize()),
+            );
+
+            p
+        };
+
+        fs::create_dir_all(index_path.parent().unwrap())?;
+
+        let data = serde_json::to_string(&Key {
+            key,
+            integrity,
+            time: 0,
+            size: data.len(),
+            metadata: Metadata {
+                url,
+                options: Options { compress: true },
+            },
+        })?;
+
+        let mut file = File::options().append(true).create(true).open(index_path)?;
+
+        write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?;
+
+        Ok(())
+    }
+}

pkgs/build-support/node/fetch-npm-deps/src/main.rs

@@ -0,0 +1,427 @@
+#![warn(clippy::pedantic)]
+
+use crate::cacache::{Cache, Key};
+use anyhow::{anyhow, bail};
+use rayon::prelude::*;
+use serde_json::{Map, Value};
+use std::{
+    collections::HashMap,
+    env, fs,
+    path::{Path, PathBuf},
+    process,
+};
+use tempfile::tempdir;
+use url::Url;
+use walkdir::WalkDir;
+
+mod cacache;
+mod parse;
+mod util;
+
+fn cache_map_path() -> Option<PathBuf> {
+    env::var_os("CACHE_MAP_PATH").map(PathBuf::from)
+}
+
+/// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies.
+///
+/// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1
+/// and SHA-512) in the lockfile. Given we only cache one version of these, the `integrity` field must be normalized to the hash
+/// we cache as (which is the strongest available one).
+///
+/// Git dependencies from specific providers can be retrieved from those providers' automatic tarball features.
+/// When these dependencies are specified with a commit identifier, npm generates a tarball, and inserts the integrity hash of that
+/// tarball into the lockfile.
+///
+/// Thus, we remove this hash, to replace it with our own determinstic copies of dependencies from hosted Git providers.
+///
+/// If no fixups were performed, `None` is returned and the lockfile structure should be left as-is. If fixups were performed, the
+/// `dependencies` key in v2 lockfiles designed for backwards compatibility with v1 parsers is removed because of inconsistent data.
+fn fixup_lockfile(
+    mut lock: Map<String, Value>,
+    cache: &Option<HashMap<String, String>>,
+) -> anyhow::Result<Option<Map<String, Value>>> {
+    let mut fixed = false;
+
+    match lock
+        .get("lockfileVersion")
+        .ok_or_else(|| anyhow!("couldn't get lockfile version"))?
+        .as_i64()
+        .ok_or_else(|| anyhow!("lockfile version isn't an int"))?
+    {
+        1 => fixup_v1_deps(
+            lock.get_mut("dependencies")
+                .unwrap()
+                .as_object_mut()
+                .unwrap(),
+            cache,
+            &mut fixed,
+        ),
+        2 | 3 => {
+            for package in lock
+                .get_mut("packages")
+                .ok_or_else(|| anyhow!("couldn't get packages"))?
+                .as_object_mut()
+                .ok_or_else(|| anyhow!("packages isn't a map"))?
+                .values_mut()
+            {
+                if let Some(Value::String(resolved)) = package.get("resolved") {
+                    if let Some(Value::String(integrity)) = package.get("integrity") {
+                        if resolved.starts_with("git+") {
+                            fixed = true;
+
+                            package
+                                .as_object_mut()
+                                .ok_or_else(|| anyhow!("package isn't a map"))?
+                                .remove("integrity");
+                        } else if let Some(cache_hashes) = cache {
+                            let cache_hash = cache_hashes
+                                .get(resolved)
+                                .expect("dependency should have a hash");
+
+                            if integrity != cache_hash {
+                                fixed = true;
+
+                                *package
+                                    .as_object_mut()
+                                    .ok_or_else(|| anyhow!("package isn't a map"))?
+                                    .get_mut("integrity")
+                                    .unwrap() = Value::String(cache_hash.clone());
+                            }
+                        }
+                    }
+                }
+            }
+
+            if fixed {
+                lock.remove("dependencies");
+            }
+        }
+        v => bail!("unsupported lockfile version {v}"),
+    }
+
+    if fixed {
+        Ok(Some(lock))
+    } else {
+        Ok(None)
+    }
+}
+
+// Recursive helper to fixup v1 lockfile deps
+fn fixup_v1_deps(
+    dependencies: &mut Map<String, Value>,
+    cache: &Option<HashMap<String, String>>,
+    fixed: &mut bool,
+) {
+    for dep in dependencies.values_mut() {
+        if let Some(Value::String(resolved)) = dep
+            .as_object()
+            .expect("v1 dep must be object")
+            .get("resolved")
+        {
+            if let Some(Value::String(integrity)) = dep
+                .as_object()
+                .expect("v1 dep must be object")
+                .get("integrity")
+            {
+                if resolved.starts_with("git+ssh://") {
+                    *fixed = true;
+
+                    dep.as_object_mut()
+                        .expect("v1 dep must be object")
+                        .remove("integrity");
+                } else if let Some(cache_hashes) = cache {
+                    let cache_hash = cache_hashes
+                        .get(resolved)
+                        .expect("dependency should have a hash");
+
+                    if integrity != cache_hash {
+                        *fixed = true;
+
+                        *dep.as_object_mut()
+                            .expect("v1 dep must be object")
+                            .get_mut("integrity")
+                            .unwrap() = Value::String(cache_hash.clone());
+                    }
+                }
+            }
+        }
+
+        if let Some(Value::Object(more_deps)) = dep.as_object_mut().unwrap().get_mut("dependencies")
+        {
+            fixup_v1_deps(more_deps, cache, fixed);
+        }
+    }
+}
+
+fn map_cache() -> anyhow::Result<HashMap<Url, String>> {
+    let mut hashes = HashMap::new();
+
+    let content_path = Path::new(&env::var_os("npmDeps").unwrap()).join("_cacache/index-v5");
+
+    for entry in WalkDir::new(content_path) {
+        let entry = entry?;
+
+        if entry.file_type().is_file() {
+            let content = fs::read_to_string(entry.path())?;
+            let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?;
+
+            hashes.insert(key.metadata.url, key.integrity);
+        }
+    }
+
+    Ok(hashes)
+}
+
+fn main() -> anyhow::Result<()> {
+    env_logger::init();
+
+    let args = env::args().collect::<Vec<_>>();
+
+    if args.len() < 2 {
+        println!("usage: {} <path/to/package-lock.json>", args[0]);
+        println!();
+        println!("Prefetches npm dependencies for usage by fetchNpmDeps.");
+
+        process::exit(1);
+    }
+
+    if let Ok(jobs) = env::var("NIX_BUILD_CORES") {
+        if !jobs.is_empty() {
+            rayon::ThreadPoolBuilder::new()
+                .num_threads(
+                    jobs.parse()
+                        .expect("NIX_BUILD_CORES must be a whole number"),
+                )
+                .build_global()
+                .unwrap();
+        }
+    }
+
+    if args[1] == "--fixup-lockfile" {
+        let lock = serde_json::from_str(&fs::read_to_string(&args[2])?)?;
+
+        let cache = cache_map_path()
+            .map(|map_path| Ok::<_, anyhow::Error>(serde_json::from_slice(&fs::read(map_path)?)?))
+            .transpose()?;
+
+        if let Some(fixed) = fixup_lockfile(lock, &cache)? {
+            println!("Fixing lockfile");
+
+            fs::write(&args[2], serde_json::to_string(&fixed)?)?;
+        }
+
+        return Ok(());
+    } else if args[1] == "--map-cache" {
+        let map = map_cache()?;
+
+        fs::write(
+            cache_map_path().expect("CACHE_MAP_PATH environment variable must be set"),
+            serde_json::to_string(&map)?,
+        )?;
+
+        return Ok(());
+    }
+
+    let lock_content = fs::read_to_string(&args[1])?;
+
+    let out_tempdir;
+
+    let (out, print_hash) = if let Some(path) = args.get(2) {
+        (Path::new(path), false)
+    } else {
+        out_tempdir = tempdir()?;
+
+        (out_tempdir.path(), true)
+    };
+
+    let packages = parse::lockfile(
+        &lock_content,
+        env::var("FORCE_GIT_DEPS").is_ok(),
+        env::var("FORCE_EMPTY_CACHE").is_ok(),
+    )?;
+
+    let cache = Cache::new(out.join("_cacache"));
+    cache.init()?;
+
+    packages.into_par_iter().try_for_each(|package| {
+        let tarball = package
+            .tarball()
+            .map_err(|e| anyhow!("couldn't fetch {} at {}: {e:?}", package.name, package.url))?;
+        let integrity = package.integrity().map(ToString::to_string);
+
+        cache
+            .put(
+                format!("make-fetch-happen:request-cache:{}", package.url),
+                package.url,
+                &tarball,
+                integrity,
+            )
+            .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?;
+
+        Ok::<_, anyhow::Error>(())
+    })?;
+
+    fs::write(out.join("package-lock.json"), lock_content)?;
+
+    if print_hash {
+        println!("{}", util::make_sri_hash(out)?);
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+
+    use super::fixup_lockfile;
+    use serde_json::json;
+
+    #[test]
+    fn lockfile_fixup() -> anyhow::Result<()> {
+        let input = json!({
+            "lockfileVersion": 2,
+            "name": "foo",
+            "packages": {
+                "": {
+
+                },
+                "foo": {
+                    "resolved": "https://github.com/NixOS/nixpkgs",
+                    "integrity": "sha1-aaa"
+                },
+                "bar": {
+                    "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
+                    "integrity": "sha512-aaa"
+                },
+                "foo-bad": {
+                    "resolved": "foo",
+                    "integrity": "sha1-foo"
+                },
+                "foo-good": {
+                    "resolved": "foo",
+                    "integrity": "sha512-foo"
+                },
+            }
+        });
+
+        let expected = json!({
+            "lockfileVersion": 2,
+            "name": "foo",
+            "packages": {
+                "": {
+
+                },
+                "foo": {
+                    "resolved": "https://github.com/NixOS/nixpkgs",
+                    "integrity": ""
+                },
+                "bar": {
+                    "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
+                },
+                "foo-bad": {
+                    "resolved": "foo",
+                    "integrity": "sha512-foo"
+                },
+                "foo-good": {
+                    "resolved": "foo",
+                    "integrity": "sha512-foo"
+                },
+            }
+        });
+
+        let mut hashes = HashMap::new();
+
+        hashes.insert(
+            String::from("https://github.com/NixOS/nixpkgs"),
+            String::new(),
+        );
+
+        hashes.insert(
+            String::from("git+ssh://git@github.com/NixOS/nixpkgs.git"),
+            String::new(),
+        );
+
+        hashes.insert(String::from("foo"), String::from("sha512-foo"));
+
+        assert_eq!(
+            fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
+            Some(expected.as_object().unwrap().clone())
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn lockfile_v1_fixup() -> anyhow::Result<()> {
+        let input = json!({
+            "lockfileVersion": 1,
+            "name": "foo",
+            "dependencies": {
+                "foo": {
+                    "resolved": "https://github.com/NixOS/nixpkgs",
+                    "integrity": "sha512-aaa"
+                },
+                "foo-good": {
+                    "resolved": "foo",
+                    "integrity": "sha512-foo"
+                },
+                "bar": {
+                    "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
+                    "integrity": "sha512-bbb",
+                    "dependencies": {
+                        "foo-bad": {
+                            "resolved": "foo",
+                            "integrity": "sha1-foo"
+                        },
+                    },
+                },
+            }
+        });
+
+        let expected = json!({
+            "lockfileVersion": 1,
+            "name": "foo",
+            "dependencies": {
+                "foo": {
+                    "resolved": "https://github.com/NixOS/nixpkgs",
+                    "integrity": ""
+                },
+                "foo-good": {
+                    "resolved": "foo",
+                    "integrity": "sha512-foo"
+                },
+                "bar": {
+                    "resolved": "git+ssh://git@github.com/NixOS/nixpkgs.git",
+                    "dependencies": {
+                        "foo-bad": {
+                            "resolved": "foo",
+                            "integrity": "sha512-foo"
+                        },
+                    },
+                },
+            }
+        });
+
+        let mut hashes = HashMap::new();
+
+        hashes.insert(
+            String::from("https://github.com/NixOS/nixpkgs"),
+            String::new(),
+        );
+
+        hashes.insert(
+            String::from("git+ssh://git@github.com/NixOS/nixpkgs.git"),
+            String::new(),
+        );
+
+        hashes.insert(String::from("foo"), String::from("sha512-foo"));
+
+        assert_eq!(
+            fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
+            Some(expected.as_object().unwrap().clone())
+        );
+
+        Ok(())
+    }
+}

pkgs/build-support/node/fetch-npm-deps/src/parse/lock.rs

@@ -0,0 +1,370 @@
+use anyhow::{anyhow, bail, Context};
+use rayon::slice::ParallelSliceMut;
+use serde::{
+    de::{self, Visitor},
+    Deserialize, Deserializer,
+};
+use std::{
+    cmp::Ordering,
+    collections::{HashMap, HashSet},
+    fmt,
+};
+use url::Url;
+
+pub(super) fn packages(content: &str) -> anyhow::Result<Vec<Package>> {
+    let lockfile: Lockfile = serde_json::from_str(content)?;
+
+    let mut packages = match lockfile.version {
+        1 => {
+            let initial_url = get_initial_url()?;
+
+            to_new_packages(lockfile.dependencies.unwrap_or_default(), &initial_url)?
+        }
+        2 | 3 => lockfile
+            .packages
+            .unwrap_or_default()
+            .into_iter()
+            .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_))))
+            .map(|(n, p)| Package { name: Some(n), ..p })
+            .collect(),
+        _ => bail!(
+            "We don't support lockfile version {}, please file an issue.",
+            lockfile.version
+        ),
+    };
+
+    packages.par_sort_by(|x, y| {
+        x.resolved
+            .partial_cmp(&y.resolved)
+            .expect("resolved should be comparable")
+            .then(
+                // v1 lockfiles can contain multiple references to the same version of a package, with
+                // different integrity values (e.g. a SHA-1 and a SHA-512 in one, but just a SHA-512 in another)
+                y.integrity
+                    .partial_cmp(&x.integrity)
+                    .expect("integrity should be comparable"),
+            )
+    });
+
+    packages.dedup_by(|x, y| x.resolved == y.resolved);
+
+    Ok(packages)
+}
+
+#[derive(Deserialize)]
+struct Lockfile {
+    #[serde(rename = "lockfileVersion")]
+    version: u8,
+    dependencies: Option<HashMap<String, OldPackage>>,
+    packages: Option<HashMap<String, Package>>,
+}
+
+#[derive(Deserialize)]
+struct OldPackage {
+    version: UrlOrString,
+    #[serde(default)]
+    bundled: bool,
+    resolved: Option<UrlOrString>,
+    integrity: Option<HashCollection>,
+    dependencies: Option<HashMap<String, OldPackage>>,
+}
+
+#[derive(Debug, Deserialize, PartialEq, Eq)]
+pub(super) struct Package {
+    #[serde(default)]
+    pub(super) name: Option<String>,
+    pub(super) resolved: Option<UrlOrString>,
+    pub(super) integrity: Option<HashCollection>,
+}
+
+#[derive(Debug, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
+#[serde(untagged)]
+pub(super) enum UrlOrString {
+    Url(Url),
+    String(String),
+}
+
+impl fmt::Display for UrlOrString {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            UrlOrString::Url(url) => url.fmt(f),
+            UrlOrString::String(string) => string.fmt(f),
+        }
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct HashCollection(HashSet<Hash>);
+
+impl HashCollection {
+    pub fn from_str(s: impl AsRef<str>) -> anyhow::Result<HashCollection> {
+        let hashes = s
+            .as_ref()
+            .split_ascii_whitespace()
+            .map(Hash::new)
+            .collect::<anyhow::Result<_>>()?;
+
+        Ok(HashCollection(hashes))
+    }
+
+    pub fn into_best(self) -> Option<Hash> {
+        self.0.into_iter().max()
+    }
+}
+
+impl PartialOrd for HashCollection {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        let lhs = self.0.iter().max()?;
+        let rhs = other.0.iter().max()?;
+
+        lhs.partial_cmp(rhs)
+    }
+}
+
+impl<'de> Deserialize<'de> for HashCollection {
+    fn deserialize<D>(deserializer: D) -> Result<HashCollection, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        deserializer.deserialize_string(HashCollectionVisitor)
+    }
+}
+
+struct HashCollectionVisitor;
+
+impl<'de> Visitor<'de> for HashCollectionVisitor {
+    type Value = HashCollection;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        formatter.write_str("a single SRI hash or a collection of them (separated by spaces)")
+    }
+
+    fn visit_str<E>(self, value: &str) -> Result<HashCollection, E>
+    where
+        E: de::Error,
+    {
+        HashCollection::from_str(value).map_err(E::custom)
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Hash)]
+pub struct Hash(String);
+
+// Hash algorithms, in ascending preference.
+const ALGOS: &[&str] = &["sha1", "sha512"];
+
+impl Hash {
+    fn new(s: impl AsRef<str>) -> anyhow::Result<Hash> {
+        let algo = s
+            .as_ref()
+            .split_once('-')
+            .ok_or_else(|| anyhow!("expected SRI hash, got {:?}", s.as_ref()))?
+            .0;
+
+        if ALGOS.iter().any(|&a| algo == a) {
+            Ok(Hash(s.as_ref().to_string()))
+        } else {
+            Err(anyhow!("unknown hash algorithm {algo:?}"))
+        }
+    }
+
+    pub fn as_str(&self) -> &str {
+        &self.0
+    }
+}
+
+impl fmt::Display for Hash {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.as_str().fmt(f)
+    }
+}
+
+#[allow(clippy::non_canonical_partial_ord_impl)]
+impl PartialOrd for Hash {
+    fn partial_cmp(&self, other: &Hash) -> Option<Ordering> {
+        let lhs = self.0.split_once('-')?.0;
+        let rhs = other.0.split_once('-')?.0;
+
+        ALGOS
+            .iter()
+            .position(|&s| lhs == s)?
+            .partial_cmp(&ALGOS.iter().position(|&s| rhs == s)?)
+    }
+}
+
+impl Ord for Hash {
+    fn cmp(&self, other: &Hash) -> Ordering {
+        self.partial_cmp(other).unwrap()
+    }
+}
+
+#[allow(clippy::case_sensitive_file_extension_comparisons)]
+fn to_new_packages(
+    old_packages: HashMap<String, OldPackage>,
+    initial_url: &Url,
+) -> anyhow::Result<Vec<Package>> {
+    let mut new = Vec::new();
+
+    for (name, mut package) in old_packages {
+        // In some cases, a bundled dependency happens to have the same version as a non-bundled one, causing
+        // the bundled one without a URL to override the entry for the non-bundled instance, which prevents the
+        // dependency from being downloaded.
+        if package.bundled {
+            continue;
+        }
+
+        if let UrlOrString::Url(v) = &package.version {
+            if v.scheme() == "npm" {
+                if let Some(UrlOrString::Url(ref url)) = &package.resolved {
+                    package.version = UrlOrString::Url(url.clone());
+                }
+            } else {
+                for (scheme, host) in [
+                    ("github", "github.com"),
+                    ("bitbucket", "bitbucket.org"),
+                    ("gitlab", "gitlab.com"),
+                ] {
+                    if v.scheme() == scheme {
+                        package.version = {
+                            let mut new_url = initial_url.clone();
+
+                            new_url.set_host(Some(host))?;
+
+                            if v.path().ends_with(".git") {
+                                new_url.set_path(v.path());
+                            } else {
+                                new_url.set_path(&format!("{}.git", v.path()));
+                            }
+
+                            new_url.set_fragment(v.fragment());
+
+                            UrlOrString::Url(new_url)
+                        };
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        new.push(Package {
+            name: Some(name),
+            resolved: if matches!(package.version, UrlOrString::Url(_)) {
+                Some(package.version)
+            } else {
+                package.resolved
+            },
+            integrity: package.integrity,
+        });
+
+        if let Some(dependencies) = package.dependencies {
+            new.append(&mut to_new_packages(dependencies, initial_url)?);
+        }
+    }
+
+    Ok(new)
+}
+
+fn get_initial_url() -> anyhow::Result<Url> {
+    Url::parse("git+ssh://git@a.b").context("initial url should be valid")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{
+        get_initial_url, packages, to_new_packages, Hash, HashCollection, OldPackage, Package,
+        UrlOrString,
+    };
+    use std::{
+        cmp::Ordering,
+        collections::{HashMap, HashSet},
+    };
+    use url::Url;
+
+    #[test]
+    fn git_shorthand_v1() -> anyhow::Result<()> {
+        let old = {
+            let mut o = HashMap::new();
+            o.insert(
+                String::from("sqlite3"),
+                OldPackage {
+                    version: UrlOrString::Url(
+                        Url::parse(
+                            "github:mapbox/node-sqlite3#593c9d498be2510d286349134537e3bf89401c4a",
+                        )
+                        .unwrap(),
+                    ),
+                    bundled: false,
+                    resolved: None,
+                    integrity: None,
+                    dependencies: None,
+                },
+            );
+            o
+        };
+
+        let initial_url = get_initial_url()?;
+
+        let new = to_new_packages(old, &initial_url)?;
+
+        assert_eq!(new.len(), 1, "new packages map should contain 1 value");
+        assert_eq!(new[0], Package {
+            name: Some(String::from("sqlite3")),
+            resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())),
+            integrity: None
+        });
+
+        Ok(())
+    }
+
+    #[test]
+    fn hash_preference() {
+        assert_eq!(
+            Hash(String::from("sha1-foo")).partial_cmp(&Hash(String::from("sha512-foo"))),
+            Some(Ordering::Less)
+        );
+
+        assert_eq!(
+            HashCollection({
+                let mut set = HashSet::new();
+                set.insert(Hash(String::from("sha512-foo")));
+                set.insert(Hash(String::from("sha1-bar")));
+                set
+            })
+            .into_best(),
+            Some(Hash(String::from("sha512-foo")))
+        );
+    }
+
+    #[test]
+    fn parse_lockfile_correctly() {
+        let packages = packages(
+            r#"{
+                "name": "node-ddr",
+                "version": "1.0.0",
+                "lockfileVersion": 1,
+                "requires": true,
+                "dependencies": {
+                    "string-width-cjs": {
+                        "version": "npm:string-width@4.2.3",
+                        "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
+                        "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
+                        "requires": {
+                            "emoji-regex": "^8.0.0",
+                            "is-fullwidth-code-point": "^3.0.0",
+                            "strip-ansi": "^6.0.1"
+                        }
+                    }
+                }
+            }"#).unwrap();
+
+        assert_eq!(packages.len(), 1);
+        assert_eq!(
+            packages[0].resolved,
+            Some(UrlOrString::Url(
+                Url::parse("https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz")
+                    .unwrap()
+            ))
+        );
+    }
+}

pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs

@@ -0,0 +1,353 @@
+use anyhow::{anyhow, bail, Context};
+use lock::UrlOrString;
+use log::{debug, info};
+use rayon::prelude::*;
+use serde_json::{Map, Value};
+use std::{
+    fs,
+    io::Write,
+    process::{Command, Stdio},
+};
+use tempfile::{tempdir, TempDir};
+use url::Url;
+
+use crate::util;
+
+pub mod lock;
+
+pub fn lockfile(
+    content: &str,
+    force_git_deps: bool,
+    force_empty_cache: bool,
+) -> anyhow::Result<Vec<Package>> {
+    debug!("parsing lockfile with contents:\n{content}");
+
+    let mut packages = lock::packages(content)
+        .context("failed to extract packages from lockfile")?
+        .into_par_iter()
+        .map(|p| {
+            let n = p.name.clone().unwrap();
+
+            Package::from_lock(p).with_context(|| format!("failed to parse data for {n}"))
+        })
+        .collect::<anyhow::Result<Vec<_>>>()?;
+
+    if packages.is_empty() && !force_empty_cache {
+        bail!("No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like <https://github.com/jeslie0/npm-lockfile-fix>. If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`.");
+    }
+
+    let mut new = Vec::new();
+
+    for pkg in packages
+        .iter()
+        .filter(|p| matches!(p.specifics, Specifics::Git { .. }))
+    {
+        let dir = match &pkg.specifics {
+            Specifics::Git { workdir } => workdir,
+            Specifics::Registry { .. } => unimplemented!(),
+        };
+
+        let path = dir.path().join("package");
+
+        info!("recursively parsing lockfile for {} at {path:?}", pkg.name);
+
+        let lockfile_contents = fs::read_to_string(path.join("package-lock.json"));
+
+        let package_json_path = path.join("package.json");
+        let mut package_json: Map<String, Value> =
+            serde_json::from_str(&fs::read_to_string(package_json_path)?)?;
+
+        if let Some(scripts) = package_json
+            .get_mut("scripts")
+            .and_then(Value::as_object_mut)
+        {
+            // https://github.com/npm/pacote/blob/272edc1bac06991fc5f95d06342334bbacfbaa4b/lib/git.js#L166-L172
+            for typ in [
+                "postinstall",
+                "build",
+                "preinstall",
+                "install",
+                "prepack",
+                "prepare",
+            ] {
+                if scripts.contains_key(typ) && lockfile_contents.is_err() && !force_git_deps {
+                    bail!("Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", pkg.name);
+                }
+            }
+        }
+
+        if let Ok(lockfile_contents) = lockfile_contents {
+            new.append(&mut lockfile(
+                &lockfile_contents,
+                force_git_deps,
+                // force_empty_cache is turned on here since recursively parsed lockfiles should be
+                // allowed to have an empty cache without erroring by default
+                true,
+            )?);
+        }
+    }
+
+    packages.append(&mut new);
+
+    packages.par_sort_by(|x, y| {
+        x.url
+            .partial_cmp(&y.url)
+            .expect("resolved should be comparable")
+    });
+
+    packages.dedup_by(|x, y| x.url == y.url);
+
+    Ok(packages)
+}
+
+#[derive(Debug)]
+pub struct Package {
+    pub name: String,
+    pub url: Url,
+    specifics: Specifics,
+}
+
+#[derive(Debug)]
+enum Specifics {
+    Registry { integrity: lock::Hash },
+    Git { workdir: TempDir },
+}
+
+impl Package {
+    fn from_lock(pkg: lock::Package) -> anyhow::Result<Package> {
+        let mut resolved = match pkg
+            .resolved
+            .expect("at this point, packages should have URLs")
+        {
+            UrlOrString::Url(u) => u,
+            UrlOrString::String(_) => panic!("at this point, all packages should have URLs"),
+        };
+
+        let specifics = match get_hosted_git_url(&resolved)? {
+            Some(hosted) => {
+                let body = util::get_url_body_with_retry(&hosted)?;
+
+                let workdir = tempdir()?;
+
+                let tar_path = workdir.path().join("package");
+
+                fs::create_dir(&tar_path)?;
+
+                let mut cmd = Command::new("tar")
+                    .args(["--extract", "--gzip", "--strip-components=1", "-C"])
+                    .arg(&tar_path)
+                    .stdin(Stdio::piped())
+                    .spawn()?;
+
+                cmd.stdin.take().unwrap().write_all(&body)?;
+
+                let exit = cmd.wait()?;
+
+                if !exit.success() {
+                    bail!(
+                        "failed to extract tarball for {}: tar exited with status code {}",
+                        pkg.name.unwrap(),
+                        exit.code().unwrap()
+                    );
+                }
+
+                resolved = hosted;
+
+                Specifics::Git { workdir }
+            }
+            None => Specifics::Registry {
+                integrity: pkg
+                    .integrity
+                    .expect("non-git dependencies should have associated integrity")
+                    .into_best()
+                    .expect("non-git dependencies should have non-empty associated integrity"),
+            },
+        };
+
+        Ok(Package {
+            name: pkg.name.unwrap(),
+            url: resolved,
+            specifics,
+        })
+    }
+
+    pub fn tarball(&self) -> anyhow::Result<Vec<u8>> {
+        match &self.specifics {
+            Specifics::Registry { .. } => Ok(util::get_url_body_with_retry(&self.url)?),
+            Specifics::Git { workdir } => Ok(Command::new("tar")
+                .args([
+                    "--sort=name",
+                    "--mtime=@0",
+                    "--owner=0",
+                    "--group=0",
+                    "--numeric-owner",
+                    "--format=gnu",
+                    "-I",
+                    "gzip -n -9",
+                    "--create",
+                    "-C",
+                ])
+                .arg(workdir.path())
+                .arg("package")
+                .output()?
+                .stdout),
+        }
+    }
+
+    pub fn integrity(&self) -> Option<&lock::Hash> {
+        match &self.specifics {
+            Specifics::Registry { integrity } => Some(integrity),
+            Specifics::Git { .. } => None,
+        }
+    }
+}
+
+#[allow(clippy::case_sensitive_file_extension_comparisons)]
+fn get_hosted_git_url(url: &Url) -> anyhow::Result<Option<Url>> {
+    if ["git", "git+ssh", "git+https", "ssh"].contains(&url.scheme()) {
+        let mut s = url
+            .path_segments()
+            .ok_or_else(|| anyhow!("bad URL: {url}"))?;
+
+        let mut get_url = || match url.host_str()? {
+            "github.com" => {
+                let user = s.next()?;
+                let mut project = s.next()?;
+                let typ = s.next();
+                let mut commit = s.next();
+
+                if typ.is_none() {
+                    commit = url.fragment();
+                } else if typ.is_some() && typ != Some("tree") {
+                    return None;
+                }
+
+                if project.ends_with(".git") {
+                    project = project.strip_suffix(".git")?;
+                }
+
+                let commit = commit.unwrap();
+
+                Some(
+                    Url::parse(&format!(
+                        "https://codeload.github.com/{user}/{project}/tar.gz/{commit}"
+                    ))
+                    .ok()?,
+                )
+            }
+            "bitbucket.org" => {
+                let user = s.next()?;
+                let mut project = s.next()?;
+                let aux = s.next();
+
+                if aux == Some("get") {
+                    return None;
+                }
+
+                if project.ends_with(".git") {
+                    project = project.strip_suffix(".git")?;
+                }
+
+                let commit = url.fragment()?;
+
+                Some(
+                    Url::parse(&format!(
+                        "https://bitbucket.org/{user}/{project}/get/{commit}.tar.gz"
+                    ))
+                    .ok()?,
+                )
+            }
+            "gitlab.com" => {
+                /* let path = &url.path()[1..];
+
+                if path.contains("/~/") || path.contains("/archive.tar.gz") {
+                    return None;
+                }
+
+                let user = s.next()?;
+                let mut project = s.next()?;
+
+                if project.ends_with(".git") {
+                    project = project.strip_suffix(".git")?;
+                }
+
+                let commit = url.fragment()?;
+
+                Some(
+                    Url::parse(&format!(
+                    "https://gitlab.com/{user}/{project}/repository/archive.tar.gz?ref={commit}"
+                ))
+                    .ok()?,
+                ) */
+
+                // lmao: https://github.com/npm/hosted-git-info/pull/109
+                None
+            }
+            "git.sr.ht" => {
+                let user = s.next()?;
+                let mut project = s.next()?;
+                let aux = s.next();
+
+                if aux == Some("archive") {
+                    return None;
+                }
+
+                if project.ends_with(".git") {
+                    project = project.strip_suffix(".git")?;
+                }
+
+                let commit = url.fragment()?;
+
+                Some(
+                    Url::parse(&format!(
+                        "https://git.sr.ht/{user}/{project}/archive/{commit}.tar.gz"
+                    ))
+                    .ok()?,
+                )
+            }
+            _ => None,
+        };
+
+        match get_url() {
+            Some(u) => Ok(Some(u)),
+            None => Err(anyhow!("This lockfile either contains a Git dependency with an unsupported host, or a malformed URL in the lockfile: {url}"))
+        }
+    } else {
+        Ok(None)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::get_hosted_git_url;
+    use url::Url;
+
+    #[test]
+    fn hosted_git_urls() {
+        for (input, expected) in [
+            (
+                "git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525",
+                Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"),
+            ),
+            (
+                "git+ssh://bitbucket.org/foo/bar#branch",
+                Some("https://bitbucket.org/foo/bar/get/branch.tar.gz")
+            ),
+            (
+                "git+ssh://git.sr.ht/~foo/bar#branch",
+                Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz")
+            ),
+        ] {
+            assert_eq!(
+                get_hosted_git_url(&Url::parse(input).unwrap()).unwrap(),
+                expected.map(|u| Url::parse(u).unwrap())
+            );
+        }
+
+        assert!(
+            get_hosted_git_url(&Url::parse("ssh://git@gitlab.com/foo/bar.git#fix/bug").unwrap())
+                .is_err(),
+            "GitLab URLs should be marked as invalid (lol)"
+        );
+    }
+}

pkgs/build-support/node/fetch-npm-deps/src/util.rs

@@ -0,0 +1,99 @@
+use anyhow::bail;
+use backoff::{retry, ExponentialBackoff};
+use data_encoding::BASE64;
+use digest::Digest;
+use isahc::{
+    config::{CaCertificate, Configurable, RedirectPolicy, SslOption},
+    Body, Request, RequestExt,
+};
+use log::info;
+use nix_nar::{Encoder, NarError};
+use serde_json::{Map, Value};
+use sha2::Sha256;
+use std::{
+    env,
+    io::{self, Read},
+    path::Path,
+};
+use url::Url;
+
+pub fn get_url(url: &Url) -> Result<Body, anyhow::Error> {
+    let mut request = Request::get(url.as_str()).redirect_policy(RedirectPolicy::Limit(10));
+
+    // Respect SSL_CERT_FILE if environment variable exists
+    if let Ok(ssl_cert_file) = env::var("SSL_CERT_FILE") {
+        if Path::new(&ssl_cert_file).exists() {
+            // When file exists, use it. NIX_SSL_CERT_FILE will still override.
+            request = request.ssl_ca_certificate(CaCertificate::file(ssl_cert_file));
+        } else if env::var("outputHash").is_ok() {
+            // When file does not exist, assume we are downloading in a FOD and
+            // therefore do not need to check certificates, since the output is
+            // already hashed.
+            request = request.ssl_options(SslOption::DANGER_ACCEPT_INVALID_CERTS);
+        }
+    }
+
+    // Respect NIX_NPM_TOKENS environment variable, which should be a JSON mapping in the shape of:
+    // `{ "registry.example.com": "example-registry-bearer-token", ... }`
+    if let Some(host) = url.host_str() {
+        if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") {
+            if let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens) {
+                if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) {
+                    info!("Found NPM token for {}. Adding authorization header to request.", host);
+                    request = request.header("Authorization", format!("Bearer {token}"));
+                }
+            }
+        }
+    }
+
+    let res = request.body(())?.send()?;
+    if !res.status().is_success() {
+        if res.status().is_client_error() {
+            bail!("Client error: {}", res.status());
+        }
+        if res.status().is_server_error() {
+            bail!("Server error: {}", res.status());
+        }
+        bail!("{}", res.status());
+    }
+    Ok(res.into_body())
+}
+
+pub fn get_url_body_with_retry(url: &Url) -> Result<Vec<u8>, anyhow::Error> {
+    retry(ExponentialBackoff::default(), || {
+        get_url(url)
+            .and_then(|mut body| {
+                let mut buf = Vec::new();
+
+                body.read_to_end(&mut buf)?;
+
+                Ok(buf)
+            })
+            .map_err(|err| match err.downcast_ref::<isahc::Error>() {
+                Some(isahc_err) => {
+                    if isahc_err.is_network() || isahc_err.is_timeout() {
+                        backoff::Error::transient(err)
+                    } else {
+                        backoff::Error::permanent(err)
+                    }
+                }
+                None => backoff::Error::permanent(err),
+            })
+    })
+    .map_err(|backoff_err| match backoff_err {
+        backoff::Error::Permanent(err)
+        | backoff::Error::Transient {
+            err,
+            retry_after: _,
+        } => err,
+    })
+}
+
+pub fn make_sri_hash(path: &Path) -> Result<String, NarError> {
+    let mut encoder = Encoder::new(path)?;
+    let mut hasher = Sha256::new();
+
+    io::copy(&mut encoder, &mut hasher)?;
+
+    Ok(format!("sha256-{}", BASE64.encode(&hasher.finalize())))
+}

pkgs/build-support/node/fetch-yarn-deps/common.js

@@ -0,0 +1,19 @@
+const path = require('path')
+
+// This has to match the logic in pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js
+// so that fixup_yarn_lock produces the same paths
+const urlToName = url => {
+  const isCodeloadGitTarballUrl = url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')
+
+  if (url.startsWith('file:')) {
+    return url
+  } else if (url.startsWith('git+') || isCodeloadGitTarballUrl) {
+    return path.basename(url)
+  } else {
+    return url
+      .replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
+      .replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
+  }
+}
+
+module.exports = { urlToName };

pkgs/build-support/node/fetch-yarn-deps/default.nix

@@ -0,0 +1,199 @@
+{
+  stdenv,
+  lib,
+  makeWrapper,
+  installShellFiles,
+  nodejsInstallManuals,
+  nodejsInstallExecutables,
+  coreutils,
+  nix-prefetch-git,
+  fetchurl,
+  jq,
+  nodejs,
+  nodejs-slim,
+  prefetch-yarn-deps,
+  fixup-yarn-lock,
+  diffutils,
+  yarn,
+  makeSetupHook,
+  cacert,
+  callPackage,
+}:
+
+let
+  yarnpkg-lockfile-tar = fetchurl {
+    url = "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz";
+    hash = "sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ==";
+  };
+
+  tests = callPackage ./tests { };
+in
+{
+  prefetch-yarn-deps = stdenv.mkDerivation {
+    name = "prefetch-yarn-deps";
+
+    dontUnpack = true;
+    dontBuild = true;
+
+    nativeBuildInputs = [ makeWrapper ];
+    buildInputs = [ nodejs-slim ];
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir -p $out/bin $out/libexec
+
+      tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js
+      mv index.js $out/libexec/yarnpkg-lockfile.js
+      cp ${./common.js} $out/libexec/common.js
+      cp ${./index.js} $out/libexec/index.js
+
+      patchShebangs $out/libexec
+      makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \
+        --prefix PATH : ${
+          lib.makeBinPath [
+            coreutils
+            nix-prefetch-git
+          ]
+        }
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit tests;
+    };
+  };
+
+  fixup-yarn-lock = stdenv.mkDerivation {
+    name = "fixup-yarn-lock";
+
+    dontUnpack = true;
+    dontBuild = true;
+
+    nativeBuildInputs = [ makeWrapper ];
+    buildInputs = [ nodejs-slim ];
+
+    installPhase = ''
+      runHook preInstall
+
+      mkdir -p $out/bin $out/libexec
+
+      tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js
+      mv index.js $out/libexec/yarnpkg-lockfile.js
+      cp ${./common.js} $out/libexec/common.js
+      cp ${./fixup.js} $out/libexec/fixup.js
+
+      patchShebangs $out/libexec
+      makeWrapper $out/libexec/fixup.js $out/bin/fixup-yarn-lock
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit tests;
+    };
+  };
+
+  fetchYarnDeps =
+    let
+      f =
+        {
+          name ? "offline",
+          src ? null,
+          hash ? "",
+          sha256 ? "",
+          ...
+        }@args:
+        let
+          hash_ =
+            if hash != "" then
+              {
+                outputHashAlgo = null;
+                outputHash = hash;
+              }
+            else if sha256 != "" then
+              {
+                outputHashAlgo = "sha256";
+                outputHash = sha256;
+              }
+            else
+              {
+                outputHashAlgo = "sha256";
+                outputHash = lib.fakeSha256;
+              };
+        in
+        stdenv.mkDerivation (
+          {
+            inherit name;
+
+            dontUnpack = src == null;
+            dontInstall = true;
+
+            nativeBuildInputs = [
+              prefetch-yarn-deps
+              cacert
+            ];
+            GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+            NODE_EXTRA_CA_CERTS = "${cacert}/etc/ssl/certs/ca-bundle.crt";
+
+            buildPhase = ''
+              runHook preBuild
+
+              yarnLock=''${yarnLock:=$PWD/yarn.lock}
+              mkdir -p $out
+              (cd $out; prefetch-yarn-deps --verbose --builder $yarnLock)
+
+              runHook postBuild
+            '';
+
+            outputHashMode = "recursive";
+          }
+          // hash_
+          // (removeAttrs args (
+            [
+              "name"
+              "hash"
+              "sha256"
+            ]
+            ++ (lib.optional (src == null) "src")
+          ))
+        );
+    in
+    lib.setFunctionArgs f (lib.functionArgs f) // { inherit tests; };
+
+  yarnConfigHook = makeSetupHook {
+    name = "yarn-config-hook";
+    propagatedBuildInputs = [
+      yarn
+      fixup-yarn-lock
+    ];
+    substitutions = {
+      # Specify `diff` by abspath to ensure that the user's build
+      # inputs do not cause us to find the wrong binaries.
+      diff = "${diffutils}/bin/diff";
+    };
+    meta = {
+      description = "Install nodejs dependencies from an offline yarn cache produced by fetchYarnDeps";
+    };
+  } ./yarn-config-hook.sh;
+
+  yarnBuildHook = makeSetupHook {
+    name = "yarn-build-hook";
+    meta = {
+      description = "Run yarn build in buildPhase";
+    };
+  } ./yarn-build-hook.sh;
+
+  yarnInstallHook = makeSetupHook {
+    name = "yarn-install-hook";
+    propagatedBuildInputs = [
+      yarn
+      nodejsInstallManuals
+      nodejsInstallExecutables
+    ];
+    substitutions = {
+      jq = lib.getExe jq;
+    };
+  } ./yarn-install-hook.sh;
+}

pkgs/build-support/node/fetch-yarn-deps/fixup.js

@@ -0,0 +1,85 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('fs')
+const process = require('process')
+const lockfile = require('./yarnpkg-lockfile.js')
+const { urlToName } = require('./common.js')
+
+const fixupYarnLock = async (lockContents, verbose) => {
+  const lockData = lockfile.parse(lockContents)
+
+  const fixedData = Object.fromEntries(
+    Object.entries(lockData.object)
+    .map(([dep, pkg]) => {
+      if (pkg.resolved === undefined) {
+        console.warn(`no resolved URL for package ${dep}`)
+        var maybeFile = dep.split("@", 2)[1]
+        if (maybeFile.startsWith("file:")) {
+          console.log(`Rewriting URL for local file dependency ${dep}`)
+          pkg.resolved = maybeFile
+        }
+        return [dep, pkg]
+      }
+      const [ url, hash ] = pkg.resolved.split("#", 2)
+
+      if (hash || url.startsWith("https://codeload.github.com/")) {
+        if (verbose) console.log(`Removing integrity for git dependency ${dep}`)
+        delete pkg.integrity
+      }
+
+      if (verbose) console.log(`Rewriting URL ${url} for dependency ${dep}`)
+      pkg.resolved = urlToName(url)
+      if (hash)
+        pkg.resolved += `#${hash}`
+
+      return [dep, pkg]
+    })
+  )
+
+  if (verbose) console.log('Done')
+
+  return fixedData
+}
+
+const showUsage = async () => {
+  process.stderr.write(`
+syntax: fixup-yarn-lock [path to yarn.lock] [options]
+
+Options:
+  -h --help         Show this help
+  -v --verbose      Verbose output
+`)
+  process.exit(1)
+}
+
+const main = async () => {
+  const args = process.argv.slice(2)
+  let next, lockFile, verbose
+  while (next = args.shift()) {
+    if (next == '--verbose' || next == '-v') {
+      verbose = true
+    } else if (next == '--help' || next == '-h') {
+      showUsage()
+    } else if (!lockFile) {
+      lockFile = next
+    } else {
+      showUsage()
+    }
+  }
+  let lockContents
+  try {
+    lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
+  } catch {
+    showUsage()
+  }
+
+  const fixedData = await fixupYarnLock(lockContents, verbose)
+  await fs.promises.writeFile(lockFile || 'yarn.lock', lockfile.stringify(fixedData))
+}
+
+main()
+  .catch(e => {
+    console.error(e)
+    process.exit(1)
+  })

pkgs/build-support/node/fetch-yarn-deps/index.js

@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('fs')
+const crypto = require('crypto')
+const process = require('process')
+const https = require('https')
+const child_process = require('child_process')
+const path = require('path')
+const lockfile = require('./yarnpkg-lockfile.js')
+const { promisify } = require('util')
+const url = require('url')
+const { URL } = url;
+const { urlToName } = require('./common.js')
+
+const execFile = promisify(child_process.execFile)
+
+const exec = async (...args) => {
+  const res = await execFile(...args)
+  if (res.error) throw new Error(res.stderr)
+  return res
+}
+
+const downloadFileHttps = (fileName, url, expectedHash, verbose, hashType = 'sha1') => {
+  return new Promise((resolve, reject) => {
+    const get = (url, redirects = 0) => https.get(url, (res) => {
+      if(redirects > 10) {
+        reject('Too many redirects!');
+        return;
+      }
+      if(res.statusCode === 301 || res.statusCode === 302) {
+        const location = new URL(res.headers.location, url);
+        if (verbose) console.log('following redirect to ' + location);
+        return get(location, redirects + 1);
+      }
+      const file = fs.createWriteStream(fileName)
+      const hash = crypto.createHash(hashType)
+      res.pipe(file)
+      res.pipe(hash).setEncoding('hex')
+      res.on('end', () => {
+        file.close()
+        const h = hash.read()
+        if (expectedHash === undefined){
+          console.log(`Warning: lockfile url ${url} doesn't end in "#<hash>" to validate against. Downloaded file had hash ${h}.`);
+        } else if (h != expectedHash) return reject(new Error(`hash mismatch, expected ${expectedHash}, got ${h} for ${url}`))
+        resolve()
+      })
+      res.on('error', e => reject(e))
+    })
+    get(url)
+  })
+}
+
+const downloadGit = async (fileName, url, rev) => {
+  await exec('nix-prefetch-git', [
+    '--out', fileName + '.tmp',
+    '--url', url,
+    '--rev', rev,
+    '--builder'
+  ])
+
+  await exec('tar', [
+    // hopefully make it reproducible across runs and systems
+    '--owner=0', '--group=0', '--numeric-owner', '--format=gnu', '--sort=name', '--mtime=@1',
+
+    // Set u+w because tar-fs can't unpack archives with read-only dirs: https://github.com/mafintosh/tar-fs/issues/79
+    '--mode', 'u+w',
+
+    '-C', fileName + '.tmp',
+    '-cf', fileName, '.'
+  ])
+
+  await exec('rm', [ '-rf', fileName + '.tmp', ])
+}
+
+const isGitUrl = pattern => {
+  // https://github.com/yarnpkg/yarn/blob/3119382885ea373d3c13d6a846de743eca8c914b/src/resolvers/exotics/git-resolver.js#L15-L47
+  const GIT_HOSTS = ['github.com', 'gitlab.com', 'bitbucket.com', 'bitbucket.org']
+  const GIT_PATTERN_MATCHERS = [/^git:/, /^git\+.+:/, /^ssh:/, /^https?:.+\.git$/, /^https?:.+\.git#.+/]
+
+  for (const matcher of GIT_PATTERN_MATCHERS) if (matcher.test(pattern)) return true
+
+  const {hostname, path} = url.parse(pattern)
+  if (hostname && path && GIT_HOSTS.indexOf(hostname) >= 0
+    // only if dependency is pointing to a git repo,
+    // e.g. facebook/flow and not file in a git repo facebook/flow/archive/v1.0.0.tar.gz
+    && path.split('/').filter(p => !!p).length === 2
+  ) return true
+
+  return false
+}
+
+const downloadPkg = (pkg, verbose) => {
+  for (let marker of ['@file:', '@link:']) {
+    const split = pkg.key.split(marker)
+    if (split.length == 2) {
+      console.info(`ignoring lockfile entry "${split[0]}" which points at path "${split[1]}"`)
+      return
+    } else if (split.length > 2) {
+      throw new Error(`The lockfile entry key "${pkg.key}" contains "${marker}" more than once. Processing is not implemented.`)
+    }
+  }
+
+  if (pkg.resolved === undefined) {
+    throw new Error(`The lockfile entry with key "${pkg.key}" cannot be downloaded because it is missing the "resolved" attribute, which should contain the URL to download from. The lockfile might be invalid.`)
+  }
+
+  const [ url, hash ] = pkg.resolved.split('#')
+  if (verbose) console.log('downloading ' + url)
+  const fileName = urlToName(url)
+  const s = url.split('/')
+  if (url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')) {
+    return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1])
+  } else if (url.startsWith('https://github.com/') && url.endsWith('.tar.gz') &&
+    (
+      s.length <= 5 ||    // https://github.com/owner/repo.tgz#feedface...
+      s[5] == "archive"   // https://github.com/owner/repo/archive/refs/tags/v0.220.1.tar.gz
+    )) {
+    return downloadGit(fileName, `https://github.com/${s[3]}/${s[4]}.git`, s[s.length-1].replace(/.tar.gz$/, ''))
+  } else if (isGitUrl(url)) {
+    return downloadGit(fileName, url.replace(/^git\+/, ''), hash)
+  } else if (url.startsWith('https://')) {
+    if (typeof pkg.integrity === 'string' || pkg.integrity instanceof String) {
+      const [ type, checksum ] = pkg.integrity.split('-')
+      return downloadFileHttps(fileName, url, Buffer.from(checksum, 'base64').toString('hex'), verbose, type)
+    }
+    return downloadFileHttps(fileName, url, hash, verbose)
+  } else if (url.startsWith('file:')) {
+    console.warn(`ignoring unsupported file:path url "${url}"`)
+  } else {
+    throw new Error('don\'t know how to download "' + url + '"')
+  }
+}
+
+const performParallel = tasks => {
+  const worker = async () => {
+    while (tasks.length > 0) await tasks.shift()()
+  }
+
+  const workers = []
+  for (let i = 0; i < 4; i++) {
+    workers.push(worker())
+  }
+
+  return Promise.all(workers)
+}
+
+// This could be implemented using [`Map.groupBy`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Map/groupBy),
+// but that method is only supported starting with Node 21
+const uniqueBy = (arr, callback) => {
+  const map = new Map()
+  for (const elem of arr) {
+    map.set(callback(elem), elem)
+  }
+  return [...map.values()]
+}
+
+const prefetchYarnDeps = async (lockContents, verbose) => {
+  const lockData = lockfile.parse(lockContents)
+  await performParallel(
+    uniqueBy(Object.entries(lockData.object), ([_, value]) => value.resolved)
+    .map(([key, value]) => () => downloadPkg({ key, ...value }, verbose))
+  )
+  await fs.promises.writeFile('yarn.lock', lockContents)
+  if (verbose) console.log('Done')
+}
+
+const showUsage = async () => {
+  process.stderr.write(`
+syntax: prefetch-yarn-deps [path to yarn.lock] [options]
+
+Options:
+  -h --help         Show this help
+  -v --verbose      Verbose output
+  --builder         Only perform the download to current directory, then exit
+`)
+  process.exit(1)
+}
+
+const main = async () => {
+  const args = process.argv.slice(2)
+  let next, lockFile, verbose, isBuilder
+  while (next = args.shift()) {
+    if (next == '--builder') {
+      isBuilder = true
+    } else if (next == '--verbose' || next == '-v') {
+      verbose = true
+    } else if (next == '--help' || next == '-h') {
+      showUsage()
+    } else if (!lockFile) {
+      lockFile = next
+    } else {
+      showUsage()
+    }
+  }
+  let lockContents
+  try {
+    lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
+  } catch {
+    showUsage()
+  }
+
+  if (isBuilder) {
+    await prefetchYarnDeps(lockContents, verbose)
+  } else {
+    const { stdout: tmpDir } = await exec('mktemp', [ '-d' ])
+
+    try {
+      process.chdir(tmpDir.trim())
+      await prefetchYarnDeps(lockContents, verbose)
+      const { stdout: hash } = await exec('nix-hash', [ '--type', 'sha256', '--base32', tmpDir.trim() ])
+      console.log(hash)
+    } finally {
+      await exec('rm', [ '-rf', tmpDir.trim() ])
+    }
+  }
+}
+
+main()
+  .catch(e => {
+    console.error(e)
+    process.exit(1)
+  })

pkgs/build-support/node/fetch-yarn-deps/tests/default.nix

@@ -0,0 +1,28 @@
+{ testers, fetchYarnDeps, ... }:
+
+{
+  file = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./file.lock;
+    sha256 = "sha256-BPuyQVCbdpFL/iRhmarwWAmWO2NodlVCOY9JU+4pfa4=";
+  };
+  simple = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./simple.lock;
+    sha256 = "sha256-FRrt8BixleILmFB2ZV8RgPNLqgS+dlH5nWoPgeaaNQ8=";
+  };
+  gitDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./git.lock;
+    sha256 = "sha256-f90IiEzHDiBdswWewRBHcJfqqpPipaMg8N0DVLq2e8Q=";
+  };
+  githubDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./github.lock;
+    sha256 = "sha256-DIKrhDKoqm7tHZmcuh9eK9VTqp6BxeW0zqDUpY4F57A=";
+  };
+  githubReleaseDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./github-release.lock;
+    sha256 = "sha256-g+y/H6k8LZ+IjWvkkwV7JhKQH1ycfeqzsIonNv4fDq8=";
+  };
+  gitUrlDep = testers.invalidateFetcherByDrvHash fetchYarnDeps {
+    yarnLock = ./giturl.lock;
+    sha256 = "sha256-VPnyqN6lePQZGXwR7VhbFnP7/0/LB621RZwT1F+KzVQ=";
+  };
+}

pkgs/build-support/node/fetch-yarn-deps/tests/file.lock

@@ -0,0 +1,9 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@org/somepack@file:vendor/orgpacks/somepack/assets":
+  version "1.0.0"
+
+"otherpack@file:vendor/otherpack":
+  version "1.0.0"

pkgs/build-support/node/fetch-yarn-deps/tests/git.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"async@git+https://github.com/caolan/async":
+  version "3.2.1"
+  resolved "git+https://github.com/caolan/async#fc9ba651341af5ab974aade6b1640e345912be83"

pkgs/build-support/node/fetch-yarn-deps/tests/github-release.lock

@@ -0,0 +1,6 @@
+"libsession_util_nodejs@https://github.com/oxen-io/libsession-util-nodejs/releases/download/v0.3.19/libsession_util_nodejs-v0.3.19.tar.gz":
+  version "0.3.19"
+  resolved "https://github.com/oxen-io/libsession-util-nodejs/releases/download/v0.3.19/libsession_util_nodejs-v0.3.19.tar.gz#221c1fc34fcc18601aea4ce1b733ebfa55af66ea"
+  dependencies:
+    cmake-js "^7.2.1"
+    node-addon-api "^6.1.0"

pkgs/build-support/node/fetch-yarn-deps/tests/github.lock

@@ -0,0 +1,7 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"async@github:caolan/async":
+  version "3.2.1"
+  resolved "https://codeload.github.com/caolan/async/tar.gz/fc9ba651341af5ab974aade6b1640e345912be83"

pkgs/build-support/node/fetch-yarn-deps/tests/giturl.lock

@@ -0,0 +1,11 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"autocomplete-atom-api@https://codeload.github.com/atom/autocomplete-atom-api/legacy.tar.gz/refs/tags/v0.10.7":
+  version "0.10.7"
+  resolved "https://codeload.github.com/atom/autocomplete-atom-api/legacy.tar.gz/refs/tags/v0.10.7#c9d51fa721d543ccfc1b2189101155e81db6b97d"
+
+"find-and-replace@https://github.com/atom-community/find-and-replace/archive/refs/tags/v0.220.1.tar.gz":
+  version "0.220.1"
+  resolved "https://github.com/atom-community/find-and-replace/archive/refs/tags/v0.220.1.tar.gz#d7a0f56511e38ee72a89895a795bbbcab4a1a405"

pkgs/build-support/node/fetch-yarn-deps/tests/simple.lock

@@ -0,0 +1,8 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+lit-html@1:
+  version "1.4.1"
+  resolved "https://registry.yarnpkg.com/lit-html/-/lit-html-1.4.1.tgz#0c6f3ee4ad4eb610a49831787f0478ad8e9ae5e0"
+  integrity sha512-B9btcSgPYb1q4oSOb/PrOT6Z/H+r6xuNzfH4lFli/AWhYwdtrgQkQWBbIc6mdnf6E2IL3gDXdkkqNktpU0OZQA==

pkgs/build-support/node/fetch-yarn-deps/yarn-build-hook.sh

@@ -0,0 +1,24 @@
+yarnBuildHook() {
+    runHook preBuild
+    echo "Executing yarnBuildHook"
+
+    if [ -z "${yarnBuildScript-}" ]; then
+        yarnBuildScript="build"
+    fi
+
+    if ! type node > /dev/null 2>&1 ; then
+        echo yarnConfigHook WARNING: a node interpreter was not added to the \
+            build, and is probably required to run \'yarn $yarnBuildHook\'. \
+            A common symptom of this is getting \'command not found\' errors \
+            for Nodejs related tools.
+    fi
+
+    yarn --offline "$yarnBuildScript" $yarnBuildFlags
+
+    echo "finished yarnBuildHook"
+    runHook postBuild
+}
+
+if [[ -z "${dontYarnBuild-}" && -z "${buildPhase-}" ]]; then
+    buildPhase=yarnBuildHook
+fi

pkgs/build-support/node/fetch-yarn-deps/yarn-config-hook.sh

@@ -0,0 +1,74 @@
+yarnConfigHook() {
+    echo "Executing yarnConfigHook"
+
+    # Use a constant HOME directory
+    export HOME=$(mktemp -d)
+    if [[ -n "$yarnOfflineCache" ]]; then
+        offlineCache="$yarnOfflineCache"
+    fi
+    if [[ -z "$offlineCache" ]]; then
+        echo yarnConfigHook: No yarnOfflineCache or offlineCache were defined\! >&2
+        exit 2
+    fi
+
+    local -r cacheLockfile="$offlineCache/yarn.lock"
+    local -r srcLockfile="$PWD/yarn.lock"
+
+    echo "Validating consistency between $srcLockfile and $cacheLockfile"
+
+    if ! @diff@ "$srcLockfile" "$cacheLockfile"; then
+      # If the diff failed, first double-check that the file exists, so we can
+      # give a friendlier error msg.
+      if ! [ -e "$srcLockfile" ]; then
+        echo
+        echo "ERROR: Missing yarn.lock from src. Expected to find it at: $srcLockfile"
+        echo "Hint: You can copy a vendored yarn.lock file via postPatch."
+        echo
+
+        exit 1
+      fi
+
+      if ! [ -e "$cacheLockfile" ]; then
+        echo
+        echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile"
+        echo
+
+        exit 1
+      fi
+
+      echo
+      echo "ERROR: fetchYarnDeps hash is out of date"
+      echo
+      echo "The yarn.lock in src is not the same as the in $offlineCache."
+      echo
+      echo "To fix the issue:"
+      echo '1. Use `lib.fakeHash` as the fetchYarnDeps hash value'
+      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
+      echo "3. Copy the 'got: sha256-' value back into the fetchYarnDeps hash field"
+      echo
+
+      exit 1
+    fi
+
+    yarn config --offline set yarn-offline-mirror "$offlineCache"
+    fixup-yarn-lock yarn.lock
+    yarn install \
+        --frozen-lockfile \
+        --force \
+        --production=false \
+        --ignore-engines \
+        --ignore-platform \
+        --ignore-scripts \
+        --no-progress \
+        --non-interactive \
+        --offline
+
+    # TODO: Check if this is really needed
+    patchShebangs node_modules
+
+    echo "finished yarnConfigHook"
+}
+
+if [[ -z "${dontYarnInstallDeps-}" ]]; then
+    postConfigureHooks+=(yarnConfigHook)
+fi

pkgs/build-support/node/fetch-yarn-deps/yarn-install-hook.sh

@@ -0,0 +1,79 @@
+# shellcheck shell=bash
+
+yarnInstallHook() {
+    echo "Executing yarnInstallHook"
+
+    runHook preInstall
+
+    local -r packageOut="$out/lib/node_modules/$(@jq@ --raw-output '.name' ./package.json)"
+    mkdir -p "$packageOut"
+
+    local -ar yarnArgs=(
+        --ignore-engines
+        --ignore-platform
+        --ignore-scripts
+        --no-progress
+        --non-interactive
+        --offline
+    )
+
+    local -r tmpDir="$(mktemp -d)"
+
+    # yarn pack does not work at all with bundleDependencies.
+    # Since we are imediately unpacking, we can just remove them from package.json
+    # This will NOT be fixed in yarn v1: https://github.com/yarnpkg/yarn/issues/6794
+    mv ./package.json "$tmpDir/package.json.orig"
+    # Note: two spellings are accepted, 'bundleDependencies' and 'bundledDependencies'
+    @jq@ 'del(.bundleDependencies)|del(.bundledDependencies)' "$tmpDir/package.json.orig" > ./package.json
+
+    # TODO: figure out a way to avoid redundant compress/decompress steps
+    yarn pack \
+        --filename "$tmpDir/yarn-pack.tgz" \
+        "${yarnArgs[@]}"
+
+    tar xzf "$tmpDir/yarn-pack.tgz" \
+        -C "$packageOut" \
+        --strip-components 1 \
+        package/
+
+    mv "$tmpDir/package.json.orig" ./package.json
+
+    nodejsInstallExecutables ./package.json
+
+    nodejsInstallManuals ./package.json
+
+    local -r nodeModulesPath="$packageOut/node_modules"
+
+    if [ ! -d "$nodeModulesPath" ]; then
+        if [ -z "${yarnKeepDevDeps-}" ]; then
+            # Yarn has a 'prune' command, but it's only a stub that directs you to use install
+            if ! yarn install \
+                --frozen-lockfile \
+                --force \
+                --production=true \
+                "${yarnArgs[@]}"
+            then
+              echo
+              echo
+              echo "ERROR: yarn prune step failed"
+              echo
+              echo 'If yarn tried to download additional dependencies above, try setting `yarnKeepDevDeps = true`.'
+              echo
+
+              exit 1
+            fi
+        fi
+
+        find node_modules -maxdepth 1 -type d -empty -delete
+
+        cp -r node_modules "$nodeModulesPath"
+    fi
+
+    runHook postInstall
+
+    echo "Finished yarnInstallHook"
+}
+
+if [ -z "${dontYarnInstall-}" ] && [ -z "${installPhase-}" ]; then
+    installPhase=yarnInstallHook
+fi

pkgs/build-support/node/import-npm-lock/default.nix

@@ -0,0 +1,234 @@
+{
+  lib,
+  fetchurl,
+  stdenv,
+  callPackages,
+  runCommand,
+  cctools,
+}:
+
+let
+  inherit (builtins)
+    match
+    elemAt
+    toJSON
+    removeAttrs
+    ;
+  inherit (lib) importJSON mapAttrs;
+
+  matchGitHubReference = match "github(.com)?:.+";
+  getName = package: package.name or "unknown";
+  getVersion = package: package.version or "0.0.0";
+
+  # Fetch a module from package-lock.json -> packages
+  fetchModule =
+    {
+      module,
+      npmRoot ? null,
+      fetcherOpts,
+    }:
+    (
+      if module ? "resolved" && module.resolved != null then
+        (
+          let
+            # Parse scheme from URL
+            mUrl = match "(.+)://(.+)" module.resolved;
+            scheme = elemAt mUrl 0;
+          in
+          (
+            if mUrl == null then
+              (
+                assert npmRoot != null;
+                {
+                  outPath = npmRoot + "/${module.resolved}";
+                }
+              )
+            else if (scheme == "http" || scheme == "https") then
+              (fetchurl (
+                {
+                  url = module.resolved;
+                  hash = module.integrity;
+                }
+                // fetcherOpts
+              ))
+            else if lib.hasPrefix "git" module.resolved then
+              (fetchGit (
+                {
+                  url = module.resolved;
+                }
+                // fetcherOpts
+              ))
+            else
+              throw "Unsupported URL scheme: ${scheme}"
+          )
+        )
+      else
+        null
+    );
+
+  cleanModule = lib.flip removeAttrs [
+    "link" # Remove link not to symlink directories. These have been processed to store paths already.
+    "funding" # Remove funding to get rid sponsorship nag in build output
+  ];
+
+  # Manage node_modules outside of the store with hooks
+  hooks = callPackages ./hooks { };
+
+in
+lib.fix (self: {
+  importNpmLock =
+    {
+      npmRoot ? null,
+      package ? importJSON (npmRoot + "/package.json"),
+      packageLock ? importJSON (npmRoot + "/package-lock.json"),
+      pname ? getName package,
+      version ? getVersion package,
+      # A map of additional fetcher options forwarded to the fetcher used to download the package.
+      # Example: { "node_modules/axios" = { curlOptsList = [ "--verbose" ]; }; }
+      # This will download the axios package with curl's verbose option.
+      fetcherOpts ? { },
+      # A map from node_module path to an alternative package to use instead of fetching the source in package-lock.json.
+      # Example: { "node_modules/axios" = stdenv.mkDerivation { ... }; }
+      # This is useful if you want to inject custom sources for a specific package.
+      packageSourceOverrides ? { },
+    }:
+    let
+      mapLockDependencies = mapAttrs (
+        name: version:
+        (
+          # Substitute the constraint with the version of the dependency from the top-level of package-lock.
+          if
+            (
+              # if the version is `latest`
+              version == "latest"
+              ||
+                # Or if it's a github reference
+                matchGitHubReference version != null
+            )
+          then
+            packageLock'.packages.${"node_modules/${name}"}.version
+          # But not a regular version constraint
+          else
+            version
+        )
+      );
+
+      packageLock' = packageLock // {
+        packages = mapAttrs (
+          modulePath: module:
+          let
+            src =
+              packageSourceOverrides.${modulePath} or (fetchModule {
+                inherit module npmRoot;
+                fetcherOpts = fetcherOpts.${modulePath} or { };
+              });
+          in
+          cleanModule module
+          // lib.optionalAttrs (src != null) {
+            resolved = "file:${src}";
+          }
+          // lib.optionalAttrs (module ? dependencies) {
+            dependencies = mapLockDependencies module.dependencies;
+          }
+          // lib.optionalAttrs (module ? optionalDependencies) {
+            optionalDependencies = mapLockDependencies module.optionalDependencies;
+          }
+        ) packageLock.packages;
+      };
+
+      mapPackageDependencies = mapAttrs (
+        name: _: packageLock'.packages.${"node_modules/${name}"}.resolved
+      );
+
+      # Substitute dependency references in package.json with Nix store paths
+      packageJSON' =
+        package
+        // lib.optionalAttrs (package ? dependencies) {
+          dependencies = mapPackageDependencies package.dependencies;
+        }
+        // lib.optionalAttrs (package ? devDependencies) {
+          devDependencies = mapPackageDependencies package.devDependencies;
+        };
+
+      pname = package.name or "unknown";
+
+    in
+    runCommand "${pname}-${version}-sources"
+      {
+        inherit pname version;
+
+        passAsFile = [
+          "package"
+          "packageLock"
+        ];
+
+        package = toJSON packageJSON';
+        packageLock = toJSON packageLock';
+      }
+      ''
+        mkdir $out
+        cp "$packagePath" $out/package.json
+        cp "$packageLockPath" $out/package-lock.json
+      '';
+
+  # Build node modules from package.json & package-lock.json
+  buildNodeModules =
+    {
+      npmRoot ? null,
+      package ? importJSON (npmRoot + "/package.json"),
+      packageLock ? importJSON (npmRoot + "/package-lock.json"),
+      nodejs,
+      derivationArgs ? { },
+    }:
+    stdenv.mkDerivation (
+      {
+        pname = derivationArgs.pname or "${getName package}-node-modules";
+        version = derivationArgs.version or getVersion package;
+
+        dontUnpack = true;
+
+        npmDeps = self.importNpmLock {
+          inherit npmRoot package packageLock;
+        };
+
+        package = toJSON package;
+        packageLock = toJSON packageLock;
+
+        installPhase = ''
+          runHook preInstall
+          mkdir $out
+          cp package.json $out/
+          cp package-lock.json $out/
+          [[ -d node_modules ]] && mv node_modules $out/
+          runHook postInstall
+        '';
+      }
+      // derivationArgs
+      // {
+        nativeBuildInputs = [
+          nodejs
+          nodejs.passthru.python
+          hooks.npmConfigHook
+        ]
+        ++ lib.optionals stdenv.hostPlatform.isDarwin [ cctools ]
+        ++ derivationArgs.nativeBuildInputs or [ ];
+
+        passAsFile = [
+          "package"
+          "packageLock"
+        ]
+        ++ derivationArgs.passAsFile or [ ];
+
+        postPatch = ''
+          cp --no-preserve=mode "$packagePath" package.json
+          cp --no-preserve=mode "$packageLockPath" package-lock.json
+        ''
+        + derivationArgs.postPatch or "";
+      }
+    );
+
+  inherit hooks;
+  inherit (hooks) npmConfigHook linkNodeModulesHook;
+
+  __functor = self: self.importNpmLock;
+})

pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+// When installing files rewritten to the Nix store with npm
+// npm writes the symlinks relative to the build directory.
+//
+// This makes relocating node_modules tricky when refering to the store.
+// This script walks node_modules and canonicalizes symlinks.
+
+async function canonicalize(storePrefix, root) {
+  console.log(storePrefix, root)
+  const entries = await fs.promises.readdir(root);
+  const paths = entries.map((entry) => path.join(root, entry));
+
+  const stats = await Promise.all(
+    paths.map(async (path) => {
+      return {
+        path: path,
+        stat: await fs.promises.lstat(path),
+      };
+    })
+  );
+
+  const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink());
+  const dirs = stats.filter((stat) => stat.stat.isDirectory());
+
+  // Canonicalize symlinks to their real path
+  await Promise.all(
+    symlinks.map(async (stat) => {
+      const target = await fs.promises.realpath(stat.path);
+      if (target.startsWith(storePrefix)) {
+        await fs.promises.unlink(stat.path);
+        await fs.promises.symlink(target, stat.path);
+      }
+    })
+  );
+
+  // Recurse into directories
+  await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path)));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+
+  if (fs.existsSync("node_modules")) {
+    await canonicalize(storePrefix, "node_modules");
+  }
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/default.nix

@@ -0,0 +1,27 @@
+{
+  callPackage,
+  lib,
+  makeSetupHook,
+  srcOnly,
+  nodejs,
+}:
+{
+  npmConfigHook = makeSetupHook {
+    name = "npm-config-hook";
+    substitutions = {
+      nodeSrc = srcOnly nodejs;
+      nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+      canonicalizeSymlinksScript = ./canonicalize-symlinks.js;
+      storePrefix = builtins.storeDir;
+    };
+  } ./npm-config-hook.sh;
+
+  linkNodeModulesHook = makeSetupHook {
+    name = "node-modules-hook.sh";
+    substitutions = {
+      nodejs = lib.getExe nodejs;
+      script = ./link-node-modules.js;
+      storePrefix = builtins.storeDir;
+    };
+  } ./link-node-modules-hook.sh;
+}

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh

@@ -0,0 +1,31 @@
+linkNodeModulesHook() {
+    echo "Executing linkNodeModulesHook"
+    runHook preShellHook
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    @nodejs@ @script@ @storePrefix@ "${npmDeps}/node_modules"
+    if test -d node_modules/.bin; then
+        export PATH=$(readlink -f node_modules/.bin):$PATH
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    runHook postShellHook
+    echo "Finished executing linkNodeModulesShellHook"
+}
+
+if [ -z "${dontLinkNodeModules:-}" ] && [ -z "${shellHook-}" ]; then
+    echo "Using linkNodeModulesHook shell hook"
+    shellHook=linkNodeModulesHook
+fi
+
+
+if [ -z "${dontLinkNodeModules:-}" ]; then
+    echo "Using linkNodeModulesHook preConfigure hook"
+    preConfigureHooks+=(linkNodeModulesHook)
+fi

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+async function asyncFilter(arr, pred) {
+  const filtered = [];
+  for (const elem of arr) {
+    if (await pred(elem)) {
+      filtered.push(elem);
+    }
+  }
+  return filtered;
+}
+
+// Get a list of all _unmanaged_ files in node_modules.
+// This means every file in node_modules that is _not_ a symlink to the Nix store.
+async function getUnmanagedFiles(storePrefix, files) {
+  return await asyncFilter(files, async (file) => {
+    const filePath = path.join("node_modules", file);
+
+    // Is file a symlink
+    const stat = await fs.promises.lstat(filePath);
+    if (!stat.isSymbolicLink()) {
+      return true;
+    }
+
+    // Is file in the store
+    const linkTarget = await fs.promises.readlink(filePath);
+    return !linkTarget.startsWith(storePrefix);
+  });
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+  const sourceModules = args[1];
+
+  // Ensure node_modules exists
+  try {
+    await fs.promises.mkdir("node_modules");
+  } catch (err) {
+    if (err.code !== "EEXIST") {
+      throw err;
+    }
+  }
+
+  const files = await fs.promises.readdir("node_modules");
+
+  // Get deny list of files that we don't manage.
+  // We do manage nix store symlinks, but not other files.
+  // For example: If a .vite was present in both our
+  // source node_modules and the non-store node_modules we don't want to overwrite
+  // the non-store one.
+  const unmanaged = await getUnmanagedFiles(storePrefix, files);
+  const managed = new Set(files.filter((file) => ! unmanaged.includes(file)));
+
+  const sourceFiles = await fs.promises.readdir(sourceModules);
+  await Promise.all(
+    sourceFiles.map(async (file) => {
+      const sourcePath = path.join(sourceModules, file);
+      const targetPath = path.join("node_modules", file);
+
+      // Skip file if it's not a symlink to a store path
+      if (unmanaged.includes(file)) {
+        console.log(`'${targetPath}' exists, cowardly refusing to link.`);
+        return;
+      }
+
+      // Don't unlink this file, we just wrote it.
+      managed.delete(file);
+
+      // Link file
+      try {
+        await fs.promises.symlink(sourcePath, targetPath);
+      } catch (err) {
+        // If the target file already exists remove it and try again
+        if (err.code !== "EEXIST") {
+          throw err;
+        }
+        await fs.promises.unlink(targetPath);
+        await fs.promises.symlink(sourcePath, targetPath);
+      }
+    })
+  );
+
+  // Clean up store symlinks not included in this generation of node_modules
+  await Promise.all(
+    Array.from(managed).map((file) =>
+      fs.promises.unlink(path.join("node_modules", file)),
+    )
+  );
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh

@@ -0,0 +1,70 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    if [ -z "${npmDeps-}" ]; then
+        echo "Error: 'npmDeps' should be set when using npmConfigHook."
+        exit 1
+    fi
+
+    echo "Configuring npm"
+
+    export HOME="$TMPDIR"
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+    npm config set offline true
+    npm config set progress false
+    npm config set fund false
+
+    echo "Installing patched package.json/package-lock.json"
+
+    # Save original package.json/package-lock.json for closure size reductions.
+    # The patched one contains store paths we don't want at runtime.
+    mv package.json .package.json.orig
+    if test -f package-lock.json; then # Not all packages have package-lock.json.
+        mv package-lock.json .package-lock.json.orig
+    fi
+    cp --no-preserve=mode "${npmDeps}/package.json" package.json
+    cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json
+
+    echo "Installing dependencies"
+
+    if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
+
+    patchShebangs node_modules
+
+    # Canonicalize symlinks from relative paths to the Nix store.
+    node @canonicalizeSymlinksScript@ @storePrefix@
+
+    # Revert to pre-patched package.json/package-lock.json for closure size reductions
+    mv .package.json.orig package.json
+    if test -f ".package-lock.json.orig"; then
+        mv .package-lock.json.orig package-lock.json
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postConfigureHooks+=(npmConfigHook)