push sheeet

pkgs/build-support/node/import-npm-lock/hooks/canonicalize-symlinks.js

@@ -0,0 +1,52 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+// When installing files rewritten to the Nix store with npm
+// npm writes the symlinks relative to the build directory.
+//
+// This makes relocating node_modules tricky when refering to the store.
+// This script walks node_modules and canonicalizes symlinks.
+
+async function canonicalize(storePrefix, root) {
+  console.log(storePrefix, root)
+  const entries = await fs.promises.readdir(root);
+  const paths = entries.map((entry) => path.join(root, entry));
+
+  const stats = await Promise.all(
+    paths.map(async (path) => {
+      return {
+        path: path,
+        stat: await fs.promises.lstat(path),
+      };
+    })
+  );
+
+  const symlinks = stats.filter((stat) => stat.stat.isSymbolicLink());
+  const dirs = stats.filter((stat) => stat.stat.isDirectory());
+
+  // Canonicalize symlinks to their real path
+  await Promise.all(
+    symlinks.map(async (stat) => {
+      const target = await fs.promises.realpath(stat.path);
+      if (target.startsWith(storePrefix)) {
+        await fs.promises.unlink(stat.path);
+        await fs.promises.symlink(target, stat.path);
+      }
+    })
+  );
+
+  // Recurse into directories
+  await Promise.all(dirs.map((dir) => canonicalize(storePrefix, dir.path)));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+
+  if (fs.existsSync("node_modules")) {
+    await canonicalize(storePrefix, "node_modules");
+  }
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/default.nix

@@ -0,0 +1,27 @@
+{
+  callPackage,
+  lib,
+  makeSetupHook,
+  srcOnly,
+  nodejs,
+}:
+{
+  npmConfigHook = makeSetupHook {
+    name = "npm-config-hook";
+    substitutions = {
+      nodeSrc = srcOnly nodejs;
+      nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+      canonicalizeSymlinksScript = ./canonicalize-symlinks.js;
+      storePrefix = builtins.storeDir;
+    };
+  } ./npm-config-hook.sh;
+
+  linkNodeModulesHook = makeSetupHook {
+    name = "node-modules-hook.sh";
+    substitutions = {
+      nodejs = lib.getExe nodejs;
+      script = ./link-node-modules.js;
+      storePrefix = builtins.storeDir;
+    };
+  } ./link-node-modules-hook.sh;
+}

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules-hook.sh

@@ -0,0 +1,31 @@
+linkNodeModulesHook() {
+    echo "Executing linkNodeModulesHook"
+    runHook preShellHook
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    @nodejs@ @script@ @storePrefix@ "${npmDeps}/node_modules"
+    if test -d node_modules/.bin; then
+        export PATH=$(readlink -f node_modules/.bin):$PATH
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    runHook postShellHook
+    echo "Finished executing linkNodeModulesShellHook"
+}
+
+if [ -z "${dontLinkNodeModules:-}" ] && [ -z "${shellHook-}" ]; then
+    echo "Using linkNodeModulesHook shell hook"
+    shellHook=linkNodeModulesHook
+fi
+
+
+if [ -z "${dontLinkNodeModules:-}" ]; then
+    echo "Using linkNodeModulesHook preConfigure hook"
+    preConfigureHooks+=(linkNodeModulesHook)
+fi

pkgs/build-support/node/import-npm-lock/hooks/link-node-modules.js

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+const fs = require("fs");
+const path = require("path");
+
+async function asyncFilter(arr, pred) {
+  const filtered = [];
+  for (const elem of arr) {
+    if (await pred(elem)) {
+      filtered.push(elem);
+    }
+  }
+  return filtered;
+}
+
+// Get a list of all _unmanaged_ files in node_modules.
+// This means every file in node_modules that is _not_ a symlink to the Nix store.
+async function getUnmanagedFiles(storePrefix, files) {
+  return await asyncFilter(files, async (file) => {
+    const filePath = path.join("node_modules", file);
+
+    // Is file a symlink
+    const stat = await fs.promises.lstat(filePath);
+    if (!stat.isSymbolicLink()) {
+      return true;
+    }
+
+    // Is file in the store
+    const linkTarget = await fs.promises.readlink(filePath);
+    return !linkTarget.startsWith(storePrefix);
+  });
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const storePrefix = args[0];
+  const sourceModules = args[1];
+
+  // Ensure node_modules exists
+  try {
+    await fs.promises.mkdir("node_modules");
+  } catch (err) {
+    if (err.code !== "EEXIST") {
+      throw err;
+    }
+  }
+
+  const files = await fs.promises.readdir("node_modules");
+
+  // Get deny list of files that we don't manage.
+  // We do manage nix store symlinks, but not other files.
+  // For example: If a .vite was present in both our
+  // source node_modules and the non-store node_modules we don't want to overwrite
+  // the non-store one.
+  const unmanaged = await getUnmanagedFiles(storePrefix, files);
+  const managed = new Set(files.filter((file) => ! unmanaged.includes(file)));
+
+  const sourceFiles = await fs.promises.readdir(sourceModules);
+  await Promise.all(
+    sourceFiles.map(async (file) => {
+      const sourcePath = path.join(sourceModules, file);
+      const targetPath = path.join("node_modules", file);
+
+      // Skip file if it's not a symlink to a store path
+      if (unmanaged.includes(file)) {
+        console.log(`'${targetPath}' exists, cowardly refusing to link.`);
+        return;
+      }
+
+      // Don't unlink this file, we just wrote it.
+      managed.delete(file);
+
+      // Link file
+      try {
+        await fs.promises.symlink(sourcePath, targetPath);
+      } catch (err) {
+        // If the target file already exists remove it and try again
+        if (err.code !== "EEXIST") {
+          throw err;
+        }
+        await fs.promises.unlink(targetPath);
+        await fs.promises.symlink(sourcePath, targetPath);
+      }
+    })
+  );
+
+  // Clean up store symlinks not included in this generation of node_modules
+  await Promise.all(
+    Array.from(managed).map((file) =>
+      fs.promises.unlink(path.join("node_modules", file)),
+    )
+  );
+}
+
+main();

pkgs/build-support/node/import-npm-lock/hooks/npm-config-hook.sh

@@ -0,0 +1,70 @@
+# shellcheck shell=bash
+
+npmConfigHook() {
+    echo "Executing npmConfigHook"
+
+    if [ -n "${npmRoot-}" ]; then
+      pushd "$npmRoot"
+    fi
+
+    if [ -z "${npmDeps-}" ]; then
+        echo "Error: 'npmDeps' should be set when using npmConfigHook."
+        exit 1
+    fi
+
+    echo "Configuring npm"
+
+    export HOME="$TMPDIR"
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+    npm config set offline true
+    npm config set progress false
+    npm config set fund false
+
+    echo "Installing patched package.json/package-lock.json"
+
+    # Save original package.json/package-lock.json for closure size reductions.
+    # The patched one contains store paths we don't want at runtime.
+    mv package.json .package.json.orig
+    if test -f package-lock.json; then # Not all packages have package-lock.json.
+        mv package-lock.json .package-lock.json.orig
+    fi
+    cp --no-preserve=mode "${npmDeps}/package.json" package.json
+    cp --no-preserve=mode "${npmDeps}/package-lock.json" package-lock.json
+
+    echo "Installing dependencies"
+
+    if ! npm install --ignore-scripts $npmInstallFlags "${npmInstallFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"; then
+        echo
+        echo "ERROR: npm failed to install dependencies"
+        echo
+        echo "Here are a few things you can try, depending on the error:"
+        echo '1. Set `npmFlags = [ "--legacy-peer-deps" ]`'
+        echo
+
+        exit 1
+    fi
+
+    patchShebangs node_modules
+
+    npm rebuild $npmRebuildFlags "${npmRebuildFlagsArray[@]}" $npmFlags "${npmFlagsArray[@]}"
+
+    patchShebangs node_modules
+
+    # Canonicalize symlinks from relative paths to the Nix store.
+    node @canonicalizeSymlinksScript@ @storePrefix@
+
+    # Revert to pre-patched package.json/package-lock.json for closure size reductions
+    mv .package.json.orig package.json
+    if test -f ".package-lock.json.orig"; then
+        mv .package-lock.json.orig package-lock.json
+    fi
+
+    if [ -n "${npmRoot-}" ]; then
+      popd
+    fi
+
+    echo "Finished npmConfigHook"
+}
+
+postConfigureHooks+=(npmConfigHook)