push sheeet

pkgs/development/tools/yarn2nix-moretea/default.nix

@@ -0,0 +1,543 @@
+{
+  pkgs ? import <nixpkgs> { },
+  nodejs ? pkgs.nodejs,
+  yarn ? pkgs.yarn,
+  allowAliases ? pkgs.config.allowAliases,
+}@inputs:
+
+let
+  inherit (pkgs)
+    stdenv
+    lib
+    callPackage
+    git
+    rsync
+    runCommandLocal
+    ;
+
+  compose =
+    f: g: x:
+    f (g x);
+  id = x: x;
+  composeAll = builtins.foldl' compose id;
+
+  # https://docs.npmjs.com/files/package.json#license
+  # TODO: support expression syntax (OR, AND, etc)
+  getLicenseFromSpdxId =
+    licstr: if licstr == "UNLICENSED" then lib.licenses.unfree else lib.getLicenseFromSpdxId licstr;
+in
+rec {
+  # Export yarn again to make it easier to find out which yarn was used.
+  inherit yarn;
+
+  # Re-export pkgs
+  inherit pkgs;
+
+  unlessNull = item: alt: if item == null then alt else item;
+
+  reformatPackageName =
+    pname:
+    let
+      # regex adapted from `validate-npm-package-name`
+      # will produce 3 parts e.g.
+      # "@someorg/somepackage" -> [ "@someorg/" "someorg" "somepackage" ]
+      # "somepackage" -> [ null null "somepackage" ]
+      parts = builtins.tail (builtins.match "^(@([^/]+)/)?([^/]+)$" pname);
+      # if there is no organisation we need to filter out null values.
+      non-null = builtins.filter (x: x != null) parts;
+    in
+    builtins.concatStringsSep "-" non-null;
+
+  inherit getLicenseFromSpdxId;
+
+  # Generates the yarn.nix from the yarn.lock file
+  mkYarnNix =
+    {
+      yarnLock,
+      flags ? [ ],
+    }:
+    pkgs.runCommand "yarn.nix" { }
+      "${yarn2nix}/bin/yarn2nix --lockfile ${yarnLock} --no-patch --builtin-fetchgit ${lib.escapeShellArgs flags} > $out";
+
+  # Loads the generated offline cache. This will be used by yarn as
+  # the package source.
+  importOfflineCache =
+    yarnNix:
+    let
+      pkg = callPackage yarnNix { };
+    in
+    pkg.offline_cache;
+
+  defaultYarnFlags = [
+    "--offline"
+    "--frozen-lockfile"
+    "--ignore-engines"
+  ];
+
+  mkYarnModules =
+    {
+      name ? "${pname}-${version}", # safe name and version, e.g. testcompany-one-modules-1.0.0
+      pname, # original name, e.g @testcompany/one
+      version,
+      packageJSON,
+      yarnLock,
+      yarnNix ? mkYarnNix { inherit yarnLock; },
+      offlineCache ? importOfflineCache yarnNix,
+      yarnFlags ? [ ],
+      ignoreScripts ? true,
+      nodejs ? inputs.nodejs,
+      yarn ? inputs.yarn.override { inherit nodejs; },
+      pkgConfig ? { },
+      preBuild ? "",
+      postBuild ? "",
+      workspaceDependencies ? [ ], # List of yarn packages
+      packageResolutions ? { },
+    }:
+    let
+      extraNativeBuildInputs = lib.concatMap (key: pkgConfig.${key}.nativeBuildInputs or [ ]) (
+        builtins.attrNames pkgConfig
+      );
+      extraBuildInputs = lib.concatMap (key: pkgConfig.${key}.buildInputs or [ ]) (
+        builtins.attrNames pkgConfig
+      );
+
+      postInstall = map (
+        key:
+        if (pkgConfig.${key} ? postInstall) then
+          ''
+            for f in $(find -L -path '*/node_modules/${key}' -type d); do
+              (cd "$f" && (${pkgConfig.${key}.postInstall}))
+            done
+          ''
+        else
+          ""
+      ) (builtins.attrNames pkgConfig);
+
+      # build-time JSON generation to avoid IFD
+      # see https://wiki.nixos.org/wiki/Import_From_Derivation
+      workspaceJSON =
+        pkgs.runCommand "${name}-workspace-package.json"
+          {
+            nativeBuildInputs = [ pkgs.jq ];
+            inherit packageJSON;
+            passAsFile = [ "baseJSON" ];
+            baseJSON = builtins.toJSON {
+              private = true;
+              workspaces = [ "deps/**" ];
+              resolutions = packageResolutions;
+            };
+          }
+          ''
+            jq --slurpfile packageJSON "$packageJSON" '.resolutions = $packageJSON[0].resolutions + .resolutions' <"$baseJSONPath" >$out
+          '';
+
+      workspaceDependencyLinks = lib.concatMapStringsSep "\n" (dep: ''
+        mkdir -p "deps/${dep.pname}"
+        ln -sf ${dep.packageJSON} "deps/${dep.pname}/package.json"
+      '') workspaceDependencies;
+
+    in
+    stdenv.mkDerivation {
+      inherit preBuild postBuild name;
+      dontUnpack = true;
+      dontInstall = true;
+      nativeBuildInputs = [
+        yarn
+        nodejs
+        git
+      ]
+      ++ extraNativeBuildInputs;
+      buildInputs = extraBuildInputs;
+
+      configurePhase =
+        lib.optionalString (offlineCache ? outputHash) ''
+          if ! cmp -s ${yarnLock} ${offlineCache}/yarn.lock; then
+            echo "yarn.lock changed, you need to update the fetchYarnDeps hash"
+            exit 1
+          fi
+        ''
+        + ''
+          # Yarn writes cache directories etc to $HOME.
+          export HOME=$PWD/yarn_home
+        '';
+
+      buildPhase = ''
+        runHook preBuild
+
+        mkdir -p "deps/${pname}"
+        cp ${packageJSON} "deps/${pname}/package.json"
+        cp ${workspaceJSON} ./package.json
+        cp ${yarnLock} ./yarn.lock
+        chmod +w ./yarn.lock
+
+        yarn config --offline set yarn-offline-mirror ${offlineCache}
+
+        # Do not look up in the registry, but in the offline cache.
+        ${fixup_yarn_lock}/bin/fixup_yarn_lock yarn.lock
+
+        ${workspaceDependencyLinks}
+
+        yarn install ${
+          lib.escapeShellArgs (defaultYarnFlags ++ lib.optional ignoreScripts "--ignore-scripts" ++ yarnFlags)
+        }
+
+        ${lib.concatStringsSep "\n" postInstall}
+
+        mkdir $out
+        mv node_modules $out/
+        mv deps $out/
+        patchShebangs $out
+
+        runHook postBuild
+      '';
+
+      dontCheckForBrokenSymlinks = true;
+    };
+
+  # This can be used as a shellHook in mkYarnPackage. It brings the built node_modules into
+  # the shell-hook environment.
+  linkNodeModulesHook = ''
+    if [[ -d node_modules || -L node_modules ]]; then
+      echo "./node_modules is present. Replacing."
+      rm -rf node_modules
+    fi
+
+    ln -s "$node_modules" node_modules
+  '';
+
+  mkYarnWorkspace =
+    {
+      src,
+      packageJSON ? src + "/package.json",
+      yarnLock ? src + "/yarn.lock",
+      nodejs ? inputs.nodejs,
+      yarn ? inputs.yarn.override { inherit nodejs; },
+      packageOverrides ? { },
+      ...
+    }@attrs:
+    let
+      package = lib.importJSON packageJSON;
+
+      packageGlobs =
+        if lib.isList package.workspaces then package.workspaces else package.workspaces.packages;
+
+      packageResolutions = package.resolutions or { };
+
+      globElemToRegex = lib.replaceStrings [ "*" ] [ ".*" ];
+
+      # PathGlob -> [PathGlobElem]
+      splitGlob = lib.splitString "/";
+
+      # Path -> [PathGlobElem] -> [Path]
+      # Note: Only directories are included, everything else is filtered out
+      expandGlobList =
+        base: globElems:
+        let
+          elemRegex = globElemToRegex (lib.head globElems);
+          rest = lib.tail globElems;
+          children = lib.attrNames (
+            lib.filterAttrs (name: type: type == "directory") (builtins.readDir base)
+          );
+          matchingChildren = lib.filter (child: builtins.match elemRegex child != null) children;
+        in
+        if globElems == [ ] then
+          [ base ]
+        else
+          lib.concatMap (child: expandGlobList (base + ("/" + child)) rest) matchingChildren;
+
+      # Path -> PathGlob -> [Path]
+      expandGlob = base: glob: expandGlobList base (splitGlob glob);
+
+      packagePaths = lib.concatMap (expandGlob src) packageGlobs;
+
+      packages = lib.listToAttrs (
+        map (
+          src:
+          let
+            packageJSON = src + "/package.json";
+
+            package = lib.importJSON packageJSON;
+
+            allDependencies = lib.foldl (a: b: a // b) { } (
+              map (field: lib.attrByPath [ field ] { } package) [
+                "dependencies"
+                "devDependencies"
+              ]
+            );
+
+            # { [name: String] : { pname : String, packageJSON : String, ... } } -> { [pname: String] : version } -> [{ pname : String, packageJSON : String, ... }]
+            getWorkspaceDependencies =
+              packages: allDependencies:
+              let
+                packageList = lib.attrValues packages;
+              in
+              composeAll [
+                (lib.filter (x: x != null))
+                (lib.mapAttrsToList (
+                  pname: _version: lib.findFirst (package: package.pname == pname) null packageList
+                ))
+              ] allDependencies;
+
+            workspaceDependencies = getWorkspaceDependencies packages allDependencies;
+
+            name = reformatPackageName package.name;
+          in
+          {
+            inherit name;
+            value = mkYarnPackage (
+              removeAttrs attrs [ "packageOverrides" ]
+              // {
+                inherit
+                  src
+                  packageJSON
+                  yarnLock
+                  nodejs
+                  yarn
+                  packageResolutions
+                  workspaceDependencies
+                  ;
+              }
+              // lib.attrByPath [ name ] { } packageOverrides
+            );
+          }
+        ) packagePaths
+      );
+    in
+    packages;
+
+  mkYarnPackage =
+    {
+      name ? null,
+      src,
+      packageJSON ? src + "/package.json",
+      yarnLock ? src + "/yarn.lock",
+      yarnNix ? mkYarnNix { inherit yarnLock; },
+      offlineCache ? importOfflineCache yarnNix,
+      nodejs ? inputs.nodejs,
+      yarn ? inputs.yarn.override { inherit nodejs; },
+      yarnFlags ? [ ],
+      yarnPreBuild ? "",
+      yarnPostBuild ? "",
+      pkgConfig ? { },
+      extraBuildInputs ? [ ],
+      publishBinsFor ? null,
+      workspaceDependencies ? [ ], # List of yarnPackages
+      packageResolutions ? { },
+      ...
+    }@attrs:
+    let
+      package = lib.importJSON packageJSON;
+      pname = attrs.pname or package.name;
+      safeName = reformatPackageName package.name;
+      version = attrs.version or package.version;
+      baseName = unlessNull name "${safeName}-${version}";
+
+      workspaceDependenciesTransitive = lib.unique (
+        (lib.flatten (map (dep: dep.workspaceDependencies) workspaceDependencies)) ++ workspaceDependencies
+      );
+
+      deps = mkYarnModules {
+        pname = package.name;
+        name = "${safeName}-modules-${version}";
+        preBuild = yarnPreBuild;
+        postBuild = yarnPostBuild;
+        workspaceDependencies = workspaceDependenciesTransitive;
+        inherit
+          packageJSON
+          version
+          yarnLock
+          offlineCache
+          nodejs
+          yarn
+          yarnFlags
+          pkgConfig
+          packageResolutions
+          ;
+      };
+
+      publishBinsFor_ = unlessNull publishBinsFor [ package.name ];
+
+      linkDirFunction = ''
+        linkDirToDirLinks() {
+          target=$1
+          if [ ! -f "$target" ]; then
+            mkdir -p "$target"
+          elif [ -L "$target" ]; then
+            local new=$(mktemp -d)
+            trueSource=$(realpath "$target")
+            if [ "$(ls $trueSource | wc -l)" -gt 0 ]; then
+              ln -s $trueSource/* $new/
+            fi
+            rm -r "$target"
+            mv "$new" "$target"
+          fi
+        }
+      '';
+
+      workspaceDependencyCopy = lib.concatMapStringsSep "\n" (dep: ''
+        # ensure any existing scope directory is not a symlink
+        linkDirToDirLinks "$(dirname node_modules/${dep.package.name})"
+        mkdir -p "deps/${dep.package.name}"
+        tar -xf "${dep}/tarballs/${dep.name}.tgz" --directory "deps/${dep.package.name}" --strip-components=1
+        if [ ! -e "deps/${dep.package.name}/node_modules" ]; then
+          ln -s "${deps}/deps/${dep.package.name}/node_modules" "deps/${dep.package.name}/node_modules"
+        fi
+      '') workspaceDependenciesTransitive;
+
+    in
+    stdenv.mkDerivation (
+      removeAttrs attrs [
+        "yarnNix"
+        "pkgConfig"
+        "workspaceDependencies"
+        "packageResolutions"
+      ]
+      // {
+        inherit pname version src;
+
+        name = baseName;
+
+        buildInputs = [
+          yarn
+          nodejs
+          rsync
+        ]
+        ++ extraBuildInputs;
+
+        node_modules = deps + "/node_modules";
+
+        configurePhase =
+          attrs.configurePhase or ''
+            runHook preConfigure
+
+            for localDir in npm-packages-offline-cache node_modules; do
+              if [[ -d $localDir || -L $localDir ]]; then
+                echo "$localDir dir present. Removing."
+                rm -rf $localDir
+              fi
+            done
+
+            # move convent of . to ./deps/${package.name}
+            mv $PWD $NIX_BUILD_TOP/temp
+            mkdir -p "$PWD/deps/${package.name}"
+            rm -fd "$PWD/deps/${package.name}"
+            mv $NIX_BUILD_TOP/temp "$PWD/deps/${package.name}"
+            cd $PWD
+
+            ln -s ${deps}/deps/${package.name}/node_modules "deps/${package.name}/node_modules"
+
+            cp -r $node_modules node_modules
+            chmod -R +w node_modules
+
+            ${linkDirFunction}
+
+            linkDirToDirLinks "$(dirname node_modules/${package.name})"
+            ln -s "deps/${package.name}" "node_modules/${package.name}"
+
+            ${workspaceDependencyCopy}
+
+            # Help yarn commands run in other phases find the package
+            echo "--cwd deps/${package.name}" > .yarnrc
+            runHook postConfigure
+          '';
+
+        # Replace this phase on frontend packages where only the generated
+        # files are an interesting output.
+        installPhase =
+          attrs.installPhase or ''
+            runHook preInstall
+
+            mkdir -p $out/{bin,libexec/${package.name}}
+            mv node_modules $out/libexec/${package.name}/node_modules
+            mv deps $out/libexec/${package.name}/deps
+
+            node ${./internal/fixup_bin.js} $out/bin $out/libexec/${package.name}/node_modules ${lib.concatStringsSep " " publishBinsFor_}
+
+            runHook postInstall
+          '';
+
+        dontCheckForBrokenSymlinks = true;
+
+        doDist = attrs.doDist or true;
+
+        distPhase =
+          attrs.distPhase or ''
+            # pack command ignores cwd option
+            rm -f .yarnrc
+            cd $out/libexec/${package.name}/deps/${package.name}
+            mkdir -p $out/tarballs/
+            yarn pack --offline --ignore-scripts --filename $out/tarballs/${baseName}.tgz
+          '';
+
+        passthru = {
+          inherit package packageJSON deps;
+          workspaceDependencies = workspaceDependenciesTransitive;
+        }
+        // (attrs.passthru or { });
+
+        meta = {
+          inherit (nodejs.meta) platforms;
+        }
+        // lib.optionalAttrs (package ? description) { inherit (package) description; }
+        // lib.optionalAttrs (package ? homepage) { inherit (package) homepage; }
+        // lib.optionalAttrs (package ? license) { license = getLicenseFromSpdxId package.license; }
+        // (attrs.meta or { });
+      }
+    );
+
+  yarn2nix = mkYarnPackage {
+    src = ./yarn2nix;
+
+    # yarn2nix is the only package that requires the yarnNix option.
+    # All the other projects can auto-generate that file.
+    yarnNix = ./yarn.nix;
+
+    # Using the filter above and importing package.json from the filtered
+    # source results in an error in restricted mode. To circumvent this,
+    # we import package.json from the unfiltered source
+    packageJSON = ./yarn2nix/package.json;
+
+    yarnFlags = defaultYarnFlags ++ [
+      "--ignore-scripts"
+      "--production=true"
+    ];
+
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+
+    buildPhase = ''
+      source ${./nix/expectShFunctions.sh}
+
+      expectFilePresent ./node_modules/.yarn-integrity
+
+      # check dependencies are installed
+      expectFilePresent ./node_modules/@yarnpkg/lockfile/package.json
+
+      # check devDependencies are not installed
+      expectFileOrDirAbsent ./node_modules/.bin/eslint
+      expectFileOrDirAbsent ./node_modules/eslint/package.json
+    '';
+
+    postInstall = ''
+      wrapProgram $out/bin/yarn2nix --prefix PATH : "${pkgs.nix-prefetch-git}/bin"
+    '';
+  };
+
+  fixup_yarn_lock =
+    runCommandLocal "fixup_yarn_lock"
+      {
+        buildInputs = [ nodejs ];
+      }
+      ''
+        mkdir -p $out/lib
+        mkdir -p $out/bin
+
+        cp ${./yarn2nix/lib/urlToName.js} $out/lib/urlToName.js
+        cp ${./internal/fixup_yarn_lock.js} $out/bin/fixup_yarn_lock
+
+        patchShebangs $out
+      '';
+}
+// lib.optionalAttrs allowAliases {
+  # Aliases
+  spdxLicense = getLicenseFromSpdxId; # added 2021-12-01
+}

pkgs/development/tools/yarn2nix-moretea/internal/fixup_bin.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+/* Usage:
+ * node fixup_bin.js <bin_dir> <modules_dir> [<bin_pkg_1>, <bin_pkg_2> ... ]
+ */
+
+const fs = require("fs");
+const path = require("path");
+
+const derivationBinPath = process.argv[2];
+const nodeModules = process.argv[3];
+const packagesToPublishBin = process.argv.slice(4);
+
+function processPackage(name) {
+  console.log("fixup_bin: Processing ", name);
+
+  const packagePath = `${nodeModules}/${name}`;
+  const packageJsonPath = `${packagePath}/package.json`;
+  const packageJson = JSON.parse(fs.readFileSync(packageJsonPath));
+
+  if (!packageJson.bin) {
+    console.log("fixup_bin: No binaries provided");
+    return;
+  }
+
+  // There are two alternative syntaxes for `bin`
+  // a) just a plain string, in which case the name of the package is the name of the binary.
+  // b) an object, where key is the name of the eventual binary, and the value the path to that binary.
+  if (typeof packageJson.bin === "string") {
+    const binName = packageJson.bin;
+    packageJson.bin = {};
+    packageJson.bin[packageJson.name] = binName;
+  }
+
+  // eslint-disable-next-line no-restricted-syntax, guard-for-in
+  for (const binName in packageJson.bin) {
+    const binPath = packageJson.bin[binName];
+    const normalizedBinName = binName.replace("@", "").replace("/", "-");
+
+    const targetPath = path.normalize(`${packagePath}/${binPath}`);
+    const createdPath = `${derivationBinPath}/${normalizedBinName}`;
+
+    console.log(
+      `fixup_bin: creating link ${createdPath} that points to ${targetPath}`
+    );
+
+    fs.symlinkSync(targetPath, createdPath);
+  }
+}
+
+packagesToPublishBin.forEach(pkg => {
+  processPackage(pkg);
+});

pkgs/development/tools/yarn2nix-moretea/internal/fixup_yarn_lock.js

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+
+/* Usage:
+ * node fixup_yarn_lock.js yarn.lock
+ */
+
+const fs = require("fs");
+const readline = require("readline");
+
+const urlToName = require("../lib/urlToName");
+
+const yarnLockPath = process.argv[2];
+
+const readFile = readline.createInterface({
+  input: fs.createReadStream(yarnLockPath, { encoding: "utf8" }),
+
+  // Note: we use the crlfDelay option to recognize all instances of CR LF
+  // ('\r\n') in input.txt as a single line break.
+  crlfDelay: Infinity,
+
+  terminal: false // input and output should be treated like a TTY
+});
+
+const result = [];
+
+readFile
+  .on("line", line => {
+    const arr = line.match(/^ {2}resolved "([^#]+)(#[^"]+)?"$/);
+
+    if (arr !== null) {
+      const [_, url, shaOrRev] = arr;
+
+      const fileName = urlToName(url);
+
+      result.push(`  resolved "${fileName}${shaOrRev ?? ""}"`);
+    } else {
+      result.push(line);
+    }
+  })
+  .on("close", () => {
+    fs.writeFile(yarnLockPath, result.join("\n"), "utf8", err => {
+      if (err) {
+        console.error(
+          "fixup_yarn_lock: fatal error when trying to write to yarn.lock",
+          err
+        );
+      }
+    });
+  });

pkgs/development/tools/yarn2nix-moretea/nix/expectShFunctions.sh

@@ -0,0 +1,30 @@
+expectFilePresent () {
+  if [ -f "$1" ]; then
+    echo "Test passed: file is present - $1"
+  else
+    echo "Test failed: file is absent - $1"
+    exit 1
+  fi
+}
+
+expectFileOrDirAbsent () {
+  if [ ! -e "$1" ];
+  then
+    echo "Test passed: file or dir is absent - $1"
+  else
+    echo "Test failed: file or dir is present - $1"
+    exit 1
+  fi
+}
+
+expectEqual () {
+  if [ "$1" == "$2" ];
+  then
+    echo "Test passed: output is equal to expected_output"
+  else
+    echo "Test failed: output is not equal to expected_output:"
+    echo "  output - $1"
+    echo "  expected_output - $2"
+    exit 1
+  fi
+}

pkgs/development/tools/yarn2nix-moretea/yarn2nix/.prettierignore

@@ -0,0 +1 @@
+node_modules

pkgs/development/tools/yarn2nix-moretea/yarn2nix/bin/yarn2nix.js

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const lockfile = require("@yarnpkg/lockfile");
+const { docopt } = require("docopt");
+const deepEqual = require("deep-equal");
+const R = require("ramda");
+
+const fixPkgAddMissingSha1 = require("../lib/fixPkgAddMissingSha1");
+const mapObjIndexedReturnArray = require("../lib/mapObjIndexedReturnArray");
+const generateNix = require("../lib/generateNix");
+
+const USAGE = `
+Usage: yarn2nix [options]
+
+Options:
+  -h --help           Shows this help.
+  --no-nix            Hide the nix output
+  --no-patch          Don't patch the lockfile if hashes are missing
+  --lockfile=FILE     Specify path to the lockfile [default: ./yarn.lock].
+  --builtin-fetchgit  Use builtin fetchGit for git dependencies to support on-the-fly generation of yarn.nix without an internet connection
+`;
+
+const options = docopt(USAGE);
+
+const data = fs.readFileSync(options["--lockfile"], "utf8");
+
+// json example:
+
+// {
+//   type:'success',
+//   object:{
+//     'abbrev@1':{
+//       version:'1.0.9',
+//       resolved:'https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135'
+//     },
+//     'shell-quote@git+https://github.com/srghma/node-shell-quote.git#without_unlicenced_jsonify':{
+//       version:'1.6.0',
+//       resolved:'git+https://github.com/srghma/node-shell-quote.git#0aa381896e0cd7409ead15fd444f225807a61e0a'
+//     },
+//     '@graphile/plugin-supporter@git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git':{
+//       version:'1.6.0',
+//       resolved:'git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git#1234commit'
+//     },
+//   }
+// }
+
+const json = lockfile.parse(data);
+
+if (json.type !== "success") {
+  throw new Error("yarn.lock parse error");
+}
+
+// Check for missing hashes in the yarn.lock and patch if necessary
+
+let pkgs = R.pipe(
+  mapObjIndexedReturnArray((value, key) => ({
+    ...value,
+    nameWithVersion: key
+  })),
+  R.uniqBy(R.prop("resolved"))
+)(json.object);
+
+(async () => {
+  if (!options["--no-patch"]) {
+    pkgs = await Promise.all(R.map(fixPkgAddMissingSha1, pkgs));
+  }
+
+  const origJson = lockfile.parse(data);
+
+  if (!deepEqual(origJson, json)) {
+    console.error("found changes in the lockfile", options["--lockfile"]);
+
+    if (options["--no-patch"]) {
+      console.error("...aborting");
+      process.exit(1);
+    }
+
+    fs.writeFileSync(options["--lockfile"], lockfile.stringify(json.object));
+  }
+
+  if (!options["--no-nix"]) {
+    // print to stdout
+    console.log(generateNix(pkgs, options["--builtin-fetchgit"]));
+  }
+})().catch(error => {
+  console.error(error);
+
+  process.exit(1);
+});

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/fixPkgAddMissingSha1.js

@@ -0,0 +1,64 @@
+const https = require("https");
+const crypto = require("crypto");
+
+// TODO:
+// make test case where getSha1 function is used, i.e. the case when resolved is without sha1?
+// consider using https://github.com/request/request-promise-native
+
+function getSha1(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, res => {
+      const { statusCode } = res;
+      const hash = crypto.createHash("sha1");
+
+      if (statusCode !== 200) {
+        const err = new Error(`Request Failed.\nStatus Code: ${statusCode}`);
+
+        // consume response data to free up memory
+        res.resume();
+
+        reject(err);
+      }
+
+      res.on("data", chunk => {
+        hash.update(chunk);
+      });
+
+      res.on("end", () => {
+        resolve(hash.digest("hex"));
+      });
+
+      res.on("error", reject);
+    });
+  });
+}
+
+// Object -> Object
+async function fixPkgAddMissingSha1(pkg) {
+  // local dependency
+
+  if (!pkg.resolved) {
+    console.error(
+      `yarn2nix: can't find "resolved" field for package ${pkg.nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
+    );
+    return pkg;
+  }
+
+  const [url, sha1] = pkg.resolved.split("#", 2);
+
+  if (sha1 || url.startsWith("https://codeload.github.com/")) {
+    return pkg;
+  }
+
+  // if there is no sha1 in resolved url
+  // (this could happen if yarn.lock was generated by older version of yarn)
+  // - request it from registry by https and add it to pkg
+  const newSha1 = await getSha1(url);
+
+  return {
+    ...pkg,
+    resolved: `${url}#${newSha1}`
+  };
+}
+
+module.exports = fixPkgAddMissingSha1;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/generateNix.js

@@ -0,0 +1,179 @@
+const R = require("ramda");
+const ssri = require("ssri");
+
+const urlToName = require("./urlToName");
+const { execFileSync } = require("child_process");
+
+// fetchgit transforms
+//
+// "shell-quote@git+https://github.com/srghma/node-shell-quote.git#without_unlicenced_jsonify":
+//   version "1.6.0"
+//   resolved "git+https://github.com/srghma/node-shell-quote.git#1234commit"
+//
+// to
+//
+// fetchGit {
+//   url = "https://github.com/srghma/node-shell-quote.git";
+//   ref = "without_unlicenced_jsonify";
+//   rev = "1234commit";
+// }
+//
+// and transforms
+//
+// "@graphile/plugin-supporter@git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git":
+//   version "0.6.0"
+//   resolved "git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git#1234commit"
+//
+// to
+//
+// fetchGit {
+//   url = "https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git";
+//   ref = "master";
+//   rev = "1234commit";
+// }
+
+function prefetchgit(url, rev) {
+  return JSON.parse(
+    execFileSync(
+      "nix-prefetch-git",
+      ["--rev", rev, url, "--fetch-submodules"],
+      {
+        stdio: ["ignore", "pipe", "ignore"],
+        timeout: 60000
+      }
+    )
+  ).sha256;
+}
+
+function fetchgit(fileName, url, rev, branch, builtinFetchGit) {
+  const repo = builtinFetchGit
+    ? `fetchGit ({
+         url = "${url}";
+         ref = "${branch}";
+         rev = "${rev}";
+       } // (if builtins.compareVersions "2.4pre" builtins.nixVersion < 0 then {
+         # workaround for https://github.com/NixOS/nix/issues/5128
+         allRefs = true;
+       } else {}))`
+    : `fetchgit {
+         url = "${url}";
+         rev = "${rev}";
+         sha256 = "${prefetchgit(url, rev)}";
+       }`;
+
+  return `    {
+    name = "${fileName}";
+    path =
+      let repo = ${repo};
+      in runCommand "${fileName}" { buildInputs = [gnutar]; } ''
+        # Set u+w because tar-fs can't unpack archives with read-only dirs
+        # https://github.com/mafintosh/tar-fs/issues/79
+        tar cf $out --mode u+w -C \${repo} .
+      '';
+  }`;
+}
+
+/**
+ * Parse an integrity hash out of an SSRI string.
+ *
+ * Provides a default and uses the "best" supported algorithm if there are multiple.
+ */
+function parseIntegrity(maybeIntegrity, fallbackHash) {
+  if (!maybeIntegrity && fallbackHash) {
+    return { algo: "sha1", hash: fallbackHash };
+  }
+
+  const integrities = ssri.parse(maybeIntegrity);
+  for (const key in integrities) {
+    if (!/^sha(1|256|512)$/.test(key)) {
+      delete integrities[key];
+    }
+  }
+
+  algo = integrities.pickAlgorithm();
+  hash = integrities[algo][0].digest;
+  return { algo, hash };
+}
+
+function fetchLockedDep(builtinFetchGit) {
+  return function(pkg) {
+    const { integrity, nameWithVersion, resolved } = pkg;
+
+    if (!resolved) {
+      console.error(
+        `yarn2nix: can't find "resolved" field for package ${nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
+      );
+      return "";
+    }
+
+    const [url, sha1OrRev] = resolved.split("#");
+
+    const fileName = urlToName(url);
+
+    if (resolved.startsWith("https://codeload.github.com/")) {
+      const s = resolved.split("/");
+      const githubUrl = `https://github.com/${s[3]}/${s[4]}.git`;
+      const githubRev = s[6];
+
+      const [_, branch] = nameWithVersion.split("#");
+
+      return fetchgit(
+        fileName,
+        githubUrl,
+        githubRev,
+        branch || "master",
+        builtinFetchGit
+      );
+    }
+
+    if (url.startsWith("git+") || url.startsWith("git:")) {
+      const rev = sha1OrRev;
+
+      const [_, branch] = nameWithVersion.split("#");
+
+      const urlForGit = url.replace(/^git\+/, "");
+
+      return fetchgit(
+        fileName,
+        urlForGit,
+        rev,
+        branch || "master",
+        builtinFetchGit
+      );
+    }
+
+    const { algo, hash } = parseIntegrity(integrity, sha1OrRev);
+
+    return `    {
+      name = "${fileName}";
+      path = fetchurl {
+        name = "${fileName}";
+        url  = "${url}";
+        ${algo} = "${hash}";
+      };
+    }`;
+  };
+}
+
+const HEAD = `
+{ fetchurl, fetchgit, linkFarm, runCommand, gnutar }: rec {
+  offline_cache = linkFarm "offline" packages;
+  packages = [
+`.trim();
+
+// Object -> String
+function generateNix(pkgs, builtinFetchGit) {
+  const nameWithVersionAndPackageNix = R.map(
+    fetchLockedDep(builtinFetchGit),
+    pkgs
+  );
+
+  const packagesDefinition = R.join(
+    "\n",
+    R.values(nameWithVersionAndPackageNix)
+  );
+
+  return R.join("\n", [HEAD, packagesDefinition, "  ];", "}"]);
+}
+
+module.exports = generateNix;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/mapObjIndexedReturnArray.js

@@ -0,0 +1,21 @@
+const _curry2 = require("ramda/src/internal/_curry2");
+const _map = require("ramda/src/internal/_map");
+const keys = require("ramda/src/keys");
+
+// mapObjIndexed: ((v, k, {k: v}) → v') → {k: v} → {k: v'}
+// mapObjIndexedReturnArray: ((v, k, {k: v}) → v') → {k: v} → [v']
+
+/*
+ * @example
+ *
+ *      const xyz = { x: 1, y: 2, z: 3 };
+ *      const prependKeyAndDouble = (num, key, obj) => key + (num * 2);
+ *
+ *      mapObjIndexedReturnArray(prependKeyAndDouble, xyz); //=> ['x2', 'y4', 'z6']
+ */
+
+const mapObjIndexedReturnArray = _curry2((fn, obj) =>
+  _map(key => fn(obj[key], key, obj), keys(obj))
+);
+
+module.exports = mapObjIndexedReturnArray;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js

@@ -0,0 +1,28 @@
+const path = require("path");
+
+// String -> String
+
+// @url examples:
+// - https://registry.yarnpkg.com/acorn-es7-plugin/-/acorn-es7-plugin-1.1.7.tgz
+// - https://registry.npmjs.org/acorn-es7-plugin/-/acorn-es7-plugin-1.1.7.tgz
+// - git+https://github.com/srghma/node-shell-quote.git
+// - git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git
+// - https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf
+
+function urlToName(url) {
+  // Yarn generates `codeload.github.com` tarball URLs, where the final
+  // path component (file name) is the git hash. See #111.
+  // See also https://github.com/yarnpkg/yarn/blob/989a7406/src/resolvers/exotics/github-resolver.js#L24-L26
+  let isCodeloadGitTarballUrl =
+    url.startsWith("https://codeload.github.com/") && url.includes("/tar.gz/");
+
+  if (url.startsWith("git+") || isCodeloadGitTarballUrl) {
+    return path.basename(url);
+  }
+
+  return url
+    .replace(/https:\/\/(.)*(.com)\//g, "") // prevents having long directory names
+    .replace(/[@/%:-]/g, "_"); // replace @ and : and - and % characters with underscore
+}
+
+module.exports = urlToName;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "yarn2nix",
+  "version": "1.0.0",
+  "description": "Convert packages.json and yarn.lock into a Nix expression that downloads all the dependencies",
+  "main": "index.js",
+  "repository": ".",
+  "author": "Maarten Hoogendoorn <maarten@moretea.nl>",
+  "license": "MIT",
+  "scripts": {
+    "yarn2nix": "bin/yarn2nix.js",
+    "format": "prettier-eslint --write './**/*.{js,jsx,json}'",
+    "lint": "eslint ."
+  },
+  "bin": {
+    "yarn2nix": "bin/yarn2nix.js"
+  },
+  "engines": {
+    "node": ">=8.0.0"
+  },
+  "dependencies": {
+    "@yarnpkg/lockfile": "^1.1.0",
+    "deep-equal": "^1.0.1",
+    "docopt": "^0.6.2",
+    "ramda": "^0.26.1",
+    "ssri": "^10.0.0"
+  },
+  "devDependencies": {
+    "babel-eslint": "^10.0.1",
+    "eslint": "^5.11.1",
+    "eslint-config-airbnb": "^17.1.0",
+    "eslint-config-prettier": "^3.3.0",
+    "eslint-config-standard": "^12.0.0",
+    "eslint-plugin-import": "^2.14.0",
+    "eslint-plugin-jsx-a11y": "^6.1.2",
+    "eslint-plugin-node": "^8.0.0",
+    "eslint-plugin-promise": "^4.0.1",
+    "eslint-plugin-react": "^7.12.2",
+    "eslint-plugin-standard": "^4.0.0",
+    "husky": "^1.3.1",
+    "lint-staged": "^8.1.0",
+    "prettier-eslint-cli": "^4.7.1"
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged"
+    }
+  }
+}