push sheeet

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/fixPkgAddMissingSha1.js

@@ -0,0 +1,64 @@
+const https = require("https");
+const crypto = require("crypto");
+
+// TODO:
+// make test case where getSha1 function is used, i.e. the case when resolved is without sha1?
+// consider using https://github.com/request/request-promise-native
+
+function getSha1(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, res => {
+      const { statusCode } = res;
+      const hash = crypto.createHash("sha1");
+
+      if (statusCode !== 200) {
+        const err = new Error(`Request Failed.\nStatus Code: ${statusCode}`);
+
+        // consume response data to free up memory
+        res.resume();
+
+        reject(err);
+      }
+
+      res.on("data", chunk => {
+        hash.update(chunk);
+      });
+
+      res.on("end", () => {
+        resolve(hash.digest("hex"));
+      });
+
+      res.on("error", reject);
+    });
+  });
+}
+
+// Object -> Object
+async function fixPkgAddMissingSha1(pkg) {
+  // local dependency
+
+  if (!pkg.resolved) {
+    console.error(
+      `yarn2nix: can't find "resolved" field for package ${pkg.nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
+    );
+    return pkg;
+  }
+
+  const [url, sha1] = pkg.resolved.split("#", 2);
+
+  if (sha1 || url.startsWith("https://codeload.github.com/")) {
+    return pkg;
+  }
+
+  // if there is no sha1 in resolved url
+  // (this could happen if yarn.lock was generated by older version of yarn)
+  // - request it from registry by https and add it to pkg
+  const newSha1 = await getSha1(url);
+
+  return {
+    ...pkg,
+    resolved: `${url}#${newSha1}`
+  };
+}
+
+module.exports = fixPkgAddMissingSha1;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/generateNix.js

@@ -0,0 +1,179 @@
+const R = require("ramda");
+const ssri = require("ssri");
+
+const urlToName = require("./urlToName");
+const { execFileSync } = require("child_process");
+
+// fetchgit transforms
+//
+// "shell-quote@git+https://github.com/srghma/node-shell-quote.git#without_unlicenced_jsonify":
+//   version "1.6.0"
+//   resolved "git+https://github.com/srghma/node-shell-quote.git#1234commit"
+//
+// to
+//
+// fetchGit {
+//   url = "https://github.com/srghma/node-shell-quote.git";
+//   ref = "without_unlicenced_jsonify";
+//   rev = "1234commit";
+// }
+//
+// and transforms
+//
+// "@graphile/plugin-supporter@git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git":
+//   version "0.6.0"
+//   resolved "git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git#1234commit"
+//
+// to
+//
+// fetchGit {
+//   url = "https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git";
+//   ref = "master";
+//   rev = "1234commit";
+// }
+
+function prefetchgit(url, rev) {
+  return JSON.parse(
+    execFileSync(
+      "nix-prefetch-git",
+      ["--rev", rev, url, "--fetch-submodules"],
+      {
+        stdio: ["ignore", "pipe", "ignore"],
+        timeout: 60000
+      }
+    )
+  ).sha256;
+}
+
+function fetchgit(fileName, url, rev, branch, builtinFetchGit) {
+  const repo = builtinFetchGit
+    ? `fetchGit ({
+         url = "${url}";
+         ref = "${branch}";
+         rev = "${rev}";
+       } // (if builtins.compareVersions "2.4pre" builtins.nixVersion < 0 then {
+         # workaround for https://github.com/NixOS/nix/issues/5128
+         allRefs = true;
+       } else {}))`
+    : `fetchgit {
+         url = "${url}";
+         rev = "${rev}";
+         sha256 = "${prefetchgit(url, rev)}";
+       }`;
+
+  return `    {
+    name = "${fileName}";
+    path =
+      let repo = ${repo};
+      in runCommand "${fileName}" { buildInputs = [gnutar]; } ''
+        # Set u+w because tar-fs can't unpack archives with read-only dirs
+        # https://github.com/mafintosh/tar-fs/issues/79
+        tar cf $out --mode u+w -C \${repo} .
+      '';
+  }`;
+}
+
+/**
+ * Parse an integrity hash out of an SSRI string.
+ *
+ * Provides a default and uses the "best" supported algorithm if there are multiple.
+ */
+function parseIntegrity(maybeIntegrity, fallbackHash) {
+  if (!maybeIntegrity && fallbackHash) {
+    return { algo: "sha1", hash: fallbackHash };
+  }
+
+  const integrities = ssri.parse(maybeIntegrity);
+  for (const key in integrities) {
+    if (!/^sha(1|256|512)$/.test(key)) {
+      delete integrities[key];
+    }
+  }
+
+  algo = integrities.pickAlgorithm();
+  hash = integrities[algo][0].digest;
+  return { algo, hash };
+}
+
+function fetchLockedDep(builtinFetchGit) {
+  return function(pkg) {
+    const { integrity, nameWithVersion, resolved } = pkg;
+
+    if (!resolved) {
+      console.error(
+        `yarn2nix: can't find "resolved" field for package ${nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
+      );
+      return "";
+    }
+
+    const [url, sha1OrRev] = resolved.split("#");
+
+    const fileName = urlToName(url);
+
+    if (resolved.startsWith("https://codeload.github.com/")) {
+      const s = resolved.split("/");
+      const githubUrl = `https://github.com/${s[3]}/${s[4]}.git`;
+      const githubRev = s[6];
+
+      const [_, branch] = nameWithVersion.split("#");
+
+      return fetchgit(
+        fileName,
+        githubUrl,
+        githubRev,
+        branch || "master",
+        builtinFetchGit
+      );
+    }
+
+    if (url.startsWith("git+") || url.startsWith("git:")) {
+      const rev = sha1OrRev;
+
+      const [_, branch] = nameWithVersion.split("#");
+
+      const urlForGit = url.replace(/^git\+/, "");
+
+      return fetchgit(
+        fileName,
+        urlForGit,
+        rev,
+        branch || "master",
+        builtinFetchGit
+      );
+    }
+
+    const { algo, hash } = parseIntegrity(integrity, sha1OrRev);
+
+    return `    {
+      name = "${fileName}";
+      path = fetchurl {
+        name = "${fileName}";
+        url  = "${url}";
+        ${algo} = "${hash}";
+      };
+    }`;
+  };
+}
+
+const HEAD = `
+{ fetchurl, fetchgit, linkFarm, runCommand, gnutar }: rec {
+  offline_cache = linkFarm "offline" packages;
+  packages = [
+`.trim();
+
+// Object -> String
+function generateNix(pkgs, builtinFetchGit) {
+  const nameWithVersionAndPackageNix = R.map(
+    fetchLockedDep(builtinFetchGit),
+    pkgs
+  );
+
+  const packagesDefinition = R.join(
+    "\n",
+    R.values(nameWithVersionAndPackageNix)
+  );
+
+  return R.join("\n", [HEAD, packagesDefinition, "  ];", "}"]);
+}
+
+module.exports = generateNix;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/mapObjIndexedReturnArray.js

@@ -0,0 +1,21 @@
+const _curry2 = require("ramda/src/internal/_curry2");
+const _map = require("ramda/src/internal/_map");
+const keys = require("ramda/src/keys");
+
+// mapObjIndexed: ((v, k, {k: v}) → v') → {k: v} → {k: v'}
+// mapObjIndexedReturnArray: ((v, k, {k: v}) → v') → {k: v} → [v']
+
+/*
+ * @example
+ *
+ *      const xyz = { x: 1, y: 2, z: 3 };
+ *      const prependKeyAndDouble = (num, key, obj) => key + (num * 2);
+ *
+ *      mapObjIndexedReturnArray(prependKeyAndDouble, xyz); //=> ['x2', 'y4', 'z6']
+ */
+
+const mapObjIndexedReturnArray = _curry2((fn, obj) =>
+  _map(key => fn(obj[key], key, obj), keys(obj))
+);
+
+module.exports = mapObjIndexedReturnArray;

pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js

@@ -0,0 +1,28 @@
+const path = require("path");
+
+// String -> String
+
+// @url examples:
+// - https://registry.yarnpkg.com/acorn-es7-plugin/-/acorn-es7-plugin-1.1.7.tgz
+// - https://registry.npmjs.org/acorn-es7-plugin/-/acorn-es7-plugin-1.1.7.tgz
+// - git+https://github.com/srghma/node-shell-quote.git
+// - git+https://1234user:1234pass@git.graphile.com/git/users/1234user/postgraphile-supporter.git
+// - https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf
+
+function urlToName(url) {
+  // Yarn generates `codeload.github.com` tarball URLs, where the final
+  // path component (file name) is the git hash. See #111.
+  // See also https://github.com/yarnpkg/yarn/blob/989a7406/src/resolvers/exotics/github-resolver.js#L24-L26
+  let isCodeloadGitTarballUrl =
+    url.startsWith("https://codeload.github.com/") && url.includes("/tar.gz/");
+
+  if (url.startsWith("git+") || isCodeloadGitTarballUrl) {
+    return path.basename(url);
+  }
+
+  return url
+    .replace(/https:\/\/(.)*(.com)\//g, "") // prevents having long directory names
+    .replace(/[@/%:-]/g, "_"); // replace @ and : and - and % characters with underscore
+}
+
+module.exports = urlToName;