push sheeet

pkgs/development/julia-modules/python/dag.py

@@ -0,0 +1,59 @@
+
+# This file based on a ChatGPT reponse for the following prompt:
+# "can you write code in python to build up a DAG representing
+# a dependency tree, and then a function that can return all the
+# dependencies of a given node?"
+
+class Node:
+    def __init__(self, name):
+        self.name = name
+        self.dependencies = set()
+
+
+class DAG:
+    def __init__(self):
+        self.nodes = {}
+
+    def add_node(self, node_name, dependencies=None):
+        if node_name in self.nodes:
+            raise ValueError(f"Node '{node_name}' already exists in the graph.")
+
+        node = Node(node_name)
+        if dependencies:
+            node.dependencies.update(dependencies)
+
+        self.nodes[node_name] = node
+
+    def add_dependency(self, node_name, dependency_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        if dependency_name not in self.nodes:
+            raise ValueError(f"Dependency '{dependency_name}' does not exist in the graph.")
+
+        self.nodes[node_name].dependencies.add(dependency_name)
+
+    def get_dependencies(self, node_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        node = self.nodes[node_name]
+        dependencies = set()
+
+        def traverse_dependencies(current_node):
+            for dependency in current_node.dependencies:
+                dependencies.add(dependency)
+                if dependency in self.nodes:
+                  traverse_dependencies(self.nodes[dependency])
+
+        traverse_dependencies(node)
+        return dependencies
+
+    def has_node(self, node_name):
+        return node_name in self.nodes
+
+    def __str__(self):
+        graph_str = ""
+        for node_name, node in self.nodes.items():
+            graph_str += f"{node_name} -> {', '.join(node.dependencies)}\n"
+        return graph_str

pkgs/development/julia-modules/python/dedup_overrides.py

@@ -0,0 +1,14 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+with open(out_path, "w") as f:
+  toml.dump(overrides, f)

pkgs/development/julia-modules/python/extract_artifacts.py

@@ -0,0 +1,168 @@
+
+import json
+from pathlib import Path
+import multiprocessing
+import subprocess
+import sys
+import toml
+from urllib.parse import urlparse
+import yaml
+
+import dag
+
+# This should match the behavior of the default unpackPhase.
+# See https://github.com/NixOS/nixpkgs/blob/59fa082abdbf462515facc8800d517f5728c909d/pkgs/stdenv/generic/setup.sh#L1044
+archive_extensions = [
+  # xz extensions
+  ".tar.xz",
+  ".tar.lzma",
+  ".txz",
+
+  # *.tar or *.tar.*
+  ".tar",
+  ".tar.Z",
+  ".tar.bz2",
+  ".tar.gz",
+
+  # Other tar extensions
+  ".tgz",
+  ".tbz2",
+  ".tbz",
+
+  ".zip"
+  ]
+
+def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin):
+  depends_on = set()
+  if closure_dependencies_dag.has_node(uuid):
+    depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
+
+  other_libs = extra_libs.get(uuid, [])
+
+  if is_darwin:
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+            in ''
+
+            ''"""
+  else:
+    # We provide gcc.cc.lib by default in order to get some common libraries
+    # like libquadmath.so. A number of packages expect this to be available and
+    # will give linker errors if it isn't.
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+            in ''
+              find $out -type f -executable -exec \
+                patchelf --set-rpath \\$ORIGIN:\\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc gcc.cc.lib] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \\;
+              find $out -type f -executable -exec \
+                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \\;
+            ''"""
+
+  return f"""stdenv.mkDerivation {{
+        name = "{artifact_name}";
+        src = fetchurl {{
+          url = "{url}";
+          sha256 = "{sha256}";
+        }};
+        preUnpack = ''
+          mkdir unpacked
+          cd unpacked
+        '';
+        sourceRoot = ".";
+        dontConfigure = true;
+        dontBuild = true;
+        installPhase = "cp -r . $out";
+        {fixup};
+      }}"""
+
+def get_plain_derivation(url, sha256):
+  return f"""fetchurl {{
+        url = "{url}";
+        sha256 = "{sha256}";
+      }}"""
+
+def process_item(args):
+  item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args
+  uuid, src = item
+  lines = []
+
+  artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
+  if not artifacts:
+    return f'  uuid-{uuid} = {{}};\n'
+
+  lines.append(f'  uuid-{uuid} = {{')
+
+  for artifact_name, details in artifacts.items():
+    if len(details["download"]) == 0:
+      continue
+    download = details["download"][0]
+    url = download["url"]
+    sha256 = download["sha256"]
+
+    git_tree_sha1 = details["git-tree-sha1"]
+
+    parsed_url = urlparse(url)
+    if any(parsed_url.path.endswith(x) for x in archive_extensions):
+      derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+    else:
+      derivation = get_plain_derivation(url, sha256)
+
+    lines.append(f"""    "{artifact_name}" = {{
+      sha1 = "{git_tree_sha1}";
+      path = {derivation};
+    }};\n""")
+
+  lines.append('  };\n')
+
+  return "\n".join(lines)
+
+def main():
+  dependencies_path = Path(sys.argv[1])
+  closure_yaml_path = Path(sys.argv[2])
+  julia_path = Path(sys.argv[3])
+  extract_artifacts_script = Path(sys.argv[4])
+  extra_libs = json.loads(sys.argv[5])
+  is_darwin = json.loads(sys.argv[6])
+  out_path = Path(sys.argv[7])
+
+  with open(dependencies_path, "r") as f:
+    dependencies = yaml.safe_load(f)
+    dependency_uuids = list(dependencies.keys())  # Convert dict_keys to list
+
+  with open(closure_yaml_path, "r") as f:
+    # Build up a map of UUID -> closure information
+    closure_yaml_list = yaml.safe_load(f) or []
+    closure_yaml = {}
+    for item in closure_yaml_list:
+      closure_yaml[item["uuid"]] = item
+
+    # Build up a dependency graph of UUIDs
+    closure_dependencies_dag = dag.DAG()
+    for uuid, contents in closure_yaml.items():
+      if contents.get("depends_on"):
+        closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
+
+  with open(out_path, "w") as f:
+    if is_darwin:
+      f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
+    else:
+      f.write("{ lib, fetchurl, gcc, glibc, pkgs, stdenv }:\n\n")
+
+    f.write("rec {\n")
+
+    with multiprocessing.Pool(10) as pool:
+      # Create args tuples for each item
+      process_args = [
+        (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+        for item in dependencies.items()
+      ]
+      for s in pool.map(process_item, process_args):
+        f.write(s)
+
+    f.write(f"""
+}}\n""")
+
+if __name__ == "__main__":
+  main()

pkgs/development/julia-modules/python/find_package_implications.py

@@ -0,0 +1,24 @@
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import yaml
+
+dependencies_path = Path(sys.argv[1])
+package_implications_json = sys.argv[2]
+out_path = Path(sys.argv[3])
+
+package_implications = json.loads(package_implications_json)
+with open(dependencies_path) as f:
+  desired_packages = yaml.safe_load(f) or []
+
+extra_package_names = []
+for pkg in desired_packages:
+  if pkg["name"] in package_implications:
+    extra_package_names.extend(package_implications[pkg["name"]])
+
+if len(extra_package_names) > 0:
+  with open(out_path, "w") as f:
+    f.write("\n".join(extra_package_names))

pkgs/development/julia-modules/python/format_overrides.py

@@ -0,0 +1,22 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+result = {}
+
+for (uuid, artifacts) in overrides.items():
+  if len(artifacts) == 0: continue
+
+  for (name, info) in artifacts.items():
+    result[info["sha1"]] = info["path"]
+
+with open(out_path, "w") as f:
+  toml.dump(result, f)

pkgs/development/julia-modules/python/minimal_registry.py

@@ -0,0 +1,128 @@
+
+from collections import defaultdict
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+desired_packages_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+uuid_to_versions = defaultdict(list)
+for pkg in desired_packages:
+  uuid_to_versions[pkg["uuid"]].append(pkg["version"])
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+os.makedirs(out_path)
+
+full_registry = toml.load(registry_path / "Registry.toml")
+registry = full_registry.copy()
+registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}
+
+for (uuid, versions) in uuid_to_versions.items():
+  if uuid in package_overrides:
+    info = package_overrides[uuid]
+
+    # Make a registry entry based on the info from the package override
+    path = Path(info["name"][0].upper()) / Path(info["name"])
+    registry["packages"][uuid] = {
+      "name": info["name"],
+      "path": str(path),
+    }
+
+    os.makedirs(out_path / path)
+
+    # Read the Project.yaml from the src
+    project = toml.load(Path(info["src"]) / "Project.toml")
+
+    # Generate all the registry files
+    with open(out_path / path / Path("Compat.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      # Write nothing in Compat.toml, because we've already resolved everything
+    with open(out_path / path / Path("Deps.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      if "deps" in project:
+        toml.dump(project["deps"], f)
+    with open(out_path / path / Path("Versions.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      f.write('git-tree-sha1 = "%s"\n' % info["treehash"])
+    with open(out_path / path / Path("Package.toml"), "w") as f:
+      toml.dump({
+        "name": info["name"],
+        "uuid": uuid,
+        "repo": "file://" + info["src"],
+      }, f)
+
+  elif uuid in registry["packages"]:
+    registry_info = registry["packages"][uuid]
+    name = registry_info["name"]
+    path = registry_info["path"]
+
+    os.makedirs(out_path / path)
+
+    # Copy some files to the minimal repo unchanged
+    for f in ["Compat.toml", "Deps.toml", "WeakCompat.toml", "WeakDeps.toml"]:
+      if (registry_path / path / f).exists():
+        shutil.copy2(registry_path / path / f, out_path / path)
+
+    # Copy the Versions.toml file, trimming down to the versions we care about.
+    # In the case where versions=None, this is a weak dep, and we keep all versions.
+    all_versions = toml.load(registry_path / path / "Versions.toml")
+    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions} if versions != None else all_versions
+    for k, v in versions_to_keep.items():
+      del v["nix-sha256"]
+    with open(out_path / path / "Versions.toml", "w") as f:
+      toml.dump(versions_to_keep, f)
+
+    if versions is None:
+      # This is a weak dep; just grab the whole Package.toml
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+    elif uuid in uuid_to_store_path:
+      # Fill in the local store path for the repo
+      package_toml = toml.load(registry_path / path / "Package.toml")
+      package_toml["repo"] = "file://" + uuid_to_store_path[uuid]
+      with open(out_path / path / "Package.toml", "w") as f:
+        toml.dump(package_toml, f)
+
+# Look for missing weak deps and include them. This can happen when our initial
+# resolve step finds dependencies, but we fail to resolve them at the project.py
+# stage. Usually this happens because the package that depends on them does so
+# as a weak dep, but doesn't have a Package.toml in its repo making this clear.
+for pkg in desired_packages:
+  for dep in (pkg.get("deps", []) or []):
+    uuid = dep["uuid"]
+    if not uuid in uuid_to_versions:
+      entry = full_registry["packages"].get(uuid)
+      if not entry:
+        print(f"""WARNING: found missing UUID but couldn't resolve it: {uuid}""")
+        continue
+
+      # Add this entry back to the minimal Registry.toml
+      registry["packages"][uuid] = entry
+
+      # Bring over the Package.toml
+      path = Path(entry["path"])
+      if (out_path / path / "Package.toml").exists():
+        continue
+      Path(out_path / path).mkdir(parents=True, exist_ok=True)
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+
+# Finally, dump the Registry.toml
+with open(out_path / "Registry.toml", "w") as f:
+    toml.dump(registry, f)

pkgs/development/julia-modules/python/project.py

@@ -0,0 +1,104 @@
+
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+import sys
+import toml
+import yaml
+
+
+desired_packages_path = Path(sys.argv[1])
+stdlib_infos_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+with open(stdlib_infos_path, "r") as f:
+  stdlib_infos = yaml.safe_load(f) or []
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+result = {
+  "deps": defaultdict(list)
+}
+
+for pkg in desired_packages:
+  if pkg["uuid"] in package_overrides:
+    info = package_overrides[pkg["uuid"]]
+    result["deps"][info["name"]].append({
+      "uuid": pkg["uuid"],
+      "path": info["src"],
+    })
+    continue
+
+  path = uuid_to_store_path.get(pkg["uuid"], None)
+  isStdLib = False
+  if pkg["uuid"] in stdlib_infos["stdlibs"]:
+    path = stdlib_infos["stdlib_root"] + "/" + stdlib_infos["stdlibs"][pkg["uuid"]]["name"]
+    isStdLib = True
+
+  if path:
+    if (Path(path) / "Project.toml").exists():
+      project_toml = toml.load(Path(path) / "Project.toml")
+
+      deps = []
+      weak_deps = project_toml.get("weakdeps", {})
+      extensions = project_toml.get("extensions", {})
+
+      if "deps" in project_toml:
+        # Build up deps for the manifest, excluding weak deps
+        weak_deps_uuids = weak_deps.values()
+        for (dep_name, dep_uuid) in project_toml["deps"].items():
+          if not (dep_uuid in weak_deps_uuids):
+            deps.append(dep_name)
+    else:
+      # Not all projects have a Project.toml. In this case, use the deps we
+      # calculated from the package resolve step. This isn't perfect since it
+      # will fail to properly split out weak deps, but it's better than nothing.
+      print(f"""WARNING: package {pkg["name"]} didn't have a Project.toml in {path}""")
+      deps = [x["name"] for x in pkg.get("deps", [])]
+      weak_deps = {}
+      extensions = {}
+
+    tree_hash = pkg.get("tree_hash", "")
+
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "git-tree-sha1": (tree_hash if tree_hash != "nothing" else None) or None,
+      "deps": deps or None,
+      "weakdeps": weak_deps or None,
+      "extensions": extensions or None,
+
+      # We *don't* set "path" here, because then Julia will try to use the
+      # read-only Nix store path instead of cloning to the depot. This will
+      # cause packages like Conda.jl to fail during the Pkg.build() step.
+      #
+      # "path": None if isStdLib else path ,
+    })
+  else:
+    print("WARNING: adding a package that we didn't have a path for, and it doesn't seem to be a stdlib", pkg)
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "deps": [x["name"] for x in pkg["deps"]]
+    })
+
+os.makedirs(out_path)
+
+with open(out_path / "Manifest.toml", "w") as f:
+  f.write(f'julia_version = "{stdlib_infos["julia_version"]}"\n')
+  f.write('manifest_format = "2.0"\n\n')
+  toml.dump(result, f)
+
+with open(out_path / "Project.toml", "w") as f:
+  f.write('[deps]\n')
+
+  for pkg in desired_packages:
+    if pkg.get("is_input", False):
+      f.write(f'''{pkg["name"]} = "{pkg["uuid"]}"\n''')

pkgs/development/julia-modules/python/sources_nix.py

@@ -0,0 +1,75 @@
+
+import json
+from pathlib import Path
+import re
+import shutil
+import sys
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+package_overrides = json.loads(sys.argv[2])
+desired_packages_path = Path(sys.argv[3])
+out_path = Path(sys.argv[4])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+registry = toml.load(registry_path / "Registry.toml")
+
+def ensure_version_valid(version):
+  """
+  Ensure a version string is a valid Julia-parsable version.
+  It doesn't really matter what it looks like as it's just used for overrides.
+  """
+  return re.sub('[^0-9.]','', version)
+
+with open(out_path, "w") as f:
+  f.write("{fetchgit}:\n")
+  f.write("{\n")
+  for pkg in desired_packages:
+    uuid = pkg["uuid"]
+
+    if pkg["name"] in package_overrides:
+      treehash = util.get_commit_info(package_overrides[pkg["name"]])["tree"]
+      f.write(f"""  "{uuid}" = {{
+    src = null; # Overridden: will fill in later
+    name = "{pkg["name"]}";
+    version = "{ensure_version_valid(pkg["version"])}";
+    treehash = "{treehash}";
+  }};\n""")
+    elif uuid in registry["packages"]:
+      # The treehash is missing for stdlib packages. Don't bother downloading these.
+      if (not ("tree_hash" in pkg)) or pkg["tree_hash"] == "nothing": continue
+
+      registry_info = registry["packages"][uuid]
+      path = registry_info["path"]
+      packageToml = toml.load(registry_path / path / "Package.toml")
+
+      versions_toml = registry_path / path / "Versions.toml"
+      all_versions = toml.load(versions_toml)
+      if not pkg["version"] in all_versions: continue
+      version_to_use = all_versions[pkg["version"]]
+
+      if not "nix-sha256" in version_to_use:
+        raise KeyError(f"""Couldn't find nix-sha256 hash for {pkg["name"]} {pkg["version"]} in {versions_toml}. This might indicate that we failed to prefetch the hash when computing the augmented registry. Was there a relevant failure in {registry_path / "failures.yml"}?""")
+
+      repo = packageToml["repo"]
+      f.write(f"""  "{uuid}" = {{
+    src = fetchgit {{
+      url = "{repo}";
+      rev = "{version_to_use["git-tree-sha1"]}";
+      sha256 = "{version_to_use["nix-sha256"]}";
+    }};
+    name = "{pkg["name"]}";
+    version = "{pkg["version"]}";
+    treehash = "{version_to_use["git-tree-sha1"]}";
+  }};\n""")
+    else:
+      # This is probably a stdlib
+      # print("WARNING: couldn't figure out what to do with pkg in sources_nix.py", pkg)
+      pass
+
+  f.write("}")

pkgs/development/julia-modules/python/util.py

@@ -0,0 +1,12 @@
+
+import os
+import subprocess
+import tempfile
+
+def get_commit_info(repo):
+  with tempfile.TemporaryDirectory() as home_dir:
+    env_with_home = os.environ.copy()
+    env_with_home["HOME"] = home_dir
+    subprocess.check_output(["git", "config", "--global", "--add", "safe.directory", repo], env=env_with_home)
+    lines = subprocess.check_output(["git", "log", "--pretty=raw"], cwd=repo, env=env_with_home).decode().split("\n")
+    return dict([x.split() for x in lines if len(x.split()) == 2])