push sheeet

pkgs/development/julia-modules/default.nix

@@ -0,0 +1,317 @@
+{
+  callPackage,
+  fetchgit,
+  fontconfig,
+  git,
+  lib,
+  makeWrapper,
+  python3,
+  runCommand,
+  writeTextFile,
+
+  # Artifacts dependencies
+  fetchurl,
+  gcc,
+  glibc,
+  pkgs,
+  stdenv,
+
+  julia,
+
+  # Special registry which is equal to JuliaRegistries/General, but every Versions.toml
+  # entry is augmented with a Nix sha256 hash
+  augmentedRegistry ? callPackage ./registry.nix { },
+
+  # Other overridable arguments
+  extraLibs ? [ ],
+  juliaCpuTarget ? null,
+  makeTransitiveDependenciesImportable ? false, # Used to support symbol indexing
+  makeWrapperArgs ? "",
+  packageOverrides ? { },
+  precompile ? true,
+  setDefaultDepot ? true,
+}:
+
+packageNames:
+
+let
+  util = callPackage ./util.nix { };
+
+  # Some Julia packages require access to Python. Provide a Nixpkgs version so it
+  # doesn't try to install its own.
+  pythonToUse =
+    let
+      extraPythonPackages = (
+        (callPackage ./extra-python-packages.nix { inherit python3; }).getExtraPythonPackages packageNames
+      );
+    in
+    (
+      if extraPythonPackages == [ ] then
+        python3
+      else
+        util.addPackagesToPython python3 (map (pkg: lib.getAttr pkg python3.pkgs) extraPythonPackages)
+    );
+
+  # Start by wrapping Julia so it has access to Python and any other extra libs.
+  # Also, prevent various packages (CondaPkg.jl, PythonCall.jl) from trying to do network calls.
+  juliaWrapped =
+    runCommand "julia-${julia.version}-wrapped"
+      {
+        nativeBuildInputs = [ makeWrapper ];
+        inherit makeWrapperArgs;
+      }
+      ''
+        mkdir -p $out/bin
+        makeWrapper ${julia}/bin/julia $out/bin/julia \
+          --suffix LD_LIBRARY_PATH : "${lib.makeLibraryPath extraLibs}" \
+          --set FONTCONFIG_FILE ${fontconfig.out}/etc/fonts/fonts.conf \
+          --set PYTHONHOME "${pythonToUse}" \
+          --prefix PYTHONPATH : "${pythonToUse}/${pythonToUse.sitePackages}" \
+          --set PYTHON ${pythonToUse}/bin/python $makeWrapperArgs \
+          --set JULIA_CONDAPKG_OFFLINE yes \
+          --set JULIA_CONDAPKG_BACKEND Null \
+          --set JULIA_PYTHONCALL_EXE "@PyCall"
+      '';
+
+  # If our closure ends up with certain packages, add others.
+  packageImplications = {
+    # Because we want to put PythonCall in PyCall mode so it doesn't try to download
+    # Python packages
+    PythonCall = [ "PyCall" ];
+  };
+
+  # Invoke Julia resolution logic to determine the full dependency closure. Also
+  # gather information on the Julia standard libraries, which we'll need to
+  # generate a Manifest.toml.
+  packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;
+  closureYaml = callPackage ./package-closure.nix {
+    inherit
+      augmentedRegistry
+      julia
+      packageNames
+      packageImplications
+      ;
+    packageOverrides = packageOverridesRepoified;
+  };
+  stdlibInfos = callPackage ./stdlib-infos.nix {
+    inherit julia;
+  };
+
+  # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:
+  # {
+  #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" = {
+  #     src = fetchgit {...};
+  #     name = "...";
+  #     version = "...";
+  #     treehash = "...";
+  #   };
+  #   ...
+  # }
+  dependencies =
+    runCommand "julia-sources.nix"
+      {
+        buildInputs = [
+          (python3.withPackages (
+            ps: with ps; [
+              toml
+              pyyaml
+            ]
+          ))
+          git
+        ];
+      }
+      ''
+        python ${./python}/sources_nix.py \
+          "${augmentedRegistry}" \
+          '${lib.generators.toJSON { } packageOverridesRepoified}' \
+          "${closureYaml}" \
+          "$out"
+      '';
+
+  # Import the Nix file from the previous step (IFD) and turn each dependency repo into
+  # a dummy Git repository, as Julia expects. Format the results as a YAML map from
+  # dependency UUID -> Nix store location:
+  # {
+  #   "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3":"/nix/store/...-NaNMath.jl-0877504",
+  #   ...
+  # }
+  # This is also the point where we apply the packageOverrides.
+  dependencyUuidToInfo = import dependencies { inherit fetchgit; };
+  fillInOverrideSrc =
+    uuid: info:
+    if lib.hasAttr info.name packageOverrides then
+      (info // { src = lib.getAttr info.name packageOverrides; })
+    else
+      info;
+  dependencyUuidToRepo = lib.mapAttrs util.repoifyInfo (
+    lib.mapAttrs fillInOverrideSrc dependencyUuidToInfo
+  );
+  dependencyUuidToRepoYaml = writeTextFile {
+    name = "dependency-uuid-to-repo.yml";
+    text = lib.generators.toYAML { } dependencyUuidToRepo;
+  };
+
+  # Given the augmented registry, closure info yaml, and dependency path yaml, construct a complete
+  # Julia registry containing all the necessary packages
+  dependencyUuidToInfoYaml = writeTextFile {
+    name = "dependency-uuid-to-info.yml";
+    text = lib.generators.toYAML { } dependencyUuidToInfo;
+  };
+  fillInOverrideSrc' =
+    uuid: info:
+    if lib.hasAttr info.name packageOverridesRepoified then
+      (info // { src = lib.getAttr info.name packageOverridesRepoified; })
+    else
+      info;
+  overridesOnly = lib.mapAttrs fillInOverrideSrc' (
+    lib.filterAttrs (uuid: info: info.src == null) dependencyUuidToInfo
+  );
+  minimalRegistry =
+    runCommand "minimal-julia-registry"
+      {
+        buildInputs = [
+          (python3.withPackages (
+            ps: with ps; [
+              toml
+              pyyaml
+            ]
+          ))
+          git
+        ];
+      }
+      ''
+        python ${./python}/minimal_registry.py \
+          "${augmentedRegistry}" \
+          "${closureYaml}" \
+          '${lib.generators.toJSON { } overridesOnly}' \
+          "${dependencyUuidToRepoYaml}" \
+          "$out"
+      '';
+  project =
+    runCommand "julia-project"
+      {
+        buildInputs = [
+          (python3.withPackages (
+            ps: with ps; [
+              toml
+              pyyaml
+            ]
+          ))
+          git
+        ];
+      }
+      ''
+        python ${./python}/project.py \
+          "${closureYaml}" \
+          "${stdlibInfos}" \
+          '${lib.generators.toJSON { } overridesOnly}' \
+          "${dependencyUuidToRepoYaml}" \
+          "$out"
+      '';
+
+  # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that
+  # produces the desired Overrides.toml.
+  artifactsNix =
+    runCommand "julia-artifacts.nix"
+      {
+        buildInputs = [
+          (python3.withPackages (
+            ps: with ps; [
+              toml
+              pyyaml
+            ]
+          ))
+        ];
+      }
+      ''
+        python ${./python}/extract_artifacts.py \
+          "${dependencyUuidToRepoYaml}" \
+          "${closureYaml}" \
+          "${juliaWrapped}/bin/julia" \
+          "${
+            if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl
+          }" \
+          '${lib.generators.toJSON { } (import ./extra-libs.nix)}' \
+          '${lib.generators.toJSON { } (stdenv.hostPlatform.isDarwin)}' \
+          "$out"
+      '';
+
+  # Import the artifacts Nix to build Overrides.toml (IFD)
+  artifacts = import artifactsNix (
+    {
+      inherit
+        lib
+        fetchurl
+        pkgs
+        stdenv
+        ;
+    }
+    // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
+      inherit gcc glibc;
+    }
+  );
+  overridesJson = writeTextFile {
+    name = "Overrides.json";
+    text = lib.generators.toJSON { } artifacts;
+  };
+  overridesToml =
+    runCommand "Overrides.toml" { buildInputs = [ (python3.withPackages (ps: with ps; [ toml ])) ]; }
+      ''
+        python ${./python}/format_overrides.py \
+          "${overridesJson}" \
+          "$out"
+      '';
+
+  # Build a Julia project and depot under $out/project and $out/depot respectively
+  projectAndDepot = callPackage ./depot.nix {
+    inherit
+      closureYaml
+      extraLibs
+      juliaCpuTarget
+      overridesToml
+      packageImplications
+      precompile
+      ;
+    julia = juliaWrapped;
+    inherit project;
+    registry = minimalRegistry;
+  };
+
+in
+
+runCommand "julia-${julia.version}-env"
+  {
+    nativeBuildInputs = [ makeWrapper ];
+
+    passthru = {
+      inherit julia;
+      inherit juliaWrapped;
+      inherit (julia) pname version meta;
+
+      # Expose the steps we used along the way in case the user wants to use them, for example to build
+      # expressions and build them separately to avoid IFD.
+      inherit dependencies;
+      inherit closureYaml;
+      inherit dependencyUuidToInfoYaml;
+      inherit dependencyUuidToRepoYaml;
+      inherit minimalRegistry;
+      inherit artifactsNix;
+      inherit overridesJson;
+      inherit overridesToml;
+      inherit project;
+      inherit projectAndDepot;
+      inherit stdlibInfos;
+    };
+  }
+  (
+    ''
+      mkdir -p $out/bin
+      makeWrapper ${juliaWrapped}/bin/julia $out/bin/julia \
+        --suffix JULIA_DEPOT_PATH : "${projectAndDepot}/depot" \
+        --set-default JULIA_PROJECT "${projectAndDepot}/project" \
+        --set-default JULIA_LOAD_PATH '@:${projectAndDepot}/project/Project.toml:@v#.#:@stdlib'
+    ''
+    + lib.optionalString setDefaultDepot ''
+      sed -i '2 i\JULIA_DEPOT_PATH=''${JULIA_DEPOT_PATH-"$HOME/.julia"}' $out/bin/julia
+    ''
+  )

pkgs/development/julia-modules/depot.nix

@@ -0,0 +1,130 @@
+{
+  lib,
+  runCommand,
+
+  cacert,
+  curl,
+  git,
+  julia,
+  python3,
+  stdenv,
+
+  closureYaml,
+  extraLibs,
+  juliaCpuTarget,
+  overridesToml,
+  packageImplications,
+  project,
+  precompile,
+  registry,
+}:
+
+let
+  # On darwin, we don't want to specify JULIA_SSL_CA_ROOTS_PATH. If we do (using a -bin julia derivation, which is the
+  # only kind darwin currently supports), you get an error like this:
+  #
+  # GitError(Code:ERROR, Class:SSL, Your Julia is built with a SSL/TLS engine that libgit2 doesn't know how to configure
+  # to use a file or directory of certificate authority roots, but your environment specifies one via the SSL_CERT_FILE
+  # variable. If you believe your system's root certificates are safe to use, you can `export JULIA_SSL_CA_ROOTS_PATH=""`
+  # in your environment to use those instead.)
+  setJuliaSslCaRootsPath =
+    if stdenv.targetPlatform.isDarwin then
+      ''export JULIA_SSL_CA_ROOTS_PATH=""''
+    else
+      ''export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"'';
+
+in
+
+runCommand "julia-depot"
+  {
+    nativeBuildInputs = [
+      curl
+      git
+      julia
+      (python3.withPackages (ps: with ps; [ pyyaml ]))
+    ]
+    ++ extraLibs;
+    inherit precompile project registry;
+  }
+  (
+    ''
+      export HOME=$(pwd)
+
+      echo "Building Julia depot and project with the following inputs"
+      echo "Julia: ${julia}"
+      echo "Project: $project"
+      echo "Registry: $registry"
+      echo "Overrides ${overridesToml}"
+
+      mkdir -p $out/project
+      export JULIA_PROJECT="$out/project"
+      cp "$project/Manifest.toml" "$JULIA_PROJECT/Manifest.toml"
+      cp "$project/Project.toml" "$JULIA_PROJECT/Project.toml"
+
+      mkdir -p $out/depot/artifacts
+      export JULIA_DEPOT_PATH="$out/depot"
+      cp ${overridesToml} $out/depot/artifacts/Overrides.toml
+
+      # These can be useful to debug problems
+      # export JULIA_DEBUG=Pkg,loading
+
+      ${setJuliaSslCaRootsPath}
+
+      # Only precompile if configured to below
+      export JULIA_PKG_PRECOMPILE_AUTO=0
+    ''
+    + lib.optionalString (juliaCpuTarget != null) ''
+      export JULIA_CPU_TARGET="${juliaCpuTarget}"
+    ''
+    + ''
+      # Prevent a warning where Julia tries to download package server info
+      export JULIA_PKG_SERVER=""
+
+      # See if we need to add any extra package names based on the closure
+      # and the packageImplications. We're using the full closure YAML here since
+      # it's available, which is slightly weird, but it should work just as well
+      # for finding the extra packages we need to add
+      python ${./python}/find_package_implications.py "${closureYaml}" '${
+        lib.generators.toJSON { } packageImplications
+      }' extra_package_names.txt
+
+      # Work around new git security features added in git 2.44.1
+      # See https://github.com/NixOS/nixpkgs/issues/315890
+      git config --global --add safe.directory '*'
+
+      # Tell Julia to use the Git binary we provide, rather than internal libgit2.
+      export JULIA_PKG_USE_CLI_GIT="true"
+
+      # At time of writing, this appears to be the only way to turn precompiling's
+      # terminal output into standard logging, so opportunistically do that.
+      # (Note this is different from JULIA_CI).
+      export CI=true
+
+      julia -e ' \
+        import Pkg
+        import Pkg.Types: PRESERVE_NONE
+
+        Pkg.Registry.add(Pkg.RegistrySpec(path="${registry}"))
+
+        # No need to Pkg.activate() since we set JULIA_PROJECT above
+        println("Running Pkg.instantiate()")
+        Pkg.instantiate()
+
+        # Build is a separate step from instantiate.
+        # Needed for packages like Conda.jl to set themselves up.
+        println("Running Pkg.build()")
+        Pkg.build()
+
+        if "precompile" in keys(ENV) && ENV["precompile"] != "0" && ENV["precompile"] != ""
+          if isdefined(Sys, :CPU_NAME)
+            println("Precompiling with CPU_NAME = " * Sys.CPU_NAME)
+          end
+
+          Pkg.precompile()
+        end
+
+        # Remove the registry to save space
+        Pkg.Registry.rm("General")
+      '
+    ''
+  )

pkgs/development/julia-modules/extra-libs.nix

@@ -0,0 +1,15 @@
+# A map from a Julia package (typically a JLL package) to extra libraries
+# that they require from Nix.
+# The libraries should be strings evaluated in a "with pkgs" context.
+
+{
+  # Qt5Base_jll
+  # Needs access to dbus or you get "Cannot find libdbus-1 in your system"
+  # Repro: build environment with ["Plots"]
+  # > using Plots; plot(cos, 0, 2pi)
+  "ea2cea3b-5b76-57ae-a6ef-0a8af62496e1" = [ "dbus.lib" ];
+
+  # Qt6Base_jll
+  # Same reason as Qt5Base_jll
+  "c0090381-4147-56d7-9ebc-da0b1113ec56" = [ "dbus.lib" ];
+}

pkgs/development/julia-modules/extra-python-packages.nix

@@ -0,0 +1,24 @@
+{
+  lib,
+  python3,
+}:
+
+# This file contains an extra mapping from Julia packages to the Python packages they depend on.
+rec {
+  packageMapping = {
+    ExcelFiles = [ "xlrd" ];
+    PyPlot = [ "matplotlib" ];
+    PythonPlot = [ "matplotlib" ];
+    SymPy = [ "sympy" ];
+  };
+
+  getExtraPythonPackages =
+    names:
+    lib.concatMap (
+      name:
+      let
+        allCandidates = if lib.hasAttr name packageMapping then lib.getAttr name packageMapping else [ ];
+      in
+      lib.filter (x: lib.hasAttr x python3.pkgs) allCandidates
+    ) names;
+}

pkgs/development/julia-modules/extract_artifacts.jl

@@ -0,0 +1,63 @@
+
+import Base: UUID
+import Pkg.Artifacts: artifact_meta, artifact_names, find_artifacts_toml, load_artifacts_toml, select_downloadable_artifacts
+import Pkg.BinaryPlatforms: AbstractPlatform, platform_key_abi, triplet
+import Pkg.Operations: gen_build_code
+import TOML
+
+pkg_uuid = UUID(ARGS[1])
+dir = ARGS[2]
+
+artifacts_toml = find_artifacts_toml(dir)
+
+if artifacts_toml == nothing
+    print("")
+    exit()
+end
+
+platform = platform_key_abi()
+
+# Using collect_artifacts (from Pkg.jl) is more reliable than calling select_downloadable_artifacts directly.
+# collect_artifacts includes support for .pkg/select_artifacts.jl, which may produce different results.
+# If we use select_downloadable_artifacts here, then at depot build time it may try to download a different artifact
+# and fail.
+
+# However! The collect_artifacts from Pkg.jl doesn't allow us to pass lazy to select_downloadable_artifacts.
+# So we have to paste our own version in here :(
+
+function collect_artifacts(pkg_root::String; platform::AbstractPlatform)
+    # Check to see if this package has an (Julia)Artifacts.toml
+    artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[]
+    for f in artifact_names
+        artifacts_toml = joinpath(pkg_root, f)
+        if isfile(artifacts_toml)
+            selector_path = joinpath(pkg_root, ".pkg", "select_artifacts.jl")
+
+            # If there is a dynamic artifact selector, run that in an appropriate sandbox to select artifacts
+            if isfile(selector_path)
+                # Despite the fact that we inherit the project, since the in-memory manifest
+                # has not been updated yet, if we try to load any dependencies, it may fail.
+                # Therefore, this project inheritance is really only for Preferences, not dependencies.
+                select_cmd = Cmd(`$(gen_build_code(selector_path; inherit_project=true)) --startup-file=no $(triplet(platform))`)
+                meta_toml = String(read(select_cmd))
+                res = TOML.tryparse(meta_toml)
+                if res isa TOML.ParserError
+                    errstr = sprint(showerror, res; context=stderr)
+                    pkgerror("failed to parse TOML output from running $(repr(selector_path)), got: \n$errstr")
+                else
+                    push!(artifacts_tomls, (artifacts_toml, TOML.parse(meta_toml)))
+                end
+            else
+                # Otherwise, use the standard selector from `Artifacts`
+                artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy=true)
+                push!(artifacts_tomls, (artifacts_toml, artifacts))
+            end
+            break
+        end
+    end
+    return artifacts_tomls
+end
+
+for (artifacts_toml, artifacts) in collect_artifacts(dir; platform)
+    TOML.print(artifacts)
+end

pkgs/development/julia-modules/extract_artifacts_16.jl

@@ -0,0 +1,33 @@
+
+import Base: UUID
+import Pkg.Artifacts: artifact_meta, find_artifacts_toml, load_artifacts_toml
+import Pkg.BinaryPlatforms: platform_key_abi
+import TOML
+
+pkg_uuid = UUID(ARGS[1])
+dir = ARGS[2]
+
+artifacts_toml = find_artifacts_toml(dir)
+
+if artifacts_toml == nothing
+    print("")
+    exit()
+end
+
+platform = platform_key_abi()
+
+# Older Julia doesn't provide select_downloadable_artifacts or .pkg/select_artifacts.jl,
+# so gather the artifacts the old-fashioned way
+artifact_dict = load_artifacts_toml(artifacts_toml; pkg_uuid=pkg_uuid)
+
+results = Dict()
+for name in keys(artifact_dict)
+    # Get the metadata about this name for the requested platform
+    meta = artifact_meta(name, artifact_dict, artifacts_toml; platform=platform)
+
+    # If there are no instances of this name for the desired platform, skip it
+    meta === nothing && continue
+
+    results[name] = meta
+end
+TOML.print(results)

pkgs/development/julia-modules/package-closure.nix

@@ -0,0 +1,96 @@
+{
+  lib,
+  julia,
+  python3,
+  runCommand,
+
+  augmentedRegistry,
+  packageNames,
+  packageOverrides,
+  packageImplications,
+}:
+
+let
+  juliaExpression = packageNames: ''
+    import Pkg
+    Pkg.Registry.add(Pkg.RegistrySpec(path="${augmentedRegistry}"))
+
+    import Pkg.Types: Context, PackageSpec
+
+    input = ${lib.generators.toJSON { } packageNames}
+
+    if isfile("extra_package_names.txt")
+        append!(input, readlines("extra_package_names.txt"))
+    end
+
+    input = unique(input)
+
+    println("Resolving packages: " * join(input, " "))
+
+    pkgs = [PackageSpec(pkg) for pkg in input]
+
+    ctx = Context()
+
+    overrides = Dict{String, String}(${
+      builtins.concatStringsSep ", " (
+        lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides
+      )
+    })
+    ${builtins.readFile ./resolve_packages.jl}
+
+    open(ENV["out"], "w") do io
+        for spec in pkgs
+            println(io, "- name: " * spec.name)
+            println(io, "  uuid: " * string(spec.uuid))
+            println(io, "  version: " * string(spec.version))
+            println(io, "  tree_hash: " * string(spec.tree_hash))
+            if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid)
+                println(io, "  depends_on: ")
+                for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])
+                    println(io, "    \"$(dep_name)\": \"$(dep_uuid)\"")
+                end
+            end
+            println(io, "  deps: ")
+            for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])
+                println(io, "  - name: \"$(dep_name)\"")
+                println(io, "    uuid: \"$(dep_uuid)\"")
+            end
+            if spec.name in input
+                println(io, "  is_input: true")
+            end
+        end
+    end
+  '';
+in
+
+runCommand "julia-package-closure.yml"
+  {
+    buildInputs = [
+      julia
+      (python3.withPackages (ps: with ps; [ pyyaml ]))
+    ];
+  }
+  ''
+    mkdir home
+    export HOME=$(pwd)/home
+
+    echo "Resolving Julia packages with the following inputs"
+    echo "Julia: ${julia}"
+    echo "Registry: ${augmentedRegistry}"
+
+    # Prevent a warning where Julia tries to download package server info
+    export JULIA_PKG_SERVER=""
+
+    julia -e '${juliaExpression packageNames}';
+
+    # See if we need to add any extra package names based on the closure
+    # and the packageImplications
+    python ${./python}/find_package_implications.py "$out" '${
+      lib.generators.toJSON { } packageImplications
+    }' extra_package_names.txt
+
+    if [ -f extra_package_names.txt ]; then
+      echo "Re-resolving with additional package names"
+      julia -e '${juliaExpression packageNames}';
+    fi
+  ''

pkgs/development/julia-modules/python/dag.py

@@ -0,0 +1,59 @@
+
+# This file based on a ChatGPT reponse for the following prompt:
+# "can you write code in python to build up a DAG representing
+# a dependency tree, and then a function that can return all the
+# dependencies of a given node?"
+
+class Node:
+    def __init__(self, name):
+        self.name = name
+        self.dependencies = set()
+
+
+class DAG:
+    def __init__(self):
+        self.nodes = {}
+
+    def add_node(self, node_name, dependencies=None):
+        if node_name in self.nodes:
+            raise ValueError(f"Node '{node_name}' already exists in the graph.")
+
+        node = Node(node_name)
+        if dependencies:
+            node.dependencies.update(dependencies)
+
+        self.nodes[node_name] = node
+
+    def add_dependency(self, node_name, dependency_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        if dependency_name not in self.nodes:
+            raise ValueError(f"Dependency '{dependency_name}' does not exist in the graph.")
+
+        self.nodes[node_name].dependencies.add(dependency_name)
+
+    def get_dependencies(self, node_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        node = self.nodes[node_name]
+        dependencies = set()
+
+        def traverse_dependencies(current_node):
+            for dependency in current_node.dependencies:
+                dependencies.add(dependency)
+                if dependency in self.nodes:
+                  traverse_dependencies(self.nodes[dependency])
+
+        traverse_dependencies(node)
+        return dependencies
+
+    def has_node(self, node_name):
+        return node_name in self.nodes
+
+    def __str__(self):
+        graph_str = ""
+        for node_name, node in self.nodes.items():
+            graph_str += f"{node_name} -> {', '.join(node.dependencies)}\n"
+        return graph_str

pkgs/development/julia-modules/python/dedup_overrides.py

@@ -0,0 +1,14 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+with open(out_path, "w") as f:
+  toml.dump(overrides, f)

pkgs/development/julia-modules/python/extract_artifacts.py

@@ -0,0 +1,168 @@
+
+import json
+from pathlib import Path
+import multiprocessing
+import subprocess
+import sys
+import toml
+from urllib.parse import urlparse
+import yaml
+
+import dag
+
+# This should match the behavior of the default unpackPhase.
+# See https://github.com/NixOS/nixpkgs/blob/59fa082abdbf462515facc8800d517f5728c909d/pkgs/stdenv/generic/setup.sh#L1044
+archive_extensions = [
+  # xz extensions
+  ".tar.xz",
+  ".tar.lzma",
+  ".txz",
+
+  # *.tar or *.tar.*
+  ".tar",
+  ".tar.Z",
+  ".tar.bz2",
+  ".tar.gz",
+
+  # Other tar extensions
+  ".tgz",
+  ".tbz2",
+  ".tbz",
+
+  ".zip"
+  ]
+
+def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin):
+  depends_on = set()
+  if closure_dependencies_dag.has_node(uuid):
+    depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
+
+  other_libs = extra_libs.get(uuid, [])
+
+  if is_darwin:
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+            in ''
+
+            ''"""
+  else:
+    # We provide gcc.cc.lib by default in order to get some common libraries
+    # like libquadmath.so. A number of packages expect this to be available and
+    # will give linker errors if it isn't.
+    fixup = f"""fixupPhase = let
+            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+            in ''
+              find $out -type f -executable -exec \
+                patchelf --set-rpath \\$ORIGIN:\\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc gcc.cc.lib] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \\;
+              find $out -type f -executable -exec \
+                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \\;
+            ''"""
+
+  return f"""stdenv.mkDerivation {{
+        name = "{artifact_name}";
+        src = fetchurl {{
+          url = "{url}";
+          sha256 = "{sha256}";
+        }};
+        preUnpack = ''
+          mkdir unpacked
+          cd unpacked
+        '';
+        sourceRoot = ".";
+        dontConfigure = true;
+        dontBuild = true;
+        installPhase = "cp -r . $out";
+        {fixup};
+      }}"""
+
+def get_plain_derivation(url, sha256):
+  return f"""fetchurl {{
+        url = "{url}";
+        sha256 = "{sha256}";
+      }}"""
+
+def process_item(args):
+  item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args
+  uuid, src = item
+  lines = []
+
+  artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
+  if not artifacts:
+    return f'  uuid-{uuid} = {{}};\n'
+
+  lines.append(f'  uuid-{uuid} = {{')
+
+  for artifact_name, details in artifacts.items():
+    if len(details["download"]) == 0:
+      continue
+    download = details["download"][0]
+    url = download["url"]
+    sha256 = download["sha256"]
+
+    git_tree_sha1 = details["git-tree-sha1"]
+
+    parsed_url = urlparse(url)
+    if any(parsed_url.path.endswith(x) for x in archive_extensions):
+      derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+    else:
+      derivation = get_plain_derivation(url, sha256)
+
+    lines.append(f"""    "{artifact_name}" = {{
+      sha1 = "{git_tree_sha1}";
+      path = {derivation};
+    }};\n""")
+
+  lines.append('  };\n')
+
+  return "\n".join(lines)
+
+def main():
+  dependencies_path = Path(sys.argv[1])
+  closure_yaml_path = Path(sys.argv[2])
+  julia_path = Path(sys.argv[3])
+  extract_artifacts_script = Path(sys.argv[4])
+  extra_libs = json.loads(sys.argv[5])
+  is_darwin = json.loads(sys.argv[6])
+  out_path = Path(sys.argv[7])
+
+  with open(dependencies_path, "r") as f:
+    dependencies = yaml.safe_load(f)
+    dependency_uuids = list(dependencies.keys())  # Convert dict_keys to list
+
+  with open(closure_yaml_path, "r") as f:
+    # Build up a map of UUID -> closure information
+    closure_yaml_list = yaml.safe_load(f) or []
+    closure_yaml = {}
+    for item in closure_yaml_list:
+      closure_yaml[item["uuid"]] = item
+
+    # Build up a dependency graph of UUIDs
+    closure_dependencies_dag = dag.DAG()
+    for uuid, contents in closure_yaml.items():
+      if contents.get("depends_on"):
+        closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
+
+  with open(out_path, "w") as f:
+    if is_darwin:
+      f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
+    else:
+      f.write("{ lib, fetchurl, gcc, glibc, pkgs, stdenv }:\n\n")
+
+    f.write("rec {\n")
+
+    with multiprocessing.Pool(10) as pool:
+      # Create args tuples for each item
+      process_args = [
+        (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
+        for item in dependencies.items()
+      ]
+      for s in pool.map(process_item, process_args):
+        f.write(s)
+
+    f.write(f"""
+}}\n""")
+
+if __name__ == "__main__":
+  main()

pkgs/development/julia-modules/python/find_package_implications.py

@@ -0,0 +1,24 @@
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import yaml
+
+dependencies_path = Path(sys.argv[1])
+package_implications_json = sys.argv[2]
+out_path = Path(sys.argv[3])
+
+package_implications = json.loads(package_implications_json)
+with open(dependencies_path) as f:
+  desired_packages = yaml.safe_load(f) or []
+
+extra_package_names = []
+for pkg in desired_packages:
+  if pkg["name"] in package_implications:
+    extra_package_names.extend(package_implications[pkg["name"]])
+
+if len(extra_package_names) > 0:
+  with open(out_path, "w") as f:
+    f.write("\n".join(extra_package_names))

pkgs/development/julia-modules/python/format_overrides.py

@@ -0,0 +1,22 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+result = {}
+
+for (uuid, artifacts) in overrides.items():
+  if len(artifacts) == 0: continue
+
+  for (name, info) in artifacts.items():
+    result[info["sha1"]] = info["path"]
+
+with open(out_path, "w") as f:
+  toml.dump(result, f)

pkgs/development/julia-modules/python/minimal_registry.py

@@ -0,0 +1,128 @@
+
+from collections import defaultdict
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+desired_packages_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+uuid_to_versions = defaultdict(list)
+for pkg in desired_packages:
+  uuid_to_versions[pkg["uuid"]].append(pkg["version"])
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+os.makedirs(out_path)
+
+full_registry = toml.load(registry_path / "Registry.toml")
+registry = full_registry.copy()
+registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}
+
+for (uuid, versions) in uuid_to_versions.items():
+  if uuid in package_overrides:
+    info = package_overrides[uuid]
+
+    # Make a registry entry based on the info from the package override
+    path = Path(info["name"][0].upper()) / Path(info["name"])
+    registry["packages"][uuid] = {
+      "name": info["name"],
+      "path": str(path),
+    }
+
+    os.makedirs(out_path / path)
+
+    # Read the Project.yaml from the src
+    project = toml.load(Path(info["src"]) / "Project.toml")
+
+    # Generate all the registry files
+    with open(out_path / path / Path("Compat.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      # Write nothing in Compat.toml, because we've already resolved everything
+    with open(out_path / path / Path("Deps.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      if "deps" in project:
+        toml.dump(project["deps"], f)
+    with open(out_path / path / Path("Versions.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      f.write('git-tree-sha1 = "%s"\n' % info["treehash"])
+    with open(out_path / path / Path("Package.toml"), "w") as f:
+      toml.dump({
+        "name": info["name"],
+        "uuid": uuid,
+        "repo": "file://" + info["src"],
+      }, f)
+
+  elif uuid in registry["packages"]:
+    registry_info = registry["packages"][uuid]
+    name = registry_info["name"]
+    path = registry_info["path"]
+
+    os.makedirs(out_path / path)
+
+    # Copy some files to the minimal repo unchanged
+    for f in ["Compat.toml", "Deps.toml", "WeakCompat.toml", "WeakDeps.toml"]:
+      if (registry_path / path / f).exists():
+        shutil.copy2(registry_path / path / f, out_path / path)
+
+    # Copy the Versions.toml file, trimming down to the versions we care about.
+    # In the case where versions=None, this is a weak dep, and we keep all versions.
+    all_versions = toml.load(registry_path / path / "Versions.toml")
+    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions} if versions != None else all_versions
+    for k, v in versions_to_keep.items():
+      del v["nix-sha256"]
+    with open(out_path / path / "Versions.toml", "w") as f:
+      toml.dump(versions_to_keep, f)
+
+    if versions is None:
+      # This is a weak dep; just grab the whole Package.toml
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+    elif uuid in uuid_to_store_path:
+      # Fill in the local store path for the repo
+      package_toml = toml.load(registry_path / path / "Package.toml")
+      package_toml["repo"] = "file://" + uuid_to_store_path[uuid]
+      with open(out_path / path / "Package.toml", "w") as f:
+        toml.dump(package_toml, f)
+
+# Look for missing weak deps and include them. This can happen when our initial
+# resolve step finds dependencies, but we fail to resolve them at the project.py
+# stage. Usually this happens because the package that depends on them does so
+# as a weak dep, but doesn't have a Package.toml in its repo making this clear.
+for pkg in desired_packages:
+  for dep in (pkg.get("deps", []) or []):
+    uuid = dep["uuid"]
+    if not uuid in uuid_to_versions:
+      entry = full_registry["packages"].get(uuid)
+      if not entry:
+        print(f"""WARNING: found missing UUID but couldn't resolve it: {uuid}""")
+        continue
+
+      # Add this entry back to the minimal Registry.toml
+      registry["packages"][uuid] = entry
+
+      # Bring over the Package.toml
+      path = Path(entry["path"])
+      if (out_path / path / "Package.toml").exists():
+        continue
+      Path(out_path / path).mkdir(parents=True, exist_ok=True)
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+
+# Finally, dump the Registry.toml
+with open(out_path / "Registry.toml", "w") as f:
+    toml.dump(registry, f)

pkgs/development/julia-modules/python/project.py

@@ -0,0 +1,104 @@
+
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+import sys
+import toml
+import yaml
+
+
+desired_packages_path = Path(sys.argv[1])
+stdlib_infos_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+with open(stdlib_infos_path, "r") as f:
+  stdlib_infos = yaml.safe_load(f) or []
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+result = {
+  "deps": defaultdict(list)
+}
+
+for pkg in desired_packages:
+  if pkg["uuid"] in package_overrides:
+    info = package_overrides[pkg["uuid"]]
+    result["deps"][info["name"]].append({
+      "uuid": pkg["uuid"],
+      "path": info["src"],
+    })
+    continue
+
+  path = uuid_to_store_path.get(pkg["uuid"], None)
+  isStdLib = False
+  if pkg["uuid"] in stdlib_infos["stdlibs"]:
+    path = stdlib_infos["stdlib_root"] + "/" + stdlib_infos["stdlibs"][pkg["uuid"]]["name"]
+    isStdLib = True
+
+  if path:
+    if (Path(path) / "Project.toml").exists():
+      project_toml = toml.load(Path(path) / "Project.toml")
+
+      deps = []
+      weak_deps = project_toml.get("weakdeps", {})
+      extensions = project_toml.get("extensions", {})
+
+      if "deps" in project_toml:
+        # Build up deps for the manifest, excluding weak deps
+        weak_deps_uuids = weak_deps.values()
+        for (dep_name, dep_uuid) in project_toml["deps"].items():
+          if not (dep_uuid in weak_deps_uuids):
+            deps.append(dep_name)
+    else:
+      # Not all projects have a Project.toml. In this case, use the deps we
+      # calculated from the package resolve step. This isn't perfect since it
+      # will fail to properly split out weak deps, but it's better than nothing.
+      print(f"""WARNING: package {pkg["name"]} didn't have a Project.toml in {path}""")
+      deps = [x["name"] for x in pkg.get("deps", [])]
+      weak_deps = {}
+      extensions = {}
+
+    tree_hash = pkg.get("tree_hash", "")
+
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "git-tree-sha1": (tree_hash if tree_hash != "nothing" else None) or None,
+      "deps": deps or None,
+      "weakdeps": weak_deps or None,
+      "extensions": extensions or None,
+
+      # We *don't* set "path" here, because then Julia will try to use the
+      # read-only Nix store path instead of cloning to the depot. This will
+      # cause packages like Conda.jl to fail during the Pkg.build() step.
+      #
+      # "path": None if isStdLib else path ,
+    })
+  else:
+    print("WARNING: adding a package that we didn't have a path for, and it doesn't seem to be a stdlib", pkg)
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "deps": [x["name"] for x in pkg["deps"]]
+    })
+
+os.makedirs(out_path)
+
+with open(out_path / "Manifest.toml", "w") as f:
+  f.write(f'julia_version = "{stdlib_infos["julia_version"]}"\n')
+  f.write('manifest_format = "2.0"\n\n')
+  toml.dump(result, f)
+
+with open(out_path / "Project.toml", "w") as f:
+  f.write('[deps]\n')
+
+  for pkg in desired_packages:
+    if pkg.get("is_input", False):
+      f.write(f'''{pkg["name"]} = "{pkg["uuid"]}"\n''')

pkgs/development/julia-modules/python/sources_nix.py

@@ -0,0 +1,75 @@
+
+import json
+from pathlib import Path
+import re
+import shutil
+import sys
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+package_overrides = json.loads(sys.argv[2])
+desired_packages_path = Path(sys.argv[3])
+out_path = Path(sys.argv[4])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+registry = toml.load(registry_path / "Registry.toml")
+
+def ensure_version_valid(version):
+  """
+  Ensure a version string is a valid Julia-parsable version.
+  It doesn't really matter what it looks like as it's just used for overrides.
+  """
+  return re.sub('[^0-9.]','', version)
+
+with open(out_path, "w") as f:
+  f.write("{fetchgit}:\n")
+  f.write("{\n")
+  for pkg in desired_packages:
+    uuid = pkg["uuid"]
+
+    if pkg["name"] in package_overrides:
+      treehash = util.get_commit_info(package_overrides[pkg["name"]])["tree"]
+      f.write(f"""  "{uuid}" = {{
+    src = null; # Overridden: will fill in later
+    name = "{pkg["name"]}";
+    version = "{ensure_version_valid(pkg["version"])}";
+    treehash = "{treehash}";
+  }};\n""")
+    elif uuid in registry["packages"]:
+      # The treehash is missing for stdlib packages. Don't bother downloading these.
+      if (not ("tree_hash" in pkg)) or pkg["tree_hash"] == "nothing": continue
+
+      registry_info = registry["packages"][uuid]
+      path = registry_info["path"]
+      packageToml = toml.load(registry_path / path / "Package.toml")
+
+      versions_toml = registry_path / path / "Versions.toml"
+      all_versions = toml.load(versions_toml)
+      if not pkg["version"] in all_versions: continue
+      version_to_use = all_versions[pkg["version"]]
+
+      if not "nix-sha256" in version_to_use:
+        raise KeyError(f"""Couldn't find nix-sha256 hash for {pkg["name"]} {pkg["version"]} in {versions_toml}. This might indicate that we failed to prefetch the hash when computing the augmented registry. Was there a relevant failure in {registry_path / "failures.yml"}?""")
+
+      repo = packageToml["repo"]
+      f.write(f"""  "{uuid}" = {{
+    src = fetchgit {{
+      url = "{repo}";
+      rev = "{version_to_use["git-tree-sha1"]}";
+      sha256 = "{version_to_use["nix-sha256"]}";
+    }};
+    name = "{pkg["name"]}";
+    version = "{pkg["version"]}";
+    treehash = "{version_to_use["git-tree-sha1"]}";
+  }};\n""")
+    else:
+      # This is probably a stdlib
+      # print("WARNING: couldn't figure out what to do with pkg in sources_nix.py", pkg)
+      pass
+
+  f.write("}")

pkgs/development/julia-modules/python/util.py

@@ -0,0 +1,12 @@
+
+import os
+import subprocess
+import tempfile
+
+def get_commit_info(repo):
+  with tempfile.TemporaryDirectory() as home_dir:
+    env_with_home = os.environ.copy()
+    env_with_home["HOME"] = home_dir
+    subprocess.check_output(["git", "config", "--global", "--add", "safe.directory", repo], env=env_with_home)
+    lines = subprocess.check_output(["git", "log", "--pretty=raw"], cwd=repo, env=env_with_home).decode().split("\n")
+    return dict([x.split() for x in lines if len(x.split()) == 2])

pkgs/development/julia-modules/registry.nix

@@ -0,0 +1,9 @@
+{ fetchFromGitHub }:
+
+fetchFromGitHub {
+  owner = "CodeDownIO";
+  repo = "General";
+  rev = "4b19a1dc55d2877e85a5d0e98702b75872210e9d";
+  sha256 = "sha256-mVeBTpEQnW3fvJu1+4T8z+earMjEgtdy0tnZnAxz/pk=";
+  # date = "2025-08-12T05:20:40+00:00";
+}

pkgs/development/julia-modules/resolve_packages.jl

@@ -0,0 +1,51 @@
+import Pkg.API: handle_package_input!
+import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved
+import Pkg.Operations: _resolve, assert_can_add, update_package_add
+import TOML
+
+foreach(handle_package_input!, pkgs)
+
+# The handle_package_input! call above clears pkg.path, so we have to apply package overrides after
+println("Package overrides: ")
+println(overrides)
+for pkg in pkgs
+    if pkg.name in keys(overrides)
+        pkg.path = overrides[pkg.name]
+
+        # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps.
+        # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even
+        # if it does not appear in the registry.
+        # See https://github.com/NixOS/nixpkgs/issues/279853
+        project_toml = joinpath(pkg.path, "Project.toml")
+        if isfile(project_toml)
+            toml_data = TOML.parsefile(project_toml)
+            if haskey(toml_data, "uuid")
+                ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"])
+            end
+        end
+    end
+end
+
+project_deps_resolve!(ctx.env, pkgs)
+registry_resolve!(ctx.registries, pkgs)
+stdlib_resolve!(pkgs)
+ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true)
+
+assert_can_add(ctx, pkgs)
+
+for (i, pkg) in pairs(pkgs)
+    entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid)
+    is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])
+    if VERSION >= VersionNumber("1.11")
+        pkgs[i] = update_package_add(ctx, pkg, entry, nothing, nothing, is_dep)
+    else
+        pkgs[i] = update_package_add(ctx, pkg, entry, is_dep)
+    end
+end
+
+foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs)
+
+# Save the original pkgs for later. We might need to augment it with the weak dependencies
+orig_pkgs = deepcopy(pkgs)
+
+pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version)

pkgs/development/julia-modules/stdlib-infos.nix

@@ -0,0 +1,36 @@
+{
+  julia,
+  runCommand,
+}:
+
+let
+  juliaExpression = ''
+    using Pkg
+    open(ENV["out"], "w") do io
+        println(io, "stdlib_root: \"$(Sys.STDLIB)\"")
+
+        println(io, "julia_version: \"$(string(VERSION))\"")
+
+        stdlibs = Pkg.Types.stdlibs()
+        println(io, "stdlibs:")
+        for (uuid, (name, version)) in stdlibs
+            println(io, "  \"$(uuid)\": ")
+            println(io, "    name: $name")
+            println(io, "    version: $version")
+        end
+    end
+  '';
+in
+
+runCommand "julia-stdlib-infos.yml"
+  {
+    buildInputs = [
+      julia
+    ];
+  }
+  ''
+    # Prevent a warning where Julia tries to download package server info
+    export JULIA_PKG_SERVER=""
+
+    julia -e '${juliaExpression}';
+  ''

pkgs/development/julia-modules/tests/.gitignore

@@ -0,0 +1,3 @@
+test_runs/
+.stack-work/
+*~

pkgs/development/julia-modules/tests/README.md

@@ -0,0 +1,25 @@
+
+# Testing `julia.withPackages`
+
+This folder contains a test suite for ensuring that the top N most popular Julia packages (as measured by download count) work properly. The key parts are
+
+* `top-julia-packages.nix`: an impure derivation for fetching Julia download data and processing it into a file called `top-julia-packages.yaml`. This YAML file contains an array of objects with fields "name", "uuid", and "count", and is sorted in decreasing order of count.
+* `julia-top-n`: a small Haskell program which reads `top-julia-packages.yaml` and builds a `julia.withPackages` environment for each package, with a nice interactive display and configurable parallelism. It also tests whether evaluating `using <package-name>` works in the resulting environment.
+
+> **Warning:**
+> These tests should only be run on maintainer machines, not Hydra! `julia.withPackages` uses IFD, which is not allowed in Hydra.
+
+## Quick start
+
+``` shell
+# Test the top 100 Julia packages
+./run_tests.sh -n 100
+```
+
+## Options
+
+You can run `./run_tests.sh --help` to see additional options for the test harness. The main ones are
+
+* `-n`/`--top-n`: how many of the top packages to build (default: 100).
+* `-p`/`--parallelism`: how many builds to run at once (default: 10).
+* `-c`/`--count-file`: path to `top-julia-packages.yaml`.

pkgs/development/julia-modules/tests/julia-top-n/app/Main.hs

@@ -0,0 +1,129 @@
+{-# LANGUAGE DataKinds #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE QuasiQuotes #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE RecordWildCards #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE ViewPatterns #-}
+
+module Main (main) where
+
+import Control.Monad
+import Control.Monad.IO.Class
+import Data.Aeson as A hiding (Options, defaultOptions)
+import qualified Data.Aeson.Key             as A
+import qualified Data.Aeson.KeyMap          as HM
+import qualified Data.ByteString.Lazy.Char8 as BL8
+import qualified Data.List as L
+import Data.String.Interpolate
+import Data.Text as T hiding (count)
+import qualified Data.Vector as V
+import qualified Data.Yaml as Yaml
+import GHC.Generics
+import Options.Applicative hiding (info)
+import System.Exit
+import System.FilePath
+import Test.Sandwich
+import UnliftIO.Exception
+import UnliftIO.MVar
+import UnliftIO.Process
+import UnliftIO.QSem
+
+
+data Args = Args {
+  countFilePath :: FilePath
+  , topN :: Int
+  , parallelism :: Int
+  , juliaAttr :: Text
+  }
+
+argsParser :: Parser Args
+argsParser = Args
+  <$> strOption (long "count-file" <> short 'c' <> help "YAML file containing package names and counts")
+  <*> option auto (long "top-n" <> short 'n' <> help "How many of the top packages to build" <> showDefault <> value 100 <> metavar "INT")
+  <*> option auto (long "parallelism" <> short 'p' <> help "How many builds to run at once" <> showDefault <> value 10 <> metavar "INT")
+  <*> strOption (long "julia-attr" <> short 'a' <> help "Which Julia attr to build with" <> showDefault <> value "julia" <> metavar "STRING")
+
+data NameAndCount = NameAndCount {
+  name :: Text
+  , count :: Int
+  , uuid :: Text
+  } deriving (Show, Eq, Generic, FromJSON)
+
+newtype JuliaPath = JuliaPath FilePath
+  deriving Show
+
+julia :: Label "julia" (MVar (Maybe JuliaPath))
+julia = Label
+
+main :: IO ()
+main = do
+  clo <- parseCommandLineArgs argsParser (return ())
+  let args@(Args {..}) = optUserOptions clo
+
+  namesAndCounts :: [NameAndCount] <- Yaml.decodeFileEither countFilePath >>= \case
+    Left err -> throwIO $ userError ("Couldn't decode names and counts YAML file: " <> show err)
+    Right x -> pure x
+
+  runSandwichWithCommandLineArgs' defaultOptions argsParser $ parallel $ do
+    miscTests args
+
+    describe ("Building environments for top " <> show topN <> " Julia packages") $
+      introduce "Introduce parallel semaphore" parallelSemaphore (liftIO $ newQSem parallelism) (const $ return ()) $
+      parallel $
+      forM_ (L.take topN namesAndCounts) $ \(NameAndCount {..}) ->
+      around "Claim semaphore" claimRunSlot $
+      testExpr args name [i|#{juliaAttr}.withPackages ["#{name}"]|]
+
+miscTests :: Args -> SpecFree ctx IO ()
+miscTests args@(Args {..}) = describe "Misc tests" $ do
+  describe "works for a package outside the General registry" $ do
+    testExpr args "HelloWorld" [iii|(#{juliaAttr}.withPackages.override {
+                                      packageOverrides = {
+                                        "HelloWorld" = pkgs.fetchFromGitHub {
+                                          owner = "codedownio";
+                                          repo = "HelloWorld.jl";
+                                          rev = "9b41c55df76eb87830dd3bd0b5601ee2582a37c6";
+                                          sha256 = "sha256-G+xpMRb0RopW/xWA8KCFF/S8wuHTQbpj0qwm9CihfSc=";
+                                        };
+                                      };
+                                    }) [ "HelloWorld" ]|]
+
+  describe "misc cases" $ do
+    testExpr args "Optimization" [iii|(#{juliaAttr}.withPackages) [ "Optimization" "OptimizationOptimJL" ]|]
+
+-- * Low-level
+
+testExpr :: Args -> Text -> String -> SpecFree ctx IO ()
+testExpr _args name expr = do
+  introduce' (defaultNodeOptions { nodeOptionsVisibilityThreshold = 0 }) (T.unpack name) julia (newMVar Nothing) (const $ return ()) $ do
+    it "Builds" $ do
+      let cp = proc "nix" ["build", "--impure", "--no-link", "--json", "--expr", [i|with import ../../../../. {}; #{expr}|]]
+      output <- readCreateProcessWithLogging cp ""
+      juliaPath <- case A.eitherDecode (BL8.pack output) of
+        Right (A.Array ((V.!? 0) -> Just (A.Object (aesonLookup "outputs" -> Just (A.Object (aesonLookup "out" -> Just (A.String t))))))) -> do
+          info [i|built: #{t}|]
+          pure (JuliaPath ((T.unpack t) </> "bin" </> "julia"))
+        x -> expectationFailure ("Couldn't parse output: " <> show x)
+
+      getContext julia >>= flip modifyMVar_ (const $ return (Just juliaPath))
+
+    it "Uses" $ do
+      getContext julia >>= readMVar >>= \case
+        Nothing -> expectationFailure "Build step failed."
+        Just (JuliaPath juliaPath) -> do
+          let cp = proc juliaPath ["-e", "using " <> T.unpack name]
+          createProcessWithLogging cp >>= waitForProcess >>= (`shouldBe` ExitSuccess)
+
+  where
+    aesonLookup :: Text -> HM.KeyMap v -> Maybe v
+    aesonLookup = HM.lookup . A.fromText
+
+claimRunSlot :: (HasParallelSemaphore ctx) => ExampleT ctx IO a -> ExampleT ctx IO ()
+claimRunSlot f = do
+  s <- getContext parallelSemaphore
+  bracket_ (liftIO $ waitQSem s) (liftIO $ signalQSem s) (void f)

pkgs/development/julia-modules/tests/julia-top-n/default.nix

@@ -0,0 +1,40 @@
+{
+  mkDerivation,
+  aeson,
+  base,
+  filepath,
+  lib,
+  optparse-applicative,
+  sandwich,
+  text,
+  unliftio,
+  yaml,
+}:
+mkDerivation {
+  pname = "julia-top-n";
+  version = "0.1.0.0";
+  src = lib.fileset.toSource {
+    root = ./.;
+    fileset = lib.fileset.unions [
+      ./app
+      ./julia-top-n.cabal
+      ./package.yaml
+      ./stack.yaml
+      ./stack.yaml.lock
+    ];
+  };
+  isLibrary = false;
+  isExecutable = true;
+  executableHaskellDepends = [
+    aeson
+    base
+    filepath
+    optparse-applicative
+    sandwich
+    text
+    unliftio
+    yaml
+  ];
+  license = lib.licenses.bsd3;
+  mainProgram = "julia-top-n-exe";
+}

pkgs/development/julia-modules/tests/julia-top-n/julia-top-n.cabal

@@ -0,0 +1,35 @@
+cabal-version: 2.2
+
+-- This file has been generated from package.yaml by hpack version 0.36.0.
+--
+-- see: https://github.com/sol/hpack
+
+name:           julia-top-n
+version:        0.1.0.0
+author:         Tom McLaughlin
+maintainer:     tom@codedown.io
+license:        BSD-3-Clause
+build-type:     Simple
+
+executable julia-top-n-exe
+  main-is: Main.hs
+  other-modules:
+      Paths_julia_top_n
+  autogen-modules:
+      Paths_julia_top_n
+  hs-source-dirs:
+      app
+  ghc-options: -Wall -Wcompat -Widentities -Wincomplete-record-updates -Wincomplete-uni-patterns -Wmissing-export-lists -Wmissing-home-modules -Wpartial-fields -Wredundant-constraints -threaded -rtsopts -with-rtsopts=-N
+  build-depends:
+      aeson
+    , base >=4.7 && <5
+    , bytestring
+    , filepath
+    , optparse-applicative
+    , sandwich
+    , string-interpolate
+    , text
+    , unliftio
+    , vector
+    , yaml
+  default-language: Haskell2010

pkgs/development/julia-modules/tests/julia-top-n/package.yaml

@@ -0,0 +1,38 @@
+name: julia-top-n
+version: 0.1.0.0
+license: BSD-3-Clause
+author: "Tom McLaughlin"
+maintainer: "tom@codedown.io"
+
+dependencies:
+  - aeson
+  - base >= 4.7 && < 5
+  - bytestring
+  - filepath
+  - optparse-applicative
+  - sandwich
+  - string-interpolate
+  - text
+  - unliftio
+  - vector
+  - yaml
+
+ghc-options:
+  - -Wall
+  - -Wcompat
+  - -Widentities
+  - -Wincomplete-record-updates
+  - -Wincomplete-uni-patterns
+  - -Wmissing-export-lists
+  - -Wmissing-home-modules
+  - -Wpartial-fields
+  - -Wredundant-constraints
+
+executables:
+  julia-top-n-exe:
+    main: Main.hs
+    source-dirs: app
+    ghc-options:
+      - -threaded
+      - -rtsopts
+      - -with-rtsopts=-N

pkgs/development/julia-modules/tests/julia-top-n/stack.yaml

@@ -0,0 +1,11 @@
+
+resolver:
+  url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/22/4.yaml
+
+packages:
+  - .
+
+nix:
+  pure: false
+  packages:
+    - zlib

pkgs/development/julia-modules/tests/julia-top-n/stack.yaml.lock

@@ -0,0 +1,13 @@
+# This file was autogenerated by Stack.
+# You should not edit this file by hand.
+# For more information, please see the documentation at:
+#   https://docs.haskellstack.org/en/stable/lock_files
+
+packages: []
+snapshots:
+- completed:
+    sha256: 8b211c5a6aad3787e023dfddaf7de7868968e4f240ecedf14ad1c5b2199046ca
+    size: 714097
+    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/22/4.yaml
+  original:
+    url: https://raw.githubusercontent.com/commercialhaskell/stackage-snapshots/master/lts/22/4.yaml

pkgs/development/julia-modules/tests/process_top_n.py

@@ -0,0 +1,33 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p "python3.withPackages(ps: with ps; [ pyyaml toml ])"
+
+import csv
+from pathlib import Path
+import sys
+import toml
+import yaml
+
+requests_csv_path = Path(sys.argv[1])
+registry_path = Path(sys.argv[2])
+
+# Generate list of tuples (UUID, count)
+rows = []
+with open(requests_csv_path) as f:
+  reader = csv.reader(f)
+  for row in reader:
+    if row[2] == "user":
+      # Get UUID and request_count
+      rows.append((row[0], int(row[4])))
+rows.sort(key=(lambda x: x[1]), reverse=True)
+
+# Build a map from UUID -> name
+registry = toml.load(registry_path / "Registry.toml")
+uuid_to_name = {k: v["name"] for k, v in registry["packages"].items()}
+
+results = []
+for (uuid, count) in rows:
+  name = uuid_to_name.get(uuid)
+  if not name: continue
+  results.append({ "uuid": uuid, "name": uuid_to_name.get(uuid), "count": count })
+
+yaml.dump(results, sys.stdout, default_flow_style=False)

pkgs/development/julia-modules/tests/run_tests.sh

@@ -0,0 +1,15 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p jq
+
+set -eo pipefail
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd $SCRIPTDIR
+
+TOP_N_FILE=$(nix build --impure -f top-julia-packages.nix --no-link --json | jq -r '.[0].outputs.out')
+echo "Got top Julia packages: $TOP_N_FILE"
+
+TESTER_PROGRAM=$(nix build --impure --expr 'with import ../../../../. {}; haskellPackages.callPackage ./julia-top-n {}' --no-link --json | jq -r '.[0].outputs.out')/bin/julia-top-n-exe
+echo "Built tester program: $TESTER_PROGRAM"
+
+"$TESTER_PROGRAM" --tui -c "$TOP_N_FILE" $*

pkgs/development/julia-modules/tests/top-julia-packages.nix

@@ -0,0 +1,40 @@
+with import ../../../../. { };
+
+let
+  package-requests = stdenv.mkDerivation {
+    name = "julia-package-requests.csv";
+
+    __impure = true;
+
+    buildInputs = [
+      cacert
+      gzip
+      wget
+    ];
+
+    buildCommand = ''
+      wget https://julialang-logs.s3.amazonaws.com/public_outputs/current/package_requests.csv.gz
+      gunzip package_requests.csv.gz
+      cp package_requests.csv $out
+    '';
+  };
+
+  registry = callPackage ../registry.nix { };
+
+in
+
+runCommand "top-julia-packages.yaml"
+  {
+    __impure = true;
+    nativeBuildInputs = [
+      (python3.withPackages (
+        ps: with ps; [
+          pyyaml
+          toml
+        ]
+      ))
+    ];
+  }
+  ''
+    python ${./process_top_n.py} ${package-requests} ${registry} > $out
+  ''

pkgs/development/julia-modules/util.nix

@@ -0,0 +1,55 @@
+{
+  gitMinimal,
+  lib,
+  runCommand,
+}:
+
+{
+  # Add packages to a Python environment. Works if you pass something like either
+  # a) python3
+  # b) python3.withPackages (ps: [...])
+  # See https://github.com/NixOS/nixpkgs/pull/97467#issuecomment-689315186
+  addPackagesToPython =
+    python: packages:
+    # TODO: this stopped working because "env" ended up being a key of the base
+    # derivation like "python3" as well. Is there a robust way to determine if
+    # this Python is already wrapped?
+    if python ? "env" && lib.isDerivation python.env then
+      python.override (old: {
+        extraLibs = old.extraLibs ++ packages;
+      })
+    else
+      python.withPackages (ps: packages);
+
+  # Convert an ordinary source checkout into a repo with a single commit
+  repoifySimple =
+    name: path:
+    runCommand ''${name}-repoified'' { buildInputs = [ gitMinimal ]; } ''
+      mkdir -p $out
+      cp -r ${path}/. $out
+      cd $out
+      chmod -R u+w .
+      rm -rf .git
+      git init
+      git add . -f
+      git config user.email "julia2nix@localhost"
+      git config user.name "julia2nix"
+      git commit -m "Dummy commit"
+    '';
+
+  # Convert an dependency source info into a repo with a single commit
+  repoifyInfo =
+    uuid: info:
+    runCommand ''julia-${info.name}-${info.version}'' { buildInputs = [ gitMinimal ]; } ''
+      mkdir -p $out
+      cp -r ${info.src}/. $out
+      cd $out
+      chmod -R u+w .
+      rm -rf .git
+      git init
+      git add . -f
+      git config user.email "julia2nix@localhost"
+      git config user.name "julia2nix"
+      git commit -m "Dummy commit"
+    '';
+}