push sheeet

pkgs/development/compilers/julia/README.md

@@ -0,0 +1,24 @@
+Julia
+=====
+
+[Julia][julia], as a full-fledged programming language with an extensive
+standard library that covers numerical computing, can be somewhat challenging to
+package. This file aims to provide pointers which could not easily be included
+as comments in the expressions themselves.
+
+[julia]: https://julialang.org
+
+For Nixpkgs, the manual is as always your primary reference, and for the Julia
+side of things, you probably want to familiarise yourself with the [README
+][readme], [build instructions][build], and [release process][release_process].
+Remember that these can change between Julia releases, especially if the LTS and
+release branches have deviated greatly. A lot of the build process is
+underdocumented and thus there is no substitute for digging into the code that
+controls the build process. You are very likely to need to use the test suite to
+locate and address issues and in the end passing it, while only disabling a
+minimal set of broken or incompatible tests you think you have a good reason to
+disable, is your best bet at arriving at a solid derivation.
+
+[readme]: https://github.com/JuliaLang/julia/blob/master/README.md
+[build]: https://github.com/JuliaLang/julia/tree/master/doc/src/devdocs/build
+[release_process]: https://julialang.org/blog/2019/08/release-process

pkgs/development/compilers/julia/default.nix

@@ -0,0 +1,89 @@
+{ callPackage, fetchpatch2 }:
+
+let
+  juliaWithPackages = callPackage ../../julia-modules { };
+
+  wrapJulia =
+    julia:
+    julia.overrideAttrs (oldAttrs: {
+      passthru = (oldAttrs.passthru or { }) // {
+        withPackages = juliaWithPackages.override { inherit julia; };
+      };
+    });
+
+in
+
+{
+  julia_19-bin = wrapJulia (
+    callPackage (import ./generic-bin.nix {
+      version = "1.9.4";
+      sha256 = {
+        x86_64-linux = "07d20c4c2518833e2265ca0acee15b355463361aa4efdab858dad826cf94325c";
+        aarch64-linux = "541d0c5a9378f8d2fc384bb8595fc6ffe20d61054629a6e314fb2f8dfe2f2ade";
+        x86_64-darwin = "67eec264f6afc9e9bf72c0f62c84d91c2ebdfaed6a0aa11606e3c983d278b441";
+        aarch64-darwin = "67542975e86102eec95bc4bb7c30c5d8c7ea9f9a0b388f0e10f546945363b01a";
+      };
+      patches = [
+        # https://github.com/JuliaLang/julia/commit/f5eeba35d9bf20de251bb9160cc935c71e8b19ba
+        ./patches/1.9-bin/0001-allow-skipping-internet-required-tests.patch
+      ];
+    }) { }
+  );
+  julia_110-bin = wrapJulia (
+    callPackage (import ./generic-bin.nix {
+      version = "1.10.10";
+      sha256 = {
+        x86_64-linux = "6a78a03a71c7ab792e8673dc5cedb918e037f081ceb58b50971dfb7c64c5bf81";
+        aarch64-linux = "a4b157ed68da10471ea86acc05a0ab61c1a6931ee592a9b236be227d72da50ff";
+        x86_64-darwin = "942b0d4accc9704861c7781558829b1d521df21226ad97bd01e1e43b1518d3e6";
+        aarch64-darwin = "52d3f82c50d9402e42298b52edc3d36e0f73e59f81fc8609d22fa094fbad18be";
+      };
+    }) { }
+  );
+  julia_111-bin = wrapJulia (
+    callPackage (import ./generic-bin.nix {
+      version = "1.11.7";
+      sha256 = {
+        x86_64-linux = "aa5924114ecb89fd341e59aa898cd1882b3cb622ca4972582c1518eff5f68c05";
+        aarch64-linux = "f97f80b35c12bdaf40c26f6c55dbb7617441e49c9e6b842f65e8410a388ca6f4";
+        x86_64-darwin = "b2c11315df39da478ab0fa77fb228f3fd818f1eaf42dc5cc1223c703f7122fe5";
+        aarch64-darwin = "74df9d4755a7740d141b04524a631e2485da9d65065d934e024232f7ba0790b6";
+      };
+    }) { }
+  );
+  julia_19 = wrapJulia (
+    callPackage (import ./generic.nix {
+      version = "1.9.4";
+      hash = "sha256-YYQ7lkf9BtOymU8yd6ZN4ctaWlKX2TC4yOO8DpN0ACQ=";
+      patches = [
+        ./patches/1.9/0002-skip-failing-and-flaky-tests.patch
+      ];
+    }) { }
+  );
+  julia_110 = wrapJulia (
+    callPackage (import ./generic.nix {
+      version = "1.10.10";
+      hash = "sha256-/NTIGLlcNu4sI1rICa+PS/Jn+YnWi37zFBcbfMnv3Ys=";
+      patches = [
+        ./patches/1.10/0002-skip-failing-and-flaky-tests.patch
+        # Revert https://github.com/JuliaLang/julia/pull/55354
+        # [build] Some improvements to the LLVM build system
+        # Related: https://github.com/JuliaLang/julia/issues/55617
+        (fetchpatch2 {
+          url = "https://github.com/JuliaLang/julia/commit/0be37db8c5b5a440bd9a11960ae9c998027b7337.patch";
+          revert = true;
+          hash = "sha256-gXC3LE3AuHMlSdA4dW+rbAhJpSB6ZMaz9X1qrHDPX7Y=";
+        })
+      ];
+    }) { }
+  );
+  julia_111 = wrapJulia (
+    callPackage (import ./generic.nix {
+      version = "1.11.7";
+      hash = "sha256-puluy9YAV8kdx6mfwbN1F7Nhot+P0cRv/a0dm86Jln0=";
+      patches = [
+        ./patches/1.11/0002-skip-failing-and-flaky-tests.patch
+      ];
+    }) { }
+  );
+}

pkgs/development/compilers/julia/generic-bin.nix

@@ -0,0 +1,150 @@
+{
+  version,
+  sha256,
+  patches ? [ ],
+}:
+
+{
+  autoPatchelfHook,
+  fetchurl,
+  lib,
+  stdenv,
+}:
+
+let
+  skip_tests = [
+    # Test flaky on ofborg
+    "channels"
+    # Test flaky because of our RPATH patching
+    # https://github.com/NixOS/nixpkgs/pull/230965#issuecomment-1545336489
+    "compiler/codegen"
+    # Test flaky
+    "read"
+  ]
+  ++ lib.optionals (lib.versionAtLeast version "1.10") [
+    # Test flaky
+    # https://github.com/JuliaLang/julia/issues/52739
+    "REPL"
+    # Test flaky
+    "ccall"
+  ]
+  ++ lib.optionals (lib.versionAtLeast version "1.11") [
+    # Test flaky
+    # https://github.com/JuliaLang/julia/issues/54280
+    "loading"
+    "cmdlineargs"
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isDarwin [
+    # Test flaky on ofborg
+    "FileWatching"
+    # Test requires pbcopy
+    "InteractiveUtils"
+    # Test requires network access
+    "Sockets"
+  ]
+  ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isx86_64) [
+    # Test Failed at $out/share/julia/stdlib/v1.8/LinearAlgebra/test/blas.jl:702
+    "LinearAlgebra/blas"
+    # Test Failed at $out/share/julia/test/misc.jl:724
+    "misc"
+  ];
+in
+stdenv.mkDerivation {
+  pname = "julia-bin";
+
+  inherit version patches;
+
+  src =
+    {
+      x86_64-linux = fetchurl {
+        url = "https://julialang-s3.julialang.org/bin/linux/x64/${lib.versions.majorMinor version}/julia-${version}-linux-x86_64.tar.gz";
+        sha256 = sha256.x86_64-linux;
+      };
+      aarch64-linux = fetchurl {
+        url = "https://julialang-s3.julialang.org/bin/linux/aarch64/${lib.versions.majorMinor version}/julia-${version}-linux-aarch64.tar.gz";
+        sha256 = sha256.aarch64-linux;
+      };
+      x86_64-darwin = fetchurl {
+        url = "https://julialang-s3.julialang.org/bin/mac/x64/${lib.versions.majorMinor version}/julia-${version}-mac64.tar.gz";
+        sha256 = sha256.x86_64-darwin;
+      };
+      aarch64-darwin = fetchurl {
+        url = "https://julialang-s3.julialang.org/bin/mac/aarch64/${lib.versions.majorMinor version}/julia-${version}-macaarch64.tar.gz";
+        sha256 = sha256.aarch64-darwin;
+      };
+    }
+    .${stdenv.hostPlatform.system} or (throw "Unsupported system: ${stdenv.hostPlatform.system}");
+
+  postPatch = ''
+    # Julia fails to pick up our Certification Authority root certificates, but
+    # it provides its own so we can simply disable the test. Patching in the
+    # dynamic path to ours require us to rebuild the Julia system image.
+    substituteInPlace share/julia/stdlib/v${lib.versions.majorMinor version}/NetworkOptions/test/runtests.jl \
+      --replace '@test ca_roots_path() != bundled_ca_roots()' \
+        '@test_skip ca_roots_path() != bundled_ca_roots()'
+  '';
+
+  nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [
+    autoPatchelfHook
+    # https://github.com/JuliaLang/julia/blob/v1.9.0/NEWS.md#external-dependencies
+    stdenv.cc.cc
+  ];
+
+  installPhase = ''
+    runHook preInstall
+    cp -r . $out
+  ''
+  + lib.optionalString stdenv.hostPlatform.isLinux ''
+    # "$out/share" is intentionally omitted since it contains
+    # julia package images and patchelf would break them
+    autoPatchelf "$out/bin" "$out/lib" "$out/libexec"
+  ''
+  + ''
+    runHook postInstall
+  '';
+
+  # Breaks backtraces, etc.
+  dontStrip = true;
+  dontAutoPatchelf = true;
+
+  doInstallCheck = true;
+
+  preInstallCheck = ''
+    export JULIA_TEST_USE_MULTIPLE_WORKERS=true
+    # Some tests require read/write access to $HOME.
+    # And $HOME cannot be equal to $TMPDIR as it causes test failures
+    export HOME=$(mktemp -d)
+  '';
+
+  installCheckPhase = ''
+    runHook preInstallCheck
+    # Command lifted from `test/Makefile`.
+    $out/bin/julia \
+      --check-bounds=yes \
+      --startup-file=no \
+      --depwarn=error \
+      $out/share/julia/test/runtests.jl \
+      --skip internet_required ${toString skip_tests}
+    runHook postInstallCheck
+  '';
+
+  meta = {
+    description = "High-level, high-performance, dynamic language for technical computing";
+    homepage = "https://julialang.org";
+    # Bundled and linked with various GPL code, although Julia itself is MIT.
+    license = lib.licenses.gpl2Plus;
+    maintainers = with lib.maintainers; [
+      raskin
+      nickcao
+      wegank
+      thomasjm
+    ];
+    platforms = [
+      "x86_64-linux"
+      "aarch64-linux"
+      "x86_64-darwin"
+      "aarch64-darwin"
+    ];
+    mainProgram = "julia";
+  };
+}

pkgs/development/compilers/julia/generic.nix

@@ -0,0 +1,122 @@
+{
+  version,
+  hash,
+  patches,
+}:
+
+{
+  lib,
+  stdenv,
+  fetchurl,
+  which,
+  python3,
+  gfortran,
+  cacert,
+  cmake,
+  perl,
+  gnum4,
+  openssl,
+  libxml2,
+  zlib,
+  buildPackages,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "julia";
+
+  inherit version patches;
+
+  src = fetchurl {
+    url = "https://github.com/JuliaLang/julia/releases/download/v${version}/julia-${version}-full.tar.gz";
+    inherit hash;
+  };
+
+  strictDeps = true;
+
+  nativeBuildInputs = [
+    which
+    python3
+    gfortran
+    cmake
+    perl
+    gnum4
+    openssl
+  ];
+
+  buildInputs = [
+    libxml2
+    zlib
+  ]
+  ++ lib.optionals (lib.versionAtLeast version "1.11") [
+    cacert
+  ];
+
+  dontUseCmakeConfigure = true;
+
+  postPatch = ''
+    patchShebangs .
+  ''
+  + lib.optionalString (lib.versionAtLeast version "1.11") ''
+    substituteInPlace deps/curl.mk \
+      --replace-fail 'cd $(dir $<) && $(TAR) jxf $(notdir $<)' \
+                     'cd $(dir $<) && $(TAR) jxf $(notdir $<) && sed -i "s|/usr/bin/env perl|${lib.getExe buildPackages.perl}|" curl-$(CURL_VER)/scripts/cd2nroff'
+  '';
+
+  makeFlags = [
+    "prefix=$(out)"
+    "USE_BINARYBUILDER=0"
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isx86_64 [
+    # https://github.com/JuliaCI/julia-buildkite/blob/main/utilities/build_envs.sh
+    "JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1)"
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isAarch64 [
+    "JULIA_CPU_TARGET=generic;cortex-a57;thunderx2t99;carmel,clone_all;apple-m1,base(3);neoverse-512tvb,base(3)"
+  ];
+
+  # remove forbidden reference to $TMPDIR
+  preFixup = ''
+    for file in libcurl.so libgmpxx.so libmpfr.so; do
+      patchelf --shrink-rpath --allowed-rpath-prefixes ${builtins.storeDir} "$out/lib/julia/$file"
+    done
+  '';
+
+  # tests are flaky for aarch64-linux on hydra
+  doInstallCheck = if (lib.versionOlder version "1.10") then !stdenv.hostPlatform.isAarch64 else true;
+
+  installCheckTarget = "testall";
+
+  preInstallCheck = ''
+    export JULIA_TEST_USE_MULTIPLE_WORKERS="true"
+    # Some tests require read/write access to $HOME.
+    # And $HOME cannot be equal to $TMPDIR as it causes test failures
+    export HOME=$(mktemp -d)
+  '';
+
+  dontStrip = true;
+
+  enableParallelBuilding = true;
+
+  env = lib.optionalAttrs (lib.versionOlder version "1.11" || stdenv.hostPlatform.isAarch64) {
+    NIX_CFLAGS_COMPILE = toString [
+      "-Wno-error=implicit-function-declaration"
+      "-Wno-error=incompatible-pointer-types"
+    ];
+  };
+
+  meta = with lib; {
+    description = "High-level performance-oriented dynamical language for technical computing";
+    mainProgram = "julia";
+    homepage = "https://julialang.org/";
+    license = licenses.mit;
+    maintainers = with maintainers; [
+      nickcao
+      joshniemela
+      thomasjm
+    ];
+    platforms = [
+      "x86_64-linux"
+      "aarch64-linux"
+    ];
+  };
+}

pkgs/development/compilers/julia/patches/1.10/0002-skip-failing-and-flaky-tests.patch

@@ -0,0 +1,25 @@
+From 9da2f2596db9f4f1a61825d82d9b8c3f3b2e99aa Mon Sep 17 00:00:00 2001
+From: Nick Cao <nickcao@nichi.co>
+Date: Wed, 10 Jan 2024 20:58:20 -0500
+Subject: [PATCH 2/2] skip failing and flaky tests
+
+---
+ test/Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/test/Makefile b/test/Makefile
+index 88dbe5b2b4..a2a7a55e20 100644
+--- a/test/Makefile
++++ b/test/Makefile
+@@ -28,7 +28,7 @@ default:
+ 
+ $(TESTS):
+ 	@cd $(SRCDIR) && \
+-	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
++	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --skip NetworkOptions REPL channels FileWatching ccall $@)
+ 
+ $(addprefix revise-, $(TESTS)): revise-% :
+ 	@cd $(SRCDIR) && \
+-- 
+2.43.0
+

pkgs/development/compilers/julia/patches/1.11/0002-skip-failing-and-flaky-tests.patch

@@ -0,0 +1,25 @@
+From 9da2f2596db9f4f1a61825d82d9b8c3f3b2e99aa Mon Sep 17 00:00:00 2001
+From: Nick Cao <nickcao@nichi.co>
+Date: Wed, 10 Jan 2024 20:58:20 -0500
+Subject: [PATCH 2/2] skip failing and flaky tests
+
+---
+ test/Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/test/Makefile b/test/Makefile
+index 88dbe5b2b4..a2a7a55e20 100644
+--- a/test/Makefile
++++ b/test/Makefile
+@@ -28,7 +28,7 @@ default:
+ 
+ $(TESTS):
+ 	@cd $(SRCDIR) && \
+-	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
++	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --skip NetworkOptions REPL channels FileWatching ccall loading cmdlineargs $@)
+ 
+ $(addprefix revise-, $(TESTS)): revise-% :
+ 	@cd $(SRCDIR) && \
+-- 
+2.43.0
+

pkgs/development/compilers/julia/patches/1.9-bin/0001-allow-skipping-internet-required-tests.patch

@@ -0,0 +1,50 @@
+diff --git a/share/julia/test/choosetests.jl b/share/julia/test/choosetests.jl
+index 334ef05..db5f795 100644
+--- a/share/julia/test/choosetests.jl
++++ b/share/julia/test/choosetests.jl
+@@ -31,6 +31,19 @@ const TESTNAMES = [
+         "smallarrayshrink", "opaque_closure", "filesystem", "download",
+ ]
+ 
++const INTERNET_REQUIRED_LIST = [
++    "Artifacts",
++    "Downloads",
++    "LazyArtifacts",
++    "LibCURL",
++    "LibGit2",
++    "Pkg",
++    "download",
++    "TOML",
++]
++
++const NETWORK_REQUIRED_LIST = vcat(INTERNET_REQUIRED_LIST, ["Sockets"])
++
+ """
+ `(; tests, net_on, exit_on_error, seed) = choosetests(choices)` selects a set of tests to be
+ run. `choices` should be a vector of test names; if empty or set to
+@@ -149,6 +162,7 @@ function choosetests(choices = [])
+     filtertests!(tests, "compiler/EscapeAnalysis", [
+         "compiler/EscapeAnalysis/local", "compiler/EscapeAnalysis/interprocedural"])
+     filtertests!(tests, "stdlib", STDLIBS)
++    filtertests!(tests, "internet_required", INTERNET_REQUIRED_LIST)
+     # do ambiguous first to avoid failing if ambiguities are introduced by other tests
+     filtertests!(tests, "ambiguous")
+ 
+@@ -164,16 +178,7 @@ function choosetests(choices = [])
+         filter!(x -> x != "rounding", tests)
+     end
+ 
+-    net_required_for = filter!(in(tests), [
+-        "Artifacts",
+-        "Downloads",
+-        "LazyArtifacts",
+-        "LibCURL",
+-        "LibGit2",
+-        "Sockets",
+-        "download",
+-        "TOML",
+-    ])
++    net_required_for = filter!(in(tests), NETWORK_REQUIRED_LIST)
+     net_on = true
+     JULIA_TEST_NETWORKING_AVAILABLE = get(ENV, "JULIA_TEST_NETWORKING_AVAILABLE", "") |>
+                                       strip |>

pkgs/development/compilers/julia/patches/1.9/0002-skip-failing-and-flaky-tests.patch

@@ -0,0 +1,25 @@
+From 0e1fe51ce93847ac3c4de49a003d9762b2f3d7c6 Mon Sep 17 00:00:00 2001
+From: Nick Cao <nickcao@nichi.co>
+Date: Tue, 20 Sep 2022 18:42:59 +0800
+Subject: [PATCH 2/2] skip failing and flaky tests
+
+---
+ test/Makefile | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/test/Makefile b/test/Makefile
+index 24e137a5b1..e78f12da04 100644
+--- a/test/Makefile
++++ b/test/Makefile
+@@ -23,7 +23,7 @@ default:
+ 
+ $(TESTS):
+ 	@cd $(SRCDIR) && \
+-	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@)
++	$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --skip MozillaCACerts_jll --skip NetworkOptions --skip channels $@)
+ 
+ $(addprefix revise-, $(TESTS)): revise-% :
+ 	@cd $(SRCDIR) && \
+-- 
+2.38.1
+