push sheeet

pkgs/development/cuda-modules/modules/README.md

@@ -0,0 +1,56 @@
+# Modules
+
+Modules as they are used in `modules` exist primarily to check the shape and
+content of CUDA redistributable and feature manifests. They are ultimately meant
+to reduce the repetitive nature of repackaging CUDA redistributables.
+
+Building most redistributables follows a pattern of a manifest indicating which
+packages are available at a location, their versions, and their hashes. To avoid
+creating builders for each and every derivation, modules serve as a way for us
+to use a single `genericManifestBuilder` to build all redistributables.
+
+## `generic`
+
+The modules in `generic` are reusable components meant to check the shape and
+content of NVIDIA's CUDA redistributable manifests, our feature manifests (which
+are derived from NVIDIA's manifests), or hand-crafted Nix expressions describing
+available packages. They are used by the `genericManifestBuilder` to build CUDA
+redistributables.
+
+Generally, each package which relies on manifests or Nix release expressions
+will create an alias to the relevant generic module. For example, the [module
+for CUDNN](./cudnn/default.nix) aliases the generic module for release
+expressions, while the [module for CUDA redistributables](./cuda/default.nix)
+aliases the generic module for manifests.
+
+Alternatively, additional fields or values may need to be configured to account
+for the particulars of a package. For example, while the release expressions for
+[CUDNN](../cudnn/releases.nix) and [TensorRT](../tensorrt/releases.nix) are very
+close, they differ slightly in the fields they have. The [module for
+CUDNN](./cudnn/default.nix) is able to use the generic module for
+release expressions, while the [module for
+TensorRT](./tensorrt/default.nix) must add additional fields to the
+generic module.
+
+### `manifests`
+
+The modules in `generic/manifests` define the structure of NVIDIA's CUDA
+redistributable manifests and our feature manifests.
+
+NVIDIA's redistributable manifests are retrieved from their web server, while
+the feature manifests are produced by
+[`cuda-redist-find-features`](https://github.com/connorbaker/cuda-redist-find-features).
+
+### `releases`
+
+The modules in `generic/releases` define the structure of our hand-crafted Nix
+expressions containing information necessary to download and repackage CUDA
+redistributables. These expressions are created when NVIDIA-provided manifests
+are unavailable or otherwise unusable. For example, though CUDNN has manifests,
+a bug in NVIDIA's CI/CD causes manifests for different versions of CUDA to use
+the same name, which leads to the manifests overwriting each other.
+
+### `types`
+
+The modules in `generic/types` define reusable types used in both
+`generic/manifests` and `generic/releases`.

pkgs/development/cuda-modules/modules/cuda/default.nix

@@ -0,0 +1,4 @@
+{ options, ... }:
+{
+  options.cuda.manifests = options.generic.manifests;
+}

pkgs/development/cuda-modules/modules/cudnn/default.nix

@@ -0,0 +1,12 @@
+{ options, ... }:
+{
+  options.cudnn.releases = options.generic.releases;
+  # TODO(@connorbaker): Figure out how to add additional options to the
+  # to the generic release.
+  # {
+  #   url = options.mkOption {
+  #     description = "URL to download the tarball from";
+  #     type = types.str;
+  #   };
+  # }
+}

pkgs/development/cuda-modules/modules/cusparselt/default.nix

@@ -0,0 +1,4 @@
+{ options, ... }:
+{
+  options.cusparselt.manifests = options.generic.manifests;
+}

pkgs/development/cuda-modules/modules/cutensor/default.nix

@@ -0,0 +1,4 @@
+{ options, ... }:
+{
+  options.cutensor.manifests = options.generic.manifests;
+}

pkgs/development/cuda-modules/modules/default.nix

@@ -0,0 +1,11 @@
+{
+  imports = [
+    ./generic
+    # Always after generic
+    ./cuda
+    ./cudnn
+    ./cusparselt
+    ./cutensor
+    ./tensorrt
+  ];
+}

pkgs/development/cuda-modules/modules/generic/default.nix

@@ -0,0 +1,7 @@
+{
+  imports = [
+    ./types
+    ./manifests
+    ./releases
+  ];
+}

pkgs/development/cuda-modules/modules/generic/manifests/default.nix

@@ -0,0 +1,7 @@
+{ lib, config, ... }:
+{
+  options.generic.manifests = {
+    feature = import ./feature/manifest.nix { inherit lib config; };
+    redistrib = import ./redistrib/manifest.nix { inherit lib; };
+  };
+}

pkgs/development/cuda-modules/modules/generic/manifests/feature/manifest.nix

@@ -0,0 +1,10 @@
+{ lib, config, ... }:
+let
+  inherit (lib) options trivial types;
+  Release = import ./release.nix { inherit lib config; };
+in
+options.mkOption {
+  description = "Feature manifest is an attribute set which includes a mapping from package name to release";
+  example = trivial.importJSON ../../../../cuda/manifests/feature_11.8.0.json;
+  type = types.attrsOf Release.type;
+}

pkgs/development/cuda-modules/modules/generic/manifests/feature/outputs.nix

@@ -0,0 +1,60 @@
+{ lib, ... }:
+let
+  inherit (lib) options types;
+in
+# https://github.com/ConnorBaker/cuda-redist-find-features/blob/603407bea2fab47f2dfcd88431122a505af95b42/cuda_redist_find_features/manifest/feature/package/package.py
+options.mkOption {
+  description = "Set of outputs that a package can provide";
+  example = {
+    bin = true;
+    dev = true;
+    doc = false;
+    lib = false;
+    sample = false;
+    static = false;
+  };
+  type = types.submodule {
+    options = {
+      bin = options.mkOption {
+        description = "`bin` output requires that we have a non-empty `bin` directory containing at least one file with the executable bit set";
+        type = types.bool;
+      };
+      dev = options.mkOption {
+        description = ''
+          A `dev` output requires that we have at least one of the following non-empty directories:
+
+          - `include`
+          - `lib/pkgconfig`
+          - `share/pkgconfig`
+          - `lib/cmake`
+          - `share/aclocal`
+        '';
+        type = types.bool;
+      };
+      doc = options.mkOption {
+        description = ''
+          A `doc` output requires that we have at least one of the following non-empty directories:
+
+          - `share/info`
+          - `share/doc`
+          - `share/gtk-doc`
+          - `share/devhelp`
+          - `share/man`
+        '';
+        type = types.bool;
+      };
+      lib = options.mkOption {
+        description = "`lib` output requires that we have a non-empty lib directory containing at least one shared library";
+        type = types.bool;
+      };
+      sample = options.mkOption {
+        description = "`sample` output requires that we have a non-empty `samples` directory";
+        type = types.bool;
+      };
+      static = options.mkOption {
+        description = "`static` output requires that we have a non-empty lib directory containing at least one static library";
+        type = types.bool;
+      };
+    };
+  };
+}

pkgs/development/cuda-modules/modules/generic/manifests/feature/package.nix

@@ -0,0 +1,10 @@
+{ lib, ... }:
+let
+  inherit (lib) options types;
+  Outputs = import ./outputs.nix { inherit lib; };
+in
+options.mkOption {
+  description = "Package in the manifest";
+  example = (import ./release.nix { inherit lib; }).linux-x86_64;
+  type = types.submodule { options.outputs = Outputs; };
+}

pkgs/development/cuda-modules/modules/generic/manifests/feature/release.nix

@@ -0,0 +1,10 @@
+{ lib, config, ... }:
+let
+  inherit (lib) options types;
+  Package = import ./package.nix { inherit lib config; };
+in
+options.mkOption {
+  description = "Release is an attribute set which includes a mapping from platform to package";
+  example = (import ./manifest.nix { inherit lib; }).cuda_cccl;
+  type = types.attrsOf Package.type;
+}

pkgs/development/cuda-modules/modules/generic/manifests/redistrib/manifest.nix

@@ -0,0 +1,33 @@
+{ lib, ... }:
+let
+  inherit (lib) options trivial types;
+  Release = import ./release.nix { inherit lib; };
+in
+options.mkOption {
+  description = "Redistributable manifest is an attribute set which includes a mapping from package name to release";
+  example = trivial.importJSON ../../../../cuda/manifests/redistrib_11.8.0.json;
+  type = types.submodule {
+    # Allow any attribute name as these will be the package names
+    freeformType = types.attrsOf Release.type;
+    options = {
+      release_date = options.mkOption {
+        description = "Release date of the manifest";
+        type = types.nullOr types.str;
+        default = null;
+        example = "2023-08-29";
+      };
+      release_label = options.mkOption {
+        description = "Release label of the manifest";
+        type = types.nullOr types.str;
+        default = null;
+        example = "12.2.2";
+      };
+      release_product = options.mkOption {
+        example = "cuda";
+        description = "Release product of the manifest";
+        type = types.nullOr types.str;
+        default = null;
+      };
+    };
+  };
+}

pkgs/development/cuda-modules/modules/generic/manifests/redistrib/package.nix

@@ -0,0 +1,32 @@
+{ lib, ... }:
+let
+  inherit (lib) options types;
+in
+options.mkOption {
+  description = "Package in the manifest";
+  example = (import ./release.nix { inherit lib; }).linux-x86_64;
+  type = types.submodule {
+    options = {
+      relative_path = options.mkOption {
+        description = "Relative path to the package";
+        example = "cuda_cccl/linux-x86_64/cuda_cccl-linux-x86_64-11.5.62-archive.tar.xz";
+        type = types.str;
+      };
+      sha256 = options.mkOption {
+        description = "Sha256 hash of the package";
+        example = "bbe633d6603d5a96a214dcb9f3f6f6fd2fa04d62e53694af97ae0c7afe0121b0";
+        type = types.str;
+      };
+      md5 = options.mkOption {
+        description = "Md5 hash of the package";
+        example = "e5deef4f6cb71f14aac5be5d5745dafe";
+        type = types.str;
+      };
+      size = options.mkOption {
+        description = "Size of the package as a string";
+        type = types.str;
+        example = "960968";
+      };
+    };
+  };
+}

pkgs/development/cuda-modules/modules/generic/manifests/redistrib/release.nix

@@ -0,0 +1,36 @@
+{ lib, ... }:
+let
+  inherit (lib) options types;
+  Package = import ./package.nix { inherit lib; };
+in
+options.mkOption {
+  description = "Release is an attribute set which includes a mapping from platform to package";
+  example = (import ./manifest.nix { inherit lib; }).cuda_cccl;
+  type = types.submodule {
+    # Allow any attribute name as these will be the platform names
+    freeformType = types.attrsOf Package.type;
+    options = {
+      name = options.mkOption {
+        description = "Full name of the package";
+        example = "CXX Core Compute Libraries";
+        type = types.str;
+      };
+      license = options.mkOption {
+        description = "License of the package";
+        example = "CUDA Toolkit";
+        type = types.str;
+      };
+      license_path = options.mkOption {
+        description = "Path to the license of the package";
+        example = "cuda_cccl/LICENSE.txt";
+        default = null;
+        type = types.nullOr types.str;
+      };
+      version = options.mkOption {
+        description = "Version of the package";
+        example = "11.5.62";
+        type = types.str;
+      };
+    };
+  };
+}

pkgs/development/cuda-modules/modules/generic/releases/default.nix

@@ -0,0 +1,45 @@
+{ lib, config, ... }:
+let
+  inherit (config.generic.types) majorMinorVersion majorMinorPatchBuildVersion;
+  inherit (lib) options types;
+in
+{
+  options.generic.releases = options.mkOption {
+    description = "Collection of packages targeting different platforms";
+    type =
+      let
+        Package = options.mkOption {
+          description = "Package for a specific platform";
+          example = {
+            version = "8.0.3.4";
+            minCudaVersion = "10.2";
+            maxCudaVersion = "10.2";
+            hash = "sha256-LxcXgwe1OCRfwDsEsNLIkeNsOcx3KuF5Sj+g2dY6WD0=";
+          };
+          type = types.submodule {
+            # TODO(@connorbaker): Figure out how to extend option sets.
+            freeformType = types.attrsOf types.anything;
+            options = {
+              version = options.mkOption {
+                description = "Version of the package";
+                type = majorMinorPatchBuildVersion;
+              };
+              minCudaVersion = options.mkOption {
+                description = "Minimum CUDA version supported";
+                type = majorMinorVersion;
+              };
+              maxCudaVersion = options.mkOption {
+                description = "Maximum CUDA version supported";
+                type = majorMinorVersion;
+              };
+              hash = options.mkOption {
+                description = "Hash of the tarball";
+                type = types.str;
+              };
+            };
+          };
+        };
+      in
+      types.attrsOf (types.listOf Package.type);
+  };
+}

pkgs/development/cuda-modules/modules/generic/types/default.nix

@@ -0,0 +1,39 @@
+{ lib, ... }:
+let
+  inherit (lib) options types;
+in
+{
+  options.generic.types = options.mkOption {
+    type = types.attrsOf types.optionType;
+    default = { };
+    description = "Set of generic types";
+  };
+  config.generic.types = {
+    cudaArch = types.strMatching "^sm_[[:digit:]]+[a-z]?$" // {
+      name = "cudaArch";
+      description = "CUDA architecture name";
+    };
+    # https://github.com/ConnorBaker/cuda-redist-find-features/blob/c841980e146f8664bbcd0ba1399e486b7910617b/cuda_redist_find_features/types/_lib_so_name.py
+    libSoName = types.strMatching ".*\\.so(\\.[[:digit:]]+)*$" // {
+      name = "libSoName";
+      description = "Name of a shared object file";
+    };
+
+    majorMinorVersion = types.strMatching "^([[:digit:]]+)\\.([[:digit:]]+)$" // {
+      name = "majorMinorVersion";
+      description = "Version number with a major and minor component";
+    };
+
+    majorMinorPatchVersion = types.strMatching "^([[:digit:]]+)\\.([[:digit:]]+)\\.([[:digit:]]+)$" // {
+      name = "majorMinorPatchVersion";
+      description = "Version number with a major, minor, and patch component";
+    };
+
+    majorMinorPatchBuildVersion =
+      types.strMatching "^([[:digit:]]+)\\.([[:digit:]]+)\\.([[:digit:]]+)\\.([[:digit:]]+)$"
+      // {
+        name = "majorMinorPatchBuildVersion";
+        description = "Version number with a major, minor, patch, and build component";
+      };
+  };
+}

pkgs/development/cuda-modules/modules/tensorrt/default.nix

@@ -0,0 +1,16 @@
+{ options, ... }:
+{
+  options.tensorrt.releases = options.generic.releases;
+  # TODO(@connorbaker): Figure out how to add additional options to the
+  # to the generic release.
+  # {
+  #   cudnnVersion = lib.options.mkOption {
+  #     description = "CUDNN version supported";
+  #     type = types.nullOr majorMinorVersion;
+  #   };
+  #   filename = lib.options.mkOption {
+  #     description = "Tarball name";
+  #     type = types.str;
+  #   };
+  # }
+}