push sheeet

doc/doc-support/epub.nix

@@ -0,0 +1,54 @@
+# To build this derivation, run `nix-build -A nixpkgs-manual.epub`
+{
+  lib,
+  runCommand,
+  docbook_xsl_ns,
+  libxslt,
+  zip,
+}:
+runCommand "manual.epub"
+  {
+    nativeBuildInputs = [
+      libxslt
+      zip
+    ];
+
+    epub = ''
+      <book xmlns="http://docbook.org/ns/docbook"
+            xmlns:xlink="http://www.w3.org/1999/xlink"
+            version="5.0"
+            xml:id="nixpkgs-manual">
+        <info>
+          <title>Nixpkgs Manual</title>
+          <subtitle>Version ${lib.version}</subtitle>
+        </info>
+        <chapter>
+          <title>Temporarily unavailable</title>
+          <para>
+            The Nixpkgs manual is currently not available in EPUB format,
+            please use the <link xlink:href="https://nixos.org/nixpkgs/manual">HTML manual</link>
+            instead.
+          </para>
+          <para>
+            If you've used the EPUB manual in the past and it has been useful to you, please
+            <link xlink:href="https://github.com/NixOS/nixpkgs/issues/237234">let us know</link>.
+          </para>
+        </chapter>
+      </book>
+    '';
+
+    passAsFile = [ "epub" ];
+  }
+  ''
+    mkdir scratch
+    xsltproc \
+      --param chapter.autolabel 0 \
+      --nonet \
+      --output scratch/ \
+      ${docbook_xsl_ns}/xml/xsl/docbook/epub/docbook.xsl \
+      $epubPath
+
+    echo "application/epub+zip" > mimetype
+    zip -0Xq -b "$TMPDIR" "$out" mimetype
+    cd scratch && zip -Xr9D -b "$TMPDIR" "$out" *
+  ''

doc/doc-support/lib-function-docs.nix

@@ -0,0 +1,151 @@
+# Generates the documentation for library functions via nixdoc.
+# To build this derivation, run `nix-build -A nixpkgs-manual.lib-docs`
+{
+  lib,
+  stdenvNoCC,
+  nixdoc,
+  nix,
+  nixpkgs ? { },
+  libsets ? [
+    {
+      name = "asserts";
+      description = "assertion functions";
+    }
+    {
+      name = "attrsets";
+      description = "attribute set functions";
+    }
+    {
+      name = "strings";
+      description = "string manipulation functions";
+    }
+    {
+      name = "versions";
+      description = "version string functions";
+    }
+    {
+      name = "trivial";
+      description = "miscellaneous functions";
+    }
+    {
+      name = "fixedPoints";
+      baseName = "fixed-points";
+      description = "explicit recursion functions";
+    }
+    {
+      name = "lists";
+      description = "list manipulation functions";
+    }
+    {
+      name = "debug";
+      description = "debugging functions";
+    }
+    {
+      name = "options";
+      description = "NixOS / nixpkgs option handling";
+    }
+    {
+      name = "path";
+      description = "path functions";
+    }
+    {
+      name = "fetchers";
+      description = "functions which can be reused across fetchers";
+    }
+    {
+      name = "filesystem";
+      description = "filesystem functions";
+    }
+    {
+      name = "fileset";
+      description = "file set functions";
+    }
+    {
+      name = "sources";
+      description = "source filtering functions";
+    }
+    {
+      name = "cli";
+      description = "command-line serialization functions";
+    }
+    {
+      name = "generators";
+      description = "functions that create file formats from nix data structures";
+    }
+    {
+      name = "gvariant";
+      description = "GVariant formatted string serialization functions";
+    }
+    {
+      name = "customisation";
+      description = "Functions to customise (derivation-related) functions, derivations, or attribute sets";
+    }
+    {
+      name = "meta";
+      description = "functions for derivation metadata";
+    }
+    {
+      name = "derivations";
+      description = "miscellaneous derivation-specific functions";
+    }
+  ],
+}:
+
+stdenvNoCC.mkDerivation {
+  name = "nixpkgs-lib-docs";
+
+  src = ../../lib;
+
+  nativeBuildInputs = [
+    nixdoc
+    nix
+  ];
+
+  installPhase = ''
+    runHook preInstall
+
+    cd ..
+
+    export NIX_STATE_DIR=$(mktemp -d)
+    nix-instantiate --eval --strict --json ${./lib-function-locations.nix} \
+      --arg nixpkgsPath "./." \
+      --argstr revision ${nixpkgs.rev or "master"} \
+      --argstr libsetsJSON ${lib.escapeShellArg (builtins.toJSON libsets)} \
+      --store $(mktemp -d) \
+      > locations.json
+
+    function docgen {
+      name=$1
+      baseName=$2
+      description=$3
+      # TODO: wrap lib.$name in <literal>, make nixdoc not escape it
+      if [[ -e "lib/$baseName.nix" ]]; then
+        nixdoc -c "$name" -d "lib.$name: $description" -l locations.json -f "lib/$baseName.nix" > "$out/$name.md"
+      else
+        nixdoc -c "$name" -d "lib.$name: $description" -l locations.json -f "lib/$baseName/default.nix" > "$out/$name.md"
+      fi
+      echo "$out/$name.md" >> "$out/index.md"
+    }
+
+    mkdir -p "$out"
+
+    cat > "$out/index.md" << 'EOF'
+    ```{=include=} sections auto-id-prefix=auto-generated
+    EOF
+
+    ${lib.concatMapStrings (
+      {
+        name,
+        baseName ? name,
+        description,
+      }:
+      ''
+        docgen ${name} ${baseName} ${lib.escapeShellArg description}
+      ''
+    ) libsets}
+
+    echo '```' >> "$out/index.md"
+
+    runHook postInstall
+  '';
+}

doc/doc-support/lib-function-locations.nix

@@ -0,0 +1,82 @@
+{
+  nixpkgsPath,
+  revision,
+  libsetsJSON,
+}:
+let
+  lib = import (nixpkgsPath + "/lib");
+  libsets = builtins.fromJSON libsetsJSON;
+
+  libDefPos =
+    prefix: set:
+    builtins.concatMap (
+      name:
+      [
+        {
+          name = builtins.concatStringsSep "." (prefix ++ [ name ]);
+          location = builtins.unsafeGetAttrPos name set;
+        }
+      ]
+      ++ lib.optionals (builtins.length prefix == 0 && builtins.isAttrs set.${name}) (
+        libDefPos (prefix ++ [ name ]) set.${name}
+      )
+    ) (builtins.attrNames set);
+
+  libset =
+    toplib:
+    map (subsetname: {
+      subsetname = subsetname;
+      functions = libDefPos [ ] toplib.${subsetname};
+    }) (map (x: x.name) libsets);
+
+  flattenedLibSubset =
+    { subsetname, functions }:
+    map (fn: {
+      name = "lib.${subsetname}.${fn.name}";
+      value = fn.location;
+    }) functions;
+
+  locatedlibsets = libs: map flattenedLibSubset (libset libs);
+  removeFilenamePrefix =
+    prefix: filename:
+    let
+      prefixLen = (builtins.stringLength prefix) + 1; # +1 to remove the leading /
+      filenameLen = builtins.stringLength filename;
+      substr = builtins.substring prefixLen filenameLen filename;
+    in
+    substr;
+
+  removeNixpkgs = removeFilenamePrefix (toString nixpkgsPath);
+
+  liblocations = builtins.filter (elem: elem.value != null) (lib.lists.flatten (locatedlibsets lib));
+
+  fnLocationRelative =
+    { name, value }:
+    {
+      inherit name;
+      value = value // {
+        file = removeNixpkgs value.file;
+      };
+    };
+
+  relativeLocs = (map fnLocationRelative liblocations);
+  sanitizeId = builtins.replaceStrings [ "'" ] [ "-prime" ];
+
+  urlPrefix = "https://github.com/NixOS/nixpkgs/blob/${revision}";
+  jsonLocs = builtins.listToAttrs (
+    map (
+      { name, value }:
+      {
+        name = sanitizeId name;
+        value =
+          let
+            text = "${value.file}:${toString value.line}";
+            target = "${urlPrefix}/${value.file}#L${toString value.line}";
+          in
+          "[${text}](${target}) in `<nixpkgs>`";
+      }
+    ) relativeLocs
+  );
+
+in
+jsonLocs

doc/doc-support/options-doc.nix

@@ -0,0 +1,28 @@
+# To build this derivation, run `nix-build -A nixpkgs-manual.optionsDoc`
+{ lib, nixosOptionsDoc }:
+
+let
+  modules = lib.evalModules {
+    modules = [ ../../pkgs/top-level/config.nix ];
+    class = "nixpkgsConfig";
+  };
+
+  root = toString ../..;
+
+  transformDeclaration =
+    decl:
+    let
+      declStr = toString decl;
+      subpath = lib.removePrefix "/" (lib.removePrefix root declStr);
+    in
+    assert lib.hasPrefix root declStr;
+    {
+      url = "https://github.com/NixOS/nixpkgs/blob/master/${subpath}";
+      name = subpath;
+    };
+in
+nixosOptionsDoc {
+  inherit (modules) options;
+  documentType = "none";
+  transformOptions = opt: opt // { declarations = map transformDeclaration opt.declarations; };
+}

doc/doc-support/package.nix

@@ -0,0 +1,175 @@
+# This file describes the Nixpkgs manual, which happens to use module docs infra originally
+# developed for NixOS. To build this derivation, run `nix-build -A nixpkgs-manual`.
+#
+{
+  lib,
+  stdenvNoCC,
+  callPackage,
+  devmode,
+  mkShellNoCC,
+  documentation-highlighter,
+  nixos-render-docs,
+  nixos-render-docs-redirects,
+  writeShellScriptBin,
+  nixpkgs ? { },
+  markdown-code-runner,
+  roboto,
+  treefmt,
+  nixosOptionsDoc,
+}:
+stdenvNoCC.mkDerivation (
+  finalAttrs:
+  let
+    inherit (finalAttrs.finalPackage.optionsDoc) optionsJSON;
+    inherit (finalAttrs.finalPackage) epub lib-docs pythonInterpreterTable;
+
+    # Make anything from lib (the module system internals) invisible
+    hide-lib =
+      opt:
+      opt
+      // {
+        visible = if lib.all (decl: decl == "lib/modules.nix") opt.declarations then false else opt.visible;
+      };
+
+    toURL =
+      decl:
+      let
+        declStr = toString decl;
+        root = toString ../..;
+        subpath = lib.removePrefix "/" (lib.removePrefix root declStr);
+      in
+      if lib.hasPrefix root declStr then
+        {
+          url = "https://github.com/NixOS/nixpkgs/blob/master/${subpath}";
+          name = "nixpkgs/${subpath}";
+        }
+      else
+        decl;
+
+    mapURLs = opt: opt // { declarations = map toURL opt.declarations; };
+
+    docs.generic.meta-maintainers = nixosOptionsDoc {
+      inherit (lib.evalModules { modules = [ ../../modules/generic/meta-maintainers.nix ]; }) options;
+      transformOptions = opt: hide-lib (mapURLs opt);
+    };
+  in
+  {
+    name = "nixpkgs-manual";
+
+    nativeBuildInputs = [ nixos-render-docs ];
+
+    src = lib.cleanSourceWith {
+      src = ../.;
+      filter =
+        path: type:
+        type == "directory"
+        || lib.hasSuffix ".md" path
+        || lib.hasSuffix ".md.in" path
+        || lib.elem path (
+          map toString [
+            ../style.css
+            ../anchor-use.js
+            ../anchor.min.js
+            ../manpage-urls.json
+            ../redirects.json
+          ]
+        );
+    };
+
+    postPatch = ''
+      ln -s ${optionsJSON}/share/doc/nixos/options.json ./config-options.json
+      ln -s ${treefmt.functionsDoc.markdown} ./packages/treefmt-functions.section.md
+      ln -s ${treefmt.optionsDoc.optionsJSON}/share/doc/nixos/options.json ./treefmt-options.json
+      ln -s ${docs.generic.meta-maintainers.optionsJSON}/share/doc/nixos/options.json ./options-modules-generic-meta-maintainers.json
+    '';
+
+    buildPhase = ''
+      runHook preBuild
+
+      substituteInPlace ./languages-frameworks/python.section.md \
+        --subst-var-by python-interpreter-table "$(<"${pythonInterpreterTable}")"
+
+      cat ./functions/library.md.in ${lib-docs}/index.md > ./functions/library.md
+
+      substitute ./manual.md.in ./manual.md \
+        --replace-fail '@MANUAL_VERSION@' '${lib.version}'
+
+      mkdir -p out/media
+
+      mkdir -p out/highlightjs
+      cp -t out/highlightjs \
+        ${documentation-highlighter}/highlight.pack.js \
+        ${documentation-highlighter}/LICENSE \
+        ${documentation-highlighter}/mono-blue.css \
+        ${documentation-highlighter}/loader.js
+
+      cp -t out ./style.css ./anchor.min.js ./anchor-use.js
+
+      nixos-render-docs manual html \
+        --manpage-urls ./manpage-urls.json \
+        --redirects ./redirects.json \
+        --revision ${nixpkgs.rev or "master"} \
+        --stylesheet style.css \
+        --stylesheet highlightjs/mono-blue.css \
+        --script ./highlightjs/highlight.pack.js \
+        --script ./highlightjs/loader.js \
+        --script ./anchor.min.js \
+        --script ./anchor-use.js \
+        --toc-depth 1 \
+        --section-toc-depth 1 \
+        manual.md \
+        out/index.html
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      runHook preInstall
+
+      dest="$out/share/doc/nixpkgs"
+      mkdir -p "$(dirname "$dest")"
+      mv out "$dest"
+      cp "$dest/index.html" "$dest/manual.html"
+
+      cp ${roboto.src}/web/Roboto\[ital\,wdth\,wght\].ttf "$dest/Roboto.ttf"
+
+      cp ${epub} "$dest/nixpkgs-manual.epub"
+
+      mkdir -p $out/nix-support/
+      echo "doc manual $dest index.html" >> $out/nix-support/hydra-build-products
+      echo "doc manual $dest nixpkgs-manual.epub" >> $out/nix-support/hydra-build-products
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      lib-docs = callPackage ./lib-function-docs.nix { inherit nixpkgs; };
+
+      epub = callPackage ./epub.nix { };
+
+      optionsDoc = callPackage ./options-doc.nix { };
+
+      pythonInterpreterTable = callPackage ./python-interpreter-table.nix { };
+
+      shell =
+        let
+          devmode' = devmode.override {
+            buildArgs = toString ../.;
+            open = "/share/doc/nixpkgs/index.html";
+          };
+          nixos-render-docs-redirects' = writeShellScriptBin "redirects" "${lib.getExe nixos-render-docs-redirects} --file ${toString ../redirects.json} $@";
+        in
+        mkShellNoCC {
+          packages = [
+            devmode'
+            nixos-render-docs-redirects'
+            markdown-code-runner
+          ];
+        };
+
+      tests = {
+        manpage-urls = callPackage ../tests/manpage-urls.nix { };
+      };
+    };
+  }
+)

doc/doc-support/python-interpreter-table.nix

@@ -0,0 +1,67 @@
+# To build this derivation, run `nix-build -A nixpkgs-manual.pythonInterpreterTable`
+{
+  lib,
+  writeText,
+  pkgs,
+  pythonInterpreters,
+}:
+let
+  isPythonInterpreter =
+    name:
+    /*
+      NB: Package names that don't follow the regular expression:
+      - `python-cosmopolitan` is not part of `pkgs.pythonInterpreters`.
+      - `_prebuilt` interpreters are used for bootstrapping internally.
+      - `python3Minimal` contains python packages, left behind conservatively.
+      - `rustpython` lacks `pythonVersion` and `implementation`.
+    */
+    (lib.strings.match "(pypy|python)([[:digit:]]*)" name) != null;
+
+  interpreterName =
+    pname:
+    let
+      cuteName = {
+        cpython = "CPython";
+        pypy = "PyPy";
+      };
+      interpreter = pkgs.${pname};
+    in
+    "${cuteName.${interpreter.implementation}} ${interpreter.pythonVersion}";
+
+  interpreters = lib.reverseList (
+    lib.naturalSort (lib.filter isPythonInterpreter (lib.attrNames pythonInterpreters))
+  );
+
+  aliases =
+    pname:
+    lib.attrNames (
+      lib.filterAttrs (
+        name: value:
+        # use tryEval to handle entries in aliases.nix
+        (builtins.tryEval (
+          isPythonInterpreter name && name != pname && interpreterName name == interpreterName pname
+        )).value
+      ) pkgs
+    );
+
+  result = map (pname: {
+    inherit pname;
+    aliases = aliases pname;
+    interpreter = interpreterName pname;
+  }) interpreters;
+
+  toMarkdown =
+    data:
+    let
+      line = package: ''
+        | ${package.pname} | ${lib.concatStringsSep ", " package.aliases or [ ]} | ${package.interpreter} |
+      '';
+    in
+    lib.concatStringsSep "" (map line data);
+
+in
+writeText "python-interpreter-table.md" ''
+  | Package | Aliases | Interpeter |
+  |---------|---------|------------|
+  ${toMarkdown result}
+''