push sheeet

maintainers/scripts/README.md

@@ -0,0 +1,75 @@
+# Maintainer scripts
+
+This folder contains various executable scripts for nixpkgs maintainers, and supporting data or nixlang files as needed.
+These scripts generally aren't a stable interface and may be changed or be removed.
+
+What follows is a (very incomplete) overview of available scripts.
+
+## Metadata
+
+### `get-maintainer.sh`
+
+`get-maintainer.sh [selector] value` returns a JSON object describing a given nixpkgs maintainer, equivalent to `lib.maintainers.${x} // { handle = x; }`.
+
+This allows looking up a maintainer's attrset (including GitHub and Matrix handles, email address etc.) based on any of their handles, more correctly and
+robustly than text search through `maintainer-list.nix`.
+
+```
+❯ ./get-maintainer.sh nicoo
+{
+  "email": "nicoo@debian.org",
+  "github": "nicoonoclaste",
+  "githubId": 1155801,
+  "keys": [
+    {
+      "fingerprint": "E44E 9EA5 4B8E 256A FB73 49D3 EC9D 3708 72BC 7A8C"
+    }
+  ],
+  "name": "nicoo",
+  "handle": "nicoo"
+}
+
+❯ ./get-maintainer.sh name 'Silvan Mosberger'
+{
+  "email": "contact@infinisil.com",
+  "github": "infinisil",
+  "githubId": 20525370,
+  "keys": [
+    {
+      "fingerprint": "6C2B 55D4 4E04 8266 6B7D  DA1A 422E 9EDA E015 7170"
+    }
+  ],
+  "matrix": "@infinisil:matrix.org",
+  "name": "Silvan Mosberger",
+  "handle": "infinisil"
+}
+```
+
+The maintainer is designated by a `selector` which must be one of:
+- `handle` (default): the maintainer's attribute name in `lib.maintainers`;
+- `email`, `name`, `github`, `githubId`, `matrix`, `name`:
+  attributes of the maintainer's object, matched exactly; see [`maintainer-list.nix`] for the fields' definition.
+
+[`maintainer-list.nix`]: ../maintainer-list.nix
+
+### `get-maintainer-pings-between.sh`
+
+Gets which maintainers would be pinged between two Nixpkgs revisions.
+Outputs a JSON object on stdout mapping GitHub usernames to the attributes that they would be getting pinged for.
+
+Example:
+
+```sh
+maintainers/scripts/get-maintainer-pings-between.sh HEAD^ HEAD
+```
+
+## Conventions
+
+### `sha-to-sri.py`
+
+`sha-to-sri.py path ...` (atomically) rewrites hash attributes (named `hash` or `sha(1|256|512)`) into the SRI format: `hash = "{hash name}-{base64 encoded value}"`.
+
+`path` must point to either a nix file, or a directory which will be automatically traversed.
+
+`sha-to-sri.py` automatically skips files whose first non-empty line contains `generated by` or `do not edit`.
+Moreover, when walking a directory tree, the script will skip files whose name is `yarn.nix` or contains `generated`.

maintainers/scripts/all-tarballs.nix

@@ -0,0 +1,17 @@
+/*
+  Helper expression for copy-tarballs. This returns (nearly) all
+  tarballs used the free packages in Nixpkgs.
+
+  Typical usage:
+
+  $ copy-tarballs.pl --expr 'import <nixpkgs/maintainers/scripts/all-tarballs.nix>'
+*/
+
+import ../../pkgs/top-level/release.nix {
+  # Don't apply ‘hydraJob’ to jobs, because then we can't get to the
+  # dependency graph.
+  scrubJobs = false;
+  # No need to evaluate on i686.
+  supportedSystems = [ "x86_64-linux" ];
+  bootstrapConfigs = [ ];
+}

maintainers/scripts/auto-rebase/README.md

@@ -0,0 +1,15 @@
+# Auto rebase script
+
+The [`./run.sh` script](./run.sh) in this directory rebases the current branch onto a target branch, while automatically resolving merge conflicts caused by marked commits in [`.git-blame-ignore-revs`](../../../.git-blame-ignore-revs).
+See the header comment of that file to understand how to mark commits.
+
+This is convenient for resolving merge conflicts for pull requests after e.g. treewide reformats.
+
+## Testing
+
+To run the tests in the [test directory](./test):
+```
+$ cd test
+$ nix-shell
+nix-shell> ./run.sh
+```

maintainers/scripts/auto-rebase/run.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if (( $# < 1 )); then
+    echo "Usage: $0 TARGET_BRANCH"
+    echo ""
+    echo "TARGET_BRANCH: Branch to rebase the current branch onto, e.g. master or release-24.11"
+    exit 1
+fi
+
+targetBranch=$1
+
+# Loop through all autorebase-able commits in .git-blame-ignore-revs on the base branch
+readarray -t autoLines < <(
+  git show "$targetBranch":.git-blame-ignore-revs \
+      | sed -n 's/^\([0-9a-f]\+\).*!autorebase \(.*\)$/\1 \2/p'
+)
+for line in "${autoLines[@]}"; do
+    read -r autoCommit autoCmd <<< "$line"
+
+    if ! git cat-file -e "$autoCommit"; then
+        echo "Not a valid commit: $autoCommit"
+        exit 1
+    elif git merge-base --is-ancestor "$autoCommit" HEAD; then
+        # Skip commits that we have already
+        continue
+    fi
+
+    echo -e "\e[32mAuto-rebasing commit $autoCommit with command '$autoCmd'\e[0m"
+
+    # The commit before the commit
+    parent=$(git rev-parse "$autoCommit"~)
+
+    echo "Rebasing on top of the previous commit, might need to manually resolve conflicts"
+    if ! git rebase --onto "$parent" "$(git merge-base "$targetBranch" HEAD)"; then
+      echo -e "\e[33m\e[1mRestart this script after resolving the merge conflict as described above\e[0m"
+      exit 1
+    fi
+
+    echo "Reapplying the commit on each commit of our branch"
+    # This does two things:
+    # - The parent filter inserts the auto commit between its parent and
+    #   and our first commit. By itself, this causes our first commit to
+    #   effectively "undo" the auto commit, since the tree of our first
+    #   commit is unchanged. This is why the following is also necessary:
+    # - The tree filter runs the command on each of our own commits,
+    #   effectively reapplying it.
+    FILTER_BRANCH_SQUELCH_WARNING=1 git filter-branch \
+        --parent-filter "sed 's/$parent/$autoCommit/'" \
+        --tree-filter "$autoCmd" \
+        "$autoCommit"..HEAD
+
+    # A tempting alternative is something along the lines of
+    #   git rebase --strategy-option=theirs --onto "$rev" "$parent" \
+    #     --exec '$autoCmd && git commit --all --amend --no-edit' \
+    # but this causes problems because merges are not guaranteed to maintain the formatting.
+    # The ./test.sh exercises such a case.
+done
+
+echo "Rebasing on top of the latest target branch commit"
+git rebase --onto "$targetBranch" "$(git merge-base "$targetBranch" HEAD)"

maintainers/scripts/auto-rebase/test/default.nix

@@ -0,0 +1,46 @@
+let
+  pkgs = import ../../../.. {
+    config = { };
+    overlays = [ ];
+  };
+
+  inherit (pkgs)
+    lib
+    stdenvNoCC
+    gitMinimal
+    treefmt
+    nixfmt
+    ;
+in
+
+stdenvNoCC.mkDerivation {
+  name = "test";
+  src = lib.fileset.toSource {
+    root = ./..;
+    fileset = lib.fileset.unions [
+      ../run.sh
+      ./run.sh
+      ./first.diff
+      ./second.diff
+    ];
+  };
+  nativeBuildInputs = [
+    gitMinimal
+    treefmt
+    nixfmt
+  ];
+  patchPhase = ''
+    patchShebangs .
+  '';
+
+  buildPhase = ''
+    export HOME=$(mktemp -d)
+    export PAGER=true
+    git config --global user.email "Your Name"
+    git config --global user.name "your.name@example.com"
+    ./test/run.sh
+  '';
+  installPhase = ''
+    touch $out
+  '';
+}

maintainers/scripts/auto-rebase/test/first.diff

@@ -0,0 +1,11 @@
+diff --git a/b.nix b/b.nix
+index 9d18f25..67b0466 100644
+--- a/b.nix
++++ b/b.nix
+@@ -1,5 +1,5 @@
+ {
+   this = "is";
+ 
+-  some = "set";
++  some = "value";
+ }

maintainers/scripts/auto-rebase/test/run.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# https://stackoverflow.com/a/246128/6605742
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Allows using a local directory for temporary files,
+# which can then be inspected after the run
+if (( $# > 0 )); then
+    tmp=$(realpath "$1/tmp")
+    if [[ -e "$tmp" ]]; then
+        rm -rf "$tmp"
+    fi
+    mkdir -p "$tmp"
+else
+    tmp=$(mktemp -d)
+    trap 'rm -rf "$tmp"' exit
+fi
+
+# Tests a scenario where two poorly formatted files were modified on both the
+# main branch and the feature branch, while the main branch also did a treewide
+# format.
+
+git init "$tmp/repo"
+cd "$tmp/repo" || exit
+git branch -m main
+
+# Some initial poorly-formatted files
+cat > a.nix <<EOF
+{ x
+, y
+
+, z
+}:
+null
+EOF
+
+cat > b.nix <<EOF
+{
+  this = "is";
+
+
+  some="set" ;
+ }
+EOF
+
+git add -A
+git commit -m "init"
+
+git switch -c feature
+
+# Some changes
+sed 's/set/value/' -i b.nix
+git commit -a -m "change b"
+sed '/, y/d' -i a.nix
+git commit -a -m "change a"
+
+git switch main
+
+# A change to cause a merge conflict
+sed 's/y/why/' -i a.nix
+git commit -a -m "change a"
+
+cat > treefmt.toml <<EOF
+[formatter.nix]
+command = "nixfmt"
+includes = [ "*.nix" ]
+EOF
+git add -A
+git commit -a -m "introduce treefmt"
+
+# Treewide reformat
+treefmt
+git commit -a -m "format"
+
+echo "$(git rev-parse HEAD) # !autorebase treefmt" > .git-blame-ignore-revs
+git add -A
+git commit -a -m "update ignored revs"
+
+git switch feature
+
+# Setup complete
+
+git log --graph --oneline feature main
+
+# This expectedly fails with a merge conflict that has to be manually resolved
+"$SCRIPT_DIR"/../run.sh main && exit 1
+sed '/<<</,/>>>/d' -i a.nix
+git add a.nix
+GIT_EDITOR=true git rebase --continue
+
+"$SCRIPT_DIR"/../run.sh main
+
+git log --graph --oneline feature main
+
+checkDiff() {
+    local ref=$1
+    local file=$2
+    expectedDiff=$(cat "$file")
+    actualDiff=$(git diff "$ref"~ "$ref")
+    if [[ "$expectedDiff" != "$actualDiff" ]]; then
+        echo -e "Expected this diff:\n$expectedDiff"
+        echo -e "But got this diff:\n$actualDiff"
+        exit 1
+    fi
+}
+
+checkDiff HEAD~ "$SCRIPT_DIR"/first.diff
+checkDiff HEAD "$SCRIPT_DIR"/second.diff
+
+echo "Success!"

maintainers/scripts/auto-rebase/test/second.diff

@@ -0,0 +1,11 @@
+diff --git a/a.nix b/a.nix
+index 18ba7ce..bcf38bc 100644
+--- a/a.nix
++++ b/a.nix
+@@ -1,6 +1,5 @@
+ {
+   x,
+-  why,
+ 
+   z,
+ }:

maintainers/scripts/bootstrap-files/README.md

@@ -0,0 +1,83 @@
+# Bootstrap files
+
+Currently `nixpkgs` builds most of its packages using bootstrap seed binaries (without the reliance on external inputs):
+
+- `bootstrap-tools`: an archive with the compiler toolchain and other helper tools enough to build the rest of the `nixpkgs`.
+- initial binaries needed to unpack `bootstrap-tools.*`.
+  On `linux` it's just `busybox`, on `darwin` and `freebsd` it is unpack.nar.xz which contains the binaries and script needed to unpack the tools.
+  These binaries can be executed directly from the store.
+
+These are called "bootstrap files".
+
+Bootstrap files should always be fetched from hydra and uploaded to `tarballs.nixos.org` to guarantee that all the binaries were built from the code committed into `nixpkgs` repository.
+
+The uploads to `tarballs.nixos.org` are done by `@lovesegfault` today.
+
+This document describes the procedure of updating bootstrap files in `nixpkgs`.
+
+## How to request the bootstrap seed update
+
+To get the tarballs updated let's use an example `i686-unknown-linux-gnu` target:
+
+1. Create a local update:
+
+   ```
+   $ maintainers/scripts/bootstrap-files/refresh-tarballs.bash --commit --targets=i686-unknown-linux-gnu
+   ```
+
+2. Test the update locally. I'll build local `hello` derivation with the result:
+
+   ```
+   $ nix-build -A hello --argstr system i686-linux
+   ```
+
+   To validate cross-targets `binfmt` `NixOS` helper can be useful.
+   For `riscv64-unknown-linux-gnu` the `/etc/nixos/configuration.nix` entry would be `boot.binfmt.emulatedSystems = [ "riscv64-linux" ]`.
+
+3. Propose the commit as a PR to update bootstrap tarballs, tag people who can help you test the updated architecture and once reviewed tag `@lovesegfault` to upload the tarballs.
+
+## How to add bootstrap files for a new target
+
+The procedure to add a new target is very similar to the update procedure.
+The only difference is that you need to set up a new job to build the `bootstrapFiles`.
+To do that you will need the following:
+
+1. Add your new target to `lib/systems/examples.nix`
+
+   This will populate `pkgsCross.$target` attribute set.
+   If you are dealing with `bootstrapFiles` upload you probably already have it.
+
+2. Add your new target to `pkgs/stdenv/linux/make-bootstrap-tools-cross.nix`.
+   This will add a new hydra job to `nixpkgs:cross-trunk` jobset.
+
+3. Wait for a hydra to build your bootstrap tarballs.
+
+4. Add your new target to `maintainers/scripts/bootstrap-files/refresh-tarballs.bash` around `CROSS_TARGETS=()`.
+
+5. Add your new target to `pkgs/stdenv/linux/default.nix` and follow standard bootstrap seed update procedure above.
+
+## Bootstrap files job definitions
+
+There are two types of bootstrap files:
+
+- natively built `stdenvBootstrapTools.build` hydra jobs in [`nixpkgs:trunk`](https://hydra.nixos.org/jobset/nixpkgs/trunk#tabs-jobs) jobset.
+  Incomplete list of examples is:
+
+  * `aarch64-unknown-linux-musl.nix`
+  * `i686-unknown-linux-gnu.nix`
+
+  These are Tier 1 hydra platforms.
+
+- cross-built by `bootstrapTools.build` hydra jobs in [`nixpkgs:cross-trunk`](https://hydra.nixos.org/jobset/nixpkgs/cross-trunk#tabs-jobs) jobset.
+  Incomplete list of examples is:
+
+  * `mips64el-unknown-linux-gnuabi64.nix`
+  * `mips64el-unknown-linux-gnuabin32.nix`
+  * `mipsel-unknown-linux-gnu.nix`
+  * `powerpc64le-unknown-linux-gnu.nix`
+  * `riscv64-unknown-linux-gnu.nix`
+
+  These are usually Tier 2 and lower targets.
+
+The `.build` job contains `/on-server/` subdirectory with binaries to be uploaded to `tarballs.nixos.org`.
+The files are uploaded to `tarballs.nixos.org` by writers to `S3` store.

maintainers/scripts/bootstrap-files/refresh-tarballs.bash

@@ -0,0 +1,315 @@
+#!/usr/bin/env nix-shell
+#! nix-shell --pure
+#! nix-shell -i bash
+#! nix-shell -p curl cacert
+#! nix-shell -p git
+#! nix-shell -p nix
+#! nix-shell -p jq
+
+set -o pipefail
+
+# How the refresher works:
+#
+# For a given list of <targets>:
+# 1. fetch latest successful '.build` job
+# 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit
+# 3. fetch all the `.build` artifacts from '$out/on-server/' directory
+# 4. calculate hashes and craft the commit message with the details on
+#    how to upload the result to 'tarballs.nixos.org'
+
+scratch_dir=$(mktemp -d)
+trap 'rm -rf -- "${scratch_dir}"' EXIT
+
+usage() {
+    cat >&2 <<EOF
+Usage:
+    $0 [ --commit ] --targets=<target>[,<target>,...]
+
+    The tool must be ran from the root directory of 'nixpkgs' repository.
+
+Synopsis:
+    'refresh-tarballs.bash' script fetches latest bootstrapFiles built
+    by hydra, registers them in 'nixpkgs' and provides commands to
+    upload seed files to 'tarballs.nixos.org'.
+
+    This is usually done in the following cases:
+
+    1. Single target fix: current bootstrap files for a single target
+        are problematic for some reason (target-specific bug). In this
+        case we can refresh just that target as:
+
+        \$ $0 --commit --targets=i686-unknown-linux-gnu
+
+    2. Routine refresh: all bootstrap files should be refreshed to avoid
+        debugging problems that only occur on very old binaries.
+
+        \$ $0 --commit --all-targets
+
+To get help on uploading refreshed binaries to 'tarballs.nixos.org'
+please have a look at <maintainers/scripts/bootstrap-files/README.md>.
+EOF
+    exit 1
+}
+
+# log helpers
+
+die() {
+    echo "ERROR: $*" >&2
+    exit 1
+}
+
+info() {
+    echo "INFO: $*" >&2
+}
+
+[[ ${#@} -eq 0 ]] && usage
+
+# known targets
+
+NATIVE_TARGETS=(
+    aarch64-unknown-linux-gnu
+    aarch64-unknown-linux-musl
+    i686-unknown-linux-gnu
+    x86_64-unknown-linux-gnu
+    x86_64-unknown-linux-musl
+    aarch64-apple-darwin
+    x86_64-apple-darwin
+)
+
+is_native() {
+    local t target=$1
+    for t in "${NATIVE_TARGETS[@]}"; do
+        [[ $t == $target ]] && return 0
+    done
+    return 1
+}
+
+CROSS_TARGETS=(
+    armv5tel-unknown-linux-gnueabi
+    armv6l-unknown-linux-gnueabihf
+    armv6l-unknown-linux-musleabihf
+    armv7l-unknown-linux-gnueabihf
+    mips64el-unknown-linux-gnuabi64
+    mips64el-unknown-linux-gnuabin32
+    mipsel-unknown-linux-gnu
+    powerpc64-unknown-linux-gnuabielfv1
+    powerpc64-unknown-linux-gnuabielfv2
+    powerpc64le-unknown-linux-gnu
+    riscv64-unknown-linux-gnu
+    s390x-unknown-linux-gnu
+    x86_64-unknown-freebsd
+    loongarch64-unknown-linux-gnu
+)
+
+is_cross() {
+    local t target=$1
+    for t in "${CROSS_TARGETS[@]}"; do
+        [[ $t == $target ]] && return 0
+    done
+    return 1
+}
+
+nar_sri_get() {
+    local restore_path store_path
+    ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
+    restore_path="${scratch_dir}/$2"
+    xz -d < "$1" | nix-store --restore "${restore_path}"
+    [[ $? -ne 0 ]] && die "Failed to unpack '$1'"
+
+    store_path=$(nix-store --add "${restore_path}")
+    [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
+    rm -rf -- "${restore_path}"
+
+    nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
+}
+
+# collect passed options
+
+targets=()
+commit=no
+
+for arg in "$@"; do
+    case "$arg" in
+        --all-targets)
+            targets+=(
+                ${CROSS_TARGETS[@]}
+                ${NATIVE_TARGETS[@]}
+            )
+            ;;
+        --targets=*)
+            # Convert "--targets=a,b,c" to targets=(a b c) bash array.
+            comma_targets=${arg#--targets=}
+            targets+=(${comma_targets//,/ })
+            ;;
+        --commit)
+            commit=yes
+            ;;
+        *)
+            usage
+            ;;
+    esac
+done
+
+for target in "${targets[@]}"; do
+    # Native and cross jobsets differ a bit. We'll have to pick the
+    # one based on target name:
+    if is_native $target; then
+        jobset=nixpkgs/trunk
+        job="stdenvBootstrapTools.${target}.build"
+    elif is_cross $target; then
+        jobset=nixpkgs/cross-trunk
+        job="bootstrapTools.${target}.build"
+    else
+        die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one."
+    fi
+
+    # 'nixpkgs' prefix where we will write new tarball hashes
+    case "$target" in
+        *linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;;
+        *darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;;
+        *freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;;
+        *) die "don't know where to put '$target'" ;;
+    esac
+
+    # We enforce s3 prefix for all targets here. This slightly differs
+    # from manual uploads targets where names were chosen inconsistently.
+    s3_prefix="stdenv/$target"
+
+    # resolve 'latest' build to the build 'id', construct the link.
+    latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest"
+    latest_build="$target.latest-build"
+    info "Fetching latest successful build from '${latest_build_uri}'"
+    curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build"
+    [[ $? -ne 0 ]] && die "Failed to fetch latest successful build"
+    latest_build_id=$(jq '.id' < "$latest_build")
+    [[ $? -ne 0 ]] && die "Did not find 'id' in latest build"
+    build_uri="https://hydra.nixos.org/build/${latest_build_id}"
+
+    # We pick oldest jobset evaluation and extract the 'nicpkgs' commit.
+    #
+    # We use oldest instead of latest to make the result more stable
+    # across unrelated 'nixpkgs' updates. Ideally two subsequent runs of
+    # this refresher should produce the same output (provided there are
+    # no bootstrapTools updates committed between the two runs).
+    oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build")
+    [[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build"
+    eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}"
+    eval_meta="$target.eval-meta"
+    info "Fetching oldest eval details from '${eval_uri}' (can take a minute)"
+    curl -s -H "Content-Type: application/json"  -L "${eval_uri}" > "$eval_meta"
+    [[ $? -ne 0 ]] && die "Failed to fetch eval metadata"
+    nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta")
+    [[ $? -ne 0 ]] && die "Failed to fetch revision"
+
+    # Extract the build paths out of the build metadata
+    drvpath=$(jq --raw-output '.drvpath' < "${latest_build}")
+    [[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build"
+    outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}")
+    [[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build"
+    build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}")
+    [[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build"
+    build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email)
+    [[ $? -ne 0 ]] && die "Failed to format timestamp"
+
+    info "Fetching bootstrap tools to calculate hashes from '${outpath}'"
+    nix-store --realize "$outpath"
+    [[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"
+
+    fnames=()
+
+    target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix"
+    info "Writing '${target_file}'"
+    {
+        # header
+        cat <<EOF
+# Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:
+# $ ./refresh-tarballs.bash --targets=${target}
+#
+# Metadata:
+# - nixpkgs revision: ${nixpkgs_revision}
+# - hydra build: ${latest_build_uri}
+# - resolved hydra build: ${build_uri}
+# - instantiated derivation: ${drvpath}
+# - output directory: ${outpath}
+# - build time: ${build_time}
+{
+EOF
+        for p in "${outpath}/on-server"/*; do
+            fname=$(basename "$p")
+            fnames+=("$fname")
+            case "$fname" in
+                bootstrap-tools.tar.xz) attr=bootstrapTools ;;
+                busybox) attr=$fname ;;
+                unpack.nar.xz) attr=unpack ;;
+                *) die "Don't know how to map '$fname' to attribute name. Please update me."
+            esac
+
+            executable_arg=
+            executable_nix=
+            if [[ -x "$p" ]]; then
+                executable_arg="--executable"
+                executable_nix="executable = true;"
+            fi
+            unpack_nix=
+            name_nix=
+            if [[ $fname = *.nar.xz ]]; then
+                unpack_nix="unpack = true;"
+                name_nix="name = \"${fname%.nar.xz}\";"
+                sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
+                [[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
+            else
+                sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
+                [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
+                sri=$(nix-hash --to-sri "sha256:$sha256")
+                [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form"
+            fi
+
+            # individual file entries
+            cat <<EOF
+  $attr = import <nix/fetchurl.nix> {
+    url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
+    hash = "${sri}";$(
+    [[ -n ${executable_nix} ]] && printf "\n    %s" "${executable_nix}"
+    [[ -n ${name_nix} ]]       && printf "\n    %s" "${name_nix}"
+    [[ -n ${unpack_nix} ]]     && printf "\n    %s" "${unpack_nix}"
+    )
+  };
+EOF
+    done
+    # footer
+    cat <<EOF
+}
+EOF
+    } > "${target_file}"
+
+        target_file_commit_msg=${target}.commit_message
+        cat > "$target_file_commit_msg" <<EOF
+${nixpkgs_prefix}: update ${target} bootstrap-files
+
+sha256sum of files to be uploaded:
+
+$(
+echo "$ sha256sum ${outpath}/on-server/*"
+sha256sum ${outpath}/on-server/*
+)
+
+Suggested commands to upload files to 'tarballs.nixos.org':
+
+    $ nix-store --realize ${outpath}
+    $ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision}
+    $ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./
+    $ sha256sum ${fnames[*]}
+    $ sha256sum ${outpath}/on-server/*
+EOF
+
+    cat "$target_file_commit_msg"
+    if [[ $commit == yes ]]; then
+        git commit "${target_file}" -F "$target_file_commit_msg"
+    else
+        info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg"
+    fi
+    rm -- "$target_file_commit_msg"
+
+    # delete temp files
+    rm -- "$latest_build" "$eval_meta"
+done

maintainers/scripts/build.nix

@@ -0,0 +1,80 @@
+{
+  maintainer,
+  localSystem ? {
+    system = args.system or builtins.currentSystem;
+  },
+  system ? localSystem.system,
+  crossSystem ? localSystem,
+  ...
+}@args:
+
+# based on update.nix
+# nix-build build.nix --argstr maintainer <yourname>
+
+# to build for aarch64-linux using boot.binfmt.emulatedSystems:
+# nix-build build.nix --argstr maintainer <yourname> --argstr system aarch64-linux
+
+let
+  # This avoids a common situation for maintainers, where due to Git's behavior of not tracking
+  # directories, they have an empty directory somewhere in `pkgs/by-name`. Because that directory
+  # exists, `pkgs/top-level/by-name-overlay.nix` picks it up and attempts to read `package.nix` out
+  # of it... which doesn't exist, since it's empty.
+  #
+  # We don't want to run the code below on every instantiation of `nixpkgs`, as the `pkgs/by-name`
+  # eval machinery is quite performance sensitive. So we use the internals of the `by-name` overlay
+  # to implement our own way to avoid an evaluation failure for this script.
+  #
+  # See <https://github.com/NixOS/nixpkgs/issues/338227> for more motivation for this code block.
+  overlay = self: super: {
+    _internalCallByNamePackageFile =
+      file: if builtins.pathExists file then super._internalCallByNamePackageFile file else null;
+  };
+
+  nixpkgsArgs =
+    removeAttrs args [
+      "maintainer"
+      "overlays"
+    ]
+    // {
+      overlays = args.overlays or [ ] ++ [ overlay ];
+    };
+
+  pkgs = import ./../../default.nix nixpkgsArgs;
+
+  maintainer_ = pkgs.lib.maintainers.${maintainer};
+  packagesWith =
+    cond: return: set:
+    (pkgs.lib.flatten (
+      pkgs.lib.mapAttrsToList (
+        name: pkg:
+        let
+          result = builtins.tryEval (
+            if pkgs.lib.isDerivation pkg && cond name pkg then
+              # Skip packages whose closure fails on evaluation.
+              # This happens for pkgs like `python27Packages.djangoql`
+              # that have disabled Python pkgs as dependencies.
+              builtins.seq pkg.outPath [ (return name pkg) ]
+            else if pkg.recurseForDerivations or false || pkg.recurseForRelease or false then
+              packagesWith cond return pkg
+            else
+              [ ]
+          );
+        in
+        if result.success then result.value else [ ]
+      ) set
+    ));
+in
+packagesWith (
+  name: pkg:
+  (
+    if builtins.hasAttr "meta" pkg && builtins.hasAttr "maintainers" pkg.meta then
+      (
+        if builtins.isList pkg.meta.maintainers then
+          builtins.elem maintainer_ pkg.meta.maintainers
+        else
+          maintainer_ == pkg.meta.maintainers
+      )
+    else
+      false
+  )
+) (name: pkg: pkg) pkgs

maintainers/scripts/check-by-name.sh

@@ -0,0 +1 @@
+../../ci/nixpkgs-vet.sh

maintainers/scripts/check-hydra-by-maintainer.nix

@@ -0,0 +1,68 @@
+{ maintainer }:
+let
+  pkgs = import ./../../default.nix {
+    config.allowAliases = false;
+  };
+  inherit (pkgs) lib;
+  maintainer_ = pkgs.lib.maintainers.${maintainer};
+  packagesWith =
+    cond: return: prefix: set:
+    (lib.flatten (
+      lib.mapAttrsToList (
+        name: pkg:
+        let
+          result = builtins.tryEval (
+            if lib.isDerivation pkg && cond name pkg then
+              # Skip packages whose closure fails on evaluation.
+              # This happens for pkgs like `python27Packages.djangoql`
+              # that have disabled Python pkgs as dependencies.
+              builtins.seq pkg.outPath [ (return "${prefix}${name}") ]
+            else if
+              pkg.recurseForDerivations or false || pkg.recurseForRelease or false
+            # then packagesWith cond return pkg
+            then
+              packagesWith cond return "${name}." pkg
+            else
+              [ ]
+          );
+        in
+        if result.success then result.value else [ ]
+      ) set
+    ));
+
+  packages = packagesWith (
+    name: pkg:
+    (
+      if builtins.hasAttr "meta" pkg && builtins.hasAttr "maintainers" pkg.meta then
+        (
+          if builtins.isList pkg.meta.maintainers then
+            builtins.elem maintainer_ pkg.meta.maintainers
+          else
+            maintainer_ == pkg.meta.maintainers
+        )
+      else
+        false
+    )
+  ) (name: name) "" pkgs;
+
+in
+pkgs.stdenv.mkDerivation {
+  name = "nixpkgs-update-script";
+  buildInputs = [ pkgs.hydra-check ];
+  buildCommand = ''
+    echo ""
+    echo "----------------------------------------------------------------"
+    echo ""
+    echo "nix-shell maintainers/scripts/check-hydra-by-maintainer.nix --argstr maintainer SuperSandro2000"
+    echo ""
+    echo "----------------------------------------------------------------"
+    exit 1
+  '';
+  shellHook = ''
+    unset shellHook # do not contaminate nested shells
+    echo "Please stand by"
+    echo nix-shell -p hydra-check --run "hydra-check ${builtins.concatStringsSep " " packages}"
+    nix-shell -p hydra-check --run "hydra-check ${builtins.concatStringsSep " " packages}"
+    exit $?
+  '';
+}

maintainers/scripts/check-maintainer-github-handles.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq parallel
+
+# Example how to work with the `lib.maintainers` attrset.
+# Can be used to check whether all user handles are still valid.
+
+set -o errexit -o noclobber -o nounset -o pipefail
+shopt -s failglob inherit_errexit
+
+function checkCommits {
+    local ret status tmp user
+    user="$1"
+    tmp=$(mktemp)
+    curl --silent -w "%{http_code}" \
+         "https://github.com/NixOS/nixpkgs/commits?author=$user" \
+         > "$tmp"
+    # the last line of tmp contains the http status
+    status=$(tail -n1 "$tmp")
+    ret=
+    case $status in
+        200) if <"$tmp" grep -i "no commits found" > /dev/null; then
+                 ret=1
+             else
+                 ret=0
+             fi
+             ;;
+        # because of github’s hard request limits, this can take some time
+        429) sleep 2
+             printf "."
+             checkCommits "$user"
+             ret=$?
+             ;;
+        *)   printf "BAD STATUS: $(tail -n1 "$tmp") for %s\n" "$user"; ret=1
+             ret=1
+             ;;
+    esac
+    rm "$tmp"
+    return $ret
+}
+export -f checkCommits
+
+function checkUser {
+    local user="$1"
+    local status=
+    status="$(curl --silent --head "https://github.com/${user}" | grep Status)"
+    # checks whether a user handle can be found on github
+    if [[ "$status" =~ 404 ]]; then
+        printf "%s\t\t\t\t%s\n" "$status" "$user"
+    # checks whether the user handle has any nixpkgs commits
+    elif checkCommits "$user"; then
+        printf "OK!\t\t\t\t%s\n" "$user"
+    else
+        printf "No Commits!\t\t\t%s\n" "$user"
+    fi
+}
+export -f checkUser
+
+# output the maintainers set as json
+# and filter out the github username of each maintainer (if it exists)
+# then check some at the same time
+nix-instantiate -A lib.maintainers --eval --strict --json \
+    | jq -r '.[]|.github|select(.)' \
+    | parallel -j5 checkUser
+
+# To check some arbitrary users:
+# parallel -j100 checkUser ::: "eelco" "profpatsch" "Profpatsch" "a"

maintainers/scripts/check-maintainer-usernames/.gitignore

@@ -0,0 +1 @@
+/target

maintainers/scripts/check-maintainer-usernames/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "check-maintainers"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+octocrab = "0.42.1"
+serde_json = "1.0.133"
+tokio = { version = "1.41.1", features = ["fs", "macros", "rt-multi-thread"] }

maintainers/scripts/check-maintainer-usernames/src/main.rs

@@ -0,0 +1,42 @@
+use std::{env, error::Error, io::{self, Write}};
+
+use tokio::fs;
+
+#[tokio::main]
+async fn main() {
+    real_main().await.unwrap();
+}
+
+async fn real_main() -> Result<(), Box<dyn Error>> {
+    let gh = octocrab::OctocrabBuilder::default()
+        .personal_token(env::var("GITHUB_PAT").expect("no GITHUB_PAT configured"))
+        .build()?;
+
+    // first run: nix-instantiate -A lib.maintainers --eval --strict --json > maintainers.json
+    let data: serde_json::Value = serde_json::from_str(&fs::read_to_string("maintainers.json").await?)?;
+
+    let data = data.as_object().unwrap();
+    let total = data.len();
+    for (i, (maintainer_name, v)) in data.iter().enumerate() {
+        print!("\r{i}/{total}");
+        io::stdout().flush()?;
+        let Some(github_id) = v.get("githubId") else {
+            println!("\rWARN: no id for {maintainer_name}");
+            continue;
+        };
+        let Some(github) = v.get("github") else {
+            println!("\rWARN: no username for {maintainer_name}");
+            continue;
+        };
+        let Ok(name) = gh.users_by_id(github_id.as_u64().unwrap()).profile().await else {
+            println!("\rWARN: error fetching {maintainer_name}");
+            continue;
+        };
+        let name = name.login;
+        if name.to_ascii_lowercase() != github.as_str().unwrap().to_ascii_lowercase() {
+            println!("\rERR: {maintainer_name} has github = {github}, but username really is '{name}'");
+        }
+    }
+
+    Ok(())
+}

maintainers/scripts/convert-to-import-cargo-lock.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -I nixpkgs=. -i bash -p "import ./maintainers/scripts/convert-to-import-cargo-lock" nix-prefetch-git
+
+convert-to-import-cargo-lock "$@"

maintainers/scripts/convert-to-import-cargo-lock/.gitignore

@@ -0,0 +1 @@
+/target

maintainers/scripts/convert-to-import-cargo-lock/Cargo.lock

@@ -0,0 +1,106 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "anyhow"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
+
+[[package]]
+name = "basic-toml"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e819b667739967cd44d308b8c7b71305d8bb0729ac44a248aa08f33d01950b4"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "convert-to-import-cargo-lock"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "basic-toml",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.51"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
+
+[[package]]
+name = "serde"
+version = "1.0.152"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.152"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.93"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"

maintainers/scripts/convert-to-import-cargo-lock/Cargo.toml

@@ -0,0 +1,12 @@
+[package]
+name = "convert-to-import-cargo-lock"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = { version = "1.0.69" }
+basic-toml = "0.1.1"
+serde = { version = "1.0.152", features = ["derive"] }
+serde_json = "1.0.93"

maintainers/scripts/convert-to-import-cargo-lock/default.nix

@@ -0,0 +1,17 @@
+with import ../../../. { };
+
+rustPlatform.buildRustPackage {
+  name = "convert-to-import-cargo-lock";
+
+  src = lib.cleanSourceWith {
+    src = ./.;
+    filter =
+      name: type:
+      let
+        name' = baseNameOf name;
+      in
+      name' != "default.nix" && name' != "target";
+  };
+
+  cargoLock.lockFile = ./Cargo.lock;
+}

maintainers/scripts/convert-to-import-cargo-lock/shell.nix

@@ -0,0 +1,17 @@
+{
+  pkgs ? import ../../.. { },
+}:
+let
+  inherit (pkgs) lib stdenv mkShell;
+in
+mkShell {
+  packages =
+    with pkgs;
+    [
+      rustc
+      cargo
+      clippy
+      rustfmt
+    ]
+    ++ lib.optional stdenv.hostPlatform.isDarwin pkgs.libiconv;
+}

maintainers/scripts/convert-to-import-cargo-lock/src/main.rs

@@ -0,0 +1,241 @@
+#![warn(clippy::pedantic)]
+#![allow(clippy::too_many_lines)]
+
+use anyhow::anyhow;
+use serde::Deserialize;
+use std::{collections::HashMap, env, fs, path::PathBuf, process::Command};
+
+#[derive(Deserialize)]
+struct CargoLock<'a> {
+    #[serde(rename = "package", borrow)]
+    packages: Vec<Package<'a>>,
+    metadata: Option<HashMap<&'a str, &'a str>>,
+}
+
+#[derive(Deserialize)]
+struct Package<'a> {
+    name: &'a str,
+    version: &'a str,
+    source: Option<&'a str>,
+    checksum: Option<&'a str>,
+}
+
+#[derive(Deserialize)]
+struct PrefetchOutput {
+    sha256: String,
+}
+
+fn main() -> anyhow::Result<()> {
+    let mut hashes = HashMap::new();
+
+    let attr_count = env::args().len() - 1;
+
+    for (i, attr) in env::args().skip(1).enumerate() {
+        println!("converting {attr} ({}/{attr_count})", i + 1);
+
+        convert(&attr, &mut hashes)?;
+    }
+
+    Ok(())
+}
+
+fn convert(attr: &str, hashes: &mut HashMap<String, String>) -> anyhow::Result<()> {
+    let package_path = nix_eval(format!("{attr}.meta.position"))?
+        .and_then(|p| p.split_once(':').map(|(f, _)| PathBuf::from(f)));
+
+    if package_path.is_none() {
+        eprintln!("can't automatically convert {attr}: doesn't exist");
+        return Ok(());
+    }
+
+    let package_path = package_path.unwrap();
+
+    if package_path.with_file_name("Cargo.lock").exists() {
+        eprintln!("skipping {attr}: already has a vendored Cargo.lock");
+        return Ok(());
+    }
+
+    let mut src = PathBuf::from(
+        String::from_utf8(
+            Command::new("nix-build")
+                .arg("-A")
+                .arg(format!("{attr}.src"))
+                .output()?
+                .stdout,
+        )?
+        .trim(),
+    );
+
+    if !src.exists() {
+        eprintln!("can't automatically convert {attr}: src doesn't exist (bad attr?)");
+        return Ok(());
+    } else if !src.metadata()?.is_dir() {
+        eprintln!("can't automatically convert {attr}: src isn't a directory");
+        return Ok(());
+    }
+
+    if let Some(mut source_root) = nix_eval(format!("{attr}.sourceRoot"))?.map(PathBuf::from) {
+        source_root = source_root.components().skip(1).collect();
+        src.push(source_root);
+    }
+
+    let cargo_lock_path = src.join("Cargo.lock");
+
+    if !cargo_lock_path.exists() {
+        eprintln!("can't automatically convert {attr}: src doesn't contain Cargo.lock");
+        return Ok(());
+    }
+
+    let cargo_lock_content = fs::read_to_string(cargo_lock_path)?;
+
+    let cargo_lock: CargoLock = basic_toml::from_str(&cargo_lock_content)?;
+
+    let mut git_dependencies = Vec::new();
+
+    for package in cargo_lock.packages.iter().filter(|p| {
+        p.source.is_some()
+            && p.checksum
+                .or_else(|| {
+                    cargo_lock
+                        .metadata
+                        .as_ref()?
+                        .get(
+                            format!("checksum {} {} ({})", p.name, p.version, p.source.unwrap())
+                                .as_str(),
+                        )
+                        .copied()
+                })
+                .is_none()
+    }) {
+        let (typ, original_url) = package
+            .source
+            .unwrap()
+            .split_once('+')
+            .expect("dependency should have well-formed source url");
+
+        if let Some(hash) = hashes.get(original_url) {
+            continue;
+        }
+
+        assert_eq!(
+            typ, "git",
+            "packages without checksums should be git dependencies"
+        );
+
+        let (mut url, rev) = original_url
+            .split_once('#')
+            .expect("git dependency should have commit");
+
+        // TODO: improve
+        if let Some((u, _)) = url.split_once('?') {
+            url = u;
+        }
+
+        let prefetch_output: PrefetchOutput = serde_json::from_slice(
+            &Command::new("nix-prefetch-git")
+                .args(["--url", url, "--rev", rev, "--quiet", "--fetch-submodules"])
+                .output()?
+                .stdout,
+        )?;
+
+        let output_hash = String::from_utf8(
+            Command::new("nix")
+                .args([
+                    "--extra-experimental-features",
+                    "nix-command",
+                    "hash",
+                    "to-sri",
+                    "--type",
+                    "sha256",
+                    &prefetch_output.sha256,
+                ])
+                .output()?
+                .stdout,
+        )?;
+
+        let hash = output_hash.trim().to_string();
+
+        git_dependencies.push((
+            format!("{}-{}", package.name, package.version),
+            output_hash.trim().to_string().clone(),
+        ));
+
+        hashes.insert(original_url.to_string(), hash);
+    }
+
+    fs::write(
+        package_path.with_file_name("Cargo.lock"),
+        cargo_lock_content,
+    )?;
+
+    let mut package_lines: Vec<_> = fs::read_to_string(&package_path)?
+        .lines()
+        .map(String::from)
+        .collect();
+
+    let (cargo_deps_line_index, cargo_deps_line) = package_lines
+        .iter_mut()
+        .enumerate()
+        .find(|(_, l)| {
+            l.trim_start().starts_with("cargoHash") || l.trim_start().starts_with("cargoSha256")
+        })
+        .expect("package should contain cargoHash/cargoSha256");
+
+    let spaces = " ".repeat(cargo_deps_line.len() - cargo_deps_line.trim_start().len());
+
+    if git_dependencies.is_empty() {
+        *cargo_deps_line = format!("{spaces}cargoLock.lockFile = ./Cargo.lock;");
+    } else {
+        *cargo_deps_line = format!("{spaces}cargoLock = {{");
+
+        let mut index_iter = cargo_deps_line_index + 1..;
+
+        package_lines.insert(
+            index_iter.next().unwrap(),
+            format!("{spaces}  lockFile = ./Cargo.lock;"),
+        );
+
+        package_lines.insert(
+            index_iter.next().unwrap(),
+            format!("{spaces}  outputHashes = {{"),
+        );
+
+        for ((dep, hash), index) in git_dependencies.drain(..).zip(&mut index_iter) {
+            package_lines.insert(index, format!("{spaces}    {dep:?} = {hash:?};"));
+        }
+
+        package_lines.insert(index_iter.next().unwrap(), format!("{spaces}  }};"));
+        package_lines.insert(index_iter.next().unwrap(), format!("{spaces}}};"));
+    }
+
+    if package_lines.last().map(String::as_str) != Some("") {
+        package_lines.push(String::new());
+    }
+
+    fs::write(package_path, package_lines.join("\n"))?;
+
+    Ok(())
+}
+
+fn nix_eval(attr: impl AsRef<str>) -> anyhow::Result<Option<String>> {
+    let output = String::from_utf8(
+        Command::new("nix-instantiate")
+            .args(["--eval", "-A", attr.as_ref()])
+            .output()?
+            .stdout,
+    )?;
+
+    let trimmed = output.trim();
+
+    if trimmed.is_empty() || trimmed == "null" {
+        Ok(None)
+    } else {
+        Ok(Some(
+            trimmed
+                .strip_prefix('"')
+                .and_then(|p| p.strip_suffix('"'))
+                .ok_or_else(|| anyhow!("couldn't parse nix-instantiate output: {output:?}"))?
+                .to_string(),
+        ))
+    }
+}

maintainers/scripts/copy-tarballs.pl

@@ -0,0 +1,290 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i perl -p perl perlPackages.NetAmazonS3 perlPackages.FileSlurp perlPackages.JSON perlPackages.LWPProtocolHttps nix
+
+# This command uploads tarballs to tarballs.nixos.org, the
+# content-addressed cache used by fetchurl as a fallback for when
+# upstream tarballs disappear or change. Usage:
+#
+# 1) To upload one or more files:
+#
+#    $ copy-tarballs.pl --file /path/to/tarball.tar.gz
+#
+# 2) To upload all files obtained via calls to fetchurl in a Nix derivation:
+#
+#    $ copy-tarballs.pl --expr '(import <nixpkgs> {}).hello'
+
+use strict;
+use warnings;
+use File::Basename;
+use File::Path;
+use File::Slurp;
+use JSON;
+use Net::Amazon::S3;
+
+sub usage {
+    die "Syntax: $0 [--dry-run] [--exclude REGEXP] [--expr EXPR | --file FILES...]\n";
+}
+
+sub computeFixedOutputPath {
+    my ($name, $algo, $hash) = @_;
+    my $expr = <<'EXPR';
+{ name, outputHashAlgo, outputHash }:
+builtins.toString (derivation {
+  inherit name outputHashAlgo outputHash;
+  builder = "false";
+  system = "dontcare";
+  outputHashMode = "flat";
+})
+EXPR
+    open(my $fh, "-|",
+        "nix-instantiate",
+        "--eval",
+        "--strict",
+        "-E", $expr,
+        "--argstr", "name", $name,
+        "--argstr", "outputHashAlgo", $algo,
+        "--argstr", "outputHash", $hash) or die "Failed to run nix-instantiate: $!";
+
+    my $storePathJson = <$fh>;
+    chomp $storePathJson;
+    my $storePath = decode_json($storePathJson);
+    close $fh;
+    return $storePath;
+}
+
+sub nixHash {
+    my ($algo, $base16, $path) = @_;
+    open(my $fh, "-|",
+        "nix-hash",
+        "--type", $algo,
+        "--flat",
+        ($base16 ? "--base16" : ()),
+        $path) or die "Failed to run nix-hash: $!";
+    my $hash = <$fh>;
+    chomp $hash;
+    return $hash;
+}
+
+my $dryRun = 0;
+my $expr;
+my @fileNames;
+my $exclude;
+
+while (@ARGV) {
+    my $flag = shift @ARGV;
+
+    if ($flag eq "--expr") {
+        $expr = shift @ARGV or die "--expr requires an argument";
+    } elsif ($flag eq "--file") {
+        @fileNames = @ARGV;
+        last;
+    } elsif ($flag eq "--dry-run") {
+        $dryRun = 1;
+    } elsif ($flag eq "--exclude") {
+        $exclude = shift @ARGV or die "--exclude requires an argument";
+    } else {
+        usage();
+    }
+}
+
+my $bucket;
+
+if (not defined $ENV{DEBUG}) {
+    # S3 setup.
+    my $aws_access_key_id = $ENV{'AWS_ACCESS_KEY_ID'} or die "AWS_ACCESS_KEY_ID not set\n";
+    my $aws_secret_access_key = $ENV{'AWS_SECRET_ACCESS_KEY'} or die "AWS_SECRET_ACCESS_KEY not set\n";
+
+    my $s3 = Net::Amazon::S3->new(
+        { aws_access_key_id     => $aws_access_key_id,
+          aws_secret_access_key => $aws_secret_access_key,
+          retry                 => 1,
+          host                  => "s3-eu-west-1.amazonaws.com",
+        });
+
+    $bucket = $s3->bucket("nixpkgs-tarballs") or die;
+}
+
+my $doWrite = 0;
+my $cacheFile = ($ENV{"HOME"} or die "\$HOME is not set") . "/.cache/nix/copy-tarballs";
+my %cache;
+$cache{$_} = 1 foreach read_file($cacheFile, err_mode => 'quiet', chomp => 1);
+$doWrite = 1;
+
+END() {
+    File::Path::mkpath(dirname($cacheFile), 0, 0755);
+    write_file($cacheFile, map { "$_\n" } keys %cache) if $doWrite;
+}
+
+sub alreadyMirrored {
+    my ($algo, $hash) = @_;
+    my $key = "$algo/$hash";
+    return 1 if defined $cache{$key};
+    my $res = defined $bucket->get_key($key);
+    $cache{$key} = 1 if $res;
+    return $res;
+}
+
+sub uploadFile {
+    my ($fn, $name) = @_;
+
+    my $md5_16 = nixHash("md5", 0, $fn) or die;
+    my $sha1_16 = nixHash("sha1", 0, $fn) or die;
+    my $sha256_32 = nixHash("sha256", 1, $fn) or die;
+    my $sha256_16 = nixHash("sha256", 0, $fn) or die;
+    my $sha512_32 = nixHash("sha512", 1, $fn) or die;
+    my $sha512_16 = nixHash("sha512", 0, $fn) or die;
+
+    my $mainKey = "sha512/$sha512_16";
+
+    # Create redirects from the other hash types.
+    sub redirect {
+        my ($name, $dest) = @_;
+        #print STDERR "linking $name to $dest...\n";
+        $bucket->add_key($name, "", {
+            'x-amz-website-redirect-location' => "/" . $dest,
+            'x-amz-acl' => "public-read"
+        })
+            or die "failed to create redirect from $name to $dest\n";
+        $cache{$name} = 1;
+    }
+    redirect "md5/$md5_16", $mainKey;
+    redirect "sha1/$sha1_16", $mainKey;
+    redirect "sha256/$sha256_32", $mainKey;
+    redirect "sha256/$sha256_16", $mainKey;
+    redirect "sha512/$sha512_32", $mainKey;
+
+    # Upload the file as sha512/<hash-in-base-16>.
+    print STDERR "uploading $fn to $mainKey...\n";
+    $bucket->add_key_filename($mainKey, $fn, {
+        'x-amz-meta-original-name' => $name,
+        'x-amz-acl' => "public-read"
+    })
+        or die "failed to upload $fn to $mainKey\n";
+    $cache{$mainKey} = 1;
+}
+
+if (scalar @fileNames) {
+    my $res = 0;
+    foreach my $fn (@fileNames) {
+        eval {
+            if (alreadyMirrored("sha512", nixHash("sha512", 0, $fn))) {
+                print STDERR "$fn is already mirrored\n";
+            } else {
+                uploadFile($fn, basename $fn);
+            }
+        };
+        if ($@) {
+            warn "$@";
+            $res = 1;
+        }
+    }
+    exit $res;
+}
+
+elsif (defined $expr) {
+
+    # Evaluate find-tarballs.nix.
+    my $pid = open(JSON, "-|", "nix-instantiate", "--eval", "--json", "--strict",
+                   "<nixpkgs/maintainers/scripts/find-tarballs.nix>",
+                   "--arg", "expr", $expr);
+    my $stdout = <JSON>;
+    waitpid($pid, 0);
+    die "$0: evaluation failed\n" if $?;
+    close JSON;
+
+    my $fetches = decode_json($stdout);
+
+    print STDERR "evaluation returned ", scalar(@{$fetches}), " tarballs\n";
+
+    # Check every fetchurl call discovered by find-tarballs.nix.
+    my $mirrored = 0;
+    my $have = 0;
+    foreach my $fetch (sort { $a->{urls}->[0] cmp $b->{urls}->[0] } @{$fetches}) {
+        my $urls = $fetch->{urls};
+        my $algo = $fetch->{type};
+        my $hash = $fetch->{hash};
+        my $name = $fetch->{name};
+        my $isPatch = $fetch->{isPatch};
+
+        if ($isPatch) {
+            print STDERR "skipping $urls->[0] (support for patches is missing)\n";
+            next;
+        }
+
+        if ($hash =~ /^([a-z0-9]+)-([A-Za-z0-9+\/=]+)$/) {
+            $algo = $1;
+            open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", $hash) or die;
+            $hash = <$fh>;
+            close $fh;
+            chomp $hash;
+        }
+
+        next unless $algo =~ /^[a-z0-9]+$/;
+
+        # Convert non-SRI base-64 to base-16.
+        if ($hash =~ /^[A-Za-z0-9+\/=]+$/) {
+            open(my $fh, "-|", "nix", "--extra-experimental-features", "nix-command", "hash", "convert", "--to", "base16", "--hash-algo", $algo, $hash) or die;
+            $hash = <$fh>;
+            close $fh;
+            chomp $hash;
+        }
+
+        my $storePath = computeFixedOutputPath($name, $algo, $hash);
+
+        for my $url (@$urls) {
+            if (defined $ENV{DEBUG}) {
+                print "$url $algo $hash\n";
+                next;
+            }
+
+            if ($url !~ /^http:/ && $url !~ /^https:/ && $url !~ /^ftp:/ && $url !~ /^mirror:/) {
+                print STDERR "skipping $url (unsupported scheme)\n";
+                next;
+            }
+
+            next if defined $exclude && $url =~ /$exclude/;
+
+            if (alreadyMirrored($algo, $hash)) {
+                $have++;
+                last;
+            }
+
+            print STDERR "mirroring $url ($storePath, $algo, $hash)...\n";
+
+
+            if ($dryRun) {
+                $mirrored++;
+                last;
+            }
+            my $isValidPath = system("nix-store", "-r", $storePath) == 0;
+
+            # Otherwise download the file using nix-prefetch-url.
+            if (!$isValidPath) {
+                $ENV{QUIET} = 1;
+                $ENV{PRINT_PATH} = 1;
+                my $fh;
+                my $pid = open($fh, "-|", "nix-prefetch-url", "--type", $algo, $url, $hash) or die;
+                waitpid($pid, 0) or die;
+                if ($? != 0) {
+                    print STDERR "failed to fetch $url: $?\n";
+                    next;
+                }
+                <$fh>; my $storePath2 = <$fh>; chomp $storePath2;
+                if ($storePath ne $storePath2) {
+                    warn "strange: $storePath != $storePath2\n";
+                    next;
+                }
+            }
+
+            uploadFile($storePath, $url);
+            $mirrored++;
+            last;
+        }
+    }
+
+    print STDERR "mirrored $mirrored files, already have $have files\n";
+}
+
+else {
+    usage();
+}

maintainers/scripts/db-to-md.sh

@@ -0,0 +1,88 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -I nixpkgs=. -i bash -p pandoc
+
+# This script is temporarily needed while we transition the manual to
+# CommonMark. It converts DocBook files into our CommonMark flavour.
+
+debug=
+files=()
+
+while [ "$#" -gt 0 ]; do
+    i="$1"; shift 1
+    case "$i" in
+      --debug)
+        debug=1
+        ;;
+      *)
+        files+=("$i")
+        ;;
+    esac
+done
+
+echo "WARNING: This is an experimental script and might not preserve all formatting." > /dev/stderr
+echo "Please report any issues you discover." > /dev/stderr
+
+outExtension="md"
+if [[ $debug ]]; then
+    outExtension="json"
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+# NOTE: Keep in sync with Nixpkgs manual (/doc/Makefile).
+# TODO: Remove raw-attribute when we can get rid of DocBook altogether.
+pandoc_commonmark_enabled_extensions=+attributes+fenced_divs+footnotes+bracketed_spans+definition_lists+pipe_tables+raw_attribute
+targetLang="commonmark${pandoc_commonmark_enabled_extensions}+smart"
+if [[ $debug ]]; then
+    targetLang=json
+fi
+pandoc_flags=(
+    # Not needed:
+    # - diagram-generator.lua (we do not support that in NixOS manual to limit dependencies)
+    # - media extraction (was only required for diagram generator)
+    # - myst-reader/roles.lua (only relevant for MyST → DocBook)
+    # - link-manpages.lua (links should only be added to display output)
+    # - docbook-writer/rst-roles.lua (only relevant for → DocBook)
+    # - docbook-writer/labelless-link-is-xref.lua (only relevant for → DocBook)
+    "--lua-filter=$DIR/../../doc/build-aux/pandoc-filters/docbook-reader/citerefentry-to-rst-role.lua"
+    "--lua-filter=$DIR/../../doc/build-aux/pandoc-filters/myst-writer/roles.lua"
+    "--lua-filter=$DIR/doc/unknown-code-language.lua"
+    -f docbook
+    -t "$targetLang"
+    --tab-stop=2
+    --wrap=none
+)
+
+for file in "${files[@]}"; do
+    if [[ ! -f "$file" ]]; then
+        echo "db-to-md.sh: $file does not exist" > /dev/stderr
+        exit 1
+    else
+    rootElement=$(xmllint --xpath 'name(//*)' "$file")
+
+    if [[ $rootElement = chapter ]]; then
+        extension=".chapter.$outExtension"
+    elif [[ $rootElement = section ]]; then
+        extension=".section.$outExtension"
+    else
+        echo "db-to-md.sh: $file contains an unsupported root element $rootElement" > /dev/stderr
+        exit 1
+    fi
+
+    outFile="${file%".section.xml"}"
+    outFile="${outFile%".chapter.xml"}"
+    outFile="${outFile%".xml"}$extension"
+    temp1=$(mktemp)
+    $DIR/doc/escape-code-markup.py "$file" "$temp1"
+    if [[ $debug ]]; then
+        echo "Converted $file to $temp1" > /dev/stderr
+    fi
+    temp2=$(mktemp)
+    $DIR/doc/replace-xrefs-by-empty-links.py "$temp1" "$temp2"
+    if [[ $debug ]]; then
+        echo "Converted $temp1 to $temp2" > /dev/stderr
+    fi
+    pandoc "$temp2" -o "$outFile" "${pandoc_flags[@]}"
+    echo "Converted $file to $outFile" > /dev/stderr
+  fi
+done

maintainers/scripts/debian-patches.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Download patches from debian project
+# Usage $0 debian-patches.txt debian-patches.nix
+# An example input and output files can be found in tools/graphics/plotutils
+
+DEB_URL=https://sources.debian.org/data/main
+declare -a deb_patches
+mapfile -t deb_patches < $1
+
+# First letter
+deb_prefix="${deb_patches[0]:0:1}"
+prefix="${DEB_URL}/${deb_prefix}/${deb_patches[0]}/debian/patches"
+
+if [[ -n "$2" ]]; then
+    exec 1> $2
+fi
+
+cat <<EOF
+# Generated by $(basename $0) from $(basename $1)
+let
+  prefix = "${prefix}";
+in
+[
+EOF
+for ((i=1;i < ${#deb_patches[@]}; ++i)); do
+    url="${prefix}/${deb_patches[$i]}"
+    sha256=$(nix-prefetch-url $url)
+    echo "  {"
+    echo "    url = \"\${prefix}/${deb_patches[$i]}\";"
+    echo "    sha256 = \"$sha256\";"
+    echo "  }"
+done
+echo "]"

maintainers/scripts/dep-licenses.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+attr=$1
+
+: ${NIXPKGS=/etc/nixos/nixpkgs}
+
+tmp=$(mktemp --tmpdir -d nixpkgs-dep-license.XXXXXX)
+
+exitHandler() {
+    exitCode=$?
+    rm -rf "$tmp"
+    return $exitCode
+}
+
+trap "exitHandler" EXIT
+
+# fetch the trace and the drvPath of the attribute.
+nix-instantiate $NIXPKGS -A $attr --show-trace > "$tmp/drvPath" 2> "$tmp/trace" || {
+  cat 1>&2 - "$tmp/trace" <<EOF
+An error occurred while evaluating $attr.
+EOF
+  exit 1
+}
+
+# generate a sed script based on the trace output.
+sed '
+  \,@:.*:@, {
+    # \1  *.drv file
+    # \2  License terms
+    s,.*@:drv:\(.*\):\(.*\):@.*,s!\1!\1: \2!; t;,
+    s!Str(\\\"\([^,]*\)\\\",\[\])!\1!g
+    b
+  }
+  d
+' "$tmp/trace" > "$tmp/filter.sed"
+
+if test $(wc -l "$tmp/filter.sed" | sed 's/ .*//') == 0; then
+  echo 1>&2 "
+No derivation mentionned in the stack trace.  Either your derivation does
+not use stdenv.mkDerivation or you forgot to use the stdenv adapter named
+traceDrvLicenses.
+
+-  defaultStdenv = allStdenvs.stdenv;
++  defaultStdenv = traceDrvLicenses allStdenvs.stdenv;
+"
+  exit 1
+fi
+
+
+# remove all dependencies which are using stdenv.mkDerivation
+echo '
+d
+' >> "$tmp/filter.sed"
+
+nix-store -q --tree $(cat "$tmp/drvPath") | sed -f "$tmp/filter.sed"
+
+exit 0;

maintainers/scripts/doc/escape-code-markup.py

@@ -0,0 +1,97 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -I nixpkgs=channel:nixos-unstable -i python3 -p python3 -p python3.pkgs.lxml
+
+"""
+Pandoc will strip any markup within code elements so
+let’s escape them so that they can be handled manually.
+"""
+
+import lxml.etree as ET
+import re
+import sys
+
+def replace_element_by_text(el: ET.Element, text: str) -> None:
+    """
+    Author: bernulf
+    Source: https://stackoverflow.com/a/10520552/160386
+    SPDX-License-Identifier: CC-BY-SA-3.0
+    """
+    text = text + (el.tail or "")
+    parent = el.getparent()
+    if parent is not None:
+        previous = el.getprevious()
+        if previous is not None:
+            previous.tail = (previous.tail or "") + text
+        else:
+            parent.text = (parent.text or "") + text
+        parent.remove(el)
+
+DOCBOOK_NS = "http://docbook.org/ns/docbook"
+
+# List of elements that pandoc’s DocBook reader strips markup from.
+# https://github.com/jgm/pandoc/blob/master/src/Text/Pandoc/Readers/DocBook.hs
+code_elements = [
+    # CodeBlock
+    "literallayout",
+    "screen",
+    "programlisting",
+    # Code (inline)
+    "classname",
+    "code",
+    "filename",
+    "envar",
+    "literal",
+    "computeroutput",
+    "prompt",
+    "parameter",
+    "option",
+    "markup",
+    "wordasword",
+    "command",
+    "varname",
+    "function",
+    "type",
+    "symbol",
+    "constant",
+    "userinput",
+    "systemitem",
+]
+
+XMLNS_REGEX = re.compile(r'\s+xmlns(?::[^=]+)?="[^"]*"')
+ROOT_ELEMENT_REGEX = re.compile(r'^\s*<[^>]+>')
+
+def remove_xmlns(match: re.Match) -> str:
+    """
+    Removes xmlns attributes.
+
+    Expects a match containing an opening tag.
+    """
+    return XMLNS_REGEX.sub('', match.group(0))
+
+if __name__ == '__main__':
+    assert len(sys.argv) >= 3, "usage: escape-code-markup.py <input> <output>"
+
+    tree = ET.parse(sys.argv[1])
+    name_predicate = " or ".join([f"local-name()='{el}'" for el in code_elements])
+
+    for markup in tree.xpath(f"//*[({name_predicate}) and namespace-uri()='{DOCBOOK_NS}']/*"):
+        text = ET.tostring(markup, encoding=str)
+
+        # tostring adds xmlns attributes to the element we want to stringify
+        # as if it was supposed to be usable standalone.
+        # We are just converting it to CDATA so we do not care.
+        # Let’s strip the namespace declarations to keep the code clean.
+        #
+        # Note that this removes even namespaces that were potentially
+        # in the original file. Though, that should be very rare –
+        # most of the time, we will stringify empty DocBook elements
+        # like <xref> or <co> or, at worst, <link> with xlink:href attribute.
+        #
+        # Also note that the regex expects the root element to be first
+        # thing in the string. But that should be fine, the tostring method
+        # does not produce XML declaration or doctype by default.
+        text = ROOT_ELEMENT_REGEX.sub(remove_xmlns, text)
+
+        replace_element_by_text(markup, text)
+
+    tree.write(sys.argv[2])

maintainers/scripts/doc/list-systemd-manpages.zsh

@@ -0,0 +1,33 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i zsh -p zsh
+set -euo pipefail
+
+# cd into nixpkgs' root, get the store path of `systemd.man`
+cd "$(dirname "$0")/../../.."
+SYSTEMD_MAN_DIR="$(nix-build -A systemd.man)/share/man"
+
+# For each manual section
+for section in {1..8}; do
+  sec_dir="${SYSTEMD_MAN_DIR}/man${section}"
+
+  # skip section 3 (library calls)
+  ! [[ $section -eq 3 ]] || continue
+
+  # for each manpage in that section (potentially none)
+  for manpage in ${sec_dir}/*(N); do
+    # strip the directory prefix and (compressed) manpage suffix
+    page="$(basename "$manpage" ".${section}.gz")"
+
+    # if this is the manpage of a service unit
+    if [[ "$page" =~ ".*\.service" ]]; then
+     # ... and a manpage exists without the `.service` suffix
+     potential_alias="${sec_dir}/${page%\.service}.${section}.gz"
+     ! [[ -e "${potential_alias}" &&
+              # ... which points to the same file, then skip
+              "$(gunzip -c "${potential_alias}")" == ".so ${page}.${section}" ]] || continue
+    fi
+
+    # else produce a JSON fragment, with the link to the upstream manpage (as HTML)
+    echo "  \"${page}(${section})\": \"https://www.freedesktop.org/software/systemd/man/${page}.html\","
+  done
+done

maintainers/scripts/doc/replace-xrefs-by-empty-links.py

@@ -0,0 +1,32 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -I nixpkgs=channel:nixos-unstable -i python3 -p python3 -p python3.pkgs.lxml
+
+"""
+Pandoc will try to resolve xrefs and replace them with regular links.
+let’s replace them with links with empty labels which MyST
+and our pandoc filters recognize as cross-references.
+"""
+
+import lxml.etree as ET
+import sys
+
+XLINK_NS = "http://www.w3.org/1999/xlink"
+
+ns = {
+    "db": "http://docbook.org/ns/docbook",
+}
+
+
+if __name__ == '__main__':
+    assert len(sys.argv) >= 3, "usage: replace-xrefs-by-empty-links.py <input> <output>"
+
+    tree = ET.parse(sys.argv[1])
+    for xref in tree.findall(".//db:xref", ns):
+        text = ET.tostring(xref, encoding=str)
+        parent = xref.getparent()
+        link = parent.makeelement('link')
+        target_name = xref.get("linkend")
+        link.set(f"{{{XLINK_NS}}}href", f"#{target_name}")
+        parent.replace(xref, link)
+
+    tree.write(sys.argv[2])

maintainers/scripts/doc/unknown-code-language.lua

@@ -0,0 +1,12 @@
+--[[
+Adds “unknown” class to CodeBlock AST nodes without any classes.
+
+This will cause Pandoc to use fenced code block, which we prefer.
+]]
+
+function CodeBlock(elem)
+  if #elem.classes == 0 then
+    elem.classes:insert('unknown')
+    return elem
+  end
+end

maintainers/scripts/eval-release.nix

@@ -0,0 +1,40 @@
+# Evaluate `release.nix' like Hydra would. Too bad nix-instantiate can't to do this.
+
+let
+  inherit (import ../../lib) isDerivation mapAttrs;
+
+  trace = if builtins.getEnv "VERBOSE" == "1" then builtins.trace else (x: y: y);
+
+  rel = removeAttrs (import ../../pkgs/top-level/release.nix { }) [
+    "tarball"
+    "unstable"
+  ];
+
+  # Add the ‘recurseForDerivations’ attribute to ensure that
+  # nix-instantiate recurses into nested attribute sets.
+  recurse =
+    path: attrs:
+    if (builtins.tryEval attrs).success then
+      if isDerivation attrs then
+        if (builtins.tryEval attrs.drvPath).success then
+          { inherit (attrs) name drvPath; }
+        else
+          { failed = true; }
+      else if attrs == null then
+        { }
+      else
+        {
+          recurseForDerivations = true;
+        }
+        // mapAttrs (
+          n: v:
+          let
+            path' = path ++ [ n ];
+          in
+          trace path' (recurse path' v)
+        ) attrs
+    else
+      { };
+
+in
+recurse [ ] rel

maintainers/scripts/eval-release.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+if [[ -z "$VERBOSE" ]]; then
+  echo "You may set VERBOSE=1 to see debug output or to any other non-empty string to make this script completely silent"
+fi
+unset HOME NIXPKGS_CONFIG # Force empty config
+
+# With the default heap size (380MB), nix-instantiate fails:
+# Too many heap sections: Increase MAXHINCR or MAX_HEAP_SECTS
+export GC_INITIAL_HEAP_SIZE=${GC_INITIAL_HEAP_SIZE:-2000000000} # 2GB
+nix-instantiate --strict --eval-only --xml --show-trace "$(dirname "$0")"/eval-release.nix 2>&1 > /dev/null

maintainers/scripts/feature-freeze-teams.pl

@@ -0,0 +1,96 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i perl -p perl -p perlPackages.JSON perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.TermReadKey
+
+# This script generates a list of teams to ping for the Feature Freeze announcement on Discourse.
+# It's intended to be used by Release Managers before creating such posts.
+#
+# The script interactively reads a GitHub username and a corresponding GitHub Personal Access token.
+# This is required to access the GitHub Teams API so the token needs at least the read:org privilege.
+
+## no critic (InputOutput::RequireCheckedSyscalls, InputOutput::ProhibitBacktickOperators)
+use strict;
+use warnings;
+use Carp;
+use Cwd 'abs_path';
+use File::Basename;
+use JSON qw(decode_json);
+use LWP::UserAgent;
+use Term::ReadKey qw(ReadLine ReadMode);
+
+sub github_team_members {
+    my ($team_name, $username, $token) = @_;
+    my @ret;
+
+    my $req = HTTP::Request->new('GET', "https://api.github.com/orgs/NixOS/teams/$team_name/members", [ 'Accept' => 'application/vnd.github.v3+json' ]);
+    $req->authorization_basic($username, $token);
+    my $response = LWP::UserAgent->new->request($req);
+
+    if ($response->is_success) {
+        my $content = decode_json($response->decoded_content);
+        foreach (@{$content}) {
+            push @ret, $_->{'login'};
+        }
+    } else {
+        print {*STDERR} "!! Requesting members of GitHub Team '$team_name' failed: " . $response->status_line;
+    }
+
+    return \@ret;
+}
+
+# Read GitHub credentials
+print {*STDERR} 'GitHub username: ';
+my $github_user = ReadLine(0);
+ReadMode('noecho');
+print {*STDERR} 'GitHub personal access token (no echo): ';
+my $github_token = ReadLine(0);
+ReadMode('restore');
+print {*STDERR} "\n";
+chomp $github_user;
+chomp $github_token;
+
+# Read nix output
+my $nix_version = `nix --version`;
+my $out;
+my $lib_path = abs_path(dirname(__FILE__)) . '../../../lib';
+if ($nix_version =~ m/2[.]3[.]/msx) {
+    $out = `nix eval --json '(import $lib_path).teams'` || croak 'nix eval failed';
+} else {
+    $out = `nix --extra-experimental-features nix-command eval --json --impure --expr '(import $lib_path).teams'` || croak('nix eval failed');
+}
+my $data = decode_json($out);
+
+# Process teams
+print {*STDERR} "\n";
+while (my ($team_nix_key, $team_config) = each %{$data}) {
+    # Ignore teams that don't want to be or can't be pinged
+    if (not defined $team_config->{enableFeatureFreezePing} or not $team_config->{enableFeatureFreezePing}) {
+        next;
+    }
+    if (not defined $team_config->{shortName}) {
+        print {*STDERR} "!! The team with the nix key '$team_nix_key' has no shortName set - ignoring";
+        next;
+    }
+    #  Team name
+    print {*STDERR} "$team_config->{shortName}:";
+    # GitHub Teams
+    my @github_members;
+    if (defined $team_config->{github}) {
+        print {*STDERR} " \@NixOS/$team_config->{github}";
+        push @github_members, @{github_team_members($team_config->{github}, $github_user, $github_token)};
+    }
+    my %github_members = map { $_ => 1 } @github_members;
+    # Members
+    if (defined $team_config->{members}) {
+        foreach (@{$team_config->{members}}) {
+            my %user = %{$_};
+            my $github_handle = $user{'github'};
+            # Ensure we don't ping team members twice (as team member and directly)
+            if (defined $github_members{$github_handle}) {
+                next;
+            }
+            print {*STDERR} " \@$github_handle";
+        }
+    }
+
+    print {*STDERR} "\n";
+}

maintainers/scripts/fetch-kde-qt.sh

@@ -0,0 +1,196 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils findutils gnused nix wget
+
+set -efuo pipefail
+export LC_COLLATE=C # fix sort order
+
+# parse files and folders from https://download.kde.org/ and https://download.qt.io/
+# you can override this function in fetch.sh
+function PARSE_INDEX() {
+    cat "$1" | grep -o -E -e '\s+href="[^"]+\.tar\.xz"' -e '\s+href="[-_a-zA-Z0-9]+/"' | cut -d'"' -f2 | sort | uniq
+}
+
+if [ $# != 1 ]; then
+    echo "example use:" >&2
+    echo "cd nixpkgs/" >&2
+    echo "./maintainers/scripts/fetch-kde-qt.sh pkgs/development/libraries/qt-5/5.12" >&2
+    exit 1
+fi
+
+if ! echo "$1" | grep -q '^pkgs/'; then
+    echo "error: path argument must start with pkgs/" >&2
+    exit 1
+fi
+
+# need absolute path for the pushd-popd block
+if [ -f "$1" ]; then
+    echo "ok: using fetchfile $1"
+    fetchfilerel="$1"
+    fetchfile="$(readlink -f "$fetchfilerel")" # resolve absolute path
+    basedir="$(dirname "$fetchfile")"
+    basedirrel="$(dirname "$fetchfilerel")"
+elif [ -d "$1" ]; then
+    echo "ok: using basedir $1"
+    basedirrel="$1"
+    basedir="$(readlink -f "$basedirrel")" # resolve absolute path
+    if ! [ -d "$basedir" ]; then
+        basedir="$(dirname "$basedir")"
+    fi
+    fetchfile="$basedir/fetch.sh"
+else
+    echo 'error: $1 must be file or dir' >&2
+    exit 1
+fi
+
+pkgname=$(basename "$basedir")
+SRCS="$basedir/srcs.nix"
+srcsrel="$basedirrel/srcs.nix"
+
+source "$fetchfile"
+
+if [ -n "$WGET_ARGS" ]; then # old format
+    BASE_URL="${WGET_ARGS[0]}" # convert to new format
+    # validate
+    if ! echo "$BASE_URL" | grep -q -E '^(http|https|ftp)://'; then
+        printf 'error: from WGET_ARGS, converted invalid BASE_URL: %q\n' "$BASE_URL" >&2
+        exit 1
+    fi
+    printf 'ok: from WGET_ARGS, converted BASE_URL: %q\n' "$BASE_URL"
+elif [ -n "$BASE_URL" ]; then # new format
+    :
+else
+    echo "error: fetch.sh must set either WGET_ARGS or BASE_URL" >&2
+    exit 1
+fi
+
+tmptpl=tmp.fetch-kde-qt.$pkgname.XXXXXXXXXX
+
+tmp=$(mktemp -d $tmptpl)
+pushd $tmp >/dev/null
+echo "tempdir is $tmp"
+
+wgetargs='--quiet --show-progress'
+#wgetargs='' # debug
+
+dirlist="$BASE_URL"
+filelist=""
+base_url_len=${#BASE_URL}
+
+clean_urls() {
+    # // -> /
+    sed -E 's,//+,/,g' | sed -E 's,^(http|https|ftp):/,&/,'
+}
+
+while [ -n "$dirlist" ]
+do
+    for dirurl in $dirlist
+    do
+        echo "fetching index.html from $dirurl"
+        relpath=$(echo "./${dirurl:$base_url_len}" | clean_urls)
+        mkdir -p "$relpath"
+        indexfile=$(echo "$relpath/index.html" | clean_urls)
+        wget $wgetargs -O "$indexfile" "$dirurl"
+        echo "parsing $indexfile"
+        filedirlist="$(PARSE_INDEX "$indexfile")"
+        filelist_next="$(echo "$filedirlist" | grep '\.tar\.xz$' | while read file; do echo "$dirurl/$file"; done)"
+        filelist_next="$(echo "$filelist_next" | clean_urls)"
+        [ -n "$filelist" ] && filelist+=$'\n'
+        filelist+="$filelist_next"
+        dirlist="$(echo "$filedirlist" | grep -v '\.tar\.xz$' | while read dir; do echo "$dirurl/$dir"; done || true)"
+        dirlist="$(echo "$dirlist" | clean_urls)"
+    done
+done
+
+filecount=$(echo "$filelist" | wc -l)
+
+if [ -z "$filelist" ]
+then
+    echo "error: no files parsed from $tmp/index.html"
+    exit 1
+fi
+
+echo "parsed $filecount tar.xz files:"; echo "$filelist"
+
+# most time is spent here
+echo "fetching $filecount sha256 files ..."
+urllist="$(echo "$filelist" | while read file; do echo "$file.sha256"; done)"
+# wget -r: keep directory structure
+echo "$urllist" | xargs wget $wgetargs -nH -r -c --no-parent && {
+    actual=$(find . -type f -name '*.sha256' | wc -l)
+    echo "fetching $filecount sha256 files done: got $actual files"
+} || {
+    # workaround: in rare cases, the server does not provide the sha256 files
+    # for example when the release is just a few hours old
+    # and the servers are not yet fully synced
+    actual=$(find . -type f -name '*.sha256' | wc -l)
+    echo "fetching $filecount sha256 files failed: got only $actual files"
+
+    # TODO fetch only missing tar.xz files
+    echo "fetching $filecount tar.xz files ..."
+    echo "$filelist" | xargs wget $wgetargs -nH -r -c --no-parent
+
+    echo "generating sha256 files ..."
+    find . -type f -name '*.tar.xz' | while read src; do
+        name=$(basename "$src")
+        sha256=$(sha256sum "$src" | cut -d' ' -f1)
+        echo "$sha256  $name" >"$src.sha256"
+    done
+}
+
+csv=$(mktemp $tmptpl.csv)
+echo "writing temporary file $csv ..."
+find . -type f -name '*.sha256' | while read sha256file; do
+    src="${sha256file%.*}" # remove extension
+    sha256=$(cat $sha256file | cut -d' ' -f1) # base16
+    sha256=$(nix-hash --type sha256 --to-base32 $sha256)
+    # Sanitize file name
+    filename=$(basename "$src" | tr '@' '_')
+    nameVersion="${filename%.tar.*}"
+    name=$(echo "$nameVersion" | sed -e 's,-[[:digit:]].*,,' | sed -e 's,-opensource-src$,,' | sed -e 's,-everywhere-src$,,')
+    version=$(echo "$nameVersion" | sed -e 's,^\([[:alpha:]][[:alnum:]]*-\)\+,,')
+    echo "$name,$version,$src,$filename,$sha256" >>$csv
+done
+
+files_before=$(grep -c 'src = ' "$SRCS")
+
+echo "writing output file $SRCS ..."
+cat >"$SRCS" <<EOF
+# DO NOT EDIT! This file is generated automatically.
+# Command: ./maintainers/scripts/fetch-kde-qt.sh $@
+{ fetchurl, mirror }:
+
+{
+EOF
+
+gawk -F , "{ print \$1 }" $csv | sort | uniq | while read name; do
+    versions=$(gawk -F , "/^$name,/ { print \$2 }" $csv)
+    latestVersion=$(echo "$versions" | sort -rV | head -n 1)
+    src=$(gawk -F , "/^$name,$latestVersion,/ { print \$3 }" $csv)
+    filename=$(gawk -F , "/^$name,$latestVersion,/ { print \$4 }" $csv)
+    sha256=$(gawk -F , "/^$name,$latestVersion,/ { print \$5 }" $csv)
+    url="${src:2}"
+    cat >>"$SRCS" <<EOF
+  $name = {
+    version = "$latestVersion";
+    src = fetchurl {
+      url = "\${mirror}/$url";
+      sha256 = "$sha256";
+      name = "$filename";
+    };
+  };
+EOF
+done
+
+echo "}" >>"$SRCS"
+
+files_after=$(grep -c 'src = ' "$SRCS")
+echo "files before: $files_before"
+echo "files after:  $files_after"
+
+echo "compare:"
+echo "git diff $srcsrel"
+
+popd >/dev/null
+rm -fr $tmp >/dev/null
+
+rm -f $csv >/dev/null

maintainers/scripts/find-tarballs.nix

@@ -0,0 +1,105 @@
+# This expression returns a list of all fetchurl calls used by ‘expr’.
+
+{
+  expr,
+  lib ? import ../../lib,
+}:
+
+let
+  inherit (lib)
+    addErrorContext
+    attrNames
+    concatLists
+    const
+    filter
+    genericClosure
+    isAttrs
+    isDerivation
+    isList
+    mapAttrsToList
+    optional
+    optionals
+    ;
+
+  root = expr;
+
+  uniqueFiles = map (x: x.file) (genericClosure {
+    startSet = map (file: {
+      key = with file; (if type == null then "" else type + "+") + hash;
+      inherit file;
+    }) files;
+    operator = const [ ];
+  });
+
+  files = map (drv: {
+    urls = drv.urls or [ drv.url ];
+    hash = drv.outputHash;
+    isPatch = (drv ? postFetch && drv.postFetch != "");
+    type = drv.outputHashAlgo;
+    name = drv.name;
+  }) fetchurlDependencies;
+
+  fetchurlDependencies = filter (
+    drv:
+    drv.outputHash or "" != "" && drv.outputHashMode or "flat" == "flat" && (drv ? url || drv ? urls)
+  ) dependencies;
+
+  dependencies = map (x: x.value) (genericClosure {
+    startSet = map keyDrv (derivationsIn' root);
+    operator = { key, value }: map keyDrv (immediateDependenciesOf value);
+  });
+
+  derivationsIn' =
+    x:
+    if !canEval x then
+      [ ]
+    else if isDerivation x then
+      optional (canEval x.drvPath) x
+    else if isList x then
+      concatLists (map derivationsIn' x)
+    else if isAttrs x then
+      concatLists (
+        mapAttrsToList (n: v: addErrorContext "while finding tarballs in '${n}':" (derivationsIn' v)) x
+      )
+    else
+      [ ];
+
+  keyDrv =
+    drv:
+    if canEval drv.drvPath then
+      {
+        key = drv.drvPath;
+        value = drv;
+      }
+    else
+      { };
+
+  immediateDependenciesOf =
+    drv:
+    concatLists (
+      mapAttrsToList (n: v: derivationsIn v) (
+        removeAttrs drv (
+          [
+            "meta"
+            "passthru"
+          ]
+          ++ optionals (drv ? passthru) (attrNames drv.passthru)
+        )
+      )
+    );
+
+  derivationsIn =
+    x:
+    if !canEval x then
+      [ ]
+    else if isDerivation x then
+      optional (canEval x.drvPath) x
+    else if isList x then
+      concatLists (map derivationsIn x)
+    else
+      [ ];
+
+  canEval = val: (builtins.tryEval val).success;
+
+in
+uniqueFiles

maintainers/scripts/fix-maintainers.pl

@@ -0,0 +1,58 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i perl -p perl -p perlPackages.LWP -p perlPackages.LWPProtocolHttps -p perlPackages.LWPUserAgent -p perlPackages.JSON -p perlPackages.PathTiny
+use LWP::UserAgent;
+use JSON;
+use Path::Tiny;
+use strict;
+use warnings;
+
+my $maintainers_list_nix = "../maintainer-list.nix";
+my $maintainers_json = from_json(`nix-instantiate --json --eval --expr 'builtins.fromJSON (builtins.toJSON (import $maintainers_list_nix))'`);
+
+STDOUT->autoflush(1);
+
+my $ua = LWP::UserAgent->new();
+
+if (!defined $ENV{GH_TOKEN}) {
+    die "Set GH_TOKEN before running this script";
+}
+
+keys %$maintainers_json; # reset the internal iterator so a prior each() doesn't affect the loop
+while(my($k, $v) = each %$maintainers_json) {
+    my $current_user = %$v{'github'};
+    if (!defined $current_user) {
+        print "$k has no github handle\n";
+    }
+    my $github_id = %$v{'githubId'};
+    if (!defined $github_id) {
+        print "$k has no githubId\n";
+        next;
+    }
+    my $url = 'https://api.github.com/user/' . $github_id;
+    my $resp = $ua->get(
+        $url,
+        "Authorization" => "Token $ENV{GH_TOKEN}"
+    );
+
+    if ($resp->header("X-RateLimit-Remaining") == 0) {
+        my $ratelimit_reset = $resp->header("X-RateLimit-Reset");
+        print "Request limit exceeded, waiting until " . scalar localtime $ratelimit_reset . "\n";
+        sleep($ratelimit_reset - time() + 5);
+    }
+    if ($resp->code != 200) {
+        print "$k likely deleted their github account\n";
+        next;
+    }
+    my $resp_json = from_json($resp->content);
+    my $api_user = %$resp_json{"login"};
+    if (!defined $current_user) {
+        print "$k is known on github as $api_user.\n";
+    }
+    elsif (lc($current_user) ne lc($api_user)) {
+        print "$k is now known on github as $api_user. Editing maintainer-list.nix…\n";
+        my $file = path($maintainers_list_nix);
+        my $data = $file->slurp_utf8;
+        $data =~ s/github = "$current_user";$/github = "$api_user";/m;
+        $file->spew_utf8($data);
+    }
+}

maintainers/scripts/get-maintainer-pings-between.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p git jq
+
+# Outputs a list of maintainers that would be pinged across two nixpkgs revisions.
+# Authors:
+#  Morgan Jones (@numinit)
+#  Tristan Ross (@RossComputerGuy)
+
+set -euo pipefail
+
+if [ $# -lt 2 ]; then
+    echo "Usage: $0 <rev-from> <rev-to>" >&2
+    exit 1
+fi
+
+repo="$(git rev-parse --show-toplevel)"
+system="$(nix-instantiate --eval --expr builtins.currentSystem)"
+rev1="$(git -C "$repo" rev-parse "$1")"
+rev2="$(git -C "$repo" rev-parse "$2")"
+
+echo "Touched files:" >&2
+git -C "$repo" diff --name-only "$rev1" "$rev2" \
+    | jq --raw-input --slurp 'split("\n")[:-1]' | tee "$TMPDIR/touched-files.json" >&2
+
+# Runs an eval in the given worktree, outputting the path to $TMPDIR/$1.path.
+# $1: The revision SHA.
+eval_in_worktree() (
+    mkdir -p .worktree
+    local rev="$1"
+    local tree=".worktree/$rev"
+    if [ ! -d "$tree" ]; then
+        git -C "$repo" worktree add -f -d "$tree" "$rev" >&2
+    fi
+    cd "$tree"
+
+    local workdir="$TMPDIR/$rev"
+    rm -rf "$workdir"
+    mkdir -p "$workdir"
+
+    nix-build ci -A eval.attrpathsSuperset -o "$workdir/paths" >&2
+    mkdir -p "$workdir/intermediates"
+    nix-build ci -A eval.singleSystem \
+        --arg evalSystem "$system" \
+        --arg attrpathFile "$workdir/paths/paths.json" \
+        --arg chunkSize ${CHUNK_SIZE:-10000} \
+        -o "$workdir/intermediates/.intermediate-1" >&2
+
+    # eval.combine nix-build needs a directory, not a symlink
+    cp -RL "$workdir/intermediates/.intermediate-1" "$workdir/intermediates/intermediate-1"
+    chmod -R +w "$workdir/intermediates/intermediate-1"
+    rm -rf "$workdir/intermediates/.intermediate-1"
+
+    nix-build ci -A eval.combine \
+        --arg resultsDir "$workdir/intermediates" \
+        -o "$workdir/result" >&2
+)
+
+eval_in_worktree "$rev1" &
+pid1=$!
+eval_in_worktree "$rev2" &
+pid2=$!
+
+wait $pid1
+wait $pid2
+
+path1="$TMPDIR/$rev1"
+path2="$TMPDIR/$rev2"
+
+# Use the repo this script was executed in to get accurate maintainer info
+nix-build "$repo/ci" -A eval.compare \
+    --arg beforeResultDir "$path1/result" \
+    --arg afterResultDir "$path2/result" \
+    --arg touchedFilesJson "$TMPDIR/touched-files.json" \
+    --arg byName true \
+    -o comparison
+
+echo "Pinged maintainers (check $repo/comparison for more details)" >&2
+jq < comparison/maintainers.json

maintainers/scripts/get-maintainer.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p jq ncurses
+# shellcheck shell=bash
+
+# Get a nixpkgs maintainer's metadata as a JSON object
+#  see HELP_MESSAGE just below, or README.md.
+
+set -euo pipefail
+
+declare -A SELECTORS=( [handle]= [email]= [github]= [githubId]= [matrix]= [name]= )
+HELP_MESSAGE="usage: '$0' [selector] value
+examples:
+  get-maintainer.sh nicoo
+  get-maintainer.sh githubId 1155801
+
+\`selector\` defaults to 'handle', can be one of:
+  ${!SELECTORS[*]}
+"
+
+MAINTAINERS_DIR="$(dirname "$0")/.."
+
+die() {
+  tput setaf 1 # red
+  echo "'$0': $*"
+  tput setaf 0 # back to black
+  exit 1
+}
+
+listAsJSON() {
+  nix-instantiate --eval --strict --json "${MAINTAINERS_DIR}/maintainer-list.nix"
+}
+
+parseArgs() {
+  [ $# -gt 0 -a $# -lt 3 ] || {
+      echo "$HELP_MESSAGE"
+      die "invalid number of arguments (must be 1 or 2)"
+  }
+
+  if [ $# -eq 1 ]; then
+    selector=handle
+  else
+    selector="$1"
+    shift
+  fi
+  [ -z "${SELECTORS[$selector]-n}" ] || {
+    echo "Valid selectors are:" "${!SELECTORS[@]}" >&2
+    die "invalid selector '$selector'"
+  }
+
+  value="$1"
+  shift
+}
+
+query() {
+  # explode { a: A, b: B, ... } into A + {handle: a}, B + {handle: b}, ...
+  local explode="to_entries[] | .value + { \"handle\": .key }"
+
+  # select matching items from the list
+  # TODO(nicoo): Support approximate matching for `name` ?
+  local select
+  case "$selector" in
+    githubId)
+      select="select(.${selector} == $value)"
+      ;;
+    *)
+      select="select(.${selector} == \"$value\")"
+  esac
+
+  echo "$explode | $select"
+}
+
+parseArgs "$@"
+listAsJSON | jq -e "$(query)"

maintainers/scripts/haskell/dependencies.nix

@@ -0,0 +1,17 @@
+# Nix script to calculate the Haskell dependencies of every haskellPackage. Used by ./hydra-report.hs.
+let
+  pkgs = import ../../.. { };
+  inherit (pkgs) lib;
+  getDeps =
+    _: pkg:
+    let
+      pname = pkg.pname or null;
+    in
+    {
+      deps = builtins.filter (x: x != null && x != pname) (
+        map (x: x.pname or null) (pkg.propagatedBuildInputs or [ ])
+      );
+      broken = (pkg.meta.hydraPlatforms or [ null ]) == [ ];
+    };
+in
+lib.mapAttrs getDeps pkgs.haskellPackages

maintainers/scripts/haskell/hydra-report.hs

@@ -0,0 +1,827 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -I nixpkgs=.
+#! nix-shell -p "haskellPackages.ghcWithPackages (p: [p.aeson p.req])"
+#! nix-shell -p nix-eval-jobs
+#! nix-shell -i runhaskell
+
+{-
+
+The purpose of this script is
+
+1) download the state of the nixpkgs/haskell-updates job from hydra (with get-report)
+2) print a summary of the state suitable for pasting into a github comment (with ping-maintainers)
+3) print a list of broken packages suitable for pasting into configuration-hackage2nix.yaml
+
+Because step 1) is quite expensive and takes roughly ~5 minutes the result is cached in a json file in XDG_CACHE.
+
+-}
+{-# LANGUAGE BlockArguments #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TupleSections #-}
+{-# LANGUAGE ViewPatterns #-}
+{-# OPTIONS_GHC -Wall #-}
+{-# LANGUAGE DataKinds #-}
+
+import Control.Monad (forM_, forM, (<=<))
+import Control.Monad.Trans (MonadIO (liftIO))
+import Data.Aeson (
+   FromJSON (..),
+   withObject,
+   (.:),
+   FromJSONKey,
+   ToJSON,
+   decodeFileStrict',
+   encodeFile,
+ )
+import Data.Aeson.Decoding (eitherDecodeStrictText)
+import Data.Foldable (Foldable (toList))
+import Data.Either (rights)
+import Data.Functor ((<&>))
+import Data.List.NonEmpty (NonEmpty, nonEmpty)
+import qualified Data.List.NonEmpty as NonEmpty
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as Map
+import Data.Maybe (fromMaybe, mapMaybe, isNothing, catMaybes)
+import Data.Monoid (Sum (Sum, getSum))
+import Data.Sequence (Seq)
+import qualified Data.Sequence as Seq
+import Data.Set (Set)
+import qualified Data.Set as Set
+import Data.Text (Text)
+import qualified Data.Text as Text
+import qualified Data.Text.IO as Text
+import Data.Time (defaultTimeLocale, formatTime, getCurrentTime)
+import Data.Time.Clock (UTCTime)
+import GHC.Generics (Generic)
+import Network.HTTP.Req (
+    GET (GET),
+    HttpResponse (HttpResponseBody),
+    NoReqBody (NoReqBody),
+    Option,
+    Req,
+    Scheme (Https),
+    bsResponse,
+    defaultHttpConfig,
+    header,
+    https,
+    jsonResponse,
+    req,
+    responseBody,
+    responseTimeout,
+    runReq,
+    (/:),
+ )
+import System.Directory (XdgDirectory (XdgCache), getXdgDirectory)
+import System.Environment (getArgs)
+import System.Exit (die)
+import System.Process (readProcess)
+import Prelude hiding (id)
+import Data.List (sortOn)
+import Control.Concurrent.Async (concurrently)
+import Control.Exception (evaluate)
+import qualified Data.IntMap.Lazy as IntMap
+import qualified Data.IntSet as IntSet
+import Data.Bifunctor (second)
+import Data.Data (Proxy)
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as ByteString
+import Distribution.Simple.Utils (safeLast, fromUTF8BS)
+
+newtype JobsetEvals = JobsetEvals
+   { evals :: Seq Eval
+   }
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+newtype Nixpkgs = Nixpkgs {revision :: Text}
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs}
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+data Eval = Eval
+   { id :: Int
+   , jobsetevalinputs :: JobsetEvalInputs
+   , builds :: Seq Int
+   }
+   deriving (Generic, ToJSON, FromJSON, Show)
+
+-- | Hydra job name.
+--
+-- Examples:
+-- - @"haskellPackages.lens.x86_64-linux"@
+-- - @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+-- - @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@
+-- - @"arion.aarch64-linux"@
+newtype JobName = JobName { unJobName :: Text }
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSONKey, FromJSON, Ord, ToJSON)
+
+-- | Datatype representing the result of querying the build evals of the
+-- haskell-updates Hydra jobset.
+--
+-- The URL <https://hydra.nixos.org/eval/EVAL_ID/builds> (where @EVAL_ID@ is a
+-- value like 1792418) returns a list of 'Build'.
+data Build = Build
+   { job :: JobName
+   , buildstatus :: Maybe Int
+     -- ^ Status of the build.  See 'getBuildState' for the meaning of each state.
+   , finished :: Int
+     -- ^ Whether or not the build is finished.  @0@ if finished, non-zero otherwise.
+   , id :: Int
+   , nixname :: Text
+     -- ^ Nix name of the derivation.
+     --
+     -- Examples:
+     -- - @"lens-5.2.1"@
+     -- - @"cabal-install-3.8.0.1"@
+     -- - @"lens-static-x86_64-unknown-linux-musl-5.1.1"@
+   , system :: Text
+     -- ^ System
+     --
+     -- Examples:
+     -- - @"x86_64-linux"@
+     -- - @"aarch64-darwin"@
+   , jobsetevals :: Seq Int
+   }
+   deriving (Generic, ToJSON, FromJSON, Show)
+
+data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround
+data RequestLogsFlag = RequestLogs | NoRequestLogs
+
+usage :: IO a
+usage = die "Usage: get-report [--slow] [EVAL-ID] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info"
+
+main :: IO ()
+main = do
+   args <- getArgs
+   case args of
+      "get-report":"--slow":id -> getBuildReports HydraSlownessWorkaround id
+      "get-report":id -> getBuildReports NoHydraSlownessWorkaround id
+      ["ping-maintainers"] -> printMaintainerPing
+      ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs
+      ["mark-broken-list"] -> printMarkBrokenList RequestLogs
+      ["eval-info"] -> printEvalInfo
+      _ -> usage
+
+reportFileName :: IO FilePath
+reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
+
+showT :: Show a => a -> Text
+showT = Text.pack . show
+
+getBuildReports :: HydraSlownessWorkaroundFlag -> [String] -> IO ()
+getBuildReports opt args = runReq defaultHttpConfig do
+   eval@Eval{id} <- case args of
+      [id] -> hydraJSONQuery mempty ["eval", Text.pack id]
+      [] -> do
+         evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"]
+         maybe (liftIO $ fail "No Evaluation found") pure evalMay
+      _ -> liftIO usage
+   liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..."
+   buildReports <- getEvalBuilds opt eval
+   liftIO do
+      fileName <- reportFileName
+      putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName
+      now <- getCurrentTime
+      encodeFile fileName (eval, now, buildReports)
+
+getEvalBuilds :: HydraSlownessWorkaroundFlag -> Eval -> Req (Seq Build)
+getEvalBuilds NoHydraSlownessWorkaround Eval{id} =
+  hydraJSONQuery mempty ["eval", showT id, "builds"]
+getEvalBuilds HydraSlownessWorkaround Eval{builds} = do
+  forM builds $ \buildId -> do
+    liftIO $ putStrLn $ "Querying build " <> show buildId
+    hydraJSONQuery mempty [ "build", showT buildId ]
+
+hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a)
+hydraQuery responseType option query = do
+  let customHeaderOpt =
+        header
+          "User-Agent"
+          "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)"
+      customTimeoutOpt = responseTimeout 900_000_000 -- 15 minutes
+      opts = customHeaderOpt <> customTimeoutOpt <> option
+      url = foldl' (/:) (https "hydra.nixos.org") query
+  responseBody <$> req GET url NoReqBody responseType opts
+
+hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a
+hydraJSONQuery = hydraQuery jsonResponse
+
+hydraPlainQuery :: [Text] -> Req ByteString
+hydraPlainQuery = hydraQuery bsResponse mempty
+
+nixEvalJobsCommand :: FilePath
+nixEvalJobsCommand = "nix-eval-jobs"
+
+nixEvalJobsParams :: [String]
+nixEvalJobsParams =
+  [
+    -- options necessary to make nix-eval-jobs behave like hydra-eval-jobs used to
+    -- https://github.com/NixOS/hydra/commit/d84ff32ce600204c6473889a3ff16cd6053533c9
+    "--meta",
+    "--force-recurse",
+    "--no-instantiate",
+    "--workers", "3",
+
+    "-I", ".",
+   "pkgs/top-level/release-haskell.nix"
+  ]
+
+nixExprCommand :: FilePath
+nixExprCommand = "nix-instantiate"
+
+nixExprParams :: [String]
+nixExprParams = ["--eval", "--strict", "--json"]
+
+-- | Holds a list of the GitHub handles of the maintainers of a given 'JobName'.
+--
+--   @
+--     JobMaintainers (JobName "haskellPackages.cabal-install.x86_64-linux") ["sternenseemann"]
+--   @
+data JobMaintainers = JobMaintainers JobName [Text]
+  deriving stock (Generic, Show)
+
+-- | Parse the entries produced by @nix-eval-jobs@, discarding all information
+--   except the name of the job (@attr@) and the @github@ attributes of the
+--   maintainer objects in @meta.maintainers@.
+instance FromJSON JobMaintainers where
+  parseJSON = withObject "HydraJob" $ \h -> do
+    jobName <- h .: "attr"
+    maintainers <- (h .: "meta")
+      >>= (withObject "Meta" $ \meta ->
+        meta .: "maintainers"
+        >>= mapM (withObject "Maintainer" $ \mt -> mt .: "github"))
+    pure $ JobMaintainers jobName maintainers
+
+-- | Map of maintained Hydra jobs to maintainer GitHub handles.
+--
+-- It has values similar to the following:
+--
+-- @@
+--  fromList
+--    [ ("arion.aarch64-linux", ["rob22"])
+--    , ("conduit.x86_64-darwin", ["snoyb", "webber"])
+--    ]
+-- @@
+type MaintainerMap = Map JobName (NonEmpty Text)
+
+-- | Information about a package which lists its dependencies and whether the
+-- package is marked broken.
+data DepInfo = DepInfo {
+   deps :: Set PkgName,
+   broken :: Bool
+}
+   deriving stock (Generic, Show)
+   deriving anyclass (FromJSON, ToJSON)
+
+-- | Map from package names to their DepInfo. This is the data we get out of a
+-- nix call.
+type DependencyMap = Map PkgName DepInfo
+
+-- | Map from package names to its broken state, number of reverse dependencies (fst) and
+-- unbroken reverse dependencies (snd).
+type ReverseDependencyMap = Map PkgName (Int, Int)
+
+-- | Calculate the (unbroken) reverse dependencies of a package by transitively
+-- going through all packages if it’s a dependency of them.
+calculateReverseDependencies :: DependencyMap -> ReverseDependencyMap
+calculateReverseDependencies depMap =
+   Map.fromDistinctAscList $ zip keys (zip (rdepMap False) (rdepMap True))
+ where
+    -- This code tries to efficiently invert the dependency map and calculate
+    -- its transitive closure by internally identifying every pkg with its index
+    -- in the package list and then using memoization.
+    keys :: [PkgName]
+    keys = Map.keys depMap
+
+    pkgToIndexMap :: Map PkgName Int
+    pkgToIndexMap = Map.fromDistinctAscList (zip keys [0..])
+
+    depInfos :: [DepInfo]
+    depInfos = Map.elems depMap
+
+    depInfoToIdx :: DepInfo -> (Bool, [Int])
+    depInfoToIdx DepInfo{broken,deps} =
+       (broken, mapMaybe (`Map.lookup` pkgToIndexMap) $ Set.toList deps)
+
+    intDeps :: [(Int, (Bool, [Int]))]
+    intDeps = zip [0..] (fmap depInfoToIdx depInfos)
+
+    rdepMap onlyUnbroken = IntSet.size <$> IntMap.elems resultList
+     where
+       resultList = IntMap.fromDistinctAscList [(i, go i) | i <- [0..length keys - 1]]
+       oneStepMap = IntMap.fromListWith IntSet.union $ (\(key,(_,deps)) -> (,IntSet.singleton key) <$> deps) <=< filter (\(_, (broken,_)) -> not (broken && onlyUnbroken)) $ intDeps
+       go pkg = IntSet.unions (oneStep:((resultList IntMap.!) <$> IntSet.toList oneStep))
+        where oneStep = IntMap.findWithDefault mempty pkg oneStepMap
+
+-- | Generate a mapping of Hydra job names to maintainer GitHub handles.
+getMaintainerMap :: IO MaintainerMap
+getMaintainerMap =
+  readJSONLinesProcess nixEvalJobsCommand nixEvalJobsParams
+  -- we ignore unparseable lines since fromJSON will fail on { "attr": …, "error": … }
+  -- entries since they don't have a @meta@ attribute.
+  <&> rights
+  <&> map (\(JobMaintainers name maintainers) -> (,) name <$> nonEmpty maintainers)
+  <&> catMaybes
+  <&> Map.fromList
+
+-- | Get the a map of all dependencies of every package by calling the nix
+-- script ./dependencies.nix.
+getDependencyMap :: IO DependencyMap
+getDependencyMap =
+   readJSONProcess
+      nixExprCommand
+      ("maintainers/scripts/haskell/dependencies.nix" : nixExprParams)
+      "Failed to decode nix output for lookup of dependencies: "
+
+-- | Run a process that produces JSON on stdout and and decode the JSON to a
+-- data type.
+--
+-- If the JSON-decoding fails, throw the JSON-decoding error.
+readJSONProcess
+   :: FromJSON a
+   => FilePath -- ^ Filename of executable.
+   -> [String] -- ^ Arguments
+   -> String -- ^ String to prefix to JSON-decode error.
+   -> IO a
+readJSONProcess exe args err = do
+   output <- readProcess exe args ""
+   let eitherDecodedOutput = eitherDecodeStrictText . Text.pack $ output
+   case eitherDecodedOutput of
+     Left decodeErr -> error $ err <> decodeErr <> "\nRaw: '" <> take 1000 output <> "'"
+     Right decodedOutput -> pure decodedOutput
+
+-- | Run a process that produces many JSON values, one per line.
+--   Error and success is reported per line via a list of 'Either's.
+readJSONLinesProcess
+   :: FromJSON a
+   => FilePath -- ^ Filename of executable.
+   -> [String] -- ^ Arguments
+   -> IO [Either String a]
+readJSONLinesProcess exe args = do
+  output <- readProcess exe args ""
+  -- TODO: slow, doesn't stream at all
+  pure . map (eitherDecodeStrictText . Text.pack) . lines $ output
+
+-- BuildStates are sorted by subjective importance/concerningness
+data BuildState
+  = Failed
+  | DependencyFailed
+  | OutputLimitExceeded
+  | Unknown (Maybe Int)
+  | TimedOut
+  | Canceled
+  | HydraFailure
+  | Unfinished
+  | Success
+  deriving stock (Show, Eq, Ord)
+
+icon :: BuildState -> Text
+icon = \case
+   Failed -> "❌"
+   DependencyFailed -> "❗"
+   OutputLimitExceeded -> "⚠️"
+   Unknown x -> "unknown code " <> showT x
+   TimedOut -> "⌛🚫"
+   Canceled -> "🚫"
+   Unfinished -> "⏳"
+   HydraFailure -> "🚧"
+   Success -> "✅"
+
+platformIcon :: Platform -> Text
+platformIcon (Platform x) = case x of
+   "x86_64-linux" -> "🐧"
+   "aarch64-linux" -> "📱"
+   "x86_64-darwin" -> "🍎"
+   "aarch64-darwin" -> "🍏"
+   _ -> x
+
+platformIsOS :: OS -> Platform -> Bool
+platformIsOS os (Platform x) = case (os, x) of
+   (Linux, "x86_64-linux") -> True
+   (Linux, "aarch64-linux") -> True
+   (Darwin, "x86_64-darwin") -> True
+   (Darwin, "aarch64-darwin") -> True
+   _ -> False
+
+
+-- | A package name.  This is parsed from a 'JobName'.
+--
+-- Examples:
+--
+-- - The 'JobName' @"haskellPackages.lens.x86_64-linux"@ produces the 'PkgName'
+--   @"lens"@.
+-- - The 'JobName' @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+--   produces the 'PkgName' @"cabal-install"@.
+-- - The 'JobName' @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@ produces
+--   the 'PkgName' @"ghc90"@.
+-- - The 'JobName' @"arion.aarch64-linux"@ produces the 'PkgName' @"arion"@.
+--
+-- 'PkgName' is also used as a key in 'DependencyMap' and 'ReverseDependencyMap'.
+-- In this case, 'PkgName' originally comes from attribute names in @haskellPackages@
+-- in Nixpkgs.
+newtype PkgName = PkgName Text
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSON, FromJSONKey, Ord, ToJSON)
+
+-- | A package set name.  This is parsed from a 'JobName'.
+--
+-- Examples:
+--
+-- - The 'JobName' @"haskellPackages.lens.x86_64-linux"@ produces the 'PkgSet'
+--   @"haskellPackages"@.
+-- - The 'JobName' @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+--   produces the 'PkgSet' @"haskell.packages.ghc925"@.
+-- - The 'JobName' @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@ produces
+--   the 'PkgSet' @"pkgsMusl.haskell.compiler"@.
+-- - The 'JobName' @"arion.aarch64-linux"@ produces the 'PkgSet' @""@.
+--
+-- As you can see from the last example, 'PkgSet' can be empty (@""@) for
+-- top-level jobs.
+newtype PkgSet = PkgSet Text
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSON, FromJSONKey, Ord, ToJSON)
+
+data BuildResult = BuildResult {state :: BuildState, id :: Int} deriving (Show, Eq, Ord)
+newtype Platform = Platform {platform :: Text} deriving (Show, Eq, Ord)
+data SummaryEntry = SummaryEntry {
+   summaryBuilds :: Table PkgSet Platform BuildResult,
+   summaryMaintainers :: Set Text,
+   summaryReverseDeps :: Int,
+   summaryUnbrokenReverseDeps :: Int
+}
+type StatusSummary = Map PkgName SummaryEntry
+
+data OS = Linux | Darwin
+
+newtype Table row col a = Table (Map (row, col) a)
+
+singletonTable :: row -> col -> a -> Table row col a
+singletonTable row col a = Table $ Map.singleton (row, col) a
+
+unionTable :: (Ord row, Ord col) => Table row col a -> Table row col a -> Table row col a
+unionTable (Table l) (Table r) = Table $ Map.union l r
+
+filterWithKeyTable :: (row -> col -> a -> Bool) -> Table row col a -> Table row col a
+filterWithKeyTable f (Table t) = Table $ Map.filterWithKey (\(r,c) a -> f r c a) t
+
+nullTable :: Table row col a -> Bool
+nullTable (Table t) = Map.null t
+
+instance (Ord row, Ord col, Semigroup a) => Semigroup (Table row col a) where
+   Table l <> Table r = Table (Map.unionWith (<>) l r)
+instance (Ord row, Ord col, Semigroup a) => Monoid (Table row col a) where
+   mempty = Table Map.empty
+instance Functor (Table row col) where
+   fmap f (Table a) = Table (fmap f a)
+instance Foldable (Table row col) where
+   foldMap f (Table a) = foldMap f a
+
+getBuildState :: Build -> BuildState
+getBuildState Build{finished, buildstatus} = case (finished, buildstatus) of
+   (0, _) -> Unfinished
+   (_, Just 0) -> Success
+   (_, Just 1) -> Failed
+   (_, Just 2) -> DependencyFailed
+   (_, Just 3) -> HydraFailure
+   (_, Just 4) -> Canceled
+   (_, Just 7) -> TimedOut
+   (_, Just 11) -> OutputLimitExceeded
+   (_, i) -> Unknown i
+
+combineStatusSummaries :: Seq StatusSummary -> StatusSummary
+combineStatusSummaries = foldl (Map.unionWith unionSummary) Map.empty
+  where
+   unionSummary :: SummaryEntry -> SummaryEntry -> SummaryEntry
+   unionSummary (SummaryEntry lb lm lr lu) (SummaryEntry rb rm rr ru) =
+      SummaryEntry (unionTable lb rb) (lm <> rm) (max lr rr) (max lu ru)
+
+buildToPkgNameAndSet :: Build -> (PkgName, PkgSet)
+buildToPkgNameAndSet Build{job = JobName jobName, system} = (name, set)
+  where
+   packageName :: Text
+   packageName = fromMaybe jobName (Text.stripSuffix ("." <> system) jobName)
+
+   splitted :: Maybe (NonEmpty Text)
+   splitted = nonEmpty $ Text.splitOn "." packageName
+
+   name :: PkgName
+   name = PkgName $ maybe packageName NonEmpty.last splitted
+
+   set :: PkgSet
+   set = PkgSet $ maybe "" (Text.intercalate "." . NonEmpty.init) splitted
+
+buildToStatusSummary :: MaintainerMap -> ReverseDependencyMap -> Build -> StatusSummary
+buildToStatusSummary maintainerMap reverseDependencyMap build@Build{job, id, system} =
+   Map.singleton pkgName summaryEntry
+  where
+   (pkgName, pkgSet) = buildToPkgNameAndSet build
+
+   maintainers :: Set Text
+   maintainers = maybe mempty (Set.fromList . toList) (Map.lookup job maintainerMap)
+
+   (reverseDeps, unbrokenReverseDeps) =
+      Map.findWithDefault (0,0) pkgName reverseDependencyMap
+
+   buildTable :: Table PkgSet Platform BuildResult
+   buildTable =
+      singletonTable pkgSet (Platform system) (BuildResult (getBuildState build) id)
+
+   summaryEntry = SummaryEntry buildTable maintainers reverseDeps unbrokenReverseDeps
+
+readBuildReports :: IO (Eval, UTCTime, Seq Build)
+readBuildReports = do
+   file <- reportFileName
+   fromMaybe (error $ "Could not decode " <> file) <$> decodeFileStrict' file
+
+sep :: Text
+sep = " | "
+joinTable :: [Text] -> Text
+joinTable t = sep <> Text.intercalate sep t <> sep
+
+type NumSummary = Table Platform BuildState Int
+
+printTable :: (Ord rows, Ord cols) => Text -> (rows -> Text) -> (cols -> Text) -> (entries -> Text) -> Table rows cols entries -> [Text]
+printTable name showR showC showE (Table mapping) = joinTable <$> (name : map showC cols) : replicate (length cols + sepsInName + 1) "---" : map printRow rows
+  where
+   sepsInName = Text.count "|" name
+   printRow row = showR row : map (\col -> maybe "" showE (Map.lookup (row, col) mapping)) cols
+   rows = toList $ Set.fromList (fst <$> Map.keys mapping)
+   cols = toList $ Set.fromList (snd <$> Map.keys mapping)
+
+printJob :: Int -> PkgName -> (Table PkgSet Platform BuildResult, Text) -> [Text]
+printJob evalId (PkgName name) (Table mapping, maintainers) =
+   if length sets <= 1
+      then map printSingleRow sets
+      else ["- [ ] " <> makeJobSearchLink (PkgSet "") name <> " " <> maintainers] <> map printRow sets
+  where
+   printRow :: PkgSet -> Text
+   printRow (PkgSet set) =
+      "  - " <> printState (PkgSet set) <> " " <>
+      makeJobSearchLink (PkgSet set) (if Text.null set then "toplevel" else set)
+
+   printSingleRow set =
+      "- [ ] " <> printState set <> " " <>
+      makeJobSearchLink set (makePkgName set) <> " " <> maintainers
+
+   makePkgName :: PkgSet -> Text
+   makePkgName (PkgSet set) = (if Text.null set then "" else set <> ".") <> name
+
+   printState set =
+      Text.intercalate " " $ map (\pf -> maybe "" (label pf) $ Map.lookup (set, pf) mapping) platforms
+
+   makeJobSearchLink :: PkgSet -> Text -> Text
+   makeJobSearchLink set linkLabel = makeSearchLink evalId linkLabel (makePkgName set)
+
+   sets :: [PkgSet]
+   sets = toList $ Set.fromList (fst <$> Map.keys mapping)
+
+   platforms :: [Platform]
+   platforms = toList $ Set.fromList (snd <$> Map.keys mapping)
+
+   label pf (BuildResult s i) = "[[" <> platformIcon pf <> icon s <> "]](https://hydra.nixos.org/build/" <> showT i <> ")"
+
+makeSearchLink :: Int -> Text -> Text -> Text
+makeSearchLink evalId linkLabel query = "[" <> linkLabel <> "](" <> "https://hydra.nixos.org/eval/" <> showT evalId <> "?filter=" <> query <> ")"
+
+statusToNumSummary :: StatusSummary -> NumSummary
+statusToNumSummary = fmap getSum . foldMap (fmap Sum . jobTotals)
+
+jobTotals :: SummaryEntry -> Table Platform BuildState Int
+jobTotals (summaryBuilds -> Table mapping) = getSum <$> Table (Map.foldMapWithKey (\(_, platform) (BuildResult buildstate _) -> Map.singleton (platform, buildstate) (Sum 1)) mapping)
+
+details :: Text -> [Text] -> [Text]
+details summary content = ["<details><summary>" <> summary <> " </summary>", ""] <> content <> ["</details>", ""]
+
+evalLine :: Eval -> UTCTime -> Text
+evalLine Eval{id, jobsetevalinputs = JobsetEvalInputs{nixpkgs = Nixpkgs{revision}}} fetchTime =
+   "*evaluation ["
+    <> showT id
+    <> "](https://hydra.nixos.org/eval/"
+    <> showT id
+    <> ") of nixpkgs commit ["
+    <> Text.take 7 revision
+    <> "](https://github.com/NixOS/nixpkgs/commits/"
+    <> revision
+    <> ") as of "
+    <> Text.pack (formatTime defaultTimeLocale "%Y-%m-%d %H:%M UTC" fetchTime)
+    <> "*"
+
+printBuildSummary :: Eval -> UTCTime -> StatusSummary -> [(PkgName, Int)] -> Text
+printBuildSummary eval@Eval{id} fetchTime summary topBrokenRdeps =
+   Text.unlines $
+      headline <> [""] <> tldr <> (("  * "<>) <$> (errors <> warnings)) <> [""]
+         <> totals
+         <> optionalList "#### Maintained Linux packages with build failure" (maintainedList (fails summaryLinux))
+         <> optionalList "#### Maintained Linux packages with failed dependency" (maintainedList (failedDeps summaryLinux))
+         <> optionalList "#### Maintained Linux packages with unknown error" (maintainedList (unknownErr summaryLinux))
+         <> optionalHideableList "#### Maintained Darwin packages with build failure" (maintainedList (fails summaryDarwin))
+         <> optionalHideableList "#### Maintained Darwin packages with failed dependency" (maintainedList (failedDeps summaryDarwin))
+         <> optionalHideableList "#### Maintained Darwin packages with unknown error" (maintainedList (unknownErr summaryDarwin))
+         <> optionalHideableList "#### Unmaintained packages with build failure" (unmaintainedList (fails summary))
+         <> optionalHideableList "#### Unmaintained packages with failed dependency" (unmaintainedList (failedDeps summary))
+         <> optionalHideableList "#### Unmaintained packages with unknown error" (unmaintainedList (unknownErr summary))
+         <> optionalHideableList "#### Top 50 broken packages, sorted by number of reverse dependencies" (brokenLine <$> topBrokenRdeps)
+         <> ["","*⤴️: The number of packages that depend (directly or indirectly) on this package (if any). If two numbers are shown the first (lower) number considers only packages which currently have enabled hydra jobs, i.e. are not marked broken. The second (higher) number considers all packages.*",""]
+         <> footer
+  where
+   footer = ["*Report generated with [maintainers/scripts/haskell/hydra-report.hs](https://github.com/NixOS/nixpkgs/blob/haskell-updates/maintainers/scripts/haskell/hydra-report.hs)*"]
+
+   headline =
+      [ "### [haskell-updates build report from hydra](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates)"
+      , evalLine eval fetchTime
+      ]
+
+   totals :: [Text]
+   totals =
+      [ "#### Build summary"
+      , ""
+      ] <>
+      printTable
+         "Platform"
+         (\x -> makeSearchLink id (platform x <> " " <> platformIcon x) ("." <> platform x))
+         (\x -> showT x <> " " <> icon x)
+         showT
+         numSummary
+
+   brokenLine :: (PkgName, Int) -> Text
+   brokenLine (PkgName name, rdeps) =
+      "[" <> name <> "](https://packdeps.haskellers.com/reverse/" <> name <>
+      ") ⤴️ " <> Text.pack (show rdeps) <> "  "
+
+   numSummary = statusToNumSummary summary
+
+   summaryLinux :: StatusSummary
+   summaryLinux = withOS Linux summary
+
+   summaryDarwin :: StatusSummary
+   summaryDarwin = withOS Darwin summary
+
+   -- Remove all BuildResult from the Table that have Platform that isn't for
+   -- the given OS.
+   tableForOS :: OS -> Table PkgSet Platform BuildResult -> Table PkgSet Platform BuildResult
+   tableForOS os = filterWithKeyTable (\_ platform _ -> platformIsOS os platform)
+
+   -- Remove all BuildResult from the StatusSummary that have a Platform that
+   -- isn't for the given OS.  Completely remove all PkgName from StatusSummary
+   -- that end up with no BuildResults.
+   withOS
+      :: OS
+      -> StatusSummary
+      -> StatusSummary
+   withOS os =
+      Map.mapMaybe
+         (\e@SummaryEntry{summaryBuilds} ->
+            let buildsForOS = tableForOS os summaryBuilds
+            in if nullTable buildsForOS then Nothing else Just e { summaryBuilds = buildsForOS }
+         )
+
+   jobsByState :: (BuildState -> Bool) -> StatusSummary -> StatusSummary
+   jobsByState predicate = Map.filter (predicate . worstState)
+
+   worstState :: SummaryEntry -> BuildState
+   worstState = foldl' min Success . fmap state . summaryBuilds
+
+   fails :: StatusSummary -> StatusSummary
+   fails = jobsByState (== Failed)
+
+   failedDeps :: StatusSummary -> StatusSummary
+   failedDeps = jobsByState (== DependencyFailed)
+
+   unknownErr :: StatusSummary -> StatusSummary
+   unknownErr = jobsByState (\x -> x > DependencyFailed && x < TimedOut)
+
+   withMaintainer :: StatusSummary -> Map PkgName (Table PkgSet Platform BuildResult, NonEmpty Text)
+   withMaintainer =
+      Map.mapMaybe
+         (\e -> (summaryBuilds e,) <$> nonEmpty (Set.toList (summaryMaintainers e)))
+
+   withoutMaintainer :: StatusSummary -> StatusSummary
+   withoutMaintainer = Map.mapMaybe (\e -> if Set.null (summaryMaintainers e) then Just e else Nothing)
+
+   optionalList :: Text -> [Text] -> [Text]
+   optionalList heading list = if null list then mempty else [heading] <> list
+
+   optionalHideableList :: Text -> [Text] -> [Text]
+   optionalHideableList heading list = if null list then mempty else [heading] <> details (showT (length list) <> " job(s)") list
+
+   maintainedList :: StatusSummary -> [Text]
+   maintainedList = showMaintainedBuild <=< Map.toList . withMaintainer
+
+   summaryEntryGetReverseDeps :: SummaryEntry -> (Int, Int)
+   summaryEntryGetReverseDeps sumEntry =
+      ( negate $ summaryUnbrokenReverseDeps sumEntry
+      , negate $ summaryReverseDeps sumEntry
+      )
+
+   sortOnReverseDeps :: [(PkgName, SummaryEntry)] -> [(PkgName, SummaryEntry)]
+   sortOnReverseDeps = sortOn (\(_, sumEntry) -> summaryEntryGetReverseDeps sumEntry)
+
+   unmaintainedList :: StatusSummary -> [Text]
+   unmaintainedList = showBuild <=< sortOnReverseDeps . Map.toList . withoutMaintainer
+
+   showBuild :: (PkgName, SummaryEntry) -> [Text]
+   showBuild (name, entry) =
+      printJob
+         id
+         name
+         ( summaryBuilds entry
+         , Text.pack
+            ( if summaryReverseDeps entry > 0
+               then
+                  " ⤴️ " <> show (summaryUnbrokenReverseDeps entry) <>
+                  " | " <> show (summaryReverseDeps entry)
+               else ""
+            )
+         )
+
+   showMaintainedBuild
+      :: (PkgName, (Table PkgSet Platform BuildResult, NonEmpty Text)) -> [Text]
+   showMaintainedBuild (name, (table, maintainers)) =
+      printJob
+         id
+         name
+         ( table
+         , Text.intercalate " " (fmap ("@" <>) (toList maintainers))
+         )
+
+   tldr = case (errors, warnings) of
+            ([],[]) -> ["🟢 **Ready to merge** (if there are no [evaluation errors](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates))"]
+            ([],_) -> ["🟡 **Potential issues** (and possibly [evaluation errors](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates))"]
+            _ -> ["🔴 **Branch not mergeable**"]
+   warnings =
+      if' (Unfinished > maybe Success worstState maintainedJob) "`maintained` jobset failed." <>
+      if' (Unfinished == maybe Success worstState mergeableJob) "`mergeable` jobset is not finished." <>
+      if' (Unfinished == maybe Success worstState maintainedJob) "`maintained` jobset is not finished."
+   errors =
+      if' (isNothing mergeableJob) "No `mergeable` job found." <>
+      if' (isNothing maintainedJob) "No `maintained` job found." <>
+      if' (Unfinished > maybe Success worstState mergeableJob) "`mergeable` jobset failed." <>
+      if' (outstandingJobs (Platform "x86_64-linux") > 100) "Too many outstanding jobs on x86_64-linux." <>
+      if' (outstandingJobs (Platform "aarch64-linux") > 100) "Too many outstanding jobs on aarch64-linux."
+
+   if' p e = if p then [e] else mempty
+
+   outstandingJobs platform | Table m <- numSummary = Map.findWithDefault 0 (platform, Unfinished) m
+
+   maintainedJob = Map.lookup (PkgName "maintained") summary
+   mergeableJob = Map.lookup (PkgName "mergeable") summary
+
+printEvalInfo :: IO ()
+printEvalInfo = do
+   (eval, fetchTime, _) <- readBuildReports
+   putStrLn (Text.unpack $ evalLine eval fetchTime)
+
+printMaintainerPing :: IO ()
+printMaintainerPing = do
+   (maintainerMap, (reverseDependencyMap, topBrokenRdeps)) <- concurrently getMaintainerMap do
+      depMap <- getDependencyMap
+      rdepMap <- evaluate . calculateReverseDependencies $ depMap
+      let tops = take 50 . sortOn (negate . snd) . fmap (second fst) . filter (\x -> maybe False broken $ Map.lookup (fst x) depMap) . Map.toList $ rdepMap
+      pure (rdepMap, tops)
+   (eval, fetchTime, buildReport) <- readBuildReports
+   let statusSummaries =
+          fmap (buildToStatusSummary maintainerMap reverseDependencyMap) buildReport
+       buildSum :: StatusSummary
+       buildSum = combineStatusSummaries statusSummaries
+       textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps
+   Text.putStrLn textBuildSummary
+
+printMarkBrokenList :: RequestLogsFlag -> IO ()
+printMarkBrokenList reqLogs = do
+   (_, fetchTime, buildReport) <- readBuildReports
+   runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} ->
+      case (getBuildState build, Text.splitOn "." $ unJobName job) of
+         (Failed, ["haskellPackages", name, "x86_64-linux"]) -> do
+            -- We use the last probable error cause found in the build log file.
+            error_message <- fromMaybe "failure" <$>
+              case reqLogs of
+                NoRequestLogs -> pure Nothing
+                RequestLogs -> do
+                  -- Fetch build log from hydra to figure out the cause of the error.
+                  build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
+                  pure $ safeLast $ mapMaybe probableErrorCause build_log
+            liftIO $ putStrLn $ "  - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime
+         _ -> pure ()
+
+{- | This function receives a line from a Nix Haskell builder build log and returns a possible error cause.
+ | We might need to add other causes in the future if errors happen in unusual parts of the builder.
+-}
+probableErrorCause :: ByteString -> Maybe String
+probableErrorCause "Setup: Encountered missing or private dependencies:" = Just "dependency missing"
+probableErrorCause "running tests" = Just "test failure"
+probableErrorCause build_line | ByteString.isPrefixOf "Building" build_line = Just ("failure building " <> fromUTF8BS (fst $ ByteString.breakSubstring " for" $ ByteString.drop 9 build_line))
+probableErrorCause build_line | ByteString.isSuffixOf "Phase" build_line = Just ("failure in " <> fromUTF8BS build_line)
+probableErrorCause _ = Nothing

maintainers/scripts/haskell/maintained-broken-pkgs.nix

@@ -0,0 +1,26 @@
+let
+  nixpkgs = import ../../..;
+  inherit (nixpkgs { }) haskellPackages lib;
+  maintainedPkgs = lib.filterAttrs (
+    _: v: builtins.length (v.meta.maintainers or [ ]) > 0
+  ) haskellPackages;
+  brokenPkgs = lib.filterAttrs (_: v: v.meta.broken) maintainedPkgs;
+  transitiveBrokenPkgs = lib.filterAttrs (
+    _: v: !(builtins.tryEval (v.outPath or null)).success && !v.meta.broken
+  ) maintainedPkgs;
+  infoList =
+    pkgs:
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (name: drv: "${name} ${(builtins.elemAt drv.meta.maintainers 0).github}") pkgs
+    );
+in
+{
+  report = ''
+    BROKEN:
+    ${infoList brokenPkgs}
+
+    TRANSITIVE BROKEN:
+    ${infoList transitiveBrokenPkgs}
+  '';
+  transitiveErrors = builtins.attrValues transitiveBrokenPkgs;
+}

maintainers/scripts/haskell/mark-broken.sh

@@ -0,0 +1,66 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils git -I nixpkgs=.
+
+# This script uses the data pulled with
+# maintainers/scripts/haskell/hydra-report.hs get-report to produce a list of
+# failing builds that get written to the hackage2nix config. Then
+# hackage-packages.nix gets regenerated and transitive-broken packages get
+# marked as dont-distribute in the config as well.
+# This should disable builds for most failing jobs in the haskell-updates jobset.
+
+set -euo pipefail
+
+do_commit=false
+mark_broken_list_flags=""
+
+for arg in "$@"; do
+    case "$arg" in
+        --do-commit)
+            do_commit=true
+            ;;
+        --no-request-logs)
+            mark_broken_list_flags="$mark_broken_list_flags $arg"
+            ;;
+        *)
+            echo "$0: unknown flag: $arg"
+            exit 100
+            ;;
+    esac
+done
+
+broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
+
+tmpfile=$(mktemp)
+trap "rm ${tmpfile}" 0
+
+echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script."
+echo "Generating a list of broken builds and displaying for manual confirmation ..."
+maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | LC_ALL=C.UTF-8 sort --ignore-case > "$tmpfile"
+
+$EDITOR "$tmpfile"
+
+tail -n +3 "$broken_config" >> "$tmpfile"
+
+cat > "$broken_config" << EOF
+# These packages don't compile.
+broken-packages:
+EOF
+
+# clear environment here to avoid things like allowing broken builds in
+LC_ALL=C.UTF-8 sort --ignore-case --unique "$tmpfile" >> "$broken_config"
+clear="env -u HOME -u NIXPKGS_CONFIG"
+$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh
+evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info)
+
+if $do_commit; then
+git add $broken_config
+git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+git add pkgs/development/haskell-modules/hackage-packages.nix
+git commit -F - << EOF
+haskellPackages: mark builds failing on hydra as broken
+
+This commit has been generated by maintainers/scripts/haskell/mark-broken.sh based on
+$evalline
+from the haskell-updates jobset on hydra under https://hydra.nixos.org/jobset/nixpkgs/haskell-updates
+EOF
+fi

maintainers/scripts/haskell/merge-and-open-pr.sh

@@ -0,0 +1,119 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git gh -I nixpkgs=.
+#
+# Script to merge the currently open haskell-updates PR , bump the Stackage
+# version and Hackage versions, and open the next haskell-updates PR.
+
+set -eu -o pipefail
+
+# exit after printing first argument to this function
+function die {
+  # echo the first argument
+  echo "ERROR: $1"
+  echo "Aborting!"
+
+  exit 1
+}
+
+function help {
+  echo "Usage: $0 HASKELL_UPDATES_PR_NUM"
+  echo "Merge the currently open haskell-updates PR into master, and open the next one."
+  echo
+  echo "  -h, --help                print this help"
+  echo "  HASKELL_UPDATES_PR_NUM    number of the currently open PR on NixOS/nixpkgs"
+  echo "                            for the haskell-updates branch"
+  echo
+  echo "Example:"
+  echo "  \$ $0 137340"
+
+  exit 1
+}
+
+# Read in the current haskell-updates PR number from the command line.
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+    -h|--help)
+      help
+      ;;
+    *)
+      curr_haskell_updates_pr_num="$1"
+      shift
+      ;;
+  esac
+done
+
+if [[ -z "${curr_haskell_updates_pr_num-}" ]] ; then
+  die "You must pass the current haskell-updates PR number as the first argument to this script."
+fi
+
+# Make sure you have gh authentication setup.
+if ! gh auth status 2>/dev/null ; then
+  die "You must setup the \`gh\` command.  Run \`gh auth login\`."
+fi
+
+# Make sure this is configured before we start doing anything
+push_remote="$(git config branch.haskell-updates.pushRemote)" \
+  || die 'Can'\''t determine pushRemote for haskell-updates. Please set using `git config branch.haskell-updates.pushremote <remote name>`.'
+
+# Fetch nixpkgs to get an up-to-date origin/haskell-updates branch.
+echo "Fetching origin..."
+git fetch origin >/dev/null
+
+# Make sure we are currently on a local haskell-updates branch.
+curr_branch="$(git rev-parse --abbrev-ref HEAD)"
+if [[ "$curr_branch" != "haskell-updates" ]]; then
+    die "Current branch is not called \"haskell-updates\"."
+fi
+
+# Make sure our local haskell-updates branch is on the same commit as
+# origin/haskell-updates.
+curr_branch_commit="$(git rev-parse haskell-updates)"
+origin_haskell_updates_commit="$(git rev-parse origin/haskell-updates)"
+if [[ "$curr_branch_commit" != "$origin_haskell_updates_commit" ]]; then
+    die "Current branch is not at the same commit as origin/haskell-updates"
+fi
+
+# Merge the current open haskell-updates PR.
+echo "Merging https://github.com/NixOS/nixpkgs/pull/${curr_haskell_updates_pr_num}..."
+gh pr merge --repo NixOS/nixpkgs --merge "$curr_haskell_updates_pr_num"
+
+# Update stackage, Hackage hashes, and regenerate Haskell package set
+./maintainers/scripts/haskell/update-package-set.sh
+
+# Push these new commits to the haskell-updates branch
+echo "Pushing commits just created to the remote $push_remote/haskell-updates branch..."
+git push "$push_remote" haskell-updates
+
+# Open new PR
+new_pr_body=$(cat <<EOF
+### This Merge
+
+This PR is the regular merge of the \`haskell-updates\` branch into \`staging\`.
+
+This branch is being continually built and tested by hydra at https://hydra.nixos.org/jobset/nixpkgs/haskell-updates. You may be able to find an up-to-date Hydra build report at [cdepillabout/nix-haskell-updates-status](https://github.com/cdepillabout/nix-haskell-updates-status).
+
+We roughly aim to merge these \`haskell-updates\` PRs at least once every two weeks. See the @NixOS/haskell [team calendar](https://cloud.maralorn.de/apps/calendar/p/H6migHmKX7xHoTFa) for who is currently in charge of this branch.
+
+### haskellPackages Workflow Summary
+
+Our workflow is currently described in [\`pkgs/development/haskell-modules/HACKING.md\`](https://github.com/NixOS/nixpkgs/blob/haskell-updates/pkgs/development/haskell-modules/HACKING.md).
+
+The short version is this:
+* We regularly update the Stackage and Hackage pins on \`haskell-updates\` (normally at the beginning of a merge window).
+* The community fixes builds of Haskell packages on that branch.
+* We aim at at least one merge of \`haskell-updates\` into \`staging\` every two weeks.
+* We only do the merge if the [\`mergeable\`](https://hydra.nixos.org/job/nixpkgs/haskell-updates/mergeable) job is succeeding on hydra.
+* If a [\`maintained\`](https://hydra.nixos.org/job/nixpkgs/haskell-updates/maintained) package is still broken at the time of merge, we will only merge if the maintainer has been pinged 7 days in advance. (If you care about a Haskell package, become a maintainer!)
+
+More information about Haskell packages in nixpkgs can be found [in the nixpkgs manual](https://nixos.org/manual/nixpkgs/unstable/#haskell).
+
+---
+
+This is the follow-up to #${curr_haskell_updates_pr_num}. Come to [#haskell:nixos.org](https://matrix.to/#/#haskell:nixos.org) if you have any questions.
+EOF
+)
+
+echo "Opening a PR for the next haskell-updates merge cycle..."
+gh pr create --repo NixOS/nixpkgs --base staging --head haskell-updates --title "haskellPackages: update stackage and hackage" --body "$new_pr_body"

maintainers/scripts/haskell/regenerate-hackage-packages.sh

@@ -0,0 +1,117 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils haskellPackages.cabal2nix-unstable.bin git -I nixpkgs=.
+
+set -euo pipefail
+
+self=$0
+
+print_help () {
+cat <<END_HELP
+Usage: $self [options]
+
+Options:
+   --do-commit    Commit changes to this file.
+   -f | --fast    Do not update the transitive-broken.yaml file.
+   -h | --help    Show this help.
+
+This script is used to regenerate nixpkgs' Haskell package set, using the
+tool hackage2nix from the nixos/cabal2nix repo. hackage2nix looks at the
+config files in pkgs/development/haskell-modules/configuration-hackage2nix
+and generates a Nix expression for package version specified there, using the
+Cabal files from the Hackage database (available under all-cabal-hashes) and
+its companion tool cabal2nix.
+
+Unless --fast is used, it will then use the generated nix expression by
+running regenerate-transitive-broken-packages.sh which updates the transitive-broken.yaml
+file. Then it re-runs hackage2nix.
+
+Related scripts are update-hackage.sh, for updating the snapshot of the
+Hackage database used by hackage2nix, and update-cabal2nix-unstable.sh,
+for updating the version of hackage2nix used to perform this task.
+
+Note that this script doesn't gcroot anything, so it may be broken by an
+unfortunately timed nix-store --gc.
+
+END_HELP
+}
+
+DO_COMMIT=0
+REGENERATE_TRANSITIVE=1
+
+options=$(getopt -o "fh" -l "help,fast,do-commit" -- "$@")
+
+eval set -- "$options"
+
+while true; do
+   case "$1" in
+      --do-commit)
+         DO_COMMIT=1
+         ;;
+      -f|--fast)
+         REGENERATE_TRANSITIVE=0
+         ;;
+      -h|--help)
+         print_help
+         exit 0
+         ;;
+      --)
+         break;;
+      *)
+         print_help
+         exit 1
+         ;;
+   esac
+   shift
+done
+
+HACKAGE2NIX="${HACKAGE2NIX:-hackage2nix}"
+
+config_dir=pkgs/development/haskell-modules/configuration-hackage2nix
+
+run_hackage2nix() {
+"$HACKAGE2NIX" \
+   --hackage "$unpacked_hackage" \
+   --preferred-versions <(for n in "$unpacked_hackage"/*/preferred-versions; do cat "$n"; echo; done) \
+   --nixpkgs "$PWD" \
+   --config "$compiler_config" \
+   --config "$config_dir/main.yaml" \
+   --config "$config_dir/stackage.yaml" \
+   --config "$config_dir/broken.yaml" \
+   --config "$config_dir/transitive-broken.yaml"
+}
+
+echo "Obtaining Hackage data …"
+extraction_derivation='with import ./. {}; runCommandLocal "unpacked-cabal-hashes" { } "tar xf ${all-cabal-hashes} --strip-components=1 --one-top-level=$out"'
+unpacked_hackage="$(nix-build -E "$extraction_derivation" --no-out-link)"
+
+echo "Generating compiler configuration …"
+compiler_config="$(nix-build -A haskellPackages.cabal2nix-unstable.compilerConfig --no-out-link)"
+
+echo "Running hackage2nix to regenerate pkgs/development/haskell-modules/hackage-packages.nix …"
+run_hackage2nix
+
+if [[ "$REGENERATE_TRANSITIVE" -eq 1 ]]; then
+
+echo "Regenerating transitive-broken.yaml … (pass --fast to $self to skip this step)"
+
+maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh
+
+echo "Running hackage2nix again to reflect changes in transitive-broken.yaml …"
+
+run_hackage2nix
+
+fi
+
+nixfmt pkgs/development/haskell-modules/hackage-packages.nix
+
+if [[ "$DO_COMMIT" -eq 1 ]]; then
+git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+git add pkgs/development/haskell-modules/hackage-packages.nix
+git commit --edit -F - << EOF
+haskellPackages: regenerate package set based on current config
+
+(generated by maintainers/scripts/haskell/regenerate-hackage-packages.sh)
+EOF
+fi
+
+echo "Regeneration of hackage-packages.nix finished."

maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh

@@ -0,0 +1,25 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils jq -I nixpkgs=.
+
+set -euo pipefail
+
+TMP_TEMPLATE=transitive-broken.XXXXXXX
+readonly TMP_TEMPLATE
+
+tmpfile=$(mktemp "$TMP_TEMPLATE")
+
+trap 'rm -f "${tmpfile}"' 0
+
+config_file=pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+
+cat > $tmpfile << EOF
+# This file is automatically generated by
+# maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh
+# It is supposed to list all haskellPackages that cannot evaluate because they
+# depend on a dependency marked as broken.
+dont-distribute-packages:
+EOF
+
+nix-instantiate --eval --option restrict-eval true -I . --strict --json maintainers/scripts/haskell/transitive-broken-packages.nix | jq -r . | LC_ALL=C.UTF-8 sort --ignore-case >> $tmpfile
+
+mv $tmpfile $config_file

maintainers/scripts/haskell/test-configurations.nix

@@ -0,0 +1,163 @@
+/*
+  Nix expression to test for regressions in the Haskell configuration overlays.
+
+  test-configurations.nix determines all attributes touched by given Haskell
+  configuration overlays (i. e. pkgs/development/haskell-modules/configuration-*.nix)
+  and builds all derivations (or at least a reasonable subset) affected by
+  these overrides.
+
+  By default, it checks `configuration-{common,nix,ghc-9.8.x}.nix`. You can
+  invoke it like this:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix --keep-going
+
+  It is possible to specify other configurations:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg files '[ "configuration-ghc-9.0.x.nix" "configuration-ghc-9.2.x.nix" ]' \
+      --keep-going
+
+  You can also just supply a single string:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --argstr files "configuration-arm.nix" --keep-going
+
+  You can even supply full paths which is handy, as it allows for tab-completing
+  the configurations:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --argstr files pkgs/development/haskell-modules/configuration-arm.nix \
+      --keep-going
+
+  By default, derivation that fail to evaluate are skipped, unless they are
+  “just” marked as broken. You can check for other eval errors like this:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg skipEvalErrors false --keep-going
+
+  You can also disable checking broken packages by passing a nixpkgs config:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg config '{ allowBroken = false; }' --keep-going
+
+  By default the haskell.packages.ghc*Binary sets used for bootstrapping GHC
+  are _not_ tested. You can change this using:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg skipBinaryGHCs false --keep-going
+*/
+{
+  files ? [
+    "configuration-common.nix"
+    "configuration-nix.nix"
+    "configuration-ghc-9.8.x.nix"
+  ],
+  nixpkgsPath ? ../../..,
+  config ? {
+    allowBroken = true;
+  },
+  skipEvalErrors ? true,
+  skipBinaryGHCs ? true,
+}:
+
+let
+  pkgs = import nixpkgsPath { inherit config; };
+  inherit (pkgs) lib;
+
+  # see usage explanation for the input format `files` allows
+  files' = map baseNameOf (if !builtins.isList files then [ files ] else files);
+
+  packageSetsWithVersionedHead =
+    pkgs.haskell.packages
+    // (
+      let
+        headSet = pkgs.haskell.packages.ghcHEAD;
+        # Determine the next GHC release version following GHC HEAD.
+        # GHC HEAD always has an uneven, tentative version number, e.g. 9.7.
+        # GHC releases always have even numbers, i.e. GHC 9.8 is branched off from
+        # GHC HEAD 9.7. Since we use the to be release number for GHC HEAD's
+        # configuration file, we need to calculate this here.
+        headVersion = lib.pipe headSet.ghc.version [
+          lib.versions.splitVersion
+          (lib.take 2)
+          lib.concatStrings
+          lib.strings.toInt
+          (builtins.add 1)
+          toString
+        ];
+      in
+      {
+        "ghc${headVersion}" = headSet;
+      }
+    );
+
+  setsForFile =
+    fileName:
+    let
+      # extract the unique part of the config's file name
+      configName = builtins.head (builtins.match "configuration-(.+).nix" fileName);
+      # match the major and minor version of the GHC the config is intended for, if any
+      configVersion = lib.concatStrings (builtins.match "ghc-([0-9]+).([0-9]+).x" configName);
+      # return all package sets under haskell.packages matching the version components
+      setsForVersion = map (name: packageSetsWithVersionedHead.${name}) (
+        builtins.filter (
+          setName:
+          lib.hasPrefix "ghc${configVersion}" setName && (skipBinaryGHCs -> !(lib.hasInfix "Binary" setName))
+        ) (builtins.attrNames packageSetsWithVersionedHead)
+      );
+
+      defaultSets = [ pkgs.haskellPackages ];
+    in
+    {
+      # use plain haskellPackages for the version-agnostic files
+      # TODO(@sternenseemann): also consider currently selected versioned sets
+      "common" = defaultSets;
+      "nix" = defaultSets;
+      "arm" = defaultSets;
+      "darwin" = defaultSets;
+    }
+    .${configName} or setsForVersion;
+
+  # attribute set that has all the attributes of haskellPackages set to null
+  availableHaskellPackages = builtins.listToAttrs (
+    map (attr: lib.nameValuePair attr null) (builtins.attrNames pkgs.haskellPackages)
+  );
+
+  # evaluate a configuration and only return the attributes changed by it,
+  # pass availableHaskellPackages as super in case intersectAttrs is used
+  overriddenAttrs =
+    fileName:
+    builtins.attrNames (
+      lib.fix (
+        self:
+        import (nixpkgsPath + "/pkgs/development/haskell-modules/${fileName}") {
+          haskellLib = pkgs.haskell.lib.compose;
+          inherit pkgs;
+        } self availableHaskellPackages
+      )
+    );
+
+  # list of derivations that are affected by overrides in the given configuration
+  # overlays. For common, nix, darwin etc. only the derivation from the default
+  # package set will be emitted.
+  packages =
+    builtins.filter
+      (
+        v:
+        lib.warnIf (v.meta.broken or false) "${v.pname} is marked as broken" (
+          v != null && (skipEvalErrors -> (builtins.tryEval (v.outPath or v)).success)
+        )
+      )
+      (
+        lib.concatMap (
+          fileName:
+          let
+            sets = setsForFile fileName;
+            attrs = overriddenAttrs fileName;
+          in
+          lib.concatMap (set: map (attr: set.${attr}) attrs) sets
+        ) files'
+      );
+in
+
+packages

maintainers/scripts/haskell/transitive-broken-packages.nix

@@ -0,0 +1,17 @@
+let
+  nixpkgs = import ../../..;
+  inherit (nixpkgs { }) pkgs lib;
+  getEvaluating =
+    x:
+    builtins.attrNames (
+      lib.filterAttrs (
+        _: v: (builtins.tryEval (v.outPath or null)).success && lib.isDerivation v && !v.meta.broken
+      ) x
+    );
+  brokenDeps = lib.subtractLists (getEvaluating pkgs.haskellPackages) (
+    getEvaluating (nixpkgs { config.allowBroken = true; }).haskellPackages
+  );
+in
+''
+  ${lib.concatMapStringsSep "\n" (x: " - ${x}") brokenDeps}
+''

maintainers/scripts/haskell/unbreak.nu

@@ -0,0 +1,31 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i nu -p nushell flock gnused -I nixpkgs=.
+
+# This script tests to build all packages listed in broken.yaml, expecting a build failure.
+# It will remove all packages that build fine from the list.
+
+# Attention: For unknown reasons, the script can't be easily cancelled and needs to be killed manually if it shouldn't run to completion.
+
+use std log
+
+let broken_config = "pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
+
+def is-broken [package : string]: nothing -> bool {
+    let res = with-env { NIXPKGS_ALLOW_BROKEN: "1" } {
+        # rather high timeout of half an hour, just to prevent never-ending builds
+        ^nix-build --no-out-link -j 1 --cores 1 --timeout 1800 -A $"haskellPackages.($package)" | complete
+    }
+    if $res.exit_code == 0 {
+      log warning $"($package) is not broken anymore!"
+      return false
+    } else {
+      log info $"($package) is still broken."
+      log debug $"($package) build log:\n($res.stderr)"
+      return true
+    }
+}
+
+def main [] {
+    $broken_config | open | get broken-packages
+      | par-each {|package| if not (is-broken $package) { ^flock -x $broken_config -c $"sed -i -e '/^  - ($package) /d' ($broken_config)" }}
+}

maintainers/scripts/haskell/update-cabal2nix-unstable.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils curl jq gnused haskellPackages.cabal2nix-unstable.bin nix-prefetch-scripts -I nixpkgs=.
+
+# Updates cabal2nix-unstable to the latest master of the nixos/cabal2nix repository.
+# See regenerate-hackage-packages.sh for details on the purpose of this script.
+
+set -euo pipefail
+
+# fetch current master HEAD from Github
+head_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/NixOS/cabal2nix/branches/master)"
+# extract commit hash
+commit="$(jq -r .commit.sha <<< "$head_info")"
+# extract commit timestamp and convert to date
+date="$(date "--date=$(jq -r .commit.commit.committer.date <<< "$head_info")" +%F)"
+
+# generate nix expression from cabal file, replacing the version with the commit date
+function mkPackage() {
+  output=pkgs/development/haskell-modules/cabal2nix-unstable/$1.nix
+  echo "# This file defines $1-unstable, used by maintainers/scripts/haskell/regenerate-hackage-packages.sh." > "$output"
+  cabal2nix --subpath "$1" "https://github.com/NixOS/cabal2nix/archive/$commit.tar.gz" | sed -Ee 's/version = "(.*)"/version = "\1-unstable-'"$date"'"/' >> "$output"
+  nixfmt "$output"
+}
+
+mkPackage "cabal2nix"
+mkPackage "distribution-nixpkgs"
+mkPackage "hackage-db"
+mkPackage "language-nix"

maintainers/scripts/haskell/update-hackage.sh

@@ -0,0 +1,56 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl jq git gnused -I nixpkgs=.
+#
+# SYNOPSIS
+#
+#   Update Hackage index and hashes data exposed via pkgs.all-cabal-hashes.
+#
+# DESCRIPTION
+#
+#   Find latest revision of the commercialhaskell/all-cabal-hashes repository's
+#   hackage branch and update pkgs/data/misc/hackage/pin.json accordingly.
+#
+#   This data is used by hackage2nix to generate hackage-packages.nix. Since
+#   hackage2nix uses the latest version of a package unless an explicit
+#   constraint is configured, running this script indirectly updates packages
+#   (when hackage2nix is executed afterwards).
+#
+#   Prints a version difference to stdout if the pin has been updated, nothing
+#   otherwise.
+#
+# EXIT STATUS
+#
+#   Always exit with zero (even if nothing changed) unless there was an error.
+
+set -euo pipefail
+
+if [[ "${1:-}" == "--do-commit" ]]; then
+  echo "$0: --do-commit is no longer supported. Use update-package-set.sh instead."
+  exit 100
+fi
+
+pin_file=pkgs/data/misc/hackage/pin.json
+current_commit="$(jq -r .commit $pin_file)"
+old_date="$(jq -r .msg $pin_file | sed 's/Update from Hackage at //')"
+git_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/commercialhaskell/all-cabal-hashes/branches/hackage)"
+head_commit="$(echo "$git_info" | jq -r .commit.sha)"
+commit_msg="$(echo "$git_info" | jq -r .commit.commit.message)"
+new_date="$(echo "$commit_msg" | sed 's/Update from Hackage at //')"
+
+if [ "$current_commit" != "$head_commit" ]; then
+   echo "Updating all-cabal-hashes from $old_date to $new_date" >&2
+   url="https://github.com/commercialhaskell/all-cabal-hashes/archive/$head_commit.tar.gz"
+   hash="$(nix-prefetch-url "$url")"
+   jq -n \
+     --arg commit "$head_commit" \
+     --arg hash "$hash" \
+     --arg url "$url" \
+     --arg commit_msg "$commit_msg" \
+     '{commit: $commit, url: $url, sha256: $hash, msg: $commit_msg}' \
+     > $pin_file
+else
+   echo "No new all-cabal-hashes version" >&2
+   exit 0
+fi
+
+echo "$old_date -> $new_date"

maintainers/scripts/haskell/update-package-set.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash
+#! nix-shell -p git -I nixpkgs=.
+set -euo pipefail
+
+filesToStage=(
+  'pkgs/data/misc/hackage/pin.json'
+  'pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml'
+  'pkgs/development/haskell-modules/hackage-packages.nix'
+)
+
+if ! git diff --quiet --cached; then
+  echo "Please commit staged changes before running $0" >&2
+  exit 100
+fi
+
+if ! git diff --quiet -- "${filesToStage[@]}"; then
+  echo -n "Please commit your changes to the following files before running $0: " >&2
+  echo    "${filesToStage[@]}" >&2
+  exit 100
+fi
+
+stackage_diff="$(./maintainers/scripts/haskell/update-stackage.sh)"
+hackage_diff="$(./maintainers/scripts/haskell/update-hackage.sh)"
+readonly stackage_diff hackage_diff
+
+# Prefer Stackage version diff in the commit header, fall back to Hackage
+if [[ -n "$stackage_diff" ]]; then
+  commit_message="haskellPackages: stackage $stackage_diff"
+  if [[ -n "$hackage_diff" ]]; then
+    commit_message="$commit_message
+
+all-cabal-hashes: $hackage_diff"
+  fi
+elif [[ -n "$hackage_diff" ]]; then
+  commit_message="haskellPackages: hackage $hackage_diff
+
+all-cabal-hashes: $hackage_diff"
+else
+  echo "Neither Hackage nor Stackage changed. Nothing to do." >&2
+  exit 0
+fi
+
+commit_message="$commit_message
+
+(generated by maintainers/scripts/haskell/update-package-set.sh)"
+
+# Using fast here because after the hackage-update eval errors will likely break the transitive dependencies check.
+./maintainers/scripts/haskell/regenerate-hackage-packages.sh --fast
+
+# A --do-commit flag probably doesn't make much sense
+git add -- "${filesToStage[@]}"
+git commit -m "$commit_message"

maintainers/scripts/haskell/update-stackage.sh

@@ -0,0 +1,109 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl jq git gnused gnugrep -I nixpkgs=.
+# shellcheck shell=bash
+#
+# SYNOPSIS
+#
+#   Update version constraints in hackage2nix config file from Stackage.
+#
+# DESCRIPTION
+#
+#   Fetches the latest snapshot of the configured Stackage solver which is
+#   configured via the SOLVER (either LTS or Nightly) and VERSION variables in
+#   the script.
+#
+#   VERSION is only applicable if SOLVER is LTS. SOLVER=LTS and VERSION=22
+#   will cause update-stackage.sh to fetch the latest LTS-22.XX version.
+#   If empty, the latest version of the solver is used.
+#
+#   If the configuration file has been updated, update-stackage.sh prints a
+#   version difference to stdout, e.g. 23.11 -> 23.13. Otherwise, stdout remains
+#   empty.
+#
+# EXIT STATUS
+#
+#   Always exit with zero (even if nothing changed) unless there was an error.
+
+set -eu -o pipefail
+
+if [[ "${1:-}" == "--do-commit" ]]; then
+  echo "$0: --do-commit is no longer supported. Use update-package-set.sh instead."
+  exit 100
+fi
+
+# Stackage solver to use, LTS or Nightly
+# (should be capitalized like the display name)
+SOLVER=LTS
+# Stackage solver verson, if any. Use latest if empty
+VERSION=
+TMP_TEMPLATE=update-stackage.XXXXXXX
+readonly SOLVER
+readonly VERSION
+readonly TMP_TEMPLATE
+
+toLower() {
+    printf "%s" "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+tmpfile=$(mktemp "$TMP_TEMPLATE")
+tmpfile_new=$(mktemp "$TMP_TEMPLATE")
+
+stackage_config="pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml"
+
+trap 'rm "${tmpfile}" "${tmpfile_new}"' 0
+touch "$tmpfile" "$tmpfile_new" # Creating files here so that trap creates no errors.
+
+curl -L -s "https://stackage.org/$(toLower "$SOLVER")${VERSION:+-$VERSION}/cabal.config" >"$tmpfile"
+old_version=$(grep '^# Stackage' $stackage_config | sed -e 's/.\+ \([A-Za-z]\+ [0-9.-]\+\)$/\1/g')
+version="$SOLVER $(sed -rn "s/^--.*http:..(www.)?stackage.org.snapshot.$(toLower "$SOLVER")-//p" "$tmpfile")"
+
+if [[ "$old_version" == "$version" ]]; then
+   echo "No new stackage version" >&2
+   exit 0 # Nothing to do
+fi
+
+echo "Updating Stackage from $old_version to $version." >&2
+
+# Create a simple yaml version of the file.
+sed -r \
+    -e '/^--/d' \
+    -e 's|^constraints:||' \
+    -e 's|^ +|  - |' \
+    -e 's|,$||' \
+    -e '/^with-compiler:/d' \
+    -e '/installed$/d' \
+    -e '/^$/d' \
+    < "${tmpfile}" | LC_ALL=C.UTF-8 sort --ignore-case >"${tmpfile_new}"
+
+cat > $stackage_config << EOF
+# Stackage $version
+# This file is auto-generated by
+# maintainers/scripts/haskell/update-stackage.sh
+default-package-overrides:
+EOF
+
+# Drop restrictions on some tools where we always want the latest version.
+sed -r \
+    -e '/ cabal2nix /d' \
+    -e '/ distribution-nixpkgs /d' \
+    -e '/ jailbreak-cabal /d' \
+    -e '/ language-nix /d' \
+    -e '/ hackage-db /d' \
+    -e '/ cabal-install /d' \
+    -e '/ cabal-install-solver /d' \
+    -e '/ lsp /d' \
+    -e '/ lsp-types /d' \
+    -e '/ lsp-test /d' \
+    -e '/ hie-bios /d' \
+    -e '/ ShellCheck /d' \
+    -e '/ Agda /d' \
+    -e '/ stack /d' \
+    -e '/ git-annex /d' \
+    < "${tmpfile_new}" >> $stackage_config
+# Explanations:
+# cabal2nix, distribution-nixpkgs, jailbreak-cabal, language-nix: These are our packages and we know what we are doing.
+# lsp, lsp-types, lsp-test, hie-bios: These are tightly coupled to hls which is not in stackage. They have no rdeps in stackage.
+# ShellCheck: latest version of command-line dev tool.
+# Agda: The Agda community is fast-moving; we strive to always include the newest versions of Agda and the Agda packages in nixpkgs.
+
+echo "$old_version -> $version"

maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh

@@ -0,0 +1,43 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl gnused -I nixpkgs=.
+
+# On Hackage every package description shows a category "Distributions" which
+# lists a "NixOS" version.
+# This script uploads a csv to hackage which will update the displayed versions
+# based on the current versions in nixpkgs. This happens with a simple http
+# request.
+
+# For authorization you just need to have any valid hackage account. This
+# script uses the `username` and `password-command` field from your
+# ~/.cabal/config file.
+
+# e.g. username: maralorn
+#      password-command: pass hackage.haskell.org (this can be any command, but not an arbitrary shell expression. Like cabal we only read the first output line and ignore the rest.)
+# Those fields are specified under `upload` on the `cabal` man page.
+
+if test -z "$CABAL_DIR"; then
+  dirs=(
+    "$HOME/.cabal"
+    "${XDG_CONFIG_HOME:-$HOME/.config}/cabal"
+  )
+  missing=true
+
+  for dir in "${dirs[@]}"; do
+    if test -d "$dir"; then
+      export CABAL_DIR="$dir"
+      missing=false
+      break
+    fi
+  done
+
+  if $missing; then
+    echo "Could not find the cabal configuration directory in any of: ${dirs[@]}" >&2
+    exit 101
+  fi
+fi
+
+package_list="$(nix-build -A haskell.package-list)/nixos-hackage-packages.csv"
+username=$(grep "^username:" "$CABAL_DIR/config" | sed "s/^username: //")
+password_command=$(grep "^password-command:" "$CABAL_DIR/config" | sed "s/^password-command: //")
+curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" https://hackage.haskell.org/distro/NixOS/packages.csv
+echo

maintainers/scripts/hydra-eval-failures.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: with ps; [ requests pyquery click ])"
+
+# To use, just execute this script with --help to display help.
+
+import subprocess
+import json
+import sys
+
+import click
+import requests
+from pyquery import PyQuery as pq
+
+def map_dict (f, d):
+    for k,v in d.items():
+        d[k] = f(v)
+
+maintainers_json = subprocess.check_output([
+    'nix-instantiate', '-A', 'lib.maintainers', '--eval', '--strict', '--json'
+])
+maintainers = json.loads(maintainers_json)
+MAINTAINERS = map_dict(lambda v: v.get('github', None), maintainers)
+
+def get_response_text(url):
+    return pq(requests.get(url).text)  # IO
+
+EVAL_FILE = {
+    'nixos': 'nixos/release.nix',
+    'nixpkgs': 'pkgs/top-level/release.nix',
+}
+
+
+def get_maintainers(attr_name):
+    try:
+        nixname = attr_name.split('.')
+        meta_json = subprocess.check_output([
+            'nix-instantiate',
+            '--eval',
+            '--strict',
+            '-A',
+            '.'.join(nixname[1:]) + '.meta',
+            EVAL_FILE[nixname[0]],
+            '--arg',
+            'nixpkgs',
+            './.',
+            '--json'])
+        meta = json.loads(meta_json)
+        return meta.get('maintainers', [])
+    except:
+       return []
+
+def filter_github_users(maintainers):
+    github_only = []
+    for i in maintainers:
+        if i.get('github'):
+            github_only.append(i)
+    return github_only
+
+def print_build(table_row):
+    a = pq(table_row)('a')[1]
+    print("- [ ] [{}]({})".format(a.text, a.get('href')), flush=True)
+
+    job_maintainers = filter_github_users(get_maintainers(a.text))
+    if job_maintainers:
+        print("  - maintainers: {}".format(" ".join(map(lambda u: '@' + u.get('github'), job_maintainers))))
+    # TODO: print last three persons that touched this file
+    # TODO: pinpoint the diff that broke this build, or maybe it's transient or maybe it never worked?
+
+    sys.stdout.flush()
+
+@click.command()
+@click.option(
+    '--jobset',
+    default="nixos/release-19.09",
+    help='Hydra project like nixos/release-19.09')
+def cli(jobset):
+    """
+    Given a Hydra project, inspect latest evaluation
+    and print a summary of failed builds
+    """
+
+    url = "https://hydra.nixos.org/jobset/{}".format(jobset)
+
+    # get the last evaluation
+    click.echo(click.style(
+        'Getting latest evaluation for {}'.format(url), fg='green'))
+    d = get_response_text(url)
+    evaluations = d('#tabs-evaluations').find('a[class="row-link"]')
+    latest_eval_url = evaluations[0].get('href')
+
+    # parse last evaluation page
+    click.echo(click.style(
+        'Parsing evaluation {}'.format(latest_eval_url), fg='green'))
+    d = get_response_text(latest_eval_url + '?full=1')
+
+    # TODO: aborted evaluations
+    # TODO: dependency failed without propagated builds
+    print('\nFailures:')
+    for tr in d('img[alt="Failed"]').parents('tr'):
+        print_build(tr)
+
+    print('\nDependency failures:')
+    for tr in d('img[alt="Dependency failed"]').parents('tr'):
+        print_build(tr)
+
+
+
+if __name__ == "__main__":
+    try:
+        cli()
+    except Exception as e:
+        import pdb;pdb.post_mortem()

maintainers/scripts/hydra_eval_check

@@ -0,0 +1,13 @@
+#! /bin/sh
+
+# give absolute path of release.nix as argument
+hydra_eval_jobs \
+  --argstr system x86_64-linux \
+  --argstr system i686-linux \
+  --argstr system x86_64-darwin \
+  --argstr system i686-cygwin \
+  --argstr system x86_64-cygwin \
+  --argstr system i686-freebsd \
+  --arg officialRelease false \
+  --arg nixpkgs "{ outPath = builtins.storePath ./. ; rev = 1234; }" \
+  $@

maintainers/scripts/kde/collect-licenses.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p gnutar jq reuse
+set -eu
+cd "$(dirname "$(readlink -f "$0")")"/../../..
+
+TMPDIR=$(mktemp -d)
+trap 'rm -rf $TMPDIR' EXIT
+
+echo "# Prebuilding sources..."
+nix-build -A kdePackages.sources --no-link || true
+
+echo "# Evaluating sources..."
+declare -A sources
+eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)"
+
+echo "# Collecting licenses..."
+for k in "${!sources[@]}"; do
+    echo "- Processing $k..."
+
+    if [ ! -f "${sources[$k]}" ]; then
+        echo "Not found!"
+        continue
+    fi
+
+    mkdir "$TMPDIR/$k"
+    tar -C "$TMPDIR/$k" -xf "${sources[$k]}"
+
+    (cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json"
+done
+
+jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json

maintainers/scripts/kde/collect-logs.nu

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i nu -p nushell
+cd $"($env.FILE_PWD)/../../.."
+
+mkdir logs
+nix-env -qaP -f . -A kdePackages --json --out-path | from json | values | par-each { |it|
+    echo $"Processing ($it.pname)..."
+    if "outputs" in $it {
+        nix-store --read-log $it.outputs.out | save -f $"logs/($it.pname).log"
+    }
+}

maintainers/scripts/kde/collect-metadata.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])"
+import pathlib
+
+import click
+
+import utils
+
+@click.command
+@click.argument(
+    "repo-metadata",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        path_type=pathlib.Path,
+    ),
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+@click.option(
+    "--unstable",
+    default=False,
+    is_flag=True
+)
+def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path, unstable: bool):
+    metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata, unstable)
+    out_dir = nixpkgs / "pkgs/kde/generated"
+    metadata.write_json(out_dir)
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/collect-missing-deps.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+import pathlib
+
+OK_MISSING = {
+    # we don't use precompiled QML
+    'Qt6QuickCompiler',
+    'Qt6QmlCompilerPlusPrivate',
+    # usually used for version numbers
+    'Git',
+    # useless by itself, will warn if something else is not found
+    'PkgConfig',
+    # license verification
+    'ReuseTool',
+    # dev only
+    'ClangFormat',
+    # doesn't exist
+    'Qt6X11Extras',
+}
+
+OK_MISSING_BY_PACKAGE = {
+    "angelfish": {
+        "Qt6Feedback",  # we don't have it
+    },
+    "attica": {
+        "Python3",  # only used for license checks
+    },
+    "discover": {
+        "ApkQt",  # we don't have APK (duh)
+        "rpm-ostree-1",  # we don't have rpm-ostree (duh)
+        "Snapd",  # we don't have snaps and probably never will
+        "packagekitqt6",  # intentionally disabled
+    },
+    "elisa": {
+        "UPNPQT",  # upstream says it's broken
+    },
+    "extra-cmake-modules": {
+        "Sphinx",  # only used for docs, bloats closure size
+        "QCollectionGenerator"
+    },
+    "gwenview": {
+        "Tiff",  # duplicate?
+    },
+    "kio-extras-kf5": {
+        "KDSoapWSDiscoveryClient",  # actually vendored on KF5 version
+    },
+    "kitinerary": {
+        "OsmTools",  # used for map data updates, we use prebuilt
+    },
+    "kosmindoormap": {
+        "OsmTools",  # same
+        "Protobuf",
+    },
+    "kpty": {
+        "UTEMPTER",  # we don't have it and it probably wouldn't work anyway
+    },
+    "kpublictransport": {
+        "OsmTools",  # same
+        "PolyClipping",
+        "Protobuf",
+    },
+    "krfb": {
+        "Qt6XkbCommonSupport",  # not real
+    },
+    "ksystemstats": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "kuserfeedback": {
+        "Qt6Svg",  # all used for backend console stuff we don't ship
+        "QmlLint",
+        "Qt6Charts",
+        "FLEX",
+        "BISON",
+        "Php",
+        "PhpUnit",
+    },
+    "kwin": {
+        "display-info",  # newer versions identify as libdisplay-info
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "kwin-x11": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "libksysguard": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "mlt": {
+        "Qt5",  # intentionally disabled
+        "SWIG",
+    },
+    "plasma-desktop": {
+        "scim",  # upstream is dead, not packaged in Nixpkgs
+        "KAccounts6",  # dead upstream
+        "AccountsQt6",  # dead upstream
+        "signon-oauth2plugin",  # dead upstream
+    },
+    "plasma-dialer": {
+        "KTactileFeedback",  # dead?
+    },
+    "poppler-qt6": {
+        "gobject-introspection-1.0",  # we don't actually want to build the GTK variant
+        "gdk-pixbuf-2.0",
+        "gtk+-3.0",
+    },
+    "powerdevil": {
+        "DDCUtil",  # cursed, intentionally disabled
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "print-manager": {
+        "PackageKitQt6",  # used for auto-installing drivers which does not work for obvious reasons
+    },
+    "pulseaudio-qt": {
+        "Qt6Qml",  # tests only
+        "Qt6Quick",
+    },
+    "skladnik": {
+        "POVRay",  # too expensive to rerender all the assets
+    },
+    "syntax-highlighting": {
+        "XercesC",  # only used for extra validation at build time
+    }
+}
+
+def main():
+    here = pathlib.Path(__file__).parent.parent.parent.parent
+    logs = (here / "logs").glob("*.log")
+
+    for log in sorted(logs):
+        pname = log.stem
+
+        missing = []
+        is_in_block = False
+        with log.open(errors="replace") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.startswith("--   No package '"):
+                    package = line.removeprefix("--   No package '").removesuffix("' found")
+                    missing.append(package)
+                if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:":
+                    is_in_block = True
+                elif line.startswith("--") and is_in_block:
+                    is_in_block = False
+                elif line.startswith("*") and is_in_block:
+                    package = line.removeprefix("* ")
+                    missing.append(package)
+
+        missing = {
+            package
+            for package in missing
+            if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set()))
+        }
+
+        if missing:
+            print(pname + ":")
+            for line in missing:
+                print("  -", line)
+            print()
+
+if __name__ == '__main__':
+    main()

maintainers/scripts/kde/generate-sources.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.packaging ps.pyyaml ])" nix-update
+import base64
+import binascii
+import hashlib
+import json
+import pathlib
+import subprocess
+from urllib.parse import urljoin, urlparse
+
+import bs4
+import click
+import httpx
+import jinja2
+import packaging.version as v
+
+import utils
+
+
+LEAF_TEMPLATE = jinja2.Template('''
+{ mkKdeDerivation }:
+mkKdeDerivation {
+  pname = "{{ pname }}";
+}
+'''.strip())
+
+ROOT_TEMPLATE = jinja2.Template('''
+{ callPackage }:
+{
+  {%- for p in packages %}
+  {{ p }} = callPackage ./{{ p }} { };
+  {%- endfor %}
+}
+'''.strip())
+
+PROJECTS_WITH_RUST = {
+    "akonadi-search",
+    "angelfish",
+    "kdepim-addons",
+}
+
+def to_sri(hash):
+    raw = binascii.unhexlify(hash)
+    b64 = base64.b64encode(raw).decode()
+    return f"sha256-{b64}"
+
+
+@click.command
+@click.argument(
+    "pkgset",
+    type=click.Choice(["frameworks", "gear", "plasma"]),
+    required=True
+)
+@click.argument(
+    "version",
+    type=str,
+    required=True
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+@click.option(
+    "--sources-url",
+    type=str,
+    default=None,
+)
+def main(pkgset: str, version: str, nixpkgs: pathlib.Path, sources_url: str | None):
+    root_dir = nixpkgs / "pkgs/kde"
+    set_dir = root_dir / pkgset
+    generated_dir = root_dir / "generated"
+    metadata = utils.KDERepoMetadata.from_json(generated_dir)
+
+    if sources_url is None:
+        set_url = {
+            "frameworks": f"frameworks/{version}/",
+            "gear": f"release-service/{version}/src/",
+            "plasma": f"plasma/{version}/",
+        }[pkgset]
+        sources_url = f"https://download.kde.org/stable/{set_url}"
+
+    client = httpx.Client()
+    sources = client.get(sources_url)
+    sources.raise_for_status()
+    bs = bs4.BeautifulSoup(sources.text, features="html.parser")
+
+    results = {}
+    projects_to_update_rust = set()
+    for item in bs.select("tr")[3:]:
+        link = item.select_one("td:nth-child(2) a")
+        if not link:
+            continue
+
+        project_name, version_and_ext = link.text.rsplit("-", maxsplit=1)
+
+        if project_name not in metadata.projects_by_name:
+            print(f"Warning: unknown tarball: {project_name}")
+
+        if project_name in PROJECTS_WITH_RUST:
+            projects_to_update_rust.add(project_name)
+
+        if version_and_ext.endswith(".sig"):
+            continue
+
+        version = version_and_ext.removesuffix(".tar.xz")
+
+        url = urljoin(sources_url, link.attrs["href"])
+
+        hash = client.get(url + ".sha256").text.strip()
+
+        if hash == "Hash type not supported":
+            print(f"{url} missing hash on CDN, downloading...")
+            hasher = hashlib.sha256()
+            with client.stream("GET", url, follow_redirects=True) as r:
+                for data in r.iter_bytes():
+                    hasher.update(data)
+            hash = hasher.hexdigest()
+        else:
+            hash = hash.split(" ", maxsplit=1)[0]
+
+        if existing := results.get(project_name):
+            old_version = existing["version"]
+            if v.parse(old_version) > v.parse(version):
+                print(f"{project_name} {old_version} is newer than {version}, skipping...")
+                continue
+
+        results[project_name] = {
+            "version": version,
+            "url": "mirror://kde" + urlparse(url).path,
+            "hash": to_sri(hash)
+        }
+
+        pkg_dir = set_dir / project_name
+        pkg_file = pkg_dir / "default.nix"
+
+        if not pkg_file.exists():
+            print(f"Generated new package: {pkgset}/{project_name}")
+            pkg_dir.mkdir(parents=True, exist_ok=True)
+            with pkg_file.open("w") as fd:
+                fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
+
+    set_dir.mkdir(parents=True, exist_ok=True)
+    with (set_dir / "default.nix").open("w") as fd:
+        fd.write(ROOT_TEMPLATE.render(packages=sorted(results.keys())) + "\n")
+
+    sources_dir = generated_dir / "sources"
+    sources_dir.mkdir(parents=True, exist_ok=True)
+    with (sources_dir / f"{pkgset}.json").open("w") as fd:
+        json.dump(results, fd, indent=2)
+
+    for project_name in projects_to_update_rust:
+            print(f"Updating cargoDeps hash for {pkgset}/{project_name}...")
+            subprocess.run([
+                "nix-update",
+                f"kdePackages.{project_name}",
+                "--version",
+                "skip",
+                "--override-filename",
+                pkg_file
+            ])
+
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/utils.py

@@ -0,0 +1,185 @@
+import collections
+import dataclasses
+import functools
+import json
+import pathlib
+import subprocess
+
+import yaml
+
+class DataclassEncoder(json.JSONEncoder):
+    def default(self, it):
+        if dataclasses.is_dataclass(it):
+            return dataclasses.asdict(it)
+        return super().default(it)
+
+
+@dataclasses.dataclass
+class Project:
+    name: str
+    description: str | None
+    project_path: str
+    repo_path: str | None
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    @classmethod
+    def from_yaml(cls, path: pathlib.Path):
+        data = yaml.safe_load(path.open())
+        return cls(
+            name=data["identifier"],
+            description=data["description"],
+            project_path=data["projectpath"],
+            repo_path=data["repopath"]
+        )
+
+
+def get_git_commit(path: pathlib.Path):
+    return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip()
+
+
+def validate_unique(projects: list[Project], attr: str):
+    seen = set()
+    for item in projects:
+        attr_value = getattr(item, attr)
+        if attr_value in seen:
+            raise Exception(f"Duplicate {attr}: {attr_value}")
+        seen.add(attr_value)
+
+
+THIRD_PARTY = {
+    "third-party/appstream": "appstream-qt",
+    "third-party/cmark": "cmark",
+    "third-party/gpgme": "gpgme",
+    "third-party/kdsoap": "kdsoap",
+    "third-party/libaccounts-qt": "accounts-qt",
+    "third-party/libgpg-error": "libgpg-error",
+    "third-party/libquotient": "libquotient",
+    "third-party/packagekit-qt": "packagekit-qt",
+    "third-party/poppler": "poppler",
+    "third-party/qcoro": "qcoro",
+    "third-party/qmltermwidget": "qmltermwidget",
+    "third-party/qtkeychain": "qtkeychain",
+    "third-party/signond": "signond",
+    "third-party/taglib": "taglib",
+    "third-party/wayland-protocols": "wayland-protocols",
+    "third-party/wayland": "wayland",
+    "third-party/zxing-cpp": "zxing-cpp",
+}
+
+IGNORE = {
+    "kdesupport/phonon-directshow",
+    "kdesupport/phonon-mmf",
+    "kdesupport/phonon-mplayer",
+    "kdesupport/phonon-quicktime",
+    "kdesupport/phonon-waveout",
+    "kdesupport/phonon-xine"
+}
+
+WARNED = set()
+
+
+@dataclasses.dataclass
+class KDERepoMetadata:
+    version: str
+    projects: list[Project]
+    dep_graph: dict[Project, set[Project]]
+
+    @functools.cached_property
+    def projects_by_name(self):
+        return {p.name: p for p in self.projects}
+
+    @functools.cached_property
+    def projects_by_path(self):
+        return {p.project_path: p for p in self.projects}
+
+    def try_lookup_package(self, path):
+        if path in IGNORE:
+            return None
+        project = self.projects_by_path.get(path)
+        if project is None and path not in WARNED:
+            WARNED.add(path)
+            print(f"Warning: unknown project {path}")
+        return project
+
+    @classmethod
+    def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path, unstable=False):
+        projects = [
+            Project.from_yaml(metadata_file)
+            for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml")
+        ] + [
+            Project(id, None, project_path, None)
+            for project_path, id in THIRD_PARTY.items()
+        ]
+
+        validate_unique(projects, "name")
+        validate_unique(projects, "project_path")
+
+        self = cls(
+            version=get_git_commit(repo_metadata),
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_graph = collections.defaultdict(set)
+
+        if unstable:
+            spec_name = "dependency-data-kf6-qt6"
+        else:
+            spec_name = "dependency-data-stable-kf6-qt6"
+
+        spec_path = repo_metadata / "dependencies" / spec_name
+        for line in spec_path.open():
+            line = line.strip()
+            if line.startswith("#"):
+                continue
+            if not line:
+                continue
+
+            dependent, dependency = line.split(": ")
+
+            dependent = self.try_lookup_package(dependent)
+            if dependent is None:
+                continue
+
+            dependency = self.try_lookup_package(dependency)
+            if dependency is None:
+                continue
+
+            dep_graph[dependent].add(dependency)
+
+        self.dep_graph = dep_graph
+
+        return self
+
+    def write_json(self, root: pathlib.Path):
+        root.mkdir(parents=True, exist_ok=True)
+
+        with (root / "projects.json").open("w") as fd:
+            json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+        with (root / "dependencies.json").open("w") as fd:
+            deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()}
+            json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+    @classmethod
+    def from_json(cls, root: pathlib.Path):
+        projects = [
+            Project(**v) for v in json.load((root / "projects.json").open()).values()
+        ]
+
+        deps = json.load((root / "dependencies.json").open())
+        self = cls(
+            version=deps["version"],
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_graph = collections.defaultdict(set)
+        for dependent, dependencies in deps["dependencies"].items():
+            for dependency in dependencies:
+                dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency])
+
+        self.dep_graph = dep_graph
+        return self

maintainers/scripts/luarocks-config.lua

@@ -0,0 +1,8 @@
+
+-- default of luarocks listed at src/luarocks/core/cfg.lua
+-- keep this list synced with pkgs/build-support/fetchurl/mirrors.nix
+rocks_servers = {
+	"https://luarocks.org",
+	"https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/"
+}
+version_check_on_fail = false

maintainers/scripts/luarocks-packages.csv

@@ -0,0 +1,163 @@
+name,rockspec,ref,server,version,luaversion,maintainers
+alt-getopt,,,,,,arobyn
+ansicolors,,,,,,Freed-Wu
+argparse,,,,,,
+basexx,,,,,,
+binaryheap,,,,,,vcunat
+bit32,,,,,5.1,lblasc
+busted,,,,,,
+busted-htest,,,,,,mrcjkb
+cassowary,,,,,,alerque
+cldr,,,,,,alerque
+commons.nvim,,,,,,mrcjkb
+compat53,,,,,,vcunat
+cosmo,,,,,,
+coxpcall,,,,1.17.0-1,,
+cqueues,,,,,,vcunat
+cyan,,,,,,
+datafile,,,,,,
+digestif,,,,,5.3,
+dkjson,,,,,,
+fennel,,,,,,misterio77
+fidget.nvim,,,,,,mrcjkb
+fifo,,,,,,
+fluent,,,,,,alerque
+funnyfiles.nvim,,,,,,mrcjkb
+fzf-lua,,,,,,mrcjkb
+fzy,,,,,,mrcjkb
+gitsigns.nvim,https://raw.githubusercontent.com/lewis6991/gitsigns.nvim/main/gitsigns.nvim-scm-1.rockspec,,,,5.1,
+grug-far.nvim,,,,,,teto
+haskell-tools.nvim,,,,,,mrcjkb
+http,,,,0.4-0,,vcunat
+image.nvim,,,,,,teto
+inspect,,,,,,
+jsregexp,,,,,,
+ldbus,,,http://luarocks.org/dev,,,
+ldoc,,,,,,
+lgi,,,,,,
+linenoise,https://raw.githubusercontent.com/hoelzro/lua-linenoise/master/linenoise-0.9-1.rockspec,,,,,
+ljsyscall,,,,,5.1,lblasc
+llscheck,,,,,,mrcjkb
+lmathx,,,,,5.3,alexshpilkin
+lmpfrlib,,,,,5.3,alexshpilkin
+loadkit,,,,,,alerque
+lpeg,,,,,,vyp
+lpeg_patterns,,,,,,
+lpeglabel,,,,1.6.0,,
+lrexlib-gnu,,,,,,
+lrexlib-oniguruma,,,,,,junestepp
+lrexlib-pcre,,,,,,vyp
+lrexlib-posix,,,,,,
+lsp-progress.nvim,,,,,,gepbird
+lua-cjson,,,,,,
+lua-cmsgpack,,,,,,
+lua-curl,,,,,,
+lua-ffi-zlib,,,,,,
+lua-iconv,,,,7.0.0,,
+lua-lsp,,,,,,
+lua-messagepack,,,,,,
+lua-protobuf,,,,,,lockejan
+lua-resty-http,,,,,,
+lua-resty-jwt,,,,,,
+lua-resty-openidc,,,,,,
+lua-resty-openssl,,,,,,
+lua-resty-session,,,,,,
+lua-rtoml,https://raw.githubusercontent.com/lblasc/lua-rtoml/main/lua-rtoml-0.2-0.rockspec,,,,,lblasc
+lua-subprocess,https://raw.githubusercontent.com/0x0ade/lua-subprocess/master/subprocess-scm-1.rockspec,,,,5.1,scoder12
+lua-term,,,,,,
+lua-toml,,,,,,
+lua-utils.nvim,,,,,,mrcjkb
+lua-yajl,,,,,,pstn
+lua-zlib,,,,,,koral
+lua_cliargs,,,,,,
+luabitop,https://raw.githubusercontent.com/teto/luabitop/master/luabitop-1.0.2-3.rockspec,,,,,
+luacheck,,,,,,
+luacov,,,,,,
+luadbi,,,,,,
+luadbi-mysql,,,,,,
+luadbi-postgresql,,,,,,
+luadbi-sqlite3,,,,,,
+luaepnf,,,,,,
+luaevent,,,,,,
+luaexpat,,,,1.4.1-1,,arobyn flosse
+luaffi,,,http://luarocks.org/dev,,,
+luafilesystem,,,,1.8.0-1,,flosse
+lualdap,,,,,,aanderse
+lualine.nvim,,,http://luarocks.org/dev,,,
+lualogging,,,,,,
+luaossl,,,,,5.1,
+luaposix,,,,34.1.1-1,,vyp lblasc
+luaprompt,,,,,,Freed-Wu
+luarepl,,,,,,
+luarocks,,,,,,mrcjkb teto
+luarocks-build-rust-mlua,,,,,,mrcjkb
+luarocks-build-treesitter-parser,,,,,,mrcjkb
+luarocks-build-treesitter-parser-cpp,,,,,,mrcjkb
+luasec,,,,,,flosse
+luasnip,,,,,,
+luasocket,,,,,,
+luasql-sqlite3,,,,,,vyp
+luassert,,,,,,
+luasystem,,,,,,
+luatext,,,,,,
+luaunbound,,,,,,
+luaunit,,,,,,lockejan
+luautf8,,,,,,pstn
+luazip,,,,,,
+lusc_luv,,,,,,
+lush.nvim,,,https://luarocks.org/dev,,,teto
+luuid,,,,20120509-2,,
+lyaml,,,,,,lblasc
+lz.n,,,,,,mrcjkb
+lze,,,,,,birdee
+lzextras,,,,,,birdee
+lzn-auto-require,,,,,,mrcjkb
+magick,,,,,5.1,donovanglover
+markdown,,,,,,
+mediator_lua,,,,,,
+middleclass,,,,,,
+mimetypes,,,,,,
+mini.test,,,,,,
+moonscript,https://raw.githubusercontent.com/leafo/moonscript/master/moonscript-dev-1.rockspec,,,,,arobyn
+mpack,,,,,,
+neorg,,,,,,GaetanLepage
+neotest,,,,,,mrcjkb
+nlua,,,,,,teto
+nui.nvim,,,,,,mrcjkb
+nvim-cmp,https://raw.githubusercontent.com/hrsh7th/nvim-cmp/main/nvim-cmp-scm-1.rockspec,,,,,
+nvim-nio,,,,,,mrcjkb
+nvim-web-devicons,,,,,,
+oil.nvim,,,,,,
+orgmode,,,,,,
+papis.nvim,,,,,,GaetanLepage
+pathlib.nvim,,,,,,
+penlight,,,,,,alerque
+plenary.nvim,https://raw.githubusercontent.com/nvim-lua/plenary.nvim/master/plenary.nvim-scm-1.rockspec,,,,5.1,
+psl,,,,0.3,,
+rapidjson,,,,,,
+rest.nvim,,,,,5.1,teto
+rocks-config.nvim,,,,,,mrcjkb
+rocks-dev.nvim,,,,,,mrcjkb
+rocks-git.nvim,,,,,,mrcjkb
+rocks.nvim,,,,,,mrcjkb
+rtp.nvim,,,,,,mrcjkb
+rustaceanvim,,,,,,mrcjkb
+say,,,,,,
+serpent,,,,,,lockejan
+sofa,,,,,,f4z3r
+sqlite,,,,,,
+std._debug,,,,,,
+std.normalize,,,,,,
+stdlib,,,,41.2.2,,vyp
+teal-language-server,,,,,,
+telescope-manix,,,,,,
+telescope.nvim,,,,,5.1,
+tiktoken_core,,,,,,natsukium
+tl,,,,,,mephistophiles
+toml-edit,,,,,5.1,mrcjkb
+tree-sitter-http,,,,0.0.33-1,,
+tree-sitter-norg,,,,,5.1,mrcjkb
+tree-sitter-orgmode,,,,,,
+vstruct,,,,,,
+vusted,,,,,,figsoda
+xml2lua,,,,,,teto

maintainers/scripts/nix-call-package

@@ -0,0 +1,5 @@
+#! /bin/sh
+
+echo "let pkgs = import <nixpkgs$2> {}; x = pkgs.callPackage $1 { $3 }; in ${4:-x}" |
+nix-instantiate --show-trace - |
+xargs nix-store -r -K

maintainers/scripts/nix-diff.sh

@@ -0,0 +1,277 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils gnugrep gnused
+
+################################################################################
+# nix-diff.sh                                                                  #
+################################################################################
+# This script "diffs" Nix profile generations.                                 #
+#                                                                              #
+# Example:                                                                     #
+################################################################################
+# > nix-diff.sh 90 92                                                          #
+# + gnumake-4.2.1                                                              #
+# + gnumake-4.2.1-doc                                                          #
+# - htmldoc-1.8.29                                                             #
+################################################################################
+# The example shows that as of generation 92 and since generation 90,          #
+# gnumake-4.2.1 and gnumake-4.2.1-doc have been installed, while               #
+# htmldoc-1.8.29 has been removed.                                             #
+#                                                                              #
+# The example above shows the default, minimal output mode of this script.     #
+# For more features, run `nix-diff.sh -h` for usage instructions.              #
+################################################################################
+
+usage() {
+    cat <<EOF
+usage: nix-diff.sh [-h | [-p profile | -s] [-q] [-l] [range]]
+-h:         print this message before exiting
+-q:         list the derivations installed in the parent generation
+-l:         diff every available intermediate generation between parent and
+            child
+-p profile: specify the Nix profile to use
+            * defaults to ~/.nix-profile
+-s:         use the system profile
+            * equivalent to: -p /nix/var/nix/profiles/system
+profile:    * should be something like /nix/var/nix/profiles/default, not a
+              generation link like /nix/var/nix/profiles/default-2-link
+range:      the range of generations to diff
+            * the following patterns are allowed, where A, B, and N are positive
+              integers, and G is the currently active generation:
+                A..B => diffs from generation A to generation B
+                ~N   => diffs from the Nth newest generation (older than G) to G
+                A    => diffs from generation A to G
+            * defaults to ~1
+EOF
+}
+
+usage_tip() {
+    echo 'run `nix-diff.sh -h` for usage instructions' >&2
+    exit 1
+}
+
+while getopts :hqlp:s opt; do
+    case $opt in
+        h)
+            usage
+            exit
+            ;;
+        q)
+            opt_query=1
+            ;;
+        l)
+            opt_log=1
+            ;;
+        p)
+            opt_profile=$OPTARG
+            ;;
+        s)
+            opt_profile=/nix/var/nix/profiles/system
+            ;;
+        \?)
+            echo "error: invalid option -$OPTARG" >&2
+            usage_tip
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+if [ -n "$opt_profile" ]; then
+    if ! [ -L "$opt_profile" ]; then
+        echo "error: expecting \`$opt_profile\` to be a symbolic link" >&2
+        usage_tip
+    fi
+else
+    opt_profile=$(readlink ~/.nix-profile)
+    if (( $? != 0 )); then
+        echo 'error: unable to dereference `~/.nix-profile`' >&2
+        echo 'specify the profile manually with the `-p` flag' >&2
+        usage_tip
+    fi
+fi
+
+list_gens() {
+    nix-env -p "$opt_profile" --list-generations \
+        | sed -r 's:^\s*::' \
+        | cut -d' ' -f1
+}
+
+current_gen() {
+    nix-env -p "$opt_profile" --list-generations \
+        | grep -E '\(current\)\s*$' \
+        | sed -r 's:^\s*::' \
+        | cut -d' ' -f1
+}
+
+neg_gen() {
+    local i=0 from=$1 n=$2 tmp
+    for gen in $(list_gens | sort -rn); do
+        if ((gen < from)); then
+            tmp=$gen
+            ((i++))
+            ((i == n)) && break
+        fi
+    done
+    if ((i < n)); then
+        echo -n "error: there aren't $n generation(s) older than" >&2
+        echo " generation $from" >&2
+        return 1
+    fi
+    echo $tmp
+}
+
+match() {
+    argv=("$@")
+    for i in $(seq $(($#-1))); do
+        if grep -E "^${argv[$i]}\$" <(echo "$1") >/dev/null; then
+            echo $i
+            return
+        fi
+    done
+    echo 0
+}
+
+case $(match "$1" '' '[0-9]+' '[0-9]+\.\.[0-9]+' '~[0-9]+') in
+    1)
+        diffTo=$(current_gen)
+        diffFrom=$(neg_gen $diffTo 1)
+        (($? == 1)) && usage_tip
+        ;;
+    2)
+        diffFrom=$1
+        diffTo=$(current_gen)
+        ;;
+    3)
+        diffFrom=${1%%.*}
+        diffTo=${1##*.}
+        ;;
+    4)
+        diffTo=$(current_gen)
+        diffFrom=$(neg_gen $diffTo ${1#*~})
+        (($? == 1)) && usage_tip
+        ;;
+    0)
+        echo 'error: invalid invocation' >&2
+        usage_tip
+        ;;
+esac
+
+dirA="${opt_profile}-${diffFrom}-link"
+dirB="${opt_profile}-${diffTo}-link"
+
+declare -a temp_files
+temp_length() {
+    echo -n ${#temp_files[@]}
+}
+temp_make() {
+    temp_files[$(temp_length)]=$(mktemp)
+}
+temp_clean() {
+    rm -f ${temp_files[@]}
+}
+temp_name() {
+    echo -n "${temp_files[$(($(temp_length)-1))]}"
+}
+trap 'temp_clean' EXIT
+
+temp_make
+versA=$(temp_name)
+refs=$(nix-store -q --references "$dirA")
+(( $? != 0 )) && exit 1
+echo "$refs" \
+    | grep -v env-manifest.nix \
+    | sort \
+          > "$versA"
+
+print_tag() {
+    local gen=$1
+    nix-env -p "$opt_profile" --list-generations \
+        | grep -E "^\s*${gen}" \
+        | sed -r 's:^\s*::' \
+        | sed -r 's:\s*$::'
+}
+
+if [ -n "$opt_query" ]; then
+    print_tag $diffFrom
+    cat "$versA" \
+        | sed -r 's:^[^-]+-(.*)$:    \1:'
+
+    print_line=1
+fi
+
+if [ -n "$opt_log" ]; then
+    gens=$(for gen in $(list_gens); do
+               ((diffFrom < gen && gen < diffTo)) && echo $gen
+           done)
+    # Force the $diffTo generation to be included in this list, instead of using
+    # `gen <= diffTo` in the preceding loop, so we encounter an error upon the
+    # event of its nonexistence.
+    gens=$(echo "$gens"
+           echo $diffTo)
+else
+    gens=$diffTo
+fi
+
+temp_make
+add=$(temp_name)
+temp_make
+rem=$(temp_name)
+temp_make
+out=$(temp_name)
+
+for gen in $gens; do
+
+    [ -n "$print_line" ] && echo
+
+    temp_make
+    versB=$(temp_name)
+
+    dirB="${opt_profile}-${gen}-link"
+    refs=$(nix-store -q --references "$dirB")
+    (( $? != 0 )) && exit 1
+    echo "$refs" \
+        | grep -v env-manifest.nix \
+        | sort \
+              > "$versB"
+
+    in=$(comm -3 -1 "$versA" "$versB")
+    sed -r 's:^[^-]*-(.*)$:\1+:' <(echo "$in") \
+        | sort -f \
+               > "$add"
+
+    un=$(comm -3 -2 "$versA" "$versB")
+    sed -r 's:^[^-]*-(.*)$:\1-:' <(echo "$un") \
+        | sort -f \
+               > "$rem"
+
+    cat "$rem" "$add" \
+        | sort -f \
+        | sed -r 's:(.*)-$:- \1:' \
+        | sed -r 's:(.*)\+$:\+ \1:' \
+        | grep -v '^$' \
+              > "$out"
+
+    if [ -n "$opt_query" -o -n "$opt_log" ]; then
+
+        lines=$(wc -l "$out" | cut -d' ' -f1)
+        tag=$(print_tag "$gen")
+        (( $? != 0 )) && exit 1
+        if [ $lines -eq 0 ]; then
+            echo "$tag   (no change)"
+        else
+            echo "$tag"
+        fi
+        cat "$out" \
+            | sed 's:^:    :'
+
+        print_line=1
+
+    else
+        echo "diffing from generation $diffFrom to $diffTo"
+        cat "$out"
+    fi
+
+    versA=$versB
+
+done
+
+exit 0

maintainers/scripts/nix-generate-from-cpan.nix

@@ -0,0 +1,37 @@
+{
+  stdenv,
+  lib,
+  makeWrapper,
+  perl,
+  perlPackages,
+}:
+
+stdenv.mkDerivation {
+  name = "nix-generate-from-cpan-3";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  buildInputs = with perlPackages; [
+    perl
+    GetoptLongDescriptive
+    CPANPLUS
+    Readonly
+    LogLog4perl
+  ];
+
+  dontUnpack = true;
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp ${./nix-generate-from-cpan.pl} $out/bin/nix-generate-from-cpan
+    patchShebangs $out/bin/nix-generate-from-cpan
+    wrapProgram $out/bin/nix-generate-from-cpan --set PERL5LIB $PERL5LIB
+  '';
+
+  meta = {
+    maintainers = with lib.maintainers; [ eelco ];
+    description = "Utility to generate a Nix expression for a Perl package from CPAN";
+    mainProgram = "nix-generate-from-cpan";
+    platforms = lib.platforms.unix;
+  };
+}

maintainers/scripts/nix-generate-from-cpan.pl

@@ -0,0 +1,473 @@
+#!/usr/bin/env perl
+
+use utf8;
+use strict;
+use warnings;
+
+use CPAN::Meta();
+use CPANPLUS::Backend();
+use MIME::Base64;
+use Module::CoreList;
+use Getopt::Long::Descriptive qw( describe_options );
+use JSON::PP qw( encode_json );
+use Log::Log4perl qw(:easy);
+use Readonly();
+
+# Readonly hash that maps CPAN style license strings to information
+# necessary to generate a Nixpkgs style license attribute.
+Readonly::Hash my %LICENSE_MAP => (
+
+    # The Perl 5 License (Artistic 1 & GPL 1 or later).
+    perl_5 => {
+        licenses => [qw( artistic1 gpl1Plus )]
+    },
+
+    # GNU Affero General Public License, Version 3.
+    agpl_3 => {
+        licenses => [qw( agpl3Plus )],
+        amb      => 1
+    },
+
+    # Apache Software License, Version 1.1.
+    apache_1_1 => {
+        licenses => ["Apache License 1.1"],
+        in_set   => 0
+    },
+
+    # Apache License, Version 2.0.
+    apache_2_0 => {
+        licenses => [qw( asl20 )]
+    },
+
+    # Artistic License, (Version 1).
+    artistic_1 => {
+        licenses => [qw( artistic1 )]
+    },
+
+    # Artistic License, Version 2.0.
+    artistic_2 => {
+        licenses => [qw( artistic2 )]
+    },
+
+    # BSD License (three-clause).
+    bsd => {
+        licenses => [qw( bsd3 )],
+        amb      => 1
+    },
+
+    # FreeBSD License (two-clause).
+    freebsd => {
+        licenses => [qw( bsd2 )]
+    },
+
+    # GNU Free Documentation License, Version 1.2.
+    gfdl_1_2 => {
+        licenses => [qw( fdl12Plus )]
+    },
+
+    # GNU Free Documentation License, Version 1.3.
+    gfdl_1_3 => {
+        licenses => [qw( fdl13Plus )]
+    },
+
+    # GNU General Public License, Version 1.
+    gpl_1 => {
+        licenses => [qw( gpl1Plus )],
+        amb      => 1
+    },
+
+    # GNU General Public License, Version 2. Note, we will interpret
+    # "gpl" alone as GPL v2+.
+    gpl_2 => {
+        licenses => [qw( gpl2Plus )],
+        amb      => 1
+    },
+
+    # GNU General Public License, Version 3.
+    gpl_3 => {
+        licenses => [qw( gpl3Plus )],
+        amb      => 1
+    },
+
+    # GNU Lesser General Public License, Version 2.1. Note, we will
+    # interpret "gpl" alone as LGPL v2.1+.
+    lgpl_2_1 => {
+        licenses => [qw( lgpl21Plus )],
+        amb      => 1
+    },
+
+    # GNU Lesser General Public License, Version 3.0.
+    lgpl_3_0 => {
+        licenses => [qw( lgpl3Plus )],
+        amb      => 1
+    },
+
+    # MIT (aka X11) License.
+    mit => {
+        licenses => [qw( mit )]
+    },
+
+    # Mozilla Public License, Version 1.0.
+    mozilla_1_0 => {
+        licenses => [qw( mpl10 )]
+    },
+
+    # Mozilla Public License, Version 1.1.
+    mozilla_1_1 => {
+        licenses => [qw( mpl11 )]
+    },
+
+    # OpenSSL License.
+    openssl => {
+        licenses => [qw( openssl )]
+    },
+
+    # Q Public License, Version 1.0.
+    qpl_1_0 => {
+        licenses => [qw( qpl )]
+    },
+
+    # Original SSLeay License.
+    ssleay => {
+        licenses => ["Original SSLeay License"],
+        in_set   => 0
+    },
+
+    # Sun Internet Standards Source License (SISSL).
+    sun => {
+        licenses => ["Sun Industry Standards Source License v1.1"],
+        in_set   => 0
+    },
+
+    # zlib License.
+    zlib => {
+        licenses => [qw( zlib )]
+    },
+
+    # Other Open Source Initiative (OSI) approved license.
+    open_source => {
+        licenses => [qw( free )],
+        amb      => 1
+    },
+
+    # Requires special permission from copyright holder.
+    restricted => {
+        licenses => [qw( unfree )],
+        amb      => 1
+    },
+
+    # Not an OSI approved license, but not restricted. Note, we
+    # currently map this to unfreeRedistributable, which is a
+    # conservative choice.
+    unrestricted => {
+        licenses => [qw( unfreeRedistributable )],
+        amb      => 1
+    },
+
+    # License not provided in metadata.
+    unknown => {
+        licenses => [],
+        amb      => 1
+    }
+);
+
+sub handle_opts {
+    my ( $opt, $usage ) = describe_options(
+        'usage: $0 %o MODULE',
+        [ 'maintainer|m=s', 'the package maintainer' ],
+        [ 'debug|d',        'enable debug output' ],
+        [ 'help',           'print usage message and exit' ]
+    );
+
+    if ( $opt->help ) {
+        print $usage->text;
+        exit;
+    }
+
+    my $module_name = $ARGV[0];
+
+    if ( !defined $module_name ) {
+        print STDERR "Missing module name\n";
+        print STDERR $usage->text;
+        exit 1;
+    }
+
+    return ( $opt, $module_name );
+}
+
+# Takes a Perl package attribute name and returns 1 if the name cannot
+# be referred to as a bareword. This typically happens if the package
+# name is a reserved Nix keyword.
+sub is_reserved {
+    my ($pkg) = @_;
+
+    return $pkg =~ /^(?: assert    |
+                         else      |
+                         if        |
+                         import    |
+                         in        |
+                         inherit   |
+                         let       |
+                         rec       |
+                         then      |
+                         while     |
+                         with      )$/x;
+}
+
+sub pkg_to_attr {
+    my ($module) = @_;
+    my $attr_name = $module->package_name;
+    if ( $attr_name eq "libwww-perl" ) {
+        return "LWP";
+    }
+    else {
+        $attr_name =~ s/-//g;
+        return $attr_name;
+    }
+}
+
+sub get_pkg_name {
+    my ($module) = @_;
+    return ( $module->package_name, $module->package_version =~ s/^v(\d)/$1/r );
+}
+
+sub read_meta {
+    my ($pkg_path) = @_;
+
+    my $yaml_path = "$pkg_path/META.yml";
+    my $json_path = "$pkg_path/META.json";
+    my $meta;
+
+    if ( -r $json_path ) {
+        $meta = CPAN::Meta->load_file($json_path);
+    }
+    elsif ( -r $yaml_path ) {
+        $meta = CPAN::Meta->load_file($yaml_path);
+    }
+    else {
+        WARN("package has no META.yml or META.json");
+    }
+
+    return $meta;
+}
+
+# Map a module to the attribute corresponding to its package
+# (e.g. HTML::HeadParser will be mapped to HTMLParser, because that
+# module is in the HTML-Parser package).
+sub module_to_pkg {
+    my ( $cb, $module_name ) = @_;
+    my @modules = $cb->search( type => "name", allow => [$module_name] );
+    if ( scalar @modules == 0 ) {
+
+        # Fallback.
+        $module_name =~ s/:://g;
+        return $module_name;
+    }
+    my $module    = $modules[0];
+    my $attr_name = pkg_to_attr($module);
+    DEBUG("mapped dep $module_name to $attr_name");
+    return $attr_name;
+}
+
+sub get_deps {
+    my ( $cb, $meta, $type ) = @_;
+
+    return if !defined $meta;
+
+    my $prereqs = $meta->effective_prereqs;
+    my $deps = $prereqs->requirements_for( $type, "requires" );
+    my @res;
+    foreach my $n ( $deps->required_modules ) {
+        next if $n eq "perl";
+
+        my @core = Module::CoreList->find_modules(qr/^$n$/);
+        next if (@core);
+
+        my $pkg = module_to_pkg( $cb, $n );
+
+        # If the package name is reserved then we need to refer to it
+        # through the "self" variable.
+        $pkg = "self.\"$pkg\"" if is_reserved($pkg);
+
+        push @res, $pkg;
+    }
+    return @res;
+}
+
+sub uniq {
+    return keys %{ { map { $_ => 1 } @_ } };
+}
+
+sub render_license {
+    my ($cpan_license) = @_;
+
+    return if !defined $cpan_license;
+
+    my $licenses;
+
+    # If the license is ambiguous then we'll print an extra warning.
+    # For example, "gpl_2" is ambiguous since it may refer to exactly
+    # "GPL v2" or to "GPL v2 or later".
+    my $amb = 0;
+
+    # Whether the license is available inside `lib.licenses`.
+    my $in_set = 1;
+
+    my $nix_license = $LICENSE_MAP{$cpan_license};
+    if ( !$nix_license ) {
+        WARN("Unknown license: $cpan_license");
+        $licenses = [$cpan_license];
+        $in_set   = 0;
+    }
+    else {
+        $licenses = $nix_license->{licenses};
+        $amb      = $nix_license->{amb};
+        $in_set   = !$nix_license->{in_set};
+    }
+
+    my $license_line;
+
+    if ( @$licenses == 0 ) {
+
+        # Avoid defining the license line.
+    }
+    elsif ($in_set) {
+        my $lic = 'lib.licenses';
+        if ( @$licenses == 1 ) {
+            $license_line = "$lic.$licenses->[0]";
+        }
+        else {
+            $license_line = "with $lic; [ " . join( ' ', @$licenses ) . " ]";
+        }
+    }
+    else {
+        if ( @$licenses == 1 ) {
+            $license_line = $licenses->[0];
+        }
+        else {
+            $license_line = '[ ' . join( ' ', @$licenses ) . ' ]';
+        }
+    }
+
+    INFO("license: $cpan_license");
+    WARN("License '$cpan_license' is ambiguous, please verify") if $amb;
+
+    return $license_line;
+}
+
+sub sha256_to_sri {
+    my ($sha256) = @_;
+    return "sha256-" . encode_base64(pack("H*", $sha256), '');
+}
+
+my ( $opt, $module_name ) = handle_opts();
+
+Log::Log4perl->easy_init(
+    {
+        level => $opt->debug ? $DEBUG : $INFO,
+        layout => '%m%n'
+    }
+);
+
+my $cb = CPANPLUS::Backend->new;
+
+my @modules = $cb->search( type => "name", allow => [$module_name] );
+die "module $module_name not found\n" if scalar @modules == 0;
+die "multiple packages that match module $module_name\n" if scalar @modules > 1;
+my $module = $modules[0];
+
+my ($pkg_name, $pkg_version) = get_pkg_name $module;
+my $attr_name = pkg_to_attr $module;
+
+INFO( "attribute name: ", $attr_name );
+INFO( "module: ",         $module->module );
+INFO( "version: ",        $module->version );
+INFO( "package: ", $module->package, " (", "$pkg_name-$pkg_version", ", ", $attr_name, ")" );
+INFO( "path: ",    $module->path );
+
+my $tar_path = $module->fetch();
+my $sri_hash = sha256_to_sri($module->status->checksum_value);
+INFO( "downloaded to: ", $tar_path );
+INFO( "hash: ", $sri_hash );
+
+my $pkg_path = $module->extract();
+INFO( "unpacked to: ", $pkg_path );
+
+my $meta = read_meta($pkg_path);
+
+DEBUG( "metadata: ", encode_json( $meta->as_struct ) ) if defined $meta;
+
+my @runtime_deps = sort( uniq( get_deps( $cb, $meta, "runtime" ) ) );
+INFO("runtime deps: @runtime_deps");
+
+my @build_deps = sort( uniq(
+        get_deps( $cb, $meta, "configure" ),
+        get_deps( $cb, $meta, "build" ),
+        get_deps( $cb, $meta, "test" )
+) );
+
+# Filter out runtime dependencies since those are already handled.
+my %in_runtime_deps = map { $_ => 1 } @runtime_deps;
+@build_deps = grep { not $in_runtime_deps{$_} } @build_deps;
+
+INFO("build deps: @build_deps");
+
+my $homepage = $meta ? $meta->resources->{homepage} : undef;
+INFO("homepage: $homepage") if defined $homepage;
+
+my $description = $meta ? $meta->abstract : undef;
+if ( defined $description ) {
+    $description = uc( substr( $description, 0, 1 ) )
+      . substr( $description, 1 );    # capitalise first letter
+    $description =~ s/\.$//;          # remove period at the end
+    $description =~ s/\s*$//;
+    $description =~ s/^\s*//;
+    $description =~ s/\n+/ /;         # Replace new lines by space.
+    INFO("description: $description");
+}
+
+#print(Data::Dumper::Dumper($meta->licenses) . "\n");
+my $license = $meta ? render_license( $meta->licenses ) : undef;
+
+INFO( "RSS feed: https://metacpan.org/feed/distribution/",
+    $module->package_name );
+
+my $build_fun = -e "$pkg_path/Build.PL"
+  && !-e "$pkg_path/Makefile.PL" ? "buildPerlModule" : "buildPerlPackage";
+
+print STDERR "===\n";
+
+print <<EOF;
+  ${\(is_reserved($attr_name) ? "\"$attr_name\"" : $attr_name)} = $build_fun {
+    pname = "$pkg_name";
+    version = "$pkg_version";
+    src = fetchurl {
+      url = "mirror://cpan/${\$module->path}/${\$module->package}";
+      hash = "$sri_hash";
+    };
+EOF
+print <<EOF if scalar @build_deps > 0;
+    buildInputs = [ @build_deps ];
+EOF
+print <<EOF if scalar @runtime_deps > 0;
+    propagatedBuildInputs = [ @runtime_deps ];
+EOF
+print <<EOF;
+    meta = {
+EOF
+print <<EOF if defined $homepage;
+      homepage = "$homepage";
+EOF
+print <<EOF if defined $description && $description ne "Unknown";
+      description = "$description";
+EOF
+print <<EOF if defined $license;
+      license = $license;
+EOF
+print <<EOF if $opt->maintainer;
+      maintainers = [ maintainers.${\$opt->maintainer} ];
+EOF
+print <<EOF;
+    };
+  };
+EOF

maintainers/scripts/nixpkgs-lint.nix

@@ -0,0 +1,37 @@
+{
+  stdenv,
+  lib,
+  makeWrapper,
+  perl,
+  perlPackages,
+}:
+
+stdenv.mkDerivation {
+  pname = "nixpkgs-lint";
+  version = "1";
+
+  nativeBuildInputs = [ makeWrapper ];
+  buildInputs = [
+    perl
+    perlPackages.XMLSimple
+  ];
+
+  dontUnpack = true;
+  dontBuild = true;
+
+  installPhase = ''
+    mkdir -p $out/bin
+    cp ${./nixpkgs-lint.pl} $out/bin/nixpkgs-lint
+    # make the built version hermetic
+    substituteInPlace  $out/bin/nixpkgs-lint \
+      --replace-fail "#! /usr/bin/env nix-shell" "#! ${lib.getExe perl}"
+    wrapProgram $out/bin/nixpkgs-lint --set PERL5LIB $PERL5LIB
+  '';
+
+  meta = {
+    maintainers = [ lib.maintainers.eelco ];
+    description = "A utility for Nixpkgs contributors to check Nixpkgs for common errors";
+    mainProgram = "nixpkgs-lint";
+    platforms = lib.platforms.unix;
+  };
+}

maintainers/scripts/nixpkgs-lint.pl

@@ -0,0 +1,173 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i perl -p perl perlPackages.XMLSimple
+
+use strict;
+use List::Util qw(min);
+use XML::Simple qw(:strict);
+use Getopt::Long qw(:config gnu_getopt);
+
+# Parse the command line.
+my $path = "<nixpkgs>";
+my $filter = "*";
+my $maintainer;
+
+sub showHelp {
+    print <<EOF;
+Usage: $0 [--package=NAME] [--maintainer=REGEXP] [--file=PATH]
+
+Check Nixpkgs for common errors/problems.
+
+  -p, --package        filter packages by name (default is ‘*’)
+  -m, --maintainer     filter packages by maintainer (case-insensitive regexp)
+  -f, --file           path to Nixpkgs (default is ‘<nixpkgs>’)
+
+Examples:
+  \$ nixpkgs-lint -f /my/nixpkgs -p firefox
+  \$ nixpkgs-lint -f /my/nixpkgs -m eelco
+EOF
+    exit 0;
+}
+
+GetOptions("package|p=s" => \$filter,
+           "maintainer|m=s" => \$maintainer,
+           "file|f=s" => \$path,
+           "help" => sub { showHelp() }
+    ) or exit 1;
+
+# Evaluate Nixpkgs into an XML representation.
+my $xml = `nix-env -f '$path' --arg overlays '[]' -qa '$filter' --xml --meta --drv-path`;
+die "$0: evaluation of ‘$path’ failed\n" if $? != 0;
+
+my $info = XMLin($xml, KeyAttr => { 'item' => '+attrPath', 'meta' => 'name' }, ForceArray => 1, SuppressEmpty => '' ) or die "cannot parse XML output";
+
+# Check meta information.
+print "=== Package meta information ===\n\n";
+my $nrBadNames = 0;
+my $nrMissingMaintainers = 0;
+my $nrMissingPlatforms = 0;
+my $nrMissingDescriptions = 0;
+my $nrBadDescriptions = 0;
+my $nrMissingLicenses = 0;
+
+foreach my $attr (sort keys %{$info->{item}}) {
+    my $pkg = $info->{item}->{$attr};
+
+    my $pkgName = $pkg->{name};
+    my $pkgVersion = "";
+    if ($pkgName =~ /(.*)(-[0-9].*)$/) {
+        $pkgName = $1;
+        $pkgVersion = $2;
+    }
+
+    # Check the maintainers.
+    my @maintainers;
+    my $x = $pkg->{meta}->{maintainers};
+    if (defined $x && $x->{type} eq "strings") {
+        @maintainers = map { $_->{value} } @{$x->{string}};
+    } elsif (defined $x->{value}) {
+        @maintainers = ($x->{value});
+    }
+
+    if (defined $maintainer && scalar(grep { $_ =~ /$maintainer/i } @maintainers) == 0) {
+        delete $info->{item}->{$attr};
+        next;
+    }
+
+    if (scalar @maintainers == 0) {
+        print "$attr: Lacks a maintainer\n";
+        $nrMissingMaintainers++;
+    }
+
+    # Check the platforms.
+    if (!defined $pkg->{meta}->{platforms}) {
+        print "$attr: Lacks a platform\n";
+        $nrMissingPlatforms++;
+    }
+
+    # Package names should not be capitalised.
+    if ($pkgName =~ /^[A-Z]/) {
+        print "$attr: package name ‘$pkgName’ should not be capitalised\n";
+        $nrBadNames++;
+    }
+
+    if ($pkgVersion eq "") {
+        print "$attr: package has no version\n";
+        $nrBadNames++;
+    }
+
+    # Check the license.
+    if (!defined $pkg->{meta}->{license}) {
+        print "$attr: Lacks a license\n";
+        $nrMissingLicenses++;
+    }
+
+    # Check the description.
+    my $description = $pkg->{meta}->{description}->{value};
+    if (!$description) {
+        print "$attr: Lacks a description\n";
+        $nrMissingDescriptions++;
+    } else {
+        my $bad = 0;
+        if ($description =~ /^\s/) {
+            print "$attr: Description starts with whitespace\n";
+            $bad = 1;
+        }
+        if ($description =~ /\s$/) {
+            print "$attr: Description ends with whitespace\n";
+            $bad = 1;
+        }
+        if ($description =~ /\.$/) {
+            print "$attr: Description ends with a period\n";
+            $bad = 1;
+        }
+        if (index(lc($description), lc($attr)) != -1) {
+            print "$attr: Description contains package name\n";
+            $bad = 1;
+        }
+        $nrBadDescriptions++ if $bad;
+    }
+}
+
+print "\n";
+
+# Find packages that have the same name.
+print "=== Package name collisions ===\n\n";
+
+my %pkgsByName;
+
+foreach my $attr (sort keys %{$info->{item}}) {
+    my $pkg = $info->{item}->{$attr};
+    #print STDERR "attr = $attr, name = $pkg->{name}\n";
+    $pkgsByName{$pkg->{name}} //= [];
+    push @{$pkgsByName{$pkg->{name}}}, $pkg;
+}
+
+my $nrCollisions = 0;
+foreach my $name (sort keys %pkgsByName) {
+    my @pkgs = @{$pkgsByName{$name}};
+
+    # Filter attributes that are aliases of each other (e.g. yield the
+    # same derivation path).
+    my %drvsSeen;
+    @pkgs = grep { my $x = $drvsSeen{$_->{drvPath}}; $drvsSeen{$_->{drvPath}} = 1; !defined $x } @pkgs;
+
+    # Filter packages that have a lower priority.
+    my $highest = min (map { $_->{meta}->{priority}->{value} // 0 } @pkgs);
+    @pkgs = grep { ($_->{meta}->{priority}->{value} // 0) == $highest } @pkgs;
+
+    next if scalar @pkgs == 1;
+
+    $nrCollisions++;
+    print "The following attributes evaluate to a package named ‘$name’:\n";
+    print "  ", join(", ", map { $_->{attrPath} } @pkgs), "\n\n";
+}
+
+print "=== Bottom line ===\n";
+print "Number of packages: ", scalar(keys %{$info->{item}}), "\n";
+print "Number of bad names: $nrBadNames\n";
+print "Number of missing maintainers: $nrMissingMaintainers\n";
+print "Number of missing platforms: $nrMissingPlatforms\n";
+print "Number of missing licenses: $nrMissingLicenses\n";
+print "Number of missing descriptions: $nrMissingDescriptions\n";
+print "Number of bad descriptions: $nrBadDescriptions\n";
+print "Number of name collisions: $nrCollisions\n";

maintainers/scripts/patchelf-hints.sh

@@ -0,0 +1,84 @@
+
+usage() {
+    echo "
+$0 <path to unpacked binary distribution directory>
+
+This program return the list of libraries and where to find them based on
+your currently installed programs.
+";
+    exit 1
+}
+
+if test $# -ne 1; then
+  usage
+fi
+
+binaryDist=$1
+
+hasBinaries=false
+for bin in $(find $binaryDist -executable -type f) :; do
+    if test $bin = ":"; then
+        $hasBinaries || \
+            echo "No patchable found in this directory."
+        break
+    fi
+    hasBinaries=true
+
+    echo ""
+    echo "$bin:"
+    hasLibraries=false
+    unset interpreter
+    unset addRPath
+    for lib in $(strings $bin | grep '^\(/\|\)lib.*\.so' | sort | uniq) :; do
+        if test $lib = ":"; then
+            $hasLibraries || \
+                echo "  This program is a script or it is statically linked."
+            break
+        fi
+        hasLibraries=true
+
+        echo "  $lib:";
+
+        libPath=$lib
+        lib=$(basename $lib)
+
+        #versionLessLib=$(echo $lib | sed 's,[.][.0-9]*$,,')
+
+        libs="$(
+            find /nix/store/*/lib* \( -type f -or -type l \) -name $lib |
+            grep -v '\(bootstrap-tools\|system-path\|user-environment\|extra-utils\)'
+        )"
+
+        echo "$libs" |
+        sed 's,^/nix/store/[a-z0-9]*-\([^/]*\)/.*/\([^/]*\)$,    \1 -> \2,' |
+        sort |
+        uniq;
+
+        names=$(
+            echo "$libs" |
+            sed 's,^/nix/store/[a-z0-9]*-\([^/]*\)-[.0-9]*/.*$,\1,' |
+            sort |
+            uniq;
+        )
+
+        if test "$names" = "glibc"; then names="glibc"; fi
+        if echo $names | grep -c "gcc" &> /dev/null; then names="stdenv.cc.cc"; fi
+
+        if test $lib != $libPath; then
+            interpreter="--interpreter \${$names}/lib/$lib"
+        elif echo $addRPath | grep -c "$names" &> /dev/null; then
+            :
+        else
+            addRPath=${addRPath+$addRPath:}"\${$names}/lib"
+        fi
+    done;
+    $hasLibraries && \
+        echo "
+  Patchelf command:
+
+    patchelf $interpreter \\
+      ${addRPath+--set-rpath $addRPath \\
+}      \$out/$bin
+
+"
+done;

maintainers/scripts/pluginupdate-py/pluginupdate.py

@@ -0,0 +1,973 @@
+# python library used to update plugins:
+# - pkgs/applications/editors/vim/plugins/update.py
+# - pkgs/applications/editors/kakoune/plugins/update.py
+# - pkgs/development/lua-modules/updater/updater.py
+
+# format:
+# $ nix run nixpkgs#ruff maintainers/scripts/pluginupdate.py
+# type-check:
+# $ nix run nixpkgs#python3.pkgs.mypy maintainers/scripts/pluginupdate.py
+# linted:
+# $ nix run nixpkgs#python3.pkgs.flake8 -- --ignore E501,E265 maintainers/scripts/pluginupdate.py
+
+import argparse
+import csv
+import functools
+import http
+import json
+import logging
+import os
+import re
+import subprocess
+import sys
+import time
+import traceback
+import urllib.error
+import urllib.parse
+import urllib.request
+import xml.etree.ElementTree as ET
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime
+from functools import wraps
+from multiprocessing.dummy import Pool
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+from typing import Any, Callable
+from urllib.parse import urljoin, urlparse
+
+import git
+
+ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry"  # " vim gets confused here
+ATOM_LINK = "{http://www.w3.org/2005/Atom}link"  # "
+ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated"  # "
+
+LOG_LEVELS = {
+    logging.getLevelName(level): level
+    for level in [logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR]
+}
+
+log = logging.getLogger()
+
+
+def retry(ExceptionToCheck: Any, tries: int = 4, delay: float = 3, backoff: float = 2):
+    """Retry calling the decorated function using an exponential backoff.
+    http://www.saltycrane.com/blog/2009/11/trying-out-retry-decorator-python/
+    original from: http://wiki.python.org/moin/PythonDecoratorLibrary#Retry
+    (BSD licensed)
+    :param ExceptionToCheck: the exception on which to retry
+    :param tries: number of times to try (not retry) before giving up
+    :param delay: initial delay between retries in seconds
+    :param backoff: backoff multiplier e.g. value of 2 will double the delay
+        each retry
+    """
+
+    def deco_retry(f: Callable) -> Callable:
+        @wraps(f)
+        def f_retry(*args: Any, **kwargs: Any) -> Any:
+            mtries, mdelay = tries, delay
+            while mtries > 1:
+                try:
+                    return f(*args, **kwargs)
+                except ExceptionToCheck as e:
+                    print(f"{str(e)}, Retrying in {mdelay} seconds...")
+                    time.sleep(mdelay)
+                    mtries -= 1
+                    mdelay *= backoff
+            return f(*args, **kwargs)
+
+        return f_retry  # true decorator
+
+    return deco_retry
+
+
+@dataclass
+class FetchConfig:
+    proc: int
+    github_token: str
+
+
+def make_request(url: str, token=None) -> urllib.request.Request:
+    headers = {}
+    if token is not None:
+        headers["Authorization"] = f"token {token}"
+    return urllib.request.Request(url, headers=headers)
+
+
+# a dictionary of plugins and their new repositories
+Redirects = dict["PluginDesc", "Repo"]
+
+
+class Repo:
+    def __init__(self, uri: str, branch: str) -> None:
+        self.uri = uri
+        """Url to the repo"""
+        self._branch = branch
+        # Redirect is the new Repo to use
+        self.redirect: "Repo | None" = None
+        self.token = "dummy_token"
+
+    @property
+    def name(self):
+        return self.uri.strip("/").split("/")[-1]
+
+    @property
+    def branch(self):
+        return self._branch or "HEAD"
+
+    def __str__(self) -> str:
+        return f"{self.uri}"
+
+    def __repr__(self) -> str:
+        return f"Repo({self.name}, {self.uri})"
+
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
+    def has_submodules(self) -> bool:
+        return True
+
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
+    def latest_commit(self) -> tuple[str, datetime]:
+        log.debug("Latest commit")
+        loaded = self._prefetch(None)
+        updated = datetime.strptime(loaded["date"], "%Y-%m-%dT%H:%M:%S%z")
+
+        return loaded["rev"], updated
+
+    def _prefetch(self, ref: str | None):
+        cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri]
+        if ref is not None:
+            cmd.append(ref)
+        log.debug(cmd)
+        data = subprocess.check_output(cmd)
+        loaded = json.loads(data)
+        return loaded
+
+    def prefetch(self, ref: str | None) -> str:
+        log.info("Prefetching %s", self.uri)
+        loaded = self._prefetch(ref)
+        return loaded["sha256"]
+
+    def as_nix(self, plugin: "Plugin") -> str:
+        return f"""fetchgit {{
+      url = "{self.uri}";
+      rev = "{plugin.commit}";
+      sha256 = "{plugin.sha256}";
+    }}"""
+
+
+class RepoGitHub(Repo):
+    def __init__(self, owner: str, repo: str, branch: str) -> None:
+        self.owner = owner
+        self.repo = repo
+        self.token = None
+        """Url to the repo"""
+        super().__init__(self.url(""), branch)
+        log.debug(
+            "Instantiating github repo owner=%s and repo=%s", self.owner, self.repo
+        )
+
+    @property
+    def name(self):
+        return self.repo
+
+    def url(self, path: str) -> str:
+        res = urljoin(f"https://github.com/{self.owner}/{self.repo}/", path)
+        return res
+
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
+    def has_submodules(self) -> bool:
+        try:
+            req = make_request(self.url(f"blob/{self.branch}/.gitmodules"), self.token)
+            urllib.request.urlopen(req, timeout=10).close()
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                return False
+            else:
+                raise
+        return True
+
+    @retry(urllib.error.URLError, tries=4, delay=3, backoff=2)
+    def latest_commit(self) -> tuple[str, datetime]:
+        commit_url = self.url(f"commits/{self.branch}.atom")
+        log.debug("Sending request to %s", commit_url)
+        commit_req = make_request(commit_url, self.token)
+        with urllib.request.urlopen(commit_req, timeout=10) as req:
+            self._check_for_redirect(commit_url, req)
+            xml = req.read()
+
+            # Filter out illegal XML characters
+            illegal_xml_regex = re.compile(b"[\x00-\x08\x0b-\x0c\x0e-\x1f\x7f]")
+            xml = illegal_xml_regex.sub(b"", xml)
+
+            root = ET.fromstring(xml)
+            latest_entry = root.find(ATOM_ENTRY)
+            assert latest_entry is not None, f"No commits found in repository {self}"
+            commit_link = latest_entry.find(ATOM_LINK)
+            assert commit_link is not None, f"No link tag found feed entry {xml}"
+            url = urlparse(commit_link.get("href"))
+            updated_tag = latest_entry.find(ATOM_UPDATED)
+            assert (
+                updated_tag is not None and updated_tag.text is not None
+            ), f"No updated tag found feed entry {xml}"
+            updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ")
+            return Path(str(url.path)).name, updated
+
+    def _check_for_redirect(self, url: str, req: http.client.HTTPResponse):
+        response_url = req.geturl()
+        if url != response_url:
+            new_owner, new_name = (
+                urllib.parse.urlsplit(response_url).path.strip("/").split("/")[:2]
+            )
+
+            new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch)
+            self.redirect = new_repo
+
+    def prefetch(self, commit: str) -> str:
+        if self.has_submodules():
+            sha256 = super().prefetch(commit)
+        else:
+            sha256 = self.prefetch_github(commit)
+        return sha256
+
+    def prefetch_github(self, ref: str) -> str:
+        cmd = ["nix-prefetch-url", "--unpack", self.url(f"archive/{ref}.tar.gz")]
+        log.debug("Running %s", cmd)
+        data = subprocess.check_output(cmd)
+        return data.strip().decode("utf-8")
+
+    def as_nix(self, plugin: "Plugin") -> str:
+        if plugin.has_submodules:
+            submodule_attr = "\n      fetchSubmodules = true;"
+        else:
+            submodule_attr = ""
+
+        return f"""fetchFromGitHub {{
+      owner = "{self.owner}";
+      repo = "{self.repo}";
+      rev = "{plugin.commit}";
+      sha256 = "{plugin.sha256}";{submodule_attr}
+    }}"""
+
+
+@dataclass(frozen=True)
+class PluginDesc:
+    repo: Repo
+    branch: str
+    alias: str | None
+
+    @property
+    def name(self):
+        return self.alias or self.repo.name
+
+    @staticmethod
+    def load_from_csv(config: FetchConfig, row: dict[str, str]) -> "PluginDesc":
+        log.debug("Loading row %s", row)
+        branch = row["branch"]
+        repo = make_repo(row["repo"], branch.strip())
+        repo.token = config.github_token
+        return PluginDesc(
+            repo,
+            branch.strip(),
+            # alias is usually an empty string
+            row["alias"] if row["alias"] else None,
+        )
+
+    @staticmethod
+    def load_from_string(config: FetchConfig, line: str) -> "PluginDesc":
+        branch = "HEAD"
+        alias = None
+        uri = line
+        if " as " in uri:
+            uri, alias = uri.split(" as ")
+            alias = alias.strip()
+        if "@" in uri:
+            uri, branch = uri.split("@")
+        repo = make_repo(uri.strip(), branch.strip())
+        repo.token = config.github_token
+        return PluginDesc(repo, branch.strip(), alias)
+
+
+@dataclass
+class Plugin:
+    name: str
+    commit: str
+    has_submodules: bool
+    sha256: str
+    date: datetime | None = None
+
+    @property
+    def normalized_name(self) -> str:
+        return self.name.replace(".", "-")
+
+    @property
+    def version(self) -> str:
+        assert self.date is not None
+        return self.date.strftime("%Y-%m-%d")
+
+    def as_json(self) -> dict[str, str]:
+        copy = self.__dict__.copy()
+        del copy["date"]
+        return copy
+
+
+def load_plugins_from_csv(
+    config: FetchConfig,
+    input_file: Path,
+) -> list[PluginDesc]:
+    log.debug("Load plugins from csv %s", input_file)
+    plugins = []
+    with open(input_file, newline="") as csvfile:
+        log.debug("Writing into %s", input_file)
+        reader = csv.DictReader(
+            csvfile,
+        )
+        for line in reader:
+            plugin = PluginDesc.load_from_csv(config, line)
+            plugins.append(plugin)
+
+    return plugins
+
+
+def run_nix_expr(expr, nixpkgs: str, **args):
+    """
+    :param expr nix expression to fetch current plugins
+    :param nixpkgs Path towards a nixpkgs checkout
+    """
+    with CleanEnvironment(nixpkgs) as nix_path:
+        cmd = [
+            "nix",
+            "eval",
+            "--extra-experimental-features",
+            "nix-command",
+            "--impure",
+            "--json",
+            "--expr",
+            expr,
+            "--nix-path",
+            nix_path,
+        ]
+        log.debug("Running command: %s", " ".join(cmd))
+        out = subprocess.check_output(cmd, **args)
+        data = json.loads(out)
+        return data
+
+
+class Editor:
+    """The configuration of the update script."""
+
+    def __init__(
+        self,
+        name: str,
+        root: Path,
+        get_plugins: str,
+        default_in: Path | None = None,
+        default_out: Path | None = None,
+        deprecated: Path | None = None,
+        cache_file: str | None = None,
+    ):
+        log.debug("get_plugins:", get_plugins)
+        self.name = name
+        self.root = root
+        self.get_plugins = get_plugins
+        self.default_in = default_in or root.joinpath(f"{name}-plugin-names")
+        self.default_out = default_out or root.joinpath("generated.nix")
+        self.deprecated = deprecated or root.joinpath("deprecated.json")
+        self.cache_file = cache_file or f"{name}-plugin-cache.json"
+        self.nixpkgs_repo = None
+
+    def add(self, args):
+        """CSV spec"""
+        log.debug("called the 'add' command")
+        fetch_config = FetchConfig(args.proc, args.github_token)
+        editor = self
+        for plugin_line in args.add_plugins:
+            log.debug("using plugin_line %s", plugin_line)
+            pdesc = PluginDesc.load_from_string(fetch_config, plugin_line)
+            log.debug("loaded as pdesc %s", pdesc)
+            append = [pdesc]
+            editor.rewrite_input(
+                fetch_config, args.input_file, editor.deprecated, append=append
+            )
+            plugin, _ = prefetch_plugin(pdesc)
+
+            if (  # lua updater doesn't support updating individual plugin
+                self.name != "lua"
+            ):
+                # update generated.nix
+                update = self.get_update(
+                    args.input_file,
+                    args.outfile,
+                    fetch_config,
+                    [plugin.normalized_name],
+                )
+                update()
+
+            autocommit = not args.no_commit
+            if autocommit:
+                commit(
+                    editor.nixpkgs_repo,
+                    "{drv_name}: init at {version}".format(
+                        drv_name=editor.get_drv_name(plugin.normalized_name),
+                        version=plugin.version,
+                    ),
+                    [args.outfile, args.input_file],
+                )
+
+    # Expects arguments generated by 'update' subparser
+    def update(self, args):
+        """CSV spec"""
+        print("the update member function should be overridden in subclasses")
+
+    def get_current_plugins(
+        self, config: FetchConfig, nixpkgs: str
+    ) -> list[tuple[PluginDesc, Plugin]]:
+        """To fill the cache"""
+        data = run_nix_expr(self.get_plugins, nixpkgs)
+        plugins = []
+        for name, attr in data.items():
+            checksum = attr["checksum"]
+
+            # https://github.com/NixOS/nixpkgs/blob/8a335419/pkgs/applications/editors/neovim/build-neovim-plugin.nix#L36
+            # https://github.com/NixOS/nixpkgs/pull/344478#discussion_r1786646055
+            version = re.search(r"\d\d\d\d-\d\d?-\d\d?", attr["version"])
+            if version is None:
+                raise ValueError(f"Cannot parse version: {attr['version']}")
+            date = datetime.strptime(version.group(), "%Y-%m-%d")
+
+            pdesc = PluginDesc.load_from_string(config, f'{attr["homePage"]} as {name}')
+            p = Plugin(
+                attr["pname"],
+                checksum["rev"],
+                checksum["submodules"],
+                checksum["sha256"],
+                date,
+            )
+
+            plugins.append((pdesc, p))
+        return plugins
+
+    def load_plugin_spec(self, config: FetchConfig, plugin_file) -> list[PluginDesc]:
+        """CSV spec"""
+        return load_plugins_from_csv(config, plugin_file)
+
+    def generate_nix(self, _plugins, _outfile: str):
+        """Returns nothing for now, writes directly to outfile"""
+        raise NotImplementedError()
+
+    def filter_plugins_to_update(
+        self, plugin: PluginDesc, to_update: list[str]
+    ) -> bool:
+        """Function for filtering out plugins, that user doesn't want to update.
+
+        It is mainly used for updating only specific plugins, not all of them.
+        By default it filters out plugins not present in `to_update`,
+        assuming `to_update` is a list of plugin names (the same as in the
+        result expression).
+
+        This function is never called if `to_update` is empty.
+        Feel free to override this function in derived classes.
+
+        Note:
+            Known bug: you have to use a deprecated name, instead of new one.
+            This is because we resolve deprecations later and can't get new
+            plugin URL before we request info about it.
+
+            Although, we could parse deprecated.json, but it's a whole bunch
+            of spaghetti code, which I don't want to write.
+
+        Arguments:
+            plugin: Plugin on which you decide whether to ignore or not.
+            to_update:
+                List of strings passed to via the `--update` command line parameter.
+                By default, we assume it is a list of URIs identical to what
+                is in the input file.
+
+        Returns:
+            True if we should update plugin and False if not.
+        """
+        return plugin.name.replace(".", "-") in to_update
+
+    def get_update(
+        self,
+        input_file: str,
+        output_file: str,
+        config: FetchConfig,
+        to_update: list[str] | None,
+    ):
+        if to_update is None:
+            to_update = []
+
+        current_plugins = self.get_current_plugins(config, self.nixpkgs)
+        current_plugin_specs = self.load_plugin_spec(config, input_file)
+
+        cache: Cache = Cache(
+            [plugin for _description, plugin in current_plugins], self.cache_file
+        )
+        _prefetch = functools.partial(prefetch, cache=cache)
+
+        to_update_for_filter = [x.replace(".", "-") for x in to_update]
+        plugins_to_update = (
+            current_plugin_specs
+            if len(to_update) == 0
+            else [
+                description
+                for description in current_plugin_specs
+                if self.filter_plugins_to_update(description, to_update_for_filter)
+            ]
+        )
+
+        def update() -> Redirects:
+            if len(plugins_to_update) == 0:
+                log.error(
+                    "\n\n\n\nIt seems like you provided some arguments to `--update`:\n"
+                    + ", ".join(to_update)
+                    + "\nBut after filtering, the result list of plugins is empty\n"
+                    "\n"
+                    "Are you sure you provided the same URIs as in your input file?\n"
+                    "(" + str(input_file) + ")\n\n"
+                )
+                return {}
+
+            try:
+                pool = Pool(processes=config.proc)
+                results = pool.map(_prefetch, plugins_to_update)
+            finally:
+                cache.store()
+
+            print(f"{len(results)} of {len(current_plugins)} were checked")
+            # Do only partial update of out file
+            if len(results) != len(current_plugins):
+                results = self.merge_results(current_plugins, results)
+            plugins, redirects = check_results(results)
+
+            plugins = sorted(plugins, key=lambda v: v[1].normalized_name)
+            self.generate_nix(plugins, output_file)
+
+            return redirects
+
+        return update
+
+    def merge_results(
+        self,
+        current: list[tuple[PluginDesc, Plugin]],
+        fetched: list[tuple[PluginDesc, Exception | Plugin, Repo | None]],
+    ) -> list[tuple[PluginDesc, Exception | Plugin, Repo | None]]:
+        # transforming this to dict, so lookup is O(1) instead of O(n) (n is len(current))
+        result: dict[str, tuple[PluginDesc, Exception | Plugin, Repo | None]] = {
+            # also adding redirect (third item in the result tuple)
+            pl.normalized_name: (pdesc, pl, None)
+            for pdesc, pl in current
+        }
+
+        for plugin_desc, plugin, redirect in fetched:
+            # Check if plugin is a Plugin object and has normalized_name attribute
+            if isinstance(plugin, Plugin) and hasattr(plugin, 'normalized_name'):
+                result[plugin.normalized_name] = (plugin_desc, plugin, redirect)
+            elif isinstance(plugin, Exception):
+                # For exceptions, we can't determine the normalized_name
+                # Just log the error and continue
+                log.error(f"Error fetching plugin {plugin_desc.name}: {plugin!r}")
+            else:
+                # For unexpected types, log the issue
+                log.error(f"Unexpected plugin type for {plugin_desc.name}: {type(plugin)}")
+
+        return list(result.values())
+
+    @property
+    def attr_path(self):
+        return self.name + "Plugins"
+
+    def get_drv_name(self, name: str):
+        return self.attr_path + "." + name
+
+    def rewrite_input(self, *args, **kwargs):
+        return rewrite_input(*args, **kwargs)
+
+    def create_parser(self):
+        common = argparse.ArgumentParser(
+            add_help=False,
+            description=(
+                f"""
+                Updates nix derivations for {self.name} plugins.\n
+                By default from {self.default_in} to {self.default_out}"""
+            ),
+        )
+        common.add_argument(
+            "--nixpkgs",
+            type=str,
+            default=os.getcwd(),
+            help="Adjust log level",
+        )
+        common.add_argument(
+            "--input-names",
+            "-i",
+            dest="input_file",
+            type=Path,
+            default=self.default_in,
+            help="A list of plugins in the form owner/repo",
+        )
+        common.add_argument(
+            "--out",
+            "-o",
+            dest="outfile",
+            default=self.default_out,
+            type=Path,
+            help="Filename to save generated nix code",
+        )
+        common.add_argument(
+            "--proc",
+            "-p",
+            dest="proc",
+            type=int,
+            default=30,
+            help="Number of concurrent processes to spawn. Setting --github-token allows higher values.",
+        )
+        common.add_argument(
+            "--github-token",
+            "-t",
+            type=str,
+            default=os.getenv("GITHUB_TOKEN"),
+            help="""Allows to set --proc to higher values.
+            Uses GITHUB_TOKEN environment variables as the default value.""",
+        )
+        common.add_argument(
+            "--no-commit",
+            "-n",
+            action="store_true",
+            default=False,
+            help="Whether to autocommit changes",
+        )
+        common.add_argument(
+            "--debug",
+            "-d",
+            choices=LOG_LEVELS.keys(),
+            default=logging.getLevelName(logging.WARN),
+            help="Adjust log level",
+        )
+
+        main = argparse.ArgumentParser(
+            parents=[common],
+            description=(
+                f"""
+                Updates nix derivations for {self.name} plugins.\n
+                By default from {self.default_in} to {self.default_out}"""
+            ),
+        )
+
+        subparsers = main.add_subparsers(dest="command", required=False)
+        padd = subparsers.add_parser(
+            "add",
+            parents=[],
+            description="Add new plugin",
+            add_help=False,
+        )
+        padd.set_defaults(func=self.add)
+        padd.add_argument(
+            "add_plugins",
+            default=None,
+            nargs="+",
+            help=f"Plugin to add to {self.attr_path} from Github in the form owner/repo",
+        )
+
+        pupdate = subparsers.add_parser(
+            "update",
+            description="Update all or a subset of existing plugins",
+            add_help=False,
+        )
+        pupdate.add_argument(
+            "update_only",
+            default=None,
+            nargs="*",
+            help="Plugin URLs to update (must be the same as in the input file)",
+        )
+        pupdate.set_defaults(func=self.update)
+        return main
+
+    def run(
+        self,
+    ):
+        """
+        Convenience function
+        """
+        parser = self.create_parser()
+        args = parser.parse_args()
+        command = args.command or "update"
+        logging.basicConfig()
+        log.setLevel(LOG_LEVELS[args.debug])
+        log.info("Chose to run command: %s", command)
+        self.nixpkgs = args.nixpkgs
+
+        self.nixpkgs_repo = git.Repo(args.nixpkgs, search_parent_directories=True)
+
+        getattr(self, command)(args)
+
+
+class CleanEnvironment(object):
+    def __init__(self, nixpkgs):
+        self.local_pkgs = nixpkgs
+
+    def __enter__(self) -> str:
+        """
+        local_pkgs = str(Path(__file__).parent.parent.parent)
+        """
+        self.old_environ = os.environ.copy()
+        self.empty_config = NamedTemporaryFile()
+        self.empty_config.write(b"{}")
+        self.empty_config.flush()
+        return f"localpkgs={self.local_pkgs}"
+
+    def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> None:
+        os.environ.update(self.old_environ)
+        self.empty_config.close()
+
+
+def prefetch_plugin(
+    p: PluginDesc,
+    cache: "Cache | None" = None,
+) -> tuple[Plugin, Repo | None]:
+    commit = None
+    log.info(f"Fetching last commit for plugin {p.name} from {p.repo.uri}@{p.branch}")
+    commit, date = p.repo.latest_commit()
+
+    cached_plugin = cache[commit] if cache else None
+    if cached_plugin is not None:
+        log.debug(f"Cache hit for {p.name}!")
+        cached_plugin.name = p.name
+        cached_plugin.date = date
+        return cached_plugin, p.repo.redirect
+
+    has_submodules = p.repo.has_submodules()
+    log.debug(f"prefetch {p.name}")
+    sha256 = p.repo.prefetch(commit)
+
+    return (
+        Plugin(p.name, commit, has_submodules, sha256, date=date),
+        p.repo.redirect,
+    )
+
+
+def print_download_error(plugin: PluginDesc, ex: Exception):
+    print(f"{plugin}: {ex}", file=sys.stderr)
+    ex_traceback = ex.__traceback__
+    tb_lines = [
+        line.rstrip("\n")
+        for line in traceback.format_exception(ex.__class__, ex, ex_traceback)
+    ]
+    print("\n".join(tb_lines))
+
+
+def check_results(
+    results: list[tuple[PluginDesc, Exception | Plugin, Repo | None]],
+) -> tuple[list[tuple[PluginDesc, Plugin]], Redirects]:
+    """ """
+    failures: list[tuple[PluginDesc, Exception]] = []
+    plugins = []
+    redirects: Redirects = {}
+    for pdesc, result, redirect in results:
+        if isinstance(result, Exception):
+            failures.append((pdesc, result))
+        else:
+            new_pdesc = pdesc
+            if redirect is not None:
+                redirects.update({pdesc: redirect})
+                new_pdesc = PluginDesc(redirect, pdesc.branch, pdesc.alias)
+            plugins.append((new_pdesc, result))
+
+    if len(failures) == 0:
+        return plugins, redirects
+    else:
+        log.error(f"{len(failures)} plugin(s) could not be downloaded:\n")
+
+        for plugin, exception in failures:
+            print_download_error(plugin, exception)
+
+        sys.exit(1)
+
+
+def make_repo(uri: str, branch) -> Repo:
+    """Instantiate a Repo with the correct specialization depending on server (gitub spec)"""
+    # dumb check to see if it's of the form owner/repo (=> github) or https://...
+    res = urlparse(uri)
+    if res.netloc in ["github.com", ""]:
+        res = res.path.strip("/").split("/")
+        repo = RepoGitHub(res[0], res[1], branch)
+    else:
+        repo = Repo(uri.strip(), branch)
+    return repo
+
+
+def get_cache_path(cache_file_name: str) -> Path | None:
+    xdg_cache = os.environ.get("XDG_CACHE_HOME", None)
+    if xdg_cache is None:
+        home = os.environ.get("HOME", None)
+        if home is None:
+            return None
+        xdg_cache = str(Path(home, ".cache"))
+
+    return Path(xdg_cache, cache_file_name)
+
+
+class Cache:
+    def __init__(self, initial_plugins: list[Plugin], cache_file_name: str) -> None:
+        self.cache_file = get_cache_path(cache_file_name)
+
+        downloads = {}
+        for plugin in initial_plugins:
+            downloads[plugin.commit] = plugin
+        downloads.update(self.load())
+        self.downloads = downloads
+
+    def load(self) -> dict[str, Plugin]:
+        if self.cache_file is None or not self.cache_file.exists():
+            return {}
+
+        downloads: dict[str, Plugin] = {}
+        with open(self.cache_file) as f:
+            data = json.load(f)
+            for attr in data.values():
+                p = Plugin(
+                    attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"]
+                )
+                downloads[attr["commit"]] = p
+        return downloads
+
+    def store(self) -> None:
+        if self.cache_file is None:
+            return
+
+        os.makedirs(self.cache_file.parent, exist_ok=True)
+        with open(self.cache_file, "w+") as f:
+            data = {}
+            for name, attr in self.downloads.items():
+                data[name] = attr.as_json()
+            json.dump(data, f, indent=4, sort_keys=True)
+
+    def __getitem__(self, key: str) -> Plugin | None:
+        return self.downloads.get(key, None)
+
+    def __setitem__(self, key: str, value: Plugin) -> None:
+        self.downloads[key] = value
+
+
+def prefetch(
+    pluginDesc: PluginDesc, cache: Cache
+) -> tuple[PluginDesc, Exception | Plugin, Repo | None]:
+    try:
+        plugin, redirect = prefetch_plugin(pluginDesc, cache)
+        cache[plugin.commit] = plugin
+        return (pluginDesc, plugin, redirect)
+    except Exception as e:
+        return (pluginDesc, e, None)
+
+
+def rewrite_input(
+    config: FetchConfig,
+    input_file: Path,
+    deprecated: Path,
+    # old pluginDesc and the new
+    redirects: Redirects = {},
+    append: list[PluginDesc] = [],
+):
+    log.info("Rewriting input file %s", input_file)
+    plugins = load_plugins_from_csv(config, input_file)
+
+    plugins.extend(append)
+
+    if redirects:
+        log.debug("Dealing with deprecated plugins listed in %s", deprecated)
+
+        cur_date_iso = datetime.now().strftime("%Y-%m-%d")
+        with open(deprecated, "r") as f:
+            deprecations = json.load(f)
+        # TODO parallelize this step
+        for pdesc, new_repo in redirects.items():
+            log.info("Resolving deprecated plugin %s -> %s", pdesc.name, new_repo.name)
+            new_pdesc = PluginDesc(new_repo, pdesc.branch, pdesc.alias)
+
+            old_plugin, _ = prefetch_plugin(pdesc)
+            new_plugin, _ = prefetch_plugin(new_pdesc)
+
+            if old_plugin.normalized_name != new_plugin.normalized_name:
+                deprecations[old_plugin.normalized_name] = {
+                    "new": new_plugin.normalized_name,
+                    "date": cur_date_iso,
+                }
+
+            # remove plugin from index file, so we won't add it to deprecations again
+            for i, plugin in enumerate(plugins):
+                if plugin.name == pdesc.name:
+                    plugins.pop(i)
+                    break
+            plugins.append(new_pdesc)
+
+        with open(deprecated, "w") as f:
+            json.dump(deprecations, f, indent=4, sort_keys=True)
+            f.write("\n")
+
+    with open(input_file, "w") as f:
+        log.debug("Writing into %s", input_file)
+        # fields = dataclasses.fields(PluginDesc)
+        fieldnames = ["repo", "branch", "alias"]
+        writer = csv.DictWriter(f, fieldnames, dialect="unix", quoting=csv.QUOTE_NONE)
+        writer.writeheader()
+        for plugin in sorted(plugins, key=lambda x: x.name):
+            writer.writerow(asdict(plugin))
+
+
+def commit(repo: git.Repo, message: str, files: list[Path]) -> None:
+    repo.index.add([str(f.resolve()) for f in files])
+
+    if repo.index.diff("HEAD"):
+        print(f'committing to nixpkgs "{message}"')
+        repo.index.commit(message)
+    else:
+        print("no changes in working tree to commit")
+
+
+def update_plugins(editor: Editor, args):
+    """The main entry function of this module.
+    All input arguments are grouped in the `Editor`."""
+
+    log.info("Start updating plugins")
+    if args.proc > 1 and args.github_token == None:
+        log.warning(
+            "You have enabled parallel updates but haven't set a github token.\n"
+            "You may be hit with `HTTP Error 429: too many requests` as a consequence."
+            "Either set --proc=1 or --github-token=YOUR_TOKEN. "
+        )
+
+    fetch_config = FetchConfig(args.proc, args.github_token)
+    update = editor.get_update(
+        input_file=args.input_file,
+        output_file=args.outfile,
+        config=fetch_config,
+        to_update=getattr(  # if script was called without arguments
+            args, "update_only", None
+        ),
+    )
+
+    start_time = time.time()
+    redirects = update()
+    duration = time.time() - start_time
+    print(f"The plugin update took {duration:.2f}s.")
+    editor.rewrite_input(fetch_config, args.input_file, editor.deprecated, redirects)
+
+    autocommit = not args.no_commit
+
+    if autocommit:
+        try:
+            repo = git.Repo(os.getcwd())
+            updated = datetime.now(tz=UTC).strftime("%Y-%m-%d")
+            print(args.outfile)
+            commit(repo, f"{editor.attr_path}: update on {updated}", [args.outfile])
+        except git.InvalidGitRepositoryError as e:
+            print(f"Not in a git repository: {e}", file=sys.stderr)
+            sys.exit(1)
+
+    if redirects:
+        update()
+        if autocommit:
+            commit(
+                editor.nixpkgs_repo,
+                f"{editor.attr_path}: resolve github repository redirects",
+                [args.outfile, args.input_file, editor.deprecated],
+            )

maintainers/scripts/rebuild-amount.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -e
+
+# --print: avoid dependency on environment
+optPrint=
+if [ "$1" == "--print" ]; then
+    optPrint=true
+    shift
+fi
+
+if [ "$#" != 1 ] && [ "$#" != 2 ]; then
+    cat <<EOF
+    Usage: $0 [--print] from-commit-spec [to-commit-spec]
+        You need to be in a git-controlled nixpkgs tree.
+        The current state of the tree will be used if the second commit is missing.
+
+        Examples:
+          effect of latest commit:
+              $ $0 HEAD^
+              $ $0 --print HEAD^
+          effect of the whole patch series for 'staging' branch:
+              $ $0 origin/staging staging
+EOF
+    exit 1
+fi
+
+# A slightly hacky way to get the config.
+parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
+            | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
+
+echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})."
+
+toRemove=()
+
+cleanup() {
+    rm -rf "${toRemove[@]}"
+}
+trap cleanup EXIT
+
+MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
+
+nixexpr() {
+    cat <<EONIX
+        let
+          lib = import $1/lib;
+          hydraJobs = import $1/pkgs/top-level/release.nix
+            # Compromise: accuracy vs. resources needed for evaluation.
+            { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
+          cfg = (import $1 {}).config.rebuild-amount or {};
+
+          recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
+
+          # hydraJobs leaves recurseForDerivations as empty attrmaps;
+          # that would break nix-env and we also need to recurse everywhere.
+          tweak = lib.mapAttrs
+            (name: val:
+              if name == "recurseForDerivations" then true
+              else if lib.isAttrs val && val.type or null != "derivation"
+                      then recurseIntoAttrs (tweak val)
+              else val
+            );
+
+          # Some of these contain explicit references to platform(s) we want to avoid;
+          # some even (transitively) depend on ~/.nixpkgs/config.nix (!)
+          blacklist = [
+            "tarball" "metrics" "manual"
+            "darwin-tested" "unstable" "stdenvBootstrapTools"
+            "moduleSystem" "lib-tests" # these just confuse the output
+          ];
+
+        in
+          tweak (removeAttrs hydraJobs blacklist)
+EONIX
+}
+
+# Output packages in tree $2 that weren't in $1.
+# Changing the output hash or name is taken as a change.
+# Extra nix-env parameters can be in $3
+newPkgs() {
+    # We use files instead of pipes, as running multiple nix-env processes
+    # could eat too much memory for a standard 4GiB machine.
+    local -a list
+    for i in 1 2; do
+        local l
+        l="$($MKTEMP)"
+        list[$i]="$l"
+        toRemove+=("$l")
+
+        local expr
+        expr="$($MKTEMP)"
+        toRemove+=("$expr")
+        nixexpr "${!i}" > "$expr"
+
+        nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
+            | sort > "${list[$i]}" &
+
+        if [ "$parallel" != "true" ]; then
+            wait
+        fi
+    done
+
+    wait
+    comm -13 "${list[@]}"
+}
+
+# Prepare nixpkgs trees.
+declare -a tree
+for i in 1 2; do
+    if [ -n "${!i}" ]; then # use the given commit
+        dir="$($MKTEMP -d)"
+        tree[$i]="$dir"
+        toRemove+=("$dir")
+
+        git clone --shared --no-checkout --quiet . "${tree[$i]}"
+        (cd "${tree[$i]}" && git checkout --quiet "${!i}")
+    else #use the current tree
+        tree[$i]="$(pwd)"
+    fi
+done
+
+newlist="$($MKTEMP)"
+toRemove+=("$newlist")
+# Notes:
+#    - the evaluation is done on x86_64-linux, like on Hydra.
+#    - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
+newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
+
+# Hacky: keep only the last word of each attribute path and sort.
+sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
+    | sort | uniq -c
+
+if [ -n "$optPrint" ]; then
+    echo
+    cat "$newlist"
+fi

maintainers/scripts/remove-old-aliases.py

@@ -0,0 +1,213 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: with ps; [ ])" nix
+"""
+A program to remove old aliases or convert old aliases to throws
+Example usage:
+./maintainers/scripts/remove-old-aliases.py --year 2018 --file ./pkgs/top-level/aliases.nix
+
+Check this file with mypy after every change!
+$ mypy --strict maintainers/scripts/remove-old-aliases.py
+"""
+import argparse
+import shutil
+import subprocess
+from datetime import date as datetimedate
+from datetime import datetime
+from pathlib import Path
+
+
+def process_args() -> argparse.Namespace:
+    """process args"""
+    arg_parser = argparse.ArgumentParser()
+    arg_parser.add_argument(
+        "--year", required=True, type=int, help="operate on aliases older than $year"
+    )
+    arg_parser.add_argument(
+        "--month",
+        type=int,
+        default=1,
+        help="operate on aliases older than $year-$month",
+    )
+    arg_parser.add_argument(
+        "--only-throws",
+        action="store_true",
+        help="only operate on throws. e.g remove throws older than $date",
+    )
+    arg_parser.add_argument("--file", required=True, type=Path, help="alias file")
+    arg_parser.add_argument(
+        "--dry-run", action="store_true", help="don't modify files, only print results"
+    )
+    return arg_parser.parse_args()
+
+
+def get_date_lists(
+    txt: list[str], cutoffdate: datetimedate, only_throws: bool
+) -> tuple[list[str], list[str], list[str]]:
+    """get a list of lines in which the date is older than $cutoffdate"""
+    date_older_list: list[str] = []
+    date_older_throw_list: list[str] = []
+    date_sep_line_list: list[str] = []
+
+    for lineno, line in enumerate(txt, start=1):
+        line = line.rstrip()
+        my_date = None
+        for string in line.split():
+            string = string.strip(":")
+            try:
+                # strip ':' incase there is a string like 2019-11-01:
+                my_date = datetime.strptime(string, "%Y-%m-%d").date()
+            except ValueError:
+                try:
+                    my_date = datetime.strptime(string, "%Y-%m").date()
+                except ValueError:
+                    continue
+
+        if (
+            my_date is None
+            or my_date > cutoffdate
+            or "preserve, reason:" in line.lower()
+        ):
+            continue
+
+        if "=" not in line:
+            date_sep_line_list.append(f"{lineno} {line}")
+        # 'if' lines could be complicated
+        elif "if " in line and "if =" not in line:
+            print(f"RESOLVE MANUALLY {line}")
+        elif "throw" in line:
+            date_older_throw_list.append(line)
+        elif not only_throws:
+            date_older_list.append(line)
+
+    return (
+        date_older_list,
+        date_sep_line_list,
+        date_older_throw_list,
+    )
+
+
+def convert_to_throw(date_older_list: list[str]) -> list[tuple[str, str]]:
+    """convert a list of lines to throws"""
+    converted_list = []
+    for line in date_older_list.copy():
+        indent: str = " " * (len(line) - len(line.lstrip()))
+        before_equal = ""
+        after_equal = ""
+        try:
+            before_equal, after_equal = (x.strip() for x in line.split("=", maxsplit=2))
+        except ValueError as err:
+            print(err, line, "\n")
+            date_older_list.remove(line)
+            continue
+
+        alias = before_equal
+        alias_unquoted = before_equal.strip('"')
+        replacement = next(x.strip(";:") for x in after_equal.split())
+        replacement = replacement.removeprefix("pkgs.")
+
+        converted = (
+            f"{indent}{alias} = throw \"'{alias_unquoted}' has been"
+            f" renamed to/replaced by '{replacement}'\";"
+            f" # Converted to throw {datetime.today().strftime('%Y-%m-%d')}"
+        )
+        converted_list.append((line, converted))
+
+    return converted_list
+
+
+def generate_text_to_write(
+    txt: list[str],
+    date_older_list: list[str],
+    converted_to_throw: list[tuple[str, str]],
+    date_older_throw_list: list[str],
+) -> list[str]:
+    """generate a list of text to be written to the aliasfile"""
+    text_to_write: list[str] = []
+    for line in txt:
+        text_to_append: str = ""
+        if converted_to_throw:
+            for tupl in converted_to_throw:
+                if line == tupl[0]:
+                    text_to_append = f"{tupl[1]}\n"
+        if line not in date_older_list and line not in date_older_throw_list:
+            text_to_append = f"{line}\n"
+        if text_to_append:
+            text_to_write.append(text_to_append)
+
+    return text_to_write
+
+
+def write_file(
+    aliasfile: Path,
+    text_to_write: list[str],
+) -> None:
+    """write file"""
+    temp_aliasfile = Path(f"{aliasfile}.raliases")
+    with open(temp_aliasfile, "w", encoding="utf-8") as far:
+        for line in text_to_write:
+            far.write(line)
+    print("\nChecking the syntax of the new aliasfile")
+    try:
+        subprocess.run(
+            ["nix-instantiate", "--eval", temp_aliasfile],
+            check=True,
+            stdout=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError:
+        print(
+            "\nSyntax check failed,",
+            "there may have been a line which only has\n"
+            'aliasname = "reason why";\n'
+            "when it should have been\n"
+            'aliasname = throw "reason why";',
+        )
+        temp_aliasfile.unlink()
+        return
+    shutil.move(f"{aliasfile}.raliases", aliasfile)
+    print(f"{aliasfile} modified! please verify with 'git diff'.")
+
+
+def main() -> None:
+    """main"""
+    args = process_args()
+
+    only_throws = args.only_throws
+    aliasfile = Path(args.file).absolute()
+    cutoffdate = (datetime.strptime(f"{args.year}-{args.month}-01", "%Y-%m-%d")).date()
+
+    txt: list[str] = (aliasfile.read_text(encoding="utf-8")).splitlines()
+
+    date_older_list: list[str] = []
+    date_sep_line_list: list[str] = []
+    date_older_throw_list: list[str] = []
+
+    date_older_list, date_sep_line_list, date_older_throw_list = get_date_lists(
+        txt, cutoffdate, only_throws
+    )
+
+    converted_to_throw: list[tuple[str, str]] = []
+    if date_older_list:
+        converted_to_throw = convert_to_throw(date_older_list)
+        print(" Will be converted to throws. ".center(100, "-"))
+        for l_n in date_older_list:
+            print(l_n)
+
+    if date_older_throw_list:
+        print(" Will be removed. ".center(100, "-"))
+        for l_n in date_older_throw_list:
+            print(l_n)
+
+    if date_sep_line_list:
+        print(" On separate line, resolve manually. ".center(100, "-"))
+        for l_n in date_sep_line_list:
+            print(l_n)
+
+    if not args.dry_run:
+        text_to_write = generate_text_to_write(
+            txt, date_older_list, converted_to_throw, date_older_throw_list
+        )
+        write_file(aliasfile, text_to_write)
+
+
+if __name__ == "__main__":
+    main()

maintainers/scripts/sha-to-sri.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
+
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from pathlib import Path
+from structlog.contextvars import bound_contextvars as log_context
+from typing import ClassVar, List, Tuple
+
+import hashlib, logging, re, structlog
+
+
+logger = structlog.getLogger("sha-to-SRI")
+
+
+class Encoding(ABC):
+    alphabet: ClassVar[str]
+
+    @classmethod
+    @property
+    def name(cls) -> str:
+        return cls.__name__.lower()
+
+    def toSRI(self, s: str) -> str:
+        digest = self.decode(s)
+        assert len(digest) == self.n
+
+        from base64 import b64encode
+
+        return f"{self.hashName}-{b64encode(digest).decode()}"
+
+    @classmethod
+    def all(cls, h) -> "List[Encoding]":
+        return [c(h) for c in cls.__subclasses__()]
+
+    def __init__(self, h):
+        self.n = h.digest_size
+        self.hashName = h.name
+
+    @property
+    @abstractmethod
+    def length(self) -> int: ...
+
+    @property
+    def regex(self) -> str:
+        return f"[{self.alphabet}]{{{self.length}}}"
+
+    @abstractmethod
+    def decode(self, s: str) -> bytes: ...
+
+
+class Nix32(Encoding):
+    alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
+    inverted = {c: i for i, c in enumerate(alphabet)}
+
+    @property
+    def length(self):
+        return 1 + (8 * self.n) // 5
+
+    def decode(self, s: str):
+        assert len(s) == self.length
+        out = bytearray(self.n)
+
+        for n, c in enumerate(reversed(s)):
+            digit = self.inverted[c]
+            i, j = divmod(5 * n, 8)
+            out[i] = out[i] | (digit << j) & 0xFF
+            rem = digit >> (8 - j)
+            if rem == 0:
+                continue
+            elif i < self.n:
+                out[i + 1] = rem
+            else:
+                raise ValueError(f"Invalid nix32 hash: '{s}'")
+
+        return bytes(out)
+
+
+class Hex(Encoding):
+    alphabet = "0-9A-Fa-f"
+
+    @property
+    def length(self):
+        return 2 * self.n
+
+    def decode(self, s: str):
+        from binascii import unhexlify
+
+        return unhexlify(s)
+
+
+class Base64(Encoding):
+    alphabet = "A-Za-z0-9+/"
+
+    @property
+    def format(self) -> Tuple[int, int]:
+        """Number of characters in data and padding."""
+        i, k = divmod(self.n, 3)
+        return 4 * i + (0 if k == 0 else k + 1), (3 - k) % 3
+
+    @property
+    def length(self):
+        return sum(self.format)
+
+    @property
+    def regex(self):
+        data, padding = self.format
+        return f"[{self.alphabet}]{{{data}}}={{{padding}}}"
+
+    def decode(self, s):
+        from base64 import b64decode
+
+        return b64decode(s, validate = True)
+
+
+_HASHES = (hashlib.new(n) for n in ("SHA-256", "SHA-512"))
+ENCODINGS = {h.name: Encoding.all(h) for h in _HASHES}
+
+RE = {
+    h: "|".join(
+        (f"({h}-)?" if e.name == "base64" else "") + f"(?P<{h}_{e.name}>{e.regex})"
+        for e in encodings
+    )
+    for h, encodings in ENCODINGS.items()
+}
+
+_DEF_RE = re.compile(
+    "|".join(
+        f"(?P<{h}>{h} = (?P<{h}_quote>['\"])({re})(?P={h}_quote);)"
+        for h, re in RE.items()
+    )
+)
+
+
+def defToSRI(s: str) -> str:
+    def f(m: re.Match[str]) -> str:
+        try:
+            for h, encodings in ENCODINGS.items():
+                if m.group(h) is None:
+                    continue
+
+                for e in encodings:
+                    s = m.group(f"{h}_{e.name}")
+                    if s is not None:
+                        return f'hash = "{e.toSRI(s)}";'
+
+                raise ValueError(f"Match with '{h}' but no subgroup")
+            raise ValueError("Match with no hash")
+
+        except ValueError as exn:
+            logger.error(
+                "Skipping",
+                exc_info = exn,
+            )
+            return m.group()
+
+    return _DEF_RE.sub(f, s)
+
+
+@contextmanager
+def atomicFileUpdate(target: Path):
+    """Atomically replace the contents of a file.
+
+    Guarantees that no temporary files are left behind, and `target` is either
+    left untouched, or overwritten with new content if no exception was raised.
+
+    Yields a pair `(original, new)` of open files.
+    `original` is the pre-existing file at `target`, open for reading;
+    `new` is an empty, temporary file in the same filder, open for writing.
+
+    Upon exiting the context, the files are closed; if no exception was
+    raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
+    """
+    # That's mostly copied from noto-emoji.py, should DRY it out
+    from tempfile import NamedTemporaryFile
+
+    try:
+        with target.open() as original:
+            with NamedTemporaryFile(
+                dir = target.parent,
+                prefix = target.stem,
+                suffix = target.suffix,
+                delete = False,
+                mode="w",  # otherwise the file would be opened in binary mode by default
+            ) as new:
+                tmpPath = Path(new.name)
+                yield (original, new)
+
+        tmpPath.replace(target)
+
+    except Exception:
+        tmpPath.unlink(missing_ok = True)
+        raise
+
+
+def fileToSRI(p: Path):
+    with atomicFileUpdate(p) as (og, new):
+        for i, line in enumerate(og):
+            with log_context(line = i):
+                new.write(defToSRI(line))
+
+
+_SKIP_RE = re.compile("(generated by)|(do not edit)", re.IGNORECASE)
+_IGNORE = frozenset({
+    "gemset.nix",
+    "yarn.nix",
+})
+
+if __name__ == "__main__":
+    from sys import argv
+
+    logger.info("Starting!")
+
+    def handleFile(p: Path, skipLevel = logging.INFO):
+        with log_context(file = str(p)):
+            try:
+                with p.open() as f:
+                    for line in f:
+                        if line.strip():
+                            break
+
+                    if _SKIP_RE.search(line):
+                        logger.log(skipLevel, "File looks autogenerated, skipping!")
+                        return
+
+                fileToSRI(p)
+
+            except Exception as exn:
+                logger.error(
+                    "Unhandled exception, skipping file!",
+                    exc_info = exn,
+                )
+            else:
+                logger.info("Finished processing file")
+
+    for arg in argv[1:]:
+        p = Path(arg)
+        with log_context(arg = arg):
+            if p.is_file():
+                handleFile(p, skipLevel = logging.WARNING)
+
+            elif p.is_dir():
+                logger.info("Recursing into directory")
+                for q in p.glob("**/*.nix"):
+                    if q.is_file():
+                        if q.name in _IGNORE or q.name.find("generated") != -1:
+                            logger.info("File looks autogenerated, skipping!")
+                            continue
+
+                        handleFile(q)

maintainers/scripts/update-channel-branches.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -e
+
+: ${NIXOS_CHANNELS:=https://nixos.org/channels/}
+: ${CHANNELS_NAMESPACE:=refs/heads/channels/}
+
+# List all channels which are currently in the repository which we would
+# have to remove if they are not found again.
+deadChannels=$(git for-each-ref --format="%(refname)" "$CHANNELS_NAMESPACE")
+
+updateRef() {
+    local channelName=$1
+    local newRev=$2
+
+    # if the inputs are not valid, then we do not update any branch.
+    test -z "$newRev" -o -z "$channelName" && return;
+
+    # Update the local refs/heads/channels/* branches to be in-sync with the
+    # channel references.
+    local branch=$CHANNELS_NAMESPACE$channelName
+    oldRev=$(git rev-parse --short "$branch" 2>/dev/null || true)
+    if test "$oldRev" != "$newRev"; then
+        if git update-ref "$branch" "$newRev" 2>/dev/null; then
+            if test -z "$oldRev"; then
+                echo " * [new branch]      $newRev           -> ${branch#refs/heads/}"
+            else
+                echo "                     $oldRev..$newRev  -> ${branch#refs/heads/}"
+            fi
+        else
+            if test -z "$oldRev"; then
+                echo " * [missing rev]     $newRev           -> ${branch#refs/heads/}"
+            else
+                echo "   [missing rev]     $oldRev..$newRev  -> ${branch#refs/heads/}"
+            fi
+        fi
+    fi
+
+    # Filter out the current channel from the list of dead channels.
+    deadChannels=$(grep -v "$CHANNELS_NAMESPACE$channelName" <<EOF
+$deadChannels
+EOF
+) ||true
+}
+
+# Find the name of all channels which are listed in the directory.
+echo "Fetching channels from $NIXOS_CHANNELS:"
+for channelName in : $(curl -s "$NIXOS_CHANNELS" | sed -n '/folder/ { s,.*href=",,; s,/".*,,; p }'); do
+    test "$channelName" = : && continue;
+
+    # Do not follow redirections, such that we can extract the
+    # short-changeset from the name of the directory where we are
+    # redirected to.
+    sha1=$(curl -sI "$NIXOS_CHANNELS$channelName" | sed -n '/Location/ { s,.*\.\([a-f0-9]*\)[ \r]*$,\1,; p; }')
+
+    updateRef "remotes/$channelName" "$sha1"
+done
+
+echo "Fetching channels from nixos-version:"
+if currentSystem=$(nixos-version 2>/dev/null); then
+    # If the system is entirely build from a custom nixpkgs version,
+    # then the version is not annotated in git version. This sed
+    # expression is basically matching that the expressions end with
+    # ".<sha1> (Name)" to extract the sha1.
+    sha1=$(echo "$currentSystem" | sed -n 's,^.*\.\([a-f0-9]*\) *(.*)$,\1,; T skip; p; :skip;')
+
+    updateRef current-system "$sha1"
+fi
+
+echo "Fetching channels from $HOME/.nix-defexpr:"
+for revFile in : $(find -L "$HOME/.nix-defexpr/" -maxdepth 4 -name svn-revision); do
+    test "$revFile" = : && continue;
+
+    # Deconstruct a path such as, into:
+    #
+    #   /home/luke/.nix-defexpr/channels_root/nixos/nixpkgs/svn-revision
+    #     channelName = root/nixos
+    #
+    #   /home/luke/.nix-defexpr/channels/nixpkgs/svn-revision
+    #     channelName = nixpkgs
+    #
+    user=${revFile#*.nix-defexpr/channels}
+    repo=${user#*/}
+    repo=${repo%%/*}
+    user=${user%%/*}
+    user=${user#_}
+    test -z "$user" && user=$USER
+    channelName="$user${user:+/}$repo"
+
+    sha1=$(sed -n 's,^.*\.\([a-f0-9]*\)$,\1,; T skip; p; :skip;' "$revFile")
+
+    updateRef "$channelName" "$sha1"
+done
+
+# Suggest to remove channel branches which are no longer found by this
+# script. This is to handle the cases where a local/remote channel
+# disappear. We should not attempt to remove manually any branches, as they
+# might be user branches.
+if test -n "$deadChannels"; then
+
+    echo "
+Some old channel branches are still in your repository, if you
+want to remove them, run the following command(s):
+"
+
+    while read branch; do
+        echo "    git update-ref -d $branch"
+    done <<EOF
+$deadChannels
+EOF
+
+    echo
+fi

maintainers/scripts/update-dotnet-lockfiles.nix

@@ -0,0 +1,19 @@
+/*
+  To run:
+
+      nix-shell maintainers/scripts/update-dotnet-lockfiles.nix
+
+  This script finds all the derivations in nixpkgs that have a 'fetch-deps'
+  attribute, and runs all of them sequentially. This is useful to test changes
+  to 'fetch-deps', 'nuget-to-json', or other changes to the
+  dotnet build infrastructure. Regular updates should be done through the
+  individual packages update scripts.
+*/
+{ ... }@args:
+import ./update.nix (
+  {
+    predicate = _: _: true;
+    get-script = pkg: pkg.fetch-deps or null;
+  }
+  // args
+)

maintainers/scripts/update-octave-packages

@@ -0,0 +1,468 @@
+#!/usr/bin/env nix-shell
+#!nix-shell update-octave-shell.nix -i python3
+
+"""
+Update a Octave package expression by passing in the `.nix` file, or the directory containing it.
+You can pass in multiple files or paths.
+
+You'll likely want to use
+``
+  $ ./update-octave-libraries ../../pkgs/development/octave-modules/**/default.nix
+``
+to update all non-pinned libraries in that folder.
+"""
+
+import argparse
+import os
+import pathlib
+import re
+import requests
+import yaml
+from concurrent.futures import ThreadPoolExecutor as Pool
+from packaging.version import Version as _Version
+from packaging.version import InvalidVersion
+from packaging.specifiers import SpecifierSet
+import collections
+import subprocess
+import tempfile
+
+INDEX = "https://raw.githubusercontent.com/gnu-octave/packages/main/packages"
+"""url of Octave packages' source on GitHub"""
+
+EXTENSIONS = ['tar.gz', 'tar.bz2', 'tar', 'zip']
+"""Permitted file extensions. These are evaluated from left to right and the first occurance is returned."""
+
+PRERELEASES = False
+
+GIT = "git"
+
+NIXPGKS_ROOT = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode('utf-8').strip()
+
+import logging
+logging.basicConfig(level=logging.INFO)
+
+
+class Version(_Version, collections.abc.Sequence):
+
+    def __init__(self, version):
+        super().__init__(version)
+        # We cannot use `str(Version(0.04.21))` because that becomes `0.4.21`
+        # https://github.com/avian2/unidecode/issues/13#issuecomment-354538882
+        self.raw_version = version
+
+    def __getitem__(self, i):
+        return self._version.release[i]
+
+    def __len__(self):
+        return len(self._version.release)
+
+    def __iter__(self):
+        yield from self._version.release
+
+
+def _get_values(attribute, text):
+    """Match attribute in text and return all matches.
+
+    :returns: List of matches.
+    """
+    regex = '{}\s+=\s+"(.*)";'.format(attribute)
+    regex = re.compile(regex)
+    values = regex.findall(text)
+    return values
+
+def _get_unique_value(attribute, text):
+    """Match attribute in text and return unique match.
+
+    :returns: Single match.
+    """
+    values = _get_values(attribute, text)
+    n = len(values)
+    if n > 1:
+        raise ValueError("found too many values for {}".format(attribute))
+    elif n == 1:
+        return values[0]
+    else:
+        raise ValueError("no value found for {}".format(attribute))
+
+def _get_line_and_value(attribute, text):
+    """Match attribute in text. Return the line and the value of the attribute."""
+    regex = '({}\s+=\s+"(.*)";)'.format(attribute)
+    regex = re.compile(regex)
+    value = regex.findall(text)
+    n = len(value)
+    if n > 1:
+        raise ValueError("found too many values for {}".format(attribute))
+    elif n == 1:
+        return value[0]
+    else:
+        raise ValueError("no value found for {}".format(attribute))
+
+
+def _replace_value(attribute, value, text):
+    """Search and replace value of attribute in text."""
+    old_line, old_value = _get_line_and_value(attribute, text)
+    new_line = old_line.replace(old_value, value)
+    new_text = text.replace(old_line, new_line)
+    return new_text
+
+
+def _fetch_page(url):
+    r = requests.get(url)
+    if r.status_code == requests.codes.ok:
+        return list(yaml.safe_load_all(r.content))[0]
+    else:
+        raise ValueError("request for {} failed".format(url))
+
+
+def _fetch_github(url):
+    headers = {}
+    token = os.environ.get('GITHUB_API_TOKEN')
+    if token:
+        headers["Authorization"] = f"token {token}"
+    r = requests.get(url, headers=headers)
+
+    if r.status_code == requests.codes.ok:
+        return r.json()
+    else:
+        raise ValueError("request for {} failed".format(url))
+
+
+SEMVER = {
+    'major' : 0,
+    'minor' : 1,
+    'patch' : 2,
+}
+
+
+def _determine_latest_version(current_version, target, versions):
+    """Determine latest version, given `target`, returning the more recent version.
+    """
+    current_version = Version(current_version)
+
+    def _parse_versions(versions):
+        for v in versions:
+            try:
+                yield Version(v)
+            except InvalidVersion:
+                pass
+
+    versions = _parse_versions(versions)
+
+    index = SEMVER[target]
+
+    ceiling = list(current_version[0:index])
+    if len(ceiling) == 0:
+        ceiling = None
+    else:
+        ceiling[-1]+=1
+        ceiling = Version(".".join(map(str, ceiling)))
+
+    # We do not want prereleases
+    versions = SpecifierSet(prereleases=PRERELEASES).filter(versions)
+
+    if ceiling is not None:
+        versions = SpecifierSet(f"<{ceiling}").filter(versions)
+
+    return (max(sorted(versions))).raw_version
+
+
+def _get_latest_version_octave_packages(package, extension, current_version, target):
+    """Get latest version and hash from Octave Packages."""
+    url = "{}/{}.yaml".format(INDEX, package)
+    yaml = _fetch_page(url)
+
+    versions = list(map(lambda pv: pv['id'], yaml['versions']))
+    version = _determine_latest_version(current_version, target, versions)
+
+    try:
+        releases = [v if v['id'] == version else None for v in yaml['versions']]
+    except KeyError as e:
+        raise KeyError('Could not find version {} for {}'.format(version, package)) from e
+    for release in releases:
+        if release['url'].endswith(extension):
+            sha256 = release['sha256']
+            break
+    else:
+        sha256 = None
+    return version, sha256, None
+
+
+def _get_latest_version_github(package, extension, current_version, target):
+    def strip_prefix(tag):
+        return re.sub("^[^0-9]*", "", tag)
+
+    def get_prefix(string):
+        matches = re.findall(r"^([^0-9]*)", string)
+        return next(iter(matches), "")
+
+    # when invoked as an updateScript, UPDATE_NIX_ATTR_PATH will be set
+    # this allows us to work with packages which live outside of octave-modules
+    attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"octavePackages.{package}")
+    try:
+        homepage = subprocess.check_output(
+            ["nix", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.meta.homepage"])\
+            .decode('utf-8')
+    except Exception as e:
+        raise ValueError(f"Unable to determine homepage: {e}")
+    owner_repo = homepage[len("https://github.com/"):]  # remove prefix
+    owner, repo = owner_repo.split("/")
+
+    url = f"https://api.github.com/repos/{owner}/{repo}/releases"
+    all_releases = _fetch_github(url)
+    releases = list(filter(lambda x: not x['prerelease'], all_releases))
+
+    if len(releases) == 0:
+        raise ValueError(f"{homepage} does not contain any stable releases")
+
+    versions = map(lambda x: strip_prefix(x['tag_name']), releases)
+    version = _determine_latest_version(current_version, target, versions)
+
+    release = next(filter(lambda x: strip_prefix(x['tag_name']) == version, releases))
+    prefix = get_prefix(release['tag_name'])
+    try:
+        sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", f"{release['tarball_url']}"], stderr=subprocess.DEVNULL)\
+            .decode('utf-8').strip()
+    except:
+        # this may fail if they have both a branch and a tag of the same name, attempt tag name
+        tag_url = str(release['tarball_url']).replace("tarball","tarball/refs/tags")
+        sha256 = subprocess.check_output(["nix-prefetch-url", "--type", "sha256", "--unpack", tag_url], stderr=subprocess.DEVNULL)\
+            .decode('utf-8').strip()
+
+
+    return version, sha256, prefix
+
+def _get_latest_version_git(package, extension, current_version, target):
+    """NOTE: Unimplemented!"""
+    # attr_path = os.environ.get("UPDATE_NIX_ATTR_PATH", f"octavePackages.{package}")
+    # try:
+    #     download_url = subprocess.check_output(
+    #         ["nix", "--extra-experimental-features", "nix-command", "eval", "-f", f"{NIXPGKS_ROOT}/default.nix", "--raw", f"{attr_path}.src.url"])\
+    #         .decode('utf-8')
+    # except Exception as e:
+    #     raise ValueError(f"Unable to determine download link: {e}")
+
+    # with tempfile.TemporaryDirectory(prefix=attr_path) as new_clone_location:
+    #     subprocess.run(["git", "clone", download_url, new_clone_location])
+    #     newest_commit = subprocess.check_output(
+    #         ["git" "rev-parse" "$(git branch -r)" "|" "tail" "-n" "1"]).decode('utf-8')
+    pass
+
+
+FETCHERS = {
+    'fetchFromGitHub'   :   _get_latest_version_github,
+    'fetchurl'          :   _get_latest_version_octave_packages,
+    'fetchgit'          :   _get_latest_version_git,
+}
+
+
+DEFAULT_SETUPTOOLS_EXTENSION = 'tar.gz'
+
+
+FORMATS = {
+    'setuptools'        :   DEFAULT_SETUPTOOLS_EXTENSION,
+}
+
+def _determine_fetcher(text):
+    # Count occurrences of fetchers.
+    nfetchers = sum(text.count('src = {}'.format(fetcher)) for fetcher in FETCHERS.keys())
+    if nfetchers == 0:
+        raise ValueError("no fetcher.")
+    elif nfetchers > 1:
+        raise ValueError("multiple fetchers.")
+    else:
+        # Then we check which fetcher to use.
+        for fetcher in FETCHERS.keys():
+            if 'src = {}'.format(fetcher) in text:
+                return fetcher
+
+
+def _determine_extension(text, fetcher):
+    """Determine what extension is used in the expression.
+
+    If we use:
+    - fetchPypi, we check if format is specified.
+    - fetchurl, we determine the extension from the url.
+    - fetchFromGitHub we simply use `.tar.gz`.
+    """
+    if fetcher == 'fetchurl':
+        url = _get_unique_value('url', text)
+        extension = os.path.splitext(url)[1]
+
+    elif fetcher == 'fetchFromGitHub' or fetcher == 'fetchgit':
+        if "fetchSubmodules" in text:
+            raise ValueError("fetchFromGitHub fetcher doesn't support submodules")
+        extension = "tar.gz"
+
+    return extension
+
+
+def _update_package(path, target):
+
+    # Read the expression
+    with open(path, 'r') as f:
+        text = f.read()
+
+    # Determine pname. Many files have more than one pname
+    pnames = _get_values('pname', text)
+
+    # Determine version.
+    version = _get_unique_value('version', text)
+
+    # First we check how many fetchers are mentioned.
+    fetcher = _determine_fetcher(text)
+
+    extension = _determine_extension(text, fetcher)
+
+    # Attempt a fetch using each pname, e.g. backports-zoneinfo vs backports.zoneinfo
+    successful_fetch = False
+    for pname in pnames:
+        if fetcher == "fetchgit":
+            logging.warning(f"You must update {pname} MANUALLY!")
+            return { 'path': path, 'target': target, 'pname': pname,
+                     'old_version': version, 'new_version': version }
+        try:
+            new_version, new_sha256, prefix = FETCHERS[fetcher](pname, extension, version, target)
+            successful_fetch = True
+            break
+        except ValueError:
+            continue
+
+    if not successful_fetch:
+        raise ValueError(f"Unable to find correct package using these pnames: {pnames}")
+
+    if new_version == version:
+        logging.info("Path {}: no update available for {}.".format(path, pname))
+        return False
+    elif Version(new_version) <= Version(version):
+        raise ValueError("downgrade for {}.".format(pname))
+    if not new_sha256:
+        raise ValueError("no file available for {}.".format(pname))
+
+    text = _replace_value('version', new_version, text)
+    # hashes from pypi are 16-bit encoded sha256's, normalize it to sri to avoid merge conflicts
+    # sri hashes have been the default format since nix 2.4+
+    sri_hash = subprocess.check_output(["nix", "--extra-experimental-features", "nix-command", "hash", "to-sri", "--type", "sha256", new_sha256]).decode('utf-8').strip()
+
+
+    # fetchers can specify a sha256, or a sri hash
+    try:
+        text = _replace_value('sha256', sri_hash, text)
+    except ValueError:
+        text = _replace_value('hash', sri_hash, text)
+
+    if fetcher == 'fetchFromGitHub':
+        # in the case of fetchFromGitHub, it's common to see `rev = version;` or `rev = "v${version}";`
+        # in which no string value is meant to be substituted. However, we can just overwrite the previous value.
+        regex = '(rev\s+=\s+[^;]*;)'
+        regex = re.compile(regex)
+        matches = regex.findall(text)
+        n = len(matches)
+
+        if n == 0:
+            raise ValueError("Unable to find rev value for {}.".format(pname))
+        else:
+            # forcefully rewrite rev, incase tagging conventions changed for a release
+            match = matches[0]
+            text = text.replace(match, f'rev = "refs/tags/{prefix}${{version}}";')
+            # incase there's no prefix, just rewrite without interpolation
+            text = text.replace('"${version}";', 'version;')
+
+    with open(path, 'w') as f:
+        f.write(text)
+
+        logging.info("Path {}: updated {} from {} to {}".format(path, pname, version, new_version))
+
+    result = {
+        'path'  : path,
+        'target': target,
+        'pname': pname,
+        'old_version'   : version,
+        'new_version'   : new_version,
+        #'fetcher'       : fetcher,
+        }
+
+    return result
+
+
+def _update(path, target):
+
+    # We need to read and modify a Nix expression.
+    if os.path.isdir(path):
+        path = os.path.join(path, 'default.nix')
+
+    # If a default.nix does not exist, we quit.
+    if not os.path.isfile(path):
+        logging.info("Path {}: does not exist.".format(path))
+        return False
+
+    # If file is not a Nix expression, we quit.
+    if not path.endswith(".nix"):
+        logging.info("Path {}: does not end with `.nix`.".format(path))
+        return False
+
+    try:
+        return _update_package(path, target)
+    except ValueError as e:
+        logging.warning("Path {}: {}".format(path, e))
+        return False
+
+
+def _commit(path, pname, old_version, new_version, pkgs_prefix="octave: ", **kwargs):
+    """Commit result.
+    """
+
+    msg = f'{pkgs_prefix}{pname}: {old_version} -> {new_version}'
+
+    try:
+        subprocess.check_call([GIT, 'add', path])
+        subprocess.check_call([GIT, 'commit', '-m', msg])
+    except subprocess.CalledProcessError as e:
+        subprocess.check_call([GIT, 'checkout', path])
+        raise subprocess.CalledProcessError(f'Could not commit {path}') from e
+
+    return True
+
+
+def main():
+
+    epilog = """
+environment variables:
+  GITHUB_API_TOKEN\tGitHub API token used when updating github packages
+    """
+    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, epilog=epilog)
+    parser.add_argument('package', type=str, nargs='+')
+    parser.add_argument('--target', type=str, choices=SEMVER.keys(), default='major')
+    parser.add_argument('--commit', action='store_true', help='Create a commit for each package update')
+    parser.add_argument('--use-pkgs-prefix', action='store_true', help='Use octavePackages.${pname}: instead of octave: ${pname}: when making commits')
+
+    args = parser.parse_args()
+    target = args.target
+
+    packages = list(map(os.path.abspath, args.package))
+
+    logging.info("Updating packages...")
+
+    # Use threads to update packages concurrently
+    with Pool() as p:
+        results = list(filter(bool, p.map(lambda pkg: _update(pkg, target), packages)))
+
+    logging.info("Finished updating packages.")
+
+    commit_options = {}
+    if args.use_pkgs_prefix:
+        logging.info("Using octavePackages. prefix for commits")
+        commit_options["pkgs_prefix"] = "octavePackages."
+
+    # Commits are created sequentially.
+    if args.commit:
+        logging.info("Committing updates...")
+        # list forces evaluation
+        list(map(lambda x: _commit(**x, **commit_options), results))
+        logging.info("Finished committing updates")
+
+    count = len(results)
+    logging.info("{} package(s) updated".format(count))
+
+
+if __name__ == '__main__':
+    main()

maintainers/scripts/update-octave-shell.nix

@@ -0,0 +1,20 @@
+{
+  nixpkgs ? import ../.. { },
+}:
+with nixpkgs;
+let
+  pyEnv = python3.withPackages (
+    ps: with ps; [
+      packaging
+      requests
+      toolz
+      pyyaml
+    ]
+  );
+in
+mkShell {
+  packages = [
+    pyEnv
+    nix-prefetch-scripts
+  ];
+}

maintainers/scripts/update-python-libraries

@@ -0,0 +1,3 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -I nixpkgs=channel:nixpkgs-unstable -i bash -p "python3.withPackages (ps: with ps; [ packaging requests ])" -p nix-prefetch-git
+exec python3 pkgs/by-name/up/update-python-libraries/update-python-libraries.py $@

maintainers/scripts/update-redirected-urls.sh

@@ -0,0 +1,12 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -p bash curl ripgrep jq -i bash
+
+set -euxo pipefail
+
+# Possibly also add non-https redirect, but there were non of those when I first
+# made this script to test that. Feel free to add it when it is relevant.
+curl https://repology.org/api/v1/repository/nix_unstable/problems \
+   | jq -r '.[] | select(.type == "homepage_permanent_https_redirect") | .data | "s@\(.url)@\(.target)@"' \
+   | sort | uniq | tee script.sed
+find -name '*.nix' | xargs -P4 -- sed -f script.sed -i
+rm script.sed

maintainers/scripts/update-ruby-packages

@@ -0,0 +1,16 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p bundler bundix
+
+set -euf -o pipefail
+
+(
+  cd pkgs/development/ruby-modules/with-packages
+  rm -f gemset.nix Gemfile.lock
+  # Since bundler 2+, the lock command generates a platform-dependent
+  # Gemfile.lock, hence causing to bundix to generate a gemset tied to the
+  # platform from where it was executed.
+  BUNDLE_FORCE_RUBY_PLATFORM=1 bundle lock
+  bundix
+  mv gemset.nix ../../../top-level/ruby-packages.nix
+  rm -f Gemfile.lock
+)

maintainers/scripts/update-typst-packages.py

@@ -0,0 +1,226 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -p "python3.withPackages (p: with p; [ tomli tomli-w packaging license-expression])" -i python3
+
+# This file is formatted with `ruff format`.
+
+import os
+import re
+import tomli
+import tomli_w
+import subprocess
+import concurrent.futures
+import argparse
+import tempfile
+import tarfile
+from string import punctuation
+from packaging.version import Version
+from urllib import request
+from collections import OrderedDict
+
+
+class TypstPackage:
+    def __init__(self, **kwargs):
+        self.pname = kwargs["pname"]
+        self.version = kwargs["version"]
+        self.meta = kwargs["meta"]
+        self.path = kwargs["path"]
+        self.repo = (
+            None
+            if "repository" not in self.meta["package"]
+            else self.meta["package"]["repository"]
+        )
+        self.description = self.meta["package"]["description"].rstrip(punctuation)
+        self.license = self.meta["package"]["license"]
+        self.params = "" if "params" not in kwargs else kwargs["params"]
+        self.deps = [] if "deps" not in kwargs else kwargs["deps"]
+
+    @classmethod
+    def package_name_full(cls, package_name, version):
+        version_number = map(lambda x: int(x), version.split("."))
+        version_nix = "_".join(map(lambda x: str(x), version_number))
+        return "_".join((package_name, version_nix))
+
+    def license_tokens(self):
+        import license_expression as le
+
+        try:
+            # FIXME: ad hoc conversion
+            exception_list = [("EUPL-1.2+", "EUPL-1.2")]
+
+            def sanitize_license_string(license_string, lookups):
+                if not lookups:
+                    return license_string
+                return sanitize_license_string(
+                    license_string.replace(lookups[0][0], lookups[0][1]), lookups[1:]
+                )
+
+            sanitized = sanitize_license_string(self.license, exception_list)
+            licensing = le.get_spdx_licensing()
+            parsed = licensing.parse(sanitized, validate=True)
+            return [s.key for s in licensing.license_symbols(parsed)]
+        except le.ExpressionError as e:
+            print(
+                f'Failed to parse license string "{self.license}" because of {str(e)}'
+            )
+            exit(1)
+
+    def source(self):
+        url = f"https://packages.typst.org/preview/{self.pname}-{self.version}.tar.gz"
+        cmd = [
+            "nix",
+            "store",
+            "prefetch-file",
+            "--unpack",
+            "--hash-type",
+            "sha256",
+            "--refresh",
+            "--extra-experimental-features",
+            "nix-command",
+        ]
+        result = subprocess.run(cmd + [url], capture_output=True, text=True)
+        hash = re.search(r"hash\s+\'(sha256-.{44})\'", result.stderr).groups()[0]
+        return url, hash
+
+    def to_name_full(self):
+        return self.package_name_full(self.pname, self.version)
+
+    def to_attrs(self):
+        deps = set()
+        excludes = list(map(
+            lambda e: os.path.join(self.path, e),
+            self.meta["package"]["exclude"] if "exclude" in self.meta["package"] else [],
+        ))
+        for root, _, files in os.walk(self.path):
+            for file in filter(lambda f: f.split(".")[-1] == "typ", files):
+                file_path = os.path.join(root, file)
+                if file_path in excludes:
+                    continue
+                with open(file_path, "r") as f:
+                    deps.update(
+                        set(
+                            re.findall(
+                                r"^\s*#import\s+\"@preview/([\w|-]+):(\d+.\d+.\d+)\"",
+                                f.read(),
+                                re.MULTILINE,
+                            )
+                        )
+                    )
+        self.deps = list(
+            filter(lambda p: p[0] != self.pname or p[1] != self.version, deps)
+        )
+        source_url, source_hash = self.source()
+
+        return dict(
+            url=source_url,
+            hash=source_hash,
+            typstDeps=[
+                self.package_name_full(p, v)
+                for p, v in sorted(self.deps, key=lambda x: (x[0], Version(x[1])))
+            ],
+            description=self.description,
+            license=self.license_tokens(),
+        ) | (dict(homepage=self.repo) if self.repo else dict())
+
+
+def generate_typst_packages(preview_dir, output_file):
+    package_tree = dict()
+
+    print("Parsing metadata... from", preview_dir)
+    for p in os.listdir(preview_dir):
+        package_dir = os.path.join(preview_dir, p)
+        for v in os.listdir(package_dir):
+            package_version_dir = os.path.join(package_dir, v)
+            with open(
+                os.path.join(package_version_dir, "typst.toml"), "rb"
+            ) as meta_file:
+                try:
+                    package = TypstPackage(
+                        pname=p,
+                        version=v,
+                        meta=tomli.load(meta_file),
+                        path=package_version_dir,
+                    )
+                    if package.pname in package_tree:
+                        package_tree[package.pname][v] = package
+                    else:
+                        package_tree[package.pname] = dict({v: package})
+                except tomli.TOMLDecodeError:
+                    print("Invalid typst.toml:", package_version_dir)
+
+    with open(output_file, "wb") as typst_packages:
+
+        def generate_package(pname, package_subtree):
+            sorted_keys = sorted(package_subtree.keys(), key=Version, reverse=True)
+            print(f"Generating metadata for {pname}")
+            return {
+                pname: OrderedDict(
+                    (k, package_subtree[k].to_attrs()) for k in sorted_keys
+                )
+            }
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
+            sorted_packages = sorted(package_tree.items(), key=lambda x: x[0])
+            futures = list()
+            for pname, psubtree in sorted_packages:
+                futures.append(executor.submit(generate_package, pname, psubtree))
+            packages = OrderedDict(
+                (package, subtree)
+                for future in futures
+                for package, subtree in future.result().items()
+            )
+            print(f"Writing metadata... to {output_file}")
+            tomli_w.dump(packages, typst_packages)
+
+
+def main(args):
+    PREVIEW_DIR = "packages/preview"
+    TYPST_PACKAGE_TARBALL_URL = (
+        "https://github.com/typst/packages/archive/refs/heads/main.tar.gz"
+    )
+
+    directory = args.directory
+    if not directory:
+        tempdir = tempfile.mkdtemp()
+        print(tempdir)
+        typst_tarball = os.path.join(tempdir, "main.tar.gz")
+
+        print(
+            "Downloading Typst packages source from {} to {}".format(
+                TYPST_PACKAGE_TARBALL_URL, typst_tarball
+            )
+        )
+        with request.urlopen(
+            request.Request(TYPST_PACKAGE_TARBALL_URL), timeout=15.0
+        ) as response:
+            if response.status == 200:
+                with open(typst_tarball, "wb+") as f:
+                    f.write(response.read())
+            else:
+                print("Download failed")
+                exit(1)
+        with tarfile.open(typst_tarball) as tar:
+            tar.extractall(path=tempdir, filter="data")
+        directory = os.path.join(tempdir, "packages-main")
+    directory = os.path.abspath(directory)
+
+    generate_typst_packages(
+        os.path.join(directory, PREVIEW_DIR),
+        args.output,
+    )
+
+    exit(0)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-d", "--directory", help="Local Typst Universe repository", default=None
+    )
+    parser.add_argument(
+        "-o",
+        "--output",
+        help="Output file",
+        default=os.path.join(os.path.abspath("."), "typst-packages-from-universe.toml"),
+    )
+    args = parser.parse_args()
+    main(args)

maintainers/scripts/update.nix

@@ -0,0 +1,284 @@
+/*
+  To run:
+
+      nix-shell maintainers/scripts/update.nix
+
+  See https://nixos.org/manual/nixpkgs/unstable/#var-passthru-updateScript
+*/
+{
+  package ? null,
+  maintainer ? null,
+  predicate ? null,
+  get-script ? pkg: pkg.updateScript or null,
+  path ? null,
+  max-workers ? null,
+  include-overlays ? false,
+  keep-going ? null,
+  commit ? null,
+  skip-prompt ? null,
+  order ? null,
+}:
+
+let
+  pkgs = import ./../../default.nix (
+    (
+      if include-overlays == false then
+        { overlays = [ ]; }
+      else if include-overlays == true then
+        { } # Let Nixpkgs include overlays impurely.
+      else
+        { overlays = include-overlays; }
+    )
+    // {
+      config.allowAliases = false;
+    }
+  );
+
+  inherit (pkgs) lib;
+
+  # Remove duplicate elements from the list based on some extracted value. O(n^2) complexity.
+  nubOn =
+    f: list:
+    if list == [ ] then
+      [ ]
+    else
+      let
+        x = lib.head list;
+        xs = lib.filter (p: f x != f p) (lib.drop 1 list);
+      in
+      [ x ] ++ nubOn f xs;
+
+  /*
+    Recursively find all packages (derivations) in `pkgs` matching `cond` predicate.
+
+    Type: packagesWithPath :: AttrPath → (AttrPath → derivation → bool) → AttrSet → List<AttrSet{attrPath :: str; package :: derivation; }>
+          AttrPath :: [str]
+
+    The packages will be returned as a list of named pairs comprising of:
+      - attrPath: stringified attribute path (based on `rootPath`)
+      - package: corresponding derivation
+  */
+  packagesWithPath =
+    rootPath: cond: pkgs:
+    let
+      packagesWithPathInner =
+        path: pathContent:
+        let
+          result = builtins.tryEval pathContent;
+
+          somewhatUniqueRepresentant =
+            { package, attrPath }:
+            {
+              updateScript = (get-script package);
+              # Some updaters use the same `updateScript` value for all packages.
+              # Also compare `meta.description`.
+              position = package.meta.position or null;
+              # We cannot always use `meta.position` since it might not be available
+              # or it might be shared among multiple packages.
+            };
+
+          dedupResults = lst: nubOn somewhatUniqueRepresentant (lib.concatLists lst);
+        in
+        if result.success then
+          let
+            evaluatedPathContent = result.value;
+          in
+          if lib.isDerivation evaluatedPathContent then
+            lib.optional (cond path evaluatedPathContent) {
+              attrPath = lib.concatStringsSep "." path;
+              package = evaluatedPathContent;
+            }
+          else if lib.isAttrs evaluatedPathContent then
+            # If user explicitly points to an attrSet or it is marked for recursion, we recur.
+            if
+              path == rootPath
+              || evaluatedPathContent.recurseForDerivations or false
+              || evaluatedPathContent.recurseForRelease or false
+            then
+              dedupResults (
+                lib.mapAttrsToList (name: elem: packagesWithPathInner (path ++ [ name ]) elem) evaluatedPathContent
+              )
+            else
+              [ ]
+          else
+            [ ]
+        else
+          [ ];
+    in
+    packagesWithPathInner rootPath pkgs;
+
+  # Recursively find all packages (derivations) in `pkgs` matching `cond` predicate.
+  packagesWith = packagesWithPath [ ];
+
+  # Recursively find all packages in `pkgs` with updateScript matching given predicate.
+  packagesWithUpdateScriptMatchingPredicate =
+    cond: packagesWith (path: pkg: (get-script pkg != null) && cond path pkg);
+
+  # Recursively find all packages in `pkgs` with updateScript by given maintainer.
+  packagesWithUpdateScriptAndMaintainer =
+    maintainer':
+    let
+      maintainer =
+        if !builtins.hasAttr maintainer' lib.maintainers then
+          throw "Maintainer with name `${maintainer'} does not exist in `maintainers/maintainer-list.nix`."
+        else
+          builtins.getAttr maintainer' lib.maintainers;
+    in
+    packagesWithUpdateScriptMatchingPredicate (
+      path: pkg:
+      (
+        if builtins.hasAttr "maintainers" pkg.meta then
+          (
+            if builtins.isList pkg.meta.maintainers then
+              builtins.elem maintainer pkg.meta.maintainers
+            else
+              maintainer == pkg.meta.maintainers
+          )
+        else
+          false
+      )
+    );
+
+  # Recursively find all packages under `path` in `pkgs` with updateScript.
+  packagesWithUpdateScript =
+    path: pkgs:
+    let
+      prefix = lib.splitString "." path;
+      pathContent = lib.attrByPath prefix null pkgs;
+    in
+    if pathContent == null then
+      throw "Attribute path `${path}` does not exist."
+    else
+      packagesWithPath prefix (path: pkg: (get-script pkg != null)) pathContent;
+
+  # Find a package under `path` in `pkgs` and require that it has an updateScript.
+  packageByName =
+    path: pkgs:
+    let
+      package = lib.attrByPath (lib.splitString "." path) null pkgs;
+    in
+    if package == null then
+      throw "Package with an attribute name `${path}` does not exist."
+    else if get-script package == null then
+      throw "Package with an attribute name `${path}` does not have a `passthru.updateScript` attribute defined."
+    else
+      {
+        attrPath = path;
+        inherit package;
+      };
+
+  # List of packages matched based on the CLI arguments.
+  packages =
+    if package != null then
+      [ (packageByName package pkgs) ]
+    else if predicate != null then
+      packagesWithUpdateScriptMatchingPredicate predicate pkgs
+    else if maintainer != null then
+      packagesWithUpdateScriptAndMaintainer maintainer pkgs
+    else if path != null then
+      packagesWithUpdateScript path pkgs
+    else
+      throw "No arguments provided.\n\n${helpText}";
+
+  helpText = ''
+    Please run:
+
+        % nix-shell maintainers/scripts/update.nix --argstr maintainer garbas
+
+    to run all update scripts for all packages that lists \`garbas\` as a maintainer
+    and have \`updateScript\` defined, or:
+
+        % nix-shell maintainers/scripts/update.nix --argstr package nautilus
+
+    to run update script for specific package, or
+
+        % nix-shell maintainers/scripts/update.nix --arg predicate '(path: pkg: pkg.updateScript.name or null == "gnome-update-script")'
+
+    to run update script for all packages matching given predicate, or
+
+        % nix-shell maintainers/scripts/update.nix --argstr path gnome
+
+    to run update script for all package under an attribute path.
+
+    You can also add
+
+        --argstr max-workers 8
+
+    to increase the number of jobs in parallel, or
+
+        --argstr keep-going true
+
+    to continue running when a single update fails.
+
+    You can also make the updater automatically commit on your behalf from updateScripts
+    that support it by adding
+
+        --argstr commit true
+
+    to skip prompt:
+
+        --argstr skip-prompt true
+
+    By default, the updater will update the packages in arbitrary order. Alternately, you can force a specific order based on the packages’ dependency relations:
+
+        - Reverse topological order (e.g. {"gnome-text-editor", "gimp"}, {"gtk3", "gtk4"}, {"glib"}) is useful when you want checkout each commit one by one to build each package individually but some of the packages to be updated would cause a mass rebuild for the others. Of course, this requires that none of the updated dependents require a new version of the dependency.
+
+            --argstr order reverse-topological
+
+        - Topological order (e.g. {"glib"}, {"gtk3", "gtk4"}, {"gnome-text-editor", "gimp"}) is useful when the updated dependents require a new version of updated dependency.
+
+            --argstr order topological
+
+    Note that sorting requires instantiating each package and then querying Nix store for requisites so it will be pretty slow with large number of packages.
+  '';
+
+  # Transform a matched package into an object for update.py.
+  packageData =
+    { package, attrPath }:
+    let
+      updateScript = get-script package;
+    in
+    {
+      name = package.name;
+      pname = lib.getName package;
+      oldVersion = lib.getVersion package;
+      updateScript = map toString (lib.toList (updateScript.command or updateScript));
+      supportedFeatures = updateScript.supportedFeatures or [ ];
+      attrPath = updateScript.attrPath or attrPath;
+    };
+
+  # JSON file with data for update.py.
+  packagesJson = pkgs.writeText "packages.json" (builtins.toJSON (map packageData packages));
+
+  optionalArgs =
+    lib.optional (max-workers != null) "--max-workers=${max-workers}"
+    ++ lib.optional (keep-going == "true") "--keep-going"
+    ++ lib.optional (commit == "true") "--commit"
+    ++ lib.optional (skip-prompt == "true") "--skip-prompt"
+    ++ lib.optional (order != null) "--order=${order}";
+
+  args = [ packagesJson ] ++ optionalArgs;
+
+in
+pkgs.stdenv.mkDerivation {
+  name = "nixpkgs-update-script";
+  buildCommand = ''
+    echo ""
+    echo "----------------------------------------------------------------"
+    echo ""
+    echo "Not possible to update packages using \`nix-build\`"
+    echo ""
+    echo "${helpText}"
+    echo "----------------------------------------------------------------"
+    exit 1
+  '';
+  shellHook = ''
+    unset shellHook # do not contaminate nested shells
+    exec ${pkgs.python3.interpreter} ${./update.py} ${builtins.concatStringsSep " " args}
+  '';
+  nativeBuildInputs = [
+    pkgs.git
+    pkgs.nix
+    pkgs.cacert
+  ];
+}

maintainers/scripts/update.py

@@ -0,0 +1,615 @@
+from graphlib import TopologicalSorter
+from pathlib import Path
+from typing import Any, Final, Generator, Literal
+import argparse
+import asyncio
+import contextlib
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+import tempfile
+
+
+Order = Literal["arbitrary", "reverse-topological", "topological"]
+
+
+FAKE_DEPENDENCY_FOR_INDEPENDENT_PACKAGES: Final[str] = (
+    "::fake_dependency_for_independent_packages"
+)
+
+
+class CalledProcessError(Exception):
+    process: asyncio.subprocess.Process
+    stderr: bytes | None
+
+
+class UpdateFailedException(Exception):
+    pass
+
+
+def eprint(*args: Any, **kwargs: Any) -> None:
+    print(*args, file=sys.stderr, **kwargs)
+
+
+async def check_subprocess_output(*args: str, **kwargs: Any) -> bytes:
+    """
+    Emulate check and capture_output arguments of subprocess.run function.
+    """
+    process = await asyncio.create_subprocess_exec(*args, **kwargs)
+    # We need to use communicate() instead of wait(), as the OS pipe buffers
+    # can fill up and cause a deadlock.
+    stdout, stderr = await process.communicate()
+
+    if process.returncode != 0:
+        error = CalledProcessError()
+        error.process = process
+        error.stderr = stderr
+
+        raise error
+
+    return stdout
+
+
+async def nix_instantiate(attr_path: str) -> Path:
+    out = await check_subprocess_output(
+        "nix-instantiate",
+        "-A",
+        attr_path,
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+    drv = out.decode("utf-8").strip().split("!", 1)[0]
+
+    return Path(drv)
+
+
+async def nix_query_requisites(drv: Path) -> list[Path]:
+    requisites = await check_subprocess_output(
+        "nix-store",
+        "--query",
+        "--requisites",
+        str(drv),
+        stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
+    )
+
+    drv_str = str(drv)
+
+    return [
+        Path(requisite)
+        for requisite in requisites.decode("utf-8").splitlines()
+        # Avoid self-loops.
+        if requisite != drv_str
+    ]
+
+
+async def attr_instantiation_worker(
+    semaphore: asyncio.Semaphore,
+    attr_path: str,
+) -> tuple[Path, str]:
+    async with semaphore:
+        eprint(f"Instantiating {attr_path}…")
+        return (await nix_instantiate(attr_path), attr_path)
+
+
+async def requisites_worker(
+    semaphore: asyncio.Semaphore,
+    drv: Path,
+) -> tuple[Path, list[Path]]:
+    async with semaphore:
+        eprint(f"Obtaining requisites for {drv}…")
+        return (drv, await nix_query_requisites(drv))
+
+
+def requisites_to_attrs(
+    drv_attr_paths: dict[Path, str],
+    requisites: list[Path],
+) -> set[str]:
+    """
+    Converts a set of requisite `.drv`s to a set of attribute paths.
+    Derivations that do not correspond to any of the packages we want to update will be discarded.
+    """
+    return {
+        drv_attr_paths[requisite]
+        for requisite in requisites
+        if requisite in drv_attr_paths
+    }
+
+
+def reverse_edges(graph: dict[str, set[str]]) -> dict[str, set[str]]:
+    """
+    Flips the edges of a directed graph.
+
+    Packages without any dependency relation in the updated set
+    will be added to `FAKE_DEPENDENCY_FOR_INDEPENDENT_PACKAGES` node.
+    """
+
+    reversed_graph: dict[str, set[str]] = {}
+    for dependent, dependencies in graph.items():
+        dependencies = dependencies or {FAKE_DEPENDENCY_FOR_INDEPENDENT_PACKAGES}
+        for dependency in dependencies:
+            reversed_graph.setdefault(dependency, set()).add(dependent)
+
+    return reversed_graph
+
+
+def get_independent_sorter(
+    packages: list[dict],
+) -> TopologicalSorter[str]:
+    """
+    Returns a sorter which treats all packages as independent,
+    which will allow them to be updated in parallel.
+    """
+
+    attr_deps: dict[str, set[str]] = {
+        package["attrPath"]: set() for package in packages
+    }
+    sorter = TopologicalSorter(attr_deps)
+    sorter.prepare()
+
+    return sorter
+
+
+async def get_topological_sorter(
+    max_workers: int,
+    packages: list[dict],
+    reverse_order: bool,
+) -> tuple[TopologicalSorter[str], list[dict]]:
+    """
+    Returns a sorter which returns packages in topological or reverse topological order,
+    which will ensure a package is updated before or after its dependencies, respectively.
+    """
+
+    semaphore = asyncio.Semaphore(max_workers)
+
+    drv_attr_paths = dict(
+        await asyncio.gather(
+            *(
+                attr_instantiation_worker(semaphore, package["attrPath"])
+                for package in packages
+            )
+        )
+    )
+
+    drv_requisites = await asyncio.gather(
+        *(requisites_worker(semaphore, drv) for drv in drv_attr_paths.keys())
+    )
+
+    attr_deps = {
+        drv_attr_paths[drv]: requisites_to_attrs(drv_attr_paths, requisites)
+        for drv, requisites in drv_requisites
+    }
+
+    if reverse_order:
+        attr_deps = reverse_edges(attr_deps)
+
+    # Adjust packages order based on the topological one
+    ordered = list(TopologicalSorter(attr_deps).static_order())
+    packages = sorted(packages, key=lambda package: ordered.index(package["attrPath"]))
+
+    sorter = TopologicalSorter(attr_deps)
+    sorter.prepare()
+
+    return sorter, packages
+
+
+async def run_update_script(
+    nixpkgs_root: str,
+    merge_lock: asyncio.Lock,
+    temp_dir: tuple[str, str] | None,
+    package: dict,
+    keep_going: bool,
+) -> None:
+    worktree: str | None = None
+
+    update_script_command = package["updateScript"]
+
+    if temp_dir is not None:
+        worktree, _branch = temp_dir
+
+        # Ensure the worktree is clean before update.
+        await check_subprocess_output(
+            "git",
+            "reset",
+            "--hard",
+            "--quiet",
+            "HEAD",
+            cwd=worktree,
+        )
+
+        # Update scripts can use $(dirname $0) to get their location but we want to run
+        # their clones in the git worktree, not in the main nixpkgs repo.
+        update_script_command = map(
+            lambda arg: re.sub(r"^{0}".format(re.escape(nixpkgs_root)), worktree, arg),
+            update_script_command,
+        )
+
+    eprint(f" - {package['name']}: UPDATING ...")
+
+    try:
+        update_info = await check_subprocess_output(
+            "env",
+            f"UPDATE_NIX_NAME={package['name']}",
+            f"UPDATE_NIX_PNAME={package['pname']}",
+            f"UPDATE_NIX_OLD_VERSION={package['oldVersion']}",
+            f"UPDATE_NIX_ATTR_PATH={package['attrPath']}",
+            # Run all update scripts in the Nixpkgs development shell to get access to formatters and co.
+            "nix-shell",
+            nixpkgs_root + "/shell.nix",
+            "--run",
+            " ".join([ shlex.quote(s) for s in update_script_command ]),
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+            cwd=worktree,
+        )
+        await merge_changes(merge_lock, package, update_info, temp_dir)
+    except KeyboardInterrupt as e:
+        eprint("Cancelling…")
+        raise asyncio.exceptions.CancelledError()
+    except CalledProcessError as e:
+        eprint(f" - {package['name']}: ERROR")
+        if e.stderr is not None:
+            eprint()
+            eprint(
+                f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------"
+            )
+            eprint()
+            eprint(e.stderr.decode("utf-8"))
+            with open(f"{package['pname']}.log", "wb") as logfile:
+                logfile.write(e.stderr)
+            eprint()
+            eprint(
+                f"--- SHOWING ERROR LOG FOR {package['name']} ----------------------"
+            )
+
+        if not keep_going:
+            raise UpdateFailedException(
+                f"The update script for {package['name']} failed with exit code {e.process.returncode}"
+            )
+
+
+@contextlib.contextmanager
+def make_worktree() -> Generator[tuple[str, str], None, None]:
+    with tempfile.TemporaryDirectory() as wt:
+        branch_name = f"update-{os.path.basename(wt)}"
+        target_directory = f"{wt}/nixpkgs"
+
+        subprocess.run(["git", "worktree", "add", "-b", branch_name, target_directory])
+        try:
+            yield (target_directory, branch_name)
+        finally:
+            subprocess.run(["git", "worktree", "remove", "--force", target_directory])
+            subprocess.run(["git", "branch", "-D", branch_name])
+
+
+async def commit_changes(
+    name: str,
+    merge_lock: asyncio.Lock,
+    worktree: str,
+    branch: str,
+    changes: list[dict],
+) -> None:
+    for change in changes:
+        # Git can only handle a single index operation at a time
+        async with merge_lock:
+            await check_subprocess_output("git", "add", *change["files"], cwd=worktree)
+            commit_message = "{attrPath}: {oldVersion} -> {newVersion}".format(**change)
+            if "commitMessage" in change:
+                commit_message = change["commitMessage"]
+            elif "commitBody" in change:
+                commit_message = commit_message + "\n\n" + change["commitBody"]
+            await check_subprocess_output(
+                "git",
+                "commit",
+                "--quiet",
+                "-m",
+                commit_message,
+                cwd=worktree,
+            )
+            await check_subprocess_output("git", "cherry-pick", branch)
+
+
+async def check_changes(
+    package: dict,
+    worktree: str,
+    update_info: bytes,
+) -> list[dict]:
+    if "commit" in package["supportedFeatures"]:
+        changes = json.loads(update_info)
+    else:
+        changes = [{}]
+
+    # Try to fill in missing attributes when there is just a single change.
+    if len(changes) == 1:
+        # Dynamic data from updater take precedence over static data from passthru.updateScript.
+        if "attrPath" not in changes[0]:
+            # update.nix is always passing attrPath
+            changes[0]["attrPath"] = package["attrPath"]
+
+        if "oldVersion" not in changes[0]:
+            # update.nix is always passing oldVersion
+            changes[0]["oldVersion"] = package["oldVersion"]
+
+        if "newVersion" not in changes[0]:
+            attr_path = changes[0]["attrPath"]
+            obtain_new_version_output = await check_subprocess_output(
+                "nix-instantiate",
+                "--expr",
+                f"with import ./. {{}}; lib.getVersion {attr_path}",
+                "--eval",
+                "--strict",
+                "--json",
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+                cwd=worktree,
+            )
+            changes[0]["newVersion"] = json.loads(
+                obtain_new_version_output.decode("utf-8")
+            )
+
+        if "files" not in changes[0]:
+            changed_files_output = await check_subprocess_output(
+                "git",
+                "diff",
+                "--name-only",
+                "HEAD",
+                stdout=asyncio.subprocess.PIPE,
+                cwd=worktree,
+            )
+            changed_files = changed_files_output.splitlines()
+            changes[0]["files"] = changed_files
+
+            if len(changed_files) == 0:
+                return []
+
+    return changes
+
+
+async def merge_changes(
+    merge_lock: asyncio.Lock,
+    package: dict,
+    update_info: bytes,
+    temp_dir: tuple[str, str] | None,
+) -> None:
+    if temp_dir is not None:
+        worktree, branch = temp_dir
+        changes = await check_changes(package, worktree, update_info)
+
+        if len(changes) > 0:
+            await commit_changes(package["name"], merge_lock, worktree, branch, changes)
+        else:
+            eprint(f" - {package['name']}: DONE, no changes.")
+    else:
+        eprint(f" - {package['name']}: DONE.")
+
+
+async def updater(
+    nixpkgs_root: str,
+    temp_dir: tuple[str, str] | None,
+    merge_lock: asyncio.Lock,
+    packages_to_update: asyncio.Queue[dict | None],
+    keep_going: bool,
+    commit: bool,
+) -> None:
+    while True:
+        package = await packages_to_update.get()
+        if package is None:
+            # A sentinel received, we are done.
+            return
+
+        if not ("commit" in package["supportedFeatures"] or "attrPath" in package):
+            temp_dir = None
+
+        await run_update_script(nixpkgs_root, merge_lock, temp_dir, package, keep_going)
+
+        packages_to_update.task_done()
+
+
+async def populate_queue(
+    attr_packages: dict[str, dict],
+    sorter: TopologicalSorter[str],
+    packages_to_update: asyncio.Queue[dict | None],
+    num_workers: int,
+) -> None:
+    """
+    Keeps populating the queue with packages that can be updated
+    according to ordering requirements. If topological order
+    is used, the packages will appear in waves, as packages with
+    no dependencies are processed and removed from the sorter.
+    With `order="none"`, all packages will be enqueued simultaneously.
+    """
+
+    # Fill up an update queue,
+    while sorter.is_active():
+        ready_packages = list(sorter.get_ready())
+        eprint(f"Enqueuing group of {len(ready_packages)} packages")
+        for package in ready_packages:
+            if package == FAKE_DEPENDENCY_FOR_INDEPENDENT_PACKAGES:
+                continue
+            await packages_to_update.put(attr_packages[package])
+        await packages_to_update.join()
+        sorter.done(*ready_packages)
+
+    # Add sentinels, one for each worker.
+    # A worker will terminate when it gets a sentinel from the queue.
+    for i in range(num_workers):
+        await packages_to_update.put(None)
+
+
+async def start_updates(
+    max_workers: int,
+    keep_going: bool,
+    commit: bool,
+    attr_packages: dict[str, dict],
+    sorter: TopologicalSorter[str],
+) -> None:
+    merge_lock = asyncio.Lock()
+    packages_to_update: asyncio.Queue[dict | None] = asyncio.Queue()
+
+    with contextlib.ExitStack() as stack:
+        temp_dirs: list[tuple[str, str] | None] = []
+
+        # Do not create more workers than there are packages.
+        num_workers = min(max_workers, len(attr_packages))
+
+        nixpkgs_root_output = await check_subprocess_output(
+            "git",
+            "rev-parse",
+            "--show-toplevel",
+            stdout=asyncio.subprocess.PIPE,
+        )
+        nixpkgs_root = nixpkgs_root_output.decode("utf-8").strip()
+
+        # Set up temporary directories when using auto-commit.
+        for i in range(num_workers):
+            temp_dir = stack.enter_context(make_worktree()) if commit else None
+            temp_dirs.append(temp_dir)
+
+        queue_task = populate_queue(
+            attr_packages,
+            sorter,
+            packages_to_update,
+            num_workers,
+        )
+
+        # Prepare updater workers for each temp_dir directory.
+        # At most `num_workers` instances of `run_update_script` will be running at one time.
+        updater_tasks = [
+            updater(
+                nixpkgs_root,
+                temp_dir,
+                merge_lock,
+                packages_to_update,
+                keep_going,
+                commit,
+            )
+            for temp_dir in temp_dirs
+        ]
+
+        tasks = asyncio.gather(
+            *updater_tasks,
+            queue_task,
+        )
+
+        try:
+            # Start updater workers.
+            await tasks
+        except asyncio.exceptions.CancelledError:
+            # When one worker is cancelled, cancel the others too.
+            tasks.cancel()
+        except UpdateFailedException as e:
+            # When one worker fails, cancel the others, as this exception is only thrown when keep_going is false.
+            tasks.cancel()
+            eprint(e)
+            sys.exit(1)
+
+
+async def main(
+    max_workers: int,
+    keep_going: bool,
+    commit: bool,
+    packages_path: str,
+    skip_prompt: bool,
+    order: Order,
+) -> None:
+    with open(packages_path) as f:
+        packages = json.load(f)
+
+    if order != "arbitrary":
+        eprint("Sorting packages…")
+        reverse_order = order == "reverse-topological"
+        sorter, packages = await get_topological_sorter(
+            max_workers,
+            packages,
+            reverse_order,
+        )
+    else:
+        sorter = get_independent_sorter(packages)
+
+    attr_packages = {package["attrPath"]: package for package in packages}
+
+    eprint()
+    eprint("Going to be running update for following packages:")
+    for package in packages:
+        eprint(f" - {package['name']}")
+    eprint()
+
+    confirm = "" if skip_prompt else input("Press Enter key to continue...")
+
+    if confirm == "":
+        eprint()
+        eprint("Running update for:")
+
+        await start_updates(max_workers, keep_going, commit, attr_packages, sorter)
+
+        eprint()
+        eprint("Packages updated!")
+        sys.exit()
+    else:
+        eprint("Aborting!")
+        sys.exit(130)
+
+
+parser = argparse.ArgumentParser(description="Update packages")
+parser.add_argument(
+    "--max-workers",
+    "-j",
+    dest="max_workers",
+    type=int,
+    help="Number of updates to run concurrently",
+    nargs="?",
+    default=4,
+)
+parser.add_argument(
+    "--keep-going",
+    "-k",
+    dest="keep_going",
+    action="store_true",
+    help="Do not stop after first failure",
+)
+parser.add_argument(
+    "--commit",
+    "-c",
+    dest="commit",
+    action="store_true",
+    help="Commit the changes",
+)
+parser.add_argument(
+    "--order",
+    dest="order",
+    default="arbitrary",
+    choices=["arbitrary", "reverse-topological", "topological"],
+    help="Sort the packages based on dependency relation",
+)
+parser.add_argument(
+    "packages",
+    help="JSON file containing the list of package names and their update scripts",
+)
+parser.add_argument(
+    "--skip-prompt",
+    "-s",
+    dest="skip_prompt",
+    action="store_true",
+    help="Do not stop for prompts",
+)
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    try:
+        asyncio.run(
+            main(
+                args.max_workers,
+                args.keep_going,
+                args.commit,
+                args.packages,
+                args.skip_prompt,
+                args.order,
+            )
+        )
+    except KeyboardInterrupt as e:
+        # Let’s cancel outside of the main loop too.
+        sys.exit(130)

maintainers/scripts/vanity-manual-equalities.txt

@@ -0,0 +1,7 @@
+viric	viriketo@gmail.com
+Pjotr Prins	pjotr.public01@thebird.nl
+Pjotr Prins	pjotr.public05@thebird.nl
+Wouter den Breejen	wbreejen
+MarcWeber	marcweber
+Ricardo Correia	Ricardo M. Correia
+ertesx@gmx.de	ertes

maintainers/scripts/vanity.sh

@@ -0,0 +1,122 @@
+#! /bin/sh
+
+export LANG=C LC_ALL=C LC_COLLATE=C
+
+# Load git log
+raw_git_log="$(git log)"
+git_data="$(echo "$raw_git_log" | grep 'Author:' |
+  sed -e 's/^ *Author://; s/\\//g; s/^ *//; s/ *$//;
+  s/ @ .*//; s/ *[<]/\t/; s/[>]//')"
+
+# Name - nick - email correspondence from log and from maintainer list
+# Also there are a few manual entries
+maintainers="$(cat "$(dirname "$0")/../maintainer-list.nix" |
+  grep '=' | sed -re 's/\\"/''/g;
+  s/[ 	]*([^ 	=]*)[ 	]*=[ 	]*" *(.*[^ ]) *[<](.*)[>] *".*/\1\t\2\t\3/')"
+git_lines="$( ( echo "$git_data";
+    cat "$(dirname "$0")/vanity-manual-equalities.txt") | sort |uniq)"
+
+emails="$(
+    ( echo "$maintainers" | cut -f 3; echo "$git_data" | cut -f 2 ) |
+    sort | uniq | grep -E ".+@.+[.].+"
+    )"
+
+fetchGithubName () {
+    commitid="$(
+        echo "$raw_git_log" | grep -B3 "Author: .*[<]$1[>]" | head -n 3 |
+            grep '^commit ' | tail -n 1 | sed -e 's/^commit //'
+    )"
+    userid="$(
+        curl https://github.com/NixOS/nixpkgs/commit/"$commitid" 2>/dev/null |
+        grep committed -B10 | grep 'href="/' |
+        sed -re 's@.* href="/@@; s@".*@@' |
+        grep -v "/commit/"
+    )";
+    echo "$userid"
+}
+
+[ -n "$NIXPKGS_GITHUB_NAME_CACHE" ] && {
+    echo "$emails" | while read email; do
+        line="$(grep "$email	" "$NIXPKGS_GITHUB_NAME_CACHE")"
+        [ -z "$line" ] && {
+            echo "$email	$(fetchGithubName "$email")" >> \
+                "$NIXPKGS_GITHUB_NAME_CACHE"
+        }
+    done
+}
+
+# For RDF
+normalize_name () {
+    sed -e 's/%/%25/g; s/ /%20/g; s/'\''/%27/g; s/"/%22/g; s/`/%60/g; s/\^/%5e/g; '
+}
+
+denormalize_name () {
+    sed -e 's/%20/ /g; s/%27/'\''/g; s/%22/"/g; s/%60/`/g; s/%5e/^/g; s/%25/%/g;';
+}
+
+n3="$(mktemp --suffix .n3)"
+
+# «The same person» relation and a sorting hint
+# Full name is something with a space
+(
+echo "$git_lines" | sed -re 's@(.*)\t(.*)@<my://name/\1>	<my://can-be>	<my://name/\2>.@'
+echo "$git_lines" | sed -re 's@(.*)\t(.*)@<my://name/\2>	<my://can-be>	<my://name/\1>.@'
+echo "$maintainers" | sed -re 's@(.*)\t(.*)\t(.*)@<my://name/\1>	<my://can-be>	<my://name/\2>.@'
+echo "$maintainers" | sed -re 's@(.*)\t(.*)\t(.*)@<my://name/\2>	<my://can-be>	<my://name/\3>.@'
+echo "$maintainers" | sed -re 's@(.*)\t(.*)\t(.*)@<my://name/\3>	<my://can-be>	<my://name/\1>.@'
+echo "$git_lines" | grep ' ' | cut -f 1 | sed -e 's@.*@<my://name/&>	<my://is-name>	<my://0>.@'
+echo "$git_lines" | grep -v ' ' | cut -f 1 | sed -e 's@.*@<my://name/&>	<my://is-name>	<my://1>.@'
+echo "$maintainers" | cut -f 2 | sed -e 's@.*@<my://name/&>	<my://is-name>	<my://0>.@'
+[ -n "$NIXPKGS_GITHUB_NAME_CACHE" ] && cat "$NIXPKGS_GITHUB_NAME_CACHE" |
+    grep -v "	$" |
+    sed -re 's@(.*)\t(.*)@<my://name/\1>	<my://at-github>	<my://github/\2>.@'
+) | normalize_name | grep -E '<my://[-a-z]+>' | sort | uniq > "$n3"
+
+# Get transitive closure
+sparql="$(nix-build '<nixpkgs>' -Q -A apache-jena --no-out-link)/bin/sparql"
+name_list="$(
+    "$sparql" --results=TSV --data="$n3" "
+    select ?x ?y ?g where {
+      ?x <my://can-be>+ ?y.
+      ?x <my://is-name> ?g.
+        }
+    " | tail -n +2 |
+    sed -re 's@<my://name/@@g; s@<my://@@g; s@>@@g;' |
+    sort -k 2,3 -t '	'
+)"
+github_name_list="$(
+    "$sparql" --results=TSV --data="$n3" "
+    select ?x ?y where {
+      ?x (<my://can-be>+ / <my://at-github>) ?y.
+        }
+    " | tail -n +2 |
+    sed -re 's@<my://(name|github)/@@g; s@<my://@@g; s@>@@g;'
+)"
+
+# Take first spelling option for every person
+name_list_canonical="$(echo "$name_list" | cut -f 1,2 | uniq -f1)"
+
+cleaner_script="$(echo "$name_list_canonical" | denormalize_name |
+  sed -re 's/(.*)\t(.*)/s#^\2$#\1#g/g')"
+
+# Add github usernames
+if [ -n "$NIXPKGS_GITHUB_NAME_CACHE" ]; then
+    github_adder_script="$(mktemp)"
+    echo "$github_name_list" |
+        grep -E "$(echo "$name_list_canonical" | cut -f 2 |
+        tr '\n' '|' )" |
+    sort | uniq |
+        sed -re 's/(.*)\t(.*)/s| \1$| \1\t\2|g;/' |
+    denormalize_name > "$github_adder_script"
+else
+    github_adder_script='/dev/null'
+fi
+
+echo "$name_list" | denormalize_name
+
+echo
+
+echo "$git_data" | cut -f 1 |
+    sed -e "$cleaner_script" |
+    sort | uniq -c | sort -k1n | sed -rf "$github_adder_script" |
+    sed -re 's/^ *([0-9]+) /\1\t/'