push sheeet

maintainers/scripts/haskell/dependencies.nix

@@ -0,0 +1,17 @@
+# Nix script to calculate the Haskell dependencies of every haskellPackage. Used by ./hydra-report.hs.
+let
+  pkgs = import ../../.. { };
+  inherit (pkgs) lib;
+  getDeps =
+    _: pkg:
+    let
+      pname = pkg.pname or null;
+    in
+    {
+      deps = builtins.filter (x: x != null && x != pname) (
+        map (x: x.pname or null) (pkg.propagatedBuildInputs or [ ])
+      );
+      broken = (pkg.meta.hydraPlatforms or [ null ]) == [ ];
+    };
+in
+lib.mapAttrs getDeps pkgs.haskellPackages

maintainers/scripts/haskell/hydra-report.hs

@@ -0,0 +1,827 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -I nixpkgs=.
+#! nix-shell -p "haskellPackages.ghcWithPackages (p: [p.aeson p.req])"
+#! nix-shell -p nix-eval-jobs
+#! nix-shell -i runhaskell
+
+{-
+
+The purpose of this script is
+
+1) download the state of the nixpkgs/haskell-updates job from hydra (with get-report)
+2) print a summary of the state suitable for pasting into a github comment (with ping-maintainers)
+3) print a list of broken packages suitable for pasting into configuration-hackage2nix.yaml
+
+Because step 1) is quite expensive and takes roughly ~5 minutes the result is cached in a json file in XDG_CACHE.
+
+-}
+{-# LANGUAGE BlockArguments #-}
+{-# LANGUAGE DeriveAnyClass #-}
+{-# LANGUAGE DeriveGeneric #-}
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE DuplicateRecordFields #-}
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE LambdaCase #-}
+{-# LANGUAGE NamedFieldPuns #-}
+{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE ScopedTypeVariables #-}
+{-# LANGUAGE TupleSections #-}
+{-# LANGUAGE ViewPatterns #-}
+{-# OPTIONS_GHC -Wall #-}
+{-# LANGUAGE DataKinds #-}
+
+import Control.Monad (forM_, forM, (<=<))
+import Control.Monad.Trans (MonadIO (liftIO))
+import Data.Aeson (
+   FromJSON (..),
+   withObject,
+   (.:),
+   FromJSONKey,
+   ToJSON,
+   decodeFileStrict',
+   encodeFile,
+ )
+import Data.Aeson.Decoding (eitherDecodeStrictText)
+import Data.Foldable (Foldable (toList))
+import Data.Either (rights)
+import Data.Functor ((<&>))
+import Data.List.NonEmpty (NonEmpty, nonEmpty)
+import qualified Data.List.NonEmpty as NonEmpty
+import Data.Map.Strict (Map)
+import qualified Data.Map.Strict as Map
+import Data.Maybe (fromMaybe, mapMaybe, isNothing, catMaybes)
+import Data.Monoid (Sum (Sum, getSum))
+import Data.Sequence (Seq)
+import qualified Data.Sequence as Seq
+import Data.Set (Set)
+import qualified Data.Set as Set
+import Data.Text (Text)
+import qualified Data.Text as Text
+import qualified Data.Text.IO as Text
+import Data.Time (defaultTimeLocale, formatTime, getCurrentTime)
+import Data.Time.Clock (UTCTime)
+import GHC.Generics (Generic)
+import Network.HTTP.Req (
+    GET (GET),
+    HttpResponse (HttpResponseBody),
+    NoReqBody (NoReqBody),
+    Option,
+    Req,
+    Scheme (Https),
+    bsResponse,
+    defaultHttpConfig,
+    header,
+    https,
+    jsonResponse,
+    req,
+    responseBody,
+    responseTimeout,
+    runReq,
+    (/:),
+ )
+import System.Directory (XdgDirectory (XdgCache), getXdgDirectory)
+import System.Environment (getArgs)
+import System.Exit (die)
+import System.Process (readProcess)
+import Prelude hiding (id)
+import Data.List (sortOn)
+import Control.Concurrent.Async (concurrently)
+import Control.Exception (evaluate)
+import qualified Data.IntMap.Lazy as IntMap
+import qualified Data.IntSet as IntSet
+import Data.Bifunctor (second)
+import Data.Data (Proxy)
+import Data.ByteString (ByteString)
+import qualified Data.ByteString.Char8 as ByteString
+import Distribution.Simple.Utils (safeLast, fromUTF8BS)
+
+newtype JobsetEvals = JobsetEvals
+   { evals :: Seq Eval
+   }
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+newtype Nixpkgs = Nixpkgs {revision :: Text}
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs}
+   deriving stock (Generic, Show)
+   deriving anyclass (ToJSON, FromJSON)
+
+data Eval = Eval
+   { id :: Int
+   , jobsetevalinputs :: JobsetEvalInputs
+   , builds :: Seq Int
+   }
+   deriving (Generic, ToJSON, FromJSON, Show)
+
+-- | Hydra job name.
+--
+-- Examples:
+-- - @"haskellPackages.lens.x86_64-linux"@
+-- - @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+-- - @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@
+-- - @"arion.aarch64-linux"@
+newtype JobName = JobName { unJobName :: Text }
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSONKey, FromJSON, Ord, ToJSON)
+
+-- | Datatype representing the result of querying the build evals of the
+-- haskell-updates Hydra jobset.
+--
+-- The URL <https://hydra.nixos.org/eval/EVAL_ID/builds> (where @EVAL_ID@ is a
+-- value like 1792418) returns a list of 'Build'.
+data Build = Build
+   { job :: JobName
+   , buildstatus :: Maybe Int
+     -- ^ Status of the build.  See 'getBuildState' for the meaning of each state.
+   , finished :: Int
+     -- ^ Whether or not the build is finished.  @0@ if finished, non-zero otherwise.
+   , id :: Int
+   , nixname :: Text
+     -- ^ Nix name of the derivation.
+     --
+     -- Examples:
+     -- - @"lens-5.2.1"@
+     -- - @"cabal-install-3.8.0.1"@
+     -- - @"lens-static-x86_64-unknown-linux-musl-5.1.1"@
+   , system :: Text
+     -- ^ System
+     --
+     -- Examples:
+     -- - @"x86_64-linux"@
+     -- - @"aarch64-darwin"@
+   , jobsetevals :: Seq Int
+   }
+   deriving (Generic, ToJSON, FromJSON, Show)
+
+data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround
+data RequestLogsFlag = RequestLogs | NoRequestLogs
+
+usage :: IO a
+usage = die "Usage: get-report [--slow] [EVAL-ID] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info"
+
+main :: IO ()
+main = do
+   args <- getArgs
+   case args of
+      "get-report":"--slow":id -> getBuildReports HydraSlownessWorkaround id
+      "get-report":id -> getBuildReports NoHydraSlownessWorkaround id
+      ["ping-maintainers"] -> printMaintainerPing
+      ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs
+      ["mark-broken-list"] -> printMarkBrokenList RequestLogs
+      ["eval-info"] -> printEvalInfo
+      _ -> usage
+
+reportFileName :: IO FilePath
+reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
+
+showT :: Show a => a -> Text
+showT = Text.pack . show
+
+getBuildReports :: HydraSlownessWorkaroundFlag -> [String] -> IO ()
+getBuildReports opt args = runReq defaultHttpConfig do
+   eval@Eval{id} <- case args of
+      [id] -> hydraJSONQuery mempty ["eval", Text.pack id]
+      [] -> do
+         evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"]
+         maybe (liftIO $ fail "No Evaluation found") pure evalMay
+      _ -> liftIO usage
+   liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..."
+   buildReports <- getEvalBuilds opt eval
+   liftIO do
+      fileName <- reportFileName
+      putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName
+      now <- getCurrentTime
+      encodeFile fileName (eval, now, buildReports)
+
+getEvalBuilds :: HydraSlownessWorkaroundFlag -> Eval -> Req (Seq Build)
+getEvalBuilds NoHydraSlownessWorkaround Eval{id} =
+  hydraJSONQuery mempty ["eval", showT id, "builds"]
+getEvalBuilds HydraSlownessWorkaround Eval{builds} = do
+  forM builds $ \buildId -> do
+    liftIO $ putStrLn $ "Querying build " <> show buildId
+    hydraJSONQuery mempty [ "build", showT buildId ]
+
+hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a)
+hydraQuery responseType option query = do
+  let customHeaderOpt =
+        header
+          "User-Agent"
+          "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)"
+      customTimeoutOpt = responseTimeout 900_000_000 -- 15 minutes
+      opts = customHeaderOpt <> customTimeoutOpt <> option
+      url = foldl' (/:) (https "hydra.nixos.org") query
+  responseBody <$> req GET url NoReqBody responseType opts
+
+hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a
+hydraJSONQuery = hydraQuery jsonResponse
+
+hydraPlainQuery :: [Text] -> Req ByteString
+hydraPlainQuery = hydraQuery bsResponse mempty
+
+nixEvalJobsCommand :: FilePath
+nixEvalJobsCommand = "nix-eval-jobs"
+
+nixEvalJobsParams :: [String]
+nixEvalJobsParams =
+  [
+    -- options necessary to make nix-eval-jobs behave like hydra-eval-jobs used to
+    -- https://github.com/NixOS/hydra/commit/d84ff32ce600204c6473889a3ff16cd6053533c9
+    "--meta",
+    "--force-recurse",
+    "--no-instantiate",
+    "--workers", "3",
+
+    "-I", ".",
+   "pkgs/top-level/release-haskell.nix"
+  ]
+
+nixExprCommand :: FilePath
+nixExprCommand = "nix-instantiate"
+
+nixExprParams :: [String]
+nixExprParams = ["--eval", "--strict", "--json"]
+
+-- | Holds a list of the GitHub handles of the maintainers of a given 'JobName'.
+--
+--   @
+--     JobMaintainers (JobName "haskellPackages.cabal-install.x86_64-linux") ["sternenseemann"]
+--   @
+data JobMaintainers = JobMaintainers JobName [Text]
+  deriving stock (Generic, Show)
+
+-- | Parse the entries produced by @nix-eval-jobs@, discarding all information
+--   except the name of the job (@attr@) and the @github@ attributes of the
+--   maintainer objects in @meta.maintainers@.
+instance FromJSON JobMaintainers where
+  parseJSON = withObject "HydraJob" $ \h -> do
+    jobName <- h .: "attr"
+    maintainers <- (h .: "meta")
+      >>= (withObject "Meta" $ \meta ->
+        meta .: "maintainers"
+        >>= mapM (withObject "Maintainer" $ \mt -> mt .: "github"))
+    pure $ JobMaintainers jobName maintainers
+
+-- | Map of maintained Hydra jobs to maintainer GitHub handles.
+--
+-- It has values similar to the following:
+--
+-- @@
+--  fromList
+--    [ ("arion.aarch64-linux", ["rob22"])
+--    , ("conduit.x86_64-darwin", ["snoyb", "webber"])
+--    ]
+-- @@
+type MaintainerMap = Map JobName (NonEmpty Text)
+
+-- | Information about a package which lists its dependencies and whether the
+-- package is marked broken.
+data DepInfo = DepInfo {
+   deps :: Set PkgName,
+   broken :: Bool
+}
+   deriving stock (Generic, Show)
+   deriving anyclass (FromJSON, ToJSON)
+
+-- | Map from package names to their DepInfo. This is the data we get out of a
+-- nix call.
+type DependencyMap = Map PkgName DepInfo
+
+-- | Map from package names to its broken state, number of reverse dependencies (fst) and
+-- unbroken reverse dependencies (snd).
+type ReverseDependencyMap = Map PkgName (Int, Int)
+
+-- | Calculate the (unbroken) reverse dependencies of a package by transitively
+-- going through all packages if it’s a dependency of them.
+calculateReverseDependencies :: DependencyMap -> ReverseDependencyMap
+calculateReverseDependencies depMap =
+   Map.fromDistinctAscList $ zip keys (zip (rdepMap False) (rdepMap True))
+ where
+    -- This code tries to efficiently invert the dependency map and calculate
+    -- its transitive closure by internally identifying every pkg with its index
+    -- in the package list and then using memoization.
+    keys :: [PkgName]
+    keys = Map.keys depMap
+
+    pkgToIndexMap :: Map PkgName Int
+    pkgToIndexMap = Map.fromDistinctAscList (zip keys [0..])
+
+    depInfos :: [DepInfo]
+    depInfos = Map.elems depMap
+
+    depInfoToIdx :: DepInfo -> (Bool, [Int])
+    depInfoToIdx DepInfo{broken,deps} =
+       (broken, mapMaybe (`Map.lookup` pkgToIndexMap) $ Set.toList deps)
+
+    intDeps :: [(Int, (Bool, [Int]))]
+    intDeps = zip [0..] (fmap depInfoToIdx depInfos)
+
+    rdepMap onlyUnbroken = IntSet.size <$> IntMap.elems resultList
+     where
+       resultList = IntMap.fromDistinctAscList [(i, go i) | i <- [0..length keys - 1]]
+       oneStepMap = IntMap.fromListWith IntSet.union $ (\(key,(_,deps)) -> (,IntSet.singleton key) <$> deps) <=< filter (\(_, (broken,_)) -> not (broken && onlyUnbroken)) $ intDeps
+       go pkg = IntSet.unions (oneStep:((resultList IntMap.!) <$> IntSet.toList oneStep))
+        where oneStep = IntMap.findWithDefault mempty pkg oneStepMap
+
+-- | Generate a mapping of Hydra job names to maintainer GitHub handles.
+getMaintainerMap :: IO MaintainerMap
+getMaintainerMap =
+  readJSONLinesProcess nixEvalJobsCommand nixEvalJobsParams
+  -- we ignore unparseable lines since fromJSON will fail on { "attr": …, "error": … }
+  -- entries since they don't have a @meta@ attribute.
+  <&> rights
+  <&> map (\(JobMaintainers name maintainers) -> (,) name <$> nonEmpty maintainers)
+  <&> catMaybes
+  <&> Map.fromList
+
+-- | Get the a map of all dependencies of every package by calling the nix
+-- script ./dependencies.nix.
+getDependencyMap :: IO DependencyMap
+getDependencyMap =
+   readJSONProcess
+      nixExprCommand
+      ("maintainers/scripts/haskell/dependencies.nix" : nixExprParams)
+      "Failed to decode nix output for lookup of dependencies: "
+
+-- | Run a process that produces JSON on stdout and and decode the JSON to a
+-- data type.
+--
+-- If the JSON-decoding fails, throw the JSON-decoding error.
+readJSONProcess
+   :: FromJSON a
+   => FilePath -- ^ Filename of executable.
+   -> [String] -- ^ Arguments
+   -> String -- ^ String to prefix to JSON-decode error.
+   -> IO a
+readJSONProcess exe args err = do
+   output <- readProcess exe args ""
+   let eitherDecodedOutput = eitherDecodeStrictText . Text.pack $ output
+   case eitherDecodedOutput of
+     Left decodeErr -> error $ err <> decodeErr <> "\nRaw: '" <> take 1000 output <> "'"
+     Right decodedOutput -> pure decodedOutput
+
+-- | Run a process that produces many JSON values, one per line.
+--   Error and success is reported per line via a list of 'Either's.
+readJSONLinesProcess
+   :: FromJSON a
+   => FilePath -- ^ Filename of executable.
+   -> [String] -- ^ Arguments
+   -> IO [Either String a]
+readJSONLinesProcess exe args = do
+  output <- readProcess exe args ""
+  -- TODO: slow, doesn't stream at all
+  pure . map (eitherDecodeStrictText . Text.pack) . lines $ output
+
+-- BuildStates are sorted by subjective importance/concerningness
+data BuildState
+  = Failed
+  | DependencyFailed
+  | OutputLimitExceeded
+  | Unknown (Maybe Int)
+  | TimedOut
+  | Canceled
+  | HydraFailure
+  | Unfinished
+  | Success
+  deriving stock (Show, Eq, Ord)
+
+icon :: BuildState -> Text
+icon = \case
+   Failed -> "❌"
+   DependencyFailed -> "❗"
+   OutputLimitExceeded -> "⚠️"
+   Unknown x -> "unknown code " <> showT x
+   TimedOut -> "⌛🚫"
+   Canceled -> "🚫"
+   Unfinished -> "⏳"
+   HydraFailure -> "🚧"
+   Success -> "✅"
+
+platformIcon :: Platform -> Text
+platformIcon (Platform x) = case x of
+   "x86_64-linux" -> "🐧"
+   "aarch64-linux" -> "📱"
+   "x86_64-darwin" -> "🍎"
+   "aarch64-darwin" -> "🍏"
+   _ -> x
+
+platformIsOS :: OS -> Platform -> Bool
+platformIsOS os (Platform x) = case (os, x) of
+   (Linux, "x86_64-linux") -> True
+   (Linux, "aarch64-linux") -> True
+   (Darwin, "x86_64-darwin") -> True
+   (Darwin, "aarch64-darwin") -> True
+   _ -> False
+
+
+-- | A package name.  This is parsed from a 'JobName'.
+--
+-- Examples:
+--
+-- - The 'JobName' @"haskellPackages.lens.x86_64-linux"@ produces the 'PkgName'
+--   @"lens"@.
+-- - The 'JobName' @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+--   produces the 'PkgName' @"cabal-install"@.
+-- - The 'JobName' @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@ produces
+--   the 'PkgName' @"ghc90"@.
+-- - The 'JobName' @"arion.aarch64-linux"@ produces the 'PkgName' @"arion"@.
+--
+-- 'PkgName' is also used as a key in 'DependencyMap' and 'ReverseDependencyMap'.
+-- In this case, 'PkgName' originally comes from attribute names in @haskellPackages@
+-- in Nixpkgs.
+newtype PkgName = PkgName Text
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSON, FromJSONKey, Ord, ToJSON)
+
+-- | A package set name.  This is parsed from a 'JobName'.
+--
+-- Examples:
+--
+-- - The 'JobName' @"haskellPackages.lens.x86_64-linux"@ produces the 'PkgSet'
+--   @"haskellPackages"@.
+-- - The 'JobName' @"haskell.packages.ghc925.cabal-install.aarch64-darwin"@
+--   produces the 'PkgSet' @"haskell.packages.ghc925"@.
+-- - The 'JobName' @"pkgsMusl.haskell.compiler.ghc90.x86_64-linux"@ produces
+--   the 'PkgSet' @"pkgsMusl.haskell.compiler"@.
+-- - The 'JobName' @"arion.aarch64-linux"@ produces the 'PkgSet' @""@.
+--
+-- As you can see from the last example, 'PkgSet' can be empty (@""@) for
+-- top-level jobs.
+newtype PkgSet = PkgSet Text
+   deriving stock (Generic, Show)
+   deriving newtype (Eq, FromJSON, FromJSONKey, Ord, ToJSON)
+
+data BuildResult = BuildResult {state :: BuildState, id :: Int} deriving (Show, Eq, Ord)
+newtype Platform = Platform {platform :: Text} deriving (Show, Eq, Ord)
+data SummaryEntry = SummaryEntry {
+   summaryBuilds :: Table PkgSet Platform BuildResult,
+   summaryMaintainers :: Set Text,
+   summaryReverseDeps :: Int,
+   summaryUnbrokenReverseDeps :: Int
+}
+type StatusSummary = Map PkgName SummaryEntry
+
+data OS = Linux | Darwin
+
+newtype Table row col a = Table (Map (row, col) a)
+
+singletonTable :: row -> col -> a -> Table row col a
+singletonTable row col a = Table $ Map.singleton (row, col) a
+
+unionTable :: (Ord row, Ord col) => Table row col a -> Table row col a -> Table row col a
+unionTable (Table l) (Table r) = Table $ Map.union l r
+
+filterWithKeyTable :: (row -> col -> a -> Bool) -> Table row col a -> Table row col a
+filterWithKeyTable f (Table t) = Table $ Map.filterWithKey (\(r,c) a -> f r c a) t
+
+nullTable :: Table row col a -> Bool
+nullTable (Table t) = Map.null t
+
+instance (Ord row, Ord col, Semigroup a) => Semigroup (Table row col a) where
+   Table l <> Table r = Table (Map.unionWith (<>) l r)
+instance (Ord row, Ord col, Semigroup a) => Monoid (Table row col a) where
+   mempty = Table Map.empty
+instance Functor (Table row col) where
+   fmap f (Table a) = Table (fmap f a)
+instance Foldable (Table row col) where
+   foldMap f (Table a) = foldMap f a
+
+getBuildState :: Build -> BuildState
+getBuildState Build{finished, buildstatus} = case (finished, buildstatus) of
+   (0, _) -> Unfinished
+   (_, Just 0) -> Success
+   (_, Just 1) -> Failed
+   (_, Just 2) -> DependencyFailed
+   (_, Just 3) -> HydraFailure
+   (_, Just 4) -> Canceled
+   (_, Just 7) -> TimedOut
+   (_, Just 11) -> OutputLimitExceeded
+   (_, i) -> Unknown i
+
+combineStatusSummaries :: Seq StatusSummary -> StatusSummary
+combineStatusSummaries = foldl (Map.unionWith unionSummary) Map.empty
+  where
+   unionSummary :: SummaryEntry -> SummaryEntry -> SummaryEntry
+   unionSummary (SummaryEntry lb lm lr lu) (SummaryEntry rb rm rr ru) =
+      SummaryEntry (unionTable lb rb) (lm <> rm) (max lr rr) (max lu ru)
+
+buildToPkgNameAndSet :: Build -> (PkgName, PkgSet)
+buildToPkgNameAndSet Build{job = JobName jobName, system} = (name, set)
+  where
+   packageName :: Text
+   packageName = fromMaybe jobName (Text.stripSuffix ("." <> system) jobName)
+
+   splitted :: Maybe (NonEmpty Text)
+   splitted = nonEmpty $ Text.splitOn "." packageName
+
+   name :: PkgName
+   name = PkgName $ maybe packageName NonEmpty.last splitted
+
+   set :: PkgSet
+   set = PkgSet $ maybe "" (Text.intercalate "." . NonEmpty.init) splitted
+
+buildToStatusSummary :: MaintainerMap -> ReverseDependencyMap -> Build -> StatusSummary
+buildToStatusSummary maintainerMap reverseDependencyMap build@Build{job, id, system} =
+   Map.singleton pkgName summaryEntry
+  where
+   (pkgName, pkgSet) = buildToPkgNameAndSet build
+
+   maintainers :: Set Text
+   maintainers = maybe mempty (Set.fromList . toList) (Map.lookup job maintainerMap)
+
+   (reverseDeps, unbrokenReverseDeps) =
+      Map.findWithDefault (0,0) pkgName reverseDependencyMap
+
+   buildTable :: Table PkgSet Platform BuildResult
+   buildTable =
+      singletonTable pkgSet (Platform system) (BuildResult (getBuildState build) id)
+
+   summaryEntry = SummaryEntry buildTable maintainers reverseDeps unbrokenReverseDeps
+
+readBuildReports :: IO (Eval, UTCTime, Seq Build)
+readBuildReports = do
+   file <- reportFileName
+   fromMaybe (error $ "Could not decode " <> file) <$> decodeFileStrict' file
+
+sep :: Text
+sep = " | "
+joinTable :: [Text] -> Text
+joinTable t = sep <> Text.intercalate sep t <> sep
+
+type NumSummary = Table Platform BuildState Int
+
+printTable :: (Ord rows, Ord cols) => Text -> (rows -> Text) -> (cols -> Text) -> (entries -> Text) -> Table rows cols entries -> [Text]
+printTable name showR showC showE (Table mapping) = joinTable <$> (name : map showC cols) : replicate (length cols + sepsInName + 1) "---" : map printRow rows
+  where
+   sepsInName = Text.count "|" name
+   printRow row = showR row : map (\col -> maybe "" showE (Map.lookup (row, col) mapping)) cols
+   rows = toList $ Set.fromList (fst <$> Map.keys mapping)
+   cols = toList $ Set.fromList (snd <$> Map.keys mapping)
+
+printJob :: Int -> PkgName -> (Table PkgSet Platform BuildResult, Text) -> [Text]
+printJob evalId (PkgName name) (Table mapping, maintainers) =
+   if length sets <= 1
+      then map printSingleRow sets
+      else ["- [ ] " <> makeJobSearchLink (PkgSet "") name <> " " <> maintainers] <> map printRow sets
+  where
+   printRow :: PkgSet -> Text
+   printRow (PkgSet set) =
+      "  - " <> printState (PkgSet set) <> " " <>
+      makeJobSearchLink (PkgSet set) (if Text.null set then "toplevel" else set)
+
+   printSingleRow set =
+      "- [ ] " <> printState set <> " " <>
+      makeJobSearchLink set (makePkgName set) <> " " <> maintainers
+
+   makePkgName :: PkgSet -> Text
+   makePkgName (PkgSet set) = (if Text.null set then "" else set <> ".") <> name
+
+   printState set =
+      Text.intercalate " " $ map (\pf -> maybe "" (label pf) $ Map.lookup (set, pf) mapping) platforms
+
+   makeJobSearchLink :: PkgSet -> Text -> Text
+   makeJobSearchLink set linkLabel = makeSearchLink evalId linkLabel (makePkgName set)
+
+   sets :: [PkgSet]
+   sets = toList $ Set.fromList (fst <$> Map.keys mapping)
+
+   platforms :: [Platform]
+   platforms = toList $ Set.fromList (snd <$> Map.keys mapping)
+
+   label pf (BuildResult s i) = "[[" <> platformIcon pf <> icon s <> "]](https://hydra.nixos.org/build/" <> showT i <> ")"
+
+makeSearchLink :: Int -> Text -> Text -> Text
+makeSearchLink evalId linkLabel query = "[" <> linkLabel <> "](" <> "https://hydra.nixos.org/eval/" <> showT evalId <> "?filter=" <> query <> ")"
+
+statusToNumSummary :: StatusSummary -> NumSummary
+statusToNumSummary = fmap getSum . foldMap (fmap Sum . jobTotals)
+
+jobTotals :: SummaryEntry -> Table Platform BuildState Int
+jobTotals (summaryBuilds -> Table mapping) = getSum <$> Table (Map.foldMapWithKey (\(_, platform) (BuildResult buildstate _) -> Map.singleton (platform, buildstate) (Sum 1)) mapping)
+
+details :: Text -> [Text] -> [Text]
+details summary content = ["<details><summary>" <> summary <> " </summary>", ""] <> content <> ["</details>", ""]
+
+evalLine :: Eval -> UTCTime -> Text
+evalLine Eval{id, jobsetevalinputs = JobsetEvalInputs{nixpkgs = Nixpkgs{revision}}} fetchTime =
+   "*evaluation ["
+    <> showT id
+    <> "](https://hydra.nixos.org/eval/"
+    <> showT id
+    <> ") of nixpkgs commit ["
+    <> Text.take 7 revision
+    <> "](https://github.com/NixOS/nixpkgs/commits/"
+    <> revision
+    <> ") as of "
+    <> Text.pack (formatTime defaultTimeLocale "%Y-%m-%d %H:%M UTC" fetchTime)
+    <> "*"
+
+printBuildSummary :: Eval -> UTCTime -> StatusSummary -> [(PkgName, Int)] -> Text
+printBuildSummary eval@Eval{id} fetchTime summary topBrokenRdeps =
+   Text.unlines $
+      headline <> [""] <> tldr <> (("  * "<>) <$> (errors <> warnings)) <> [""]
+         <> totals
+         <> optionalList "#### Maintained Linux packages with build failure" (maintainedList (fails summaryLinux))
+         <> optionalList "#### Maintained Linux packages with failed dependency" (maintainedList (failedDeps summaryLinux))
+         <> optionalList "#### Maintained Linux packages with unknown error" (maintainedList (unknownErr summaryLinux))
+         <> optionalHideableList "#### Maintained Darwin packages with build failure" (maintainedList (fails summaryDarwin))
+         <> optionalHideableList "#### Maintained Darwin packages with failed dependency" (maintainedList (failedDeps summaryDarwin))
+         <> optionalHideableList "#### Maintained Darwin packages with unknown error" (maintainedList (unknownErr summaryDarwin))
+         <> optionalHideableList "#### Unmaintained packages with build failure" (unmaintainedList (fails summary))
+         <> optionalHideableList "#### Unmaintained packages with failed dependency" (unmaintainedList (failedDeps summary))
+         <> optionalHideableList "#### Unmaintained packages with unknown error" (unmaintainedList (unknownErr summary))
+         <> optionalHideableList "#### Top 50 broken packages, sorted by number of reverse dependencies" (brokenLine <$> topBrokenRdeps)
+         <> ["","*⤴️: The number of packages that depend (directly or indirectly) on this package (if any). If two numbers are shown the first (lower) number considers only packages which currently have enabled hydra jobs, i.e. are not marked broken. The second (higher) number considers all packages.*",""]
+         <> footer
+  where
+   footer = ["*Report generated with [maintainers/scripts/haskell/hydra-report.hs](https://github.com/NixOS/nixpkgs/blob/haskell-updates/maintainers/scripts/haskell/hydra-report.hs)*"]
+
+   headline =
+      [ "### [haskell-updates build report from hydra](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates)"
+      , evalLine eval fetchTime
+      ]
+
+   totals :: [Text]
+   totals =
+      [ "#### Build summary"
+      , ""
+      ] <>
+      printTable
+         "Platform"
+         (\x -> makeSearchLink id (platform x <> " " <> platformIcon x) ("." <> platform x))
+         (\x -> showT x <> " " <> icon x)
+         showT
+         numSummary
+
+   brokenLine :: (PkgName, Int) -> Text
+   brokenLine (PkgName name, rdeps) =
+      "[" <> name <> "](https://packdeps.haskellers.com/reverse/" <> name <>
+      ") ⤴️ " <> Text.pack (show rdeps) <> "  "
+
+   numSummary = statusToNumSummary summary
+
+   summaryLinux :: StatusSummary
+   summaryLinux = withOS Linux summary
+
+   summaryDarwin :: StatusSummary
+   summaryDarwin = withOS Darwin summary
+
+   -- Remove all BuildResult from the Table that have Platform that isn't for
+   -- the given OS.
+   tableForOS :: OS -> Table PkgSet Platform BuildResult -> Table PkgSet Platform BuildResult
+   tableForOS os = filterWithKeyTable (\_ platform _ -> platformIsOS os platform)
+
+   -- Remove all BuildResult from the StatusSummary that have a Platform that
+   -- isn't for the given OS.  Completely remove all PkgName from StatusSummary
+   -- that end up with no BuildResults.
+   withOS
+      :: OS
+      -> StatusSummary
+      -> StatusSummary
+   withOS os =
+      Map.mapMaybe
+         (\e@SummaryEntry{summaryBuilds} ->
+            let buildsForOS = tableForOS os summaryBuilds
+            in if nullTable buildsForOS then Nothing else Just e { summaryBuilds = buildsForOS }
+         )
+
+   jobsByState :: (BuildState -> Bool) -> StatusSummary -> StatusSummary
+   jobsByState predicate = Map.filter (predicate . worstState)
+
+   worstState :: SummaryEntry -> BuildState
+   worstState = foldl' min Success . fmap state . summaryBuilds
+
+   fails :: StatusSummary -> StatusSummary
+   fails = jobsByState (== Failed)
+
+   failedDeps :: StatusSummary -> StatusSummary
+   failedDeps = jobsByState (== DependencyFailed)
+
+   unknownErr :: StatusSummary -> StatusSummary
+   unknownErr = jobsByState (\x -> x > DependencyFailed && x < TimedOut)
+
+   withMaintainer :: StatusSummary -> Map PkgName (Table PkgSet Platform BuildResult, NonEmpty Text)
+   withMaintainer =
+      Map.mapMaybe
+         (\e -> (summaryBuilds e,) <$> nonEmpty (Set.toList (summaryMaintainers e)))
+
+   withoutMaintainer :: StatusSummary -> StatusSummary
+   withoutMaintainer = Map.mapMaybe (\e -> if Set.null (summaryMaintainers e) then Just e else Nothing)
+
+   optionalList :: Text -> [Text] -> [Text]
+   optionalList heading list = if null list then mempty else [heading] <> list
+
+   optionalHideableList :: Text -> [Text] -> [Text]
+   optionalHideableList heading list = if null list then mempty else [heading] <> details (showT (length list) <> " job(s)") list
+
+   maintainedList :: StatusSummary -> [Text]
+   maintainedList = showMaintainedBuild <=< Map.toList . withMaintainer
+
+   summaryEntryGetReverseDeps :: SummaryEntry -> (Int, Int)
+   summaryEntryGetReverseDeps sumEntry =
+      ( negate $ summaryUnbrokenReverseDeps sumEntry
+      , negate $ summaryReverseDeps sumEntry
+      )
+
+   sortOnReverseDeps :: [(PkgName, SummaryEntry)] -> [(PkgName, SummaryEntry)]
+   sortOnReverseDeps = sortOn (\(_, sumEntry) -> summaryEntryGetReverseDeps sumEntry)
+
+   unmaintainedList :: StatusSummary -> [Text]
+   unmaintainedList = showBuild <=< sortOnReverseDeps . Map.toList . withoutMaintainer
+
+   showBuild :: (PkgName, SummaryEntry) -> [Text]
+   showBuild (name, entry) =
+      printJob
+         id
+         name
+         ( summaryBuilds entry
+         , Text.pack
+            ( if summaryReverseDeps entry > 0
+               then
+                  " ⤴️ " <> show (summaryUnbrokenReverseDeps entry) <>
+                  " | " <> show (summaryReverseDeps entry)
+               else ""
+            )
+         )
+
+   showMaintainedBuild
+      :: (PkgName, (Table PkgSet Platform BuildResult, NonEmpty Text)) -> [Text]
+   showMaintainedBuild (name, (table, maintainers)) =
+      printJob
+         id
+         name
+         ( table
+         , Text.intercalate " " (fmap ("@" <>) (toList maintainers))
+         )
+
+   tldr = case (errors, warnings) of
+            ([],[]) -> ["🟢 **Ready to merge** (if there are no [evaluation errors](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates))"]
+            ([],_) -> ["🟡 **Potential issues** (and possibly [evaluation errors](https://hydra.nixos.org/jobset/nixpkgs/haskell-updates))"]
+            _ -> ["🔴 **Branch not mergeable**"]
+   warnings =
+      if' (Unfinished > maybe Success worstState maintainedJob) "`maintained` jobset failed." <>
+      if' (Unfinished == maybe Success worstState mergeableJob) "`mergeable` jobset is not finished." <>
+      if' (Unfinished == maybe Success worstState maintainedJob) "`maintained` jobset is not finished."
+   errors =
+      if' (isNothing mergeableJob) "No `mergeable` job found." <>
+      if' (isNothing maintainedJob) "No `maintained` job found." <>
+      if' (Unfinished > maybe Success worstState mergeableJob) "`mergeable` jobset failed." <>
+      if' (outstandingJobs (Platform "x86_64-linux") > 100) "Too many outstanding jobs on x86_64-linux." <>
+      if' (outstandingJobs (Platform "aarch64-linux") > 100) "Too many outstanding jobs on aarch64-linux."
+
+   if' p e = if p then [e] else mempty
+
+   outstandingJobs platform | Table m <- numSummary = Map.findWithDefault 0 (platform, Unfinished) m
+
+   maintainedJob = Map.lookup (PkgName "maintained") summary
+   mergeableJob = Map.lookup (PkgName "mergeable") summary
+
+printEvalInfo :: IO ()
+printEvalInfo = do
+   (eval, fetchTime, _) <- readBuildReports
+   putStrLn (Text.unpack $ evalLine eval fetchTime)
+
+printMaintainerPing :: IO ()
+printMaintainerPing = do
+   (maintainerMap, (reverseDependencyMap, topBrokenRdeps)) <- concurrently getMaintainerMap do
+      depMap <- getDependencyMap
+      rdepMap <- evaluate . calculateReverseDependencies $ depMap
+      let tops = take 50 . sortOn (negate . snd) . fmap (second fst) . filter (\x -> maybe False broken $ Map.lookup (fst x) depMap) . Map.toList $ rdepMap
+      pure (rdepMap, tops)
+   (eval, fetchTime, buildReport) <- readBuildReports
+   let statusSummaries =
+          fmap (buildToStatusSummary maintainerMap reverseDependencyMap) buildReport
+       buildSum :: StatusSummary
+       buildSum = combineStatusSummaries statusSummaries
+       textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps
+   Text.putStrLn textBuildSummary
+
+printMarkBrokenList :: RequestLogsFlag -> IO ()
+printMarkBrokenList reqLogs = do
+   (_, fetchTime, buildReport) <- readBuildReports
+   runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} ->
+      case (getBuildState build, Text.splitOn "." $ unJobName job) of
+         (Failed, ["haskellPackages", name, "x86_64-linux"]) -> do
+            -- We use the last probable error cause found in the build log file.
+            error_message <- fromMaybe "failure" <$>
+              case reqLogs of
+                NoRequestLogs -> pure Nothing
+                RequestLogs -> do
+                  -- Fetch build log from hydra to figure out the cause of the error.
+                  build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
+                  pure $ safeLast $ mapMaybe probableErrorCause build_log
+            liftIO $ putStrLn $ "  - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime
+         _ -> pure ()
+
+{- | This function receives a line from a Nix Haskell builder build log and returns a possible error cause.
+ | We might need to add other causes in the future if errors happen in unusual parts of the builder.
+-}
+probableErrorCause :: ByteString -> Maybe String
+probableErrorCause "Setup: Encountered missing or private dependencies:" = Just "dependency missing"
+probableErrorCause "running tests" = Just "test failure"
+probableErrorCause build_line | ByteString.isPrefixOf "Building" build_line = Just ("failure building " <> fromUTF8BS (fst $ ByteString.breakSubstring " for" $ ByteString.drop 9 build_line))
+probableErrorCause build_line | ByteString.isSuffixOf "Phase" build_line = Just ("failure in " <> fromUTF8BS build_line)
+probableErrorCause _ = Nothing

maintainers/scripts/haskell/maintained-broken-pkgs.nix

@@ -0,0 +1,26 @@
+let
+  nixpkgs = import ../../..;
+  inherit (nixpkgs { }) haskellPackages lib;
+  maintainedPkgs = lib.filterAttrs (
+    _: v: builtins.length (v.meta.maintainers or [ ]) > 0
+  ) haskellPackages;
+  brokenPkgs = lib.filterAttrs (_: v: v.meta.broken) maintainedPkgs;
+  transitiveBrokenPkgs = lib.filterAttrs (
+    _: v: !(builtins.tryEval (v.outPath or null)).success && !v.meta.broken
+  ) maintainedPkgs;
+  infoList =
+    pkgs:
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (name: drv: "${name} ${(builtins.elemAt drv.meta.maintainers 0).github}") pkgs
+    );
+in
+{
+  report = ''
+    BROKEN:
+    ${infoList brokenPkgs}
+
+    TRANSITIVE BROKEN:
+    ${infoList transitiveBrokenPkgs}
+  '';
+  transitiveErrors = builtins.attrValues transitiveBrokenPkgs;
+}

maintainers/scripts/haskell/mark-broken.sh

@@ -0,0 +1,66 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils git -I nixpkgs=.
+
+# This script uses the data pulled with
+# maintainers/scripts/haskell/hydra-report.hs get-report to produce a list of
+# failing builds that get written to the hackage2nix config. Then
+# hackage-packages.nix gets regenerated and transitive-broken packages get
+# marked as dont-distribute in the config as well.
+# This should disable builds for most failing jobs in the haskell-updates jobset.
+
+set -euo pipefail
+
+do_commit=false
+mark_broken_list_flags=""
+
+for arg in "$@"; do
+    case "$arg" in
+        --do-commit)
+            do_commit=true
+            ;;
+        --no-request-logs)
+            mark_broken_list_flags="$mark_broken_list_flags $arg"
+            ;;
+        *)
+            echo "$0: unknown flag: $arg"
+            exit 100
+            ;;
+    esac
+done
+
+broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
+
+tmpfile=$(mktemp)
+trap "rm ${tmpfile}" 0
+
+echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script."
+echo "Generating a list of broken builds and displaying for manual confirmation ..."
+maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | LC_ALL=C.UTF-8 sort --ignore-case > "$tmpfile"
+
+$EDITOR "$tmpfile"
+
+tail -n +3 "$broken_config" >> "$tmpfile"
+
+cat > "$broken_config" << EOF
+# These packages don't compile.
+broken-packages:
+EOF
+
+# clear environment here to avoid things like allowing broken builds in
+LC_ALL=C.UTF-8 sort --ignore-case --unique "$tmpfile" >> "$broken_config"
+clear="env -u HOME -u NIXPKGS_CONFIG"
+$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh
+evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info)
+
+if $do_commit; then
+git add $broken_config
+git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+git add pkgs/development/haskell-modules/hackage-packages.nix
+git commit -F - << EOF
+haskellPackages: mark builds failing on hydra as broken
+
+This commit has been generated by maintainers/scripts/haskell/mark-broken.sh based on
+$evalline
+from the haskell-updates jobset on hydra under https://hydra.nixos.org/jobset/nixpkgs/haskell-updates
+EOF
+fi

maintainers/scripts/haskell/merge-and-open-pr.sh

@@ -0,0 +1,119 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p git gh -I nixpkgs=.
+#
+# Script to merge the currently open haskell-updates PR , bump the Stackage
+# version and Hackage versions, and open the next haskell-updates PR.
+
+set -eu -o pipefail
+
+# exit after printing first argument to this function
+function die {
+  # echo the first argument
+  echo "ERROR: $1"
+  echo "Aborting!"
+
+  exit 1
+}
+
+function help {
+  echo "Usage: $0 HASKELL_UPDATES_PR_NUM"
+  echo "Merge the currently open haskell-updates PR into master, and open the next one."
+  echo
+  echo "  -h, --help                print this help"
+  echo "  HASKELL_UPDATES_PR_NUM    number of the currently open PR on NixOS/nixpkgs"
+  echo "                            for the haskell-updates branch"
+  echo
+  echo "Example:"
+  echo "  \$ $0 137340"
+
+  exit 1
+}
+
+# Read in the current haskell-updates PR number from the command line.
+while [[ $# -gt 0 ]]; do
+  key="$1"
+
+  case $key in
+    -h|--help)
+      help
+      ;;
+    *)
+      curr_haskell_updates_pr_num="$1"
+      shift
+      ;;
+  esac
+done
+
+if [[ -z "${curr_haskell_updates_pr_num-}" ]] ; then
+  die "You must pass the current haskell-updates PR number as the first argument to this script."
+fi
+
+# Make sure you have gh authentication setup.
+if ! gh auth status 2>/dev/null ; then
+  die "You must setup the \`gh\` command.  Run \`gh auth login\`."
+fi
+
+# Make sure this is configured before we start doing anything
+push_remote="$(git config branch.haskell-updates.pushRemote)" \
+  || die 'Can'\''t determine pushRemote for haskell-updates. Please set using `git config branch.haskell-updates.pushremote <remote name>`.'
+
+# Fetch nixpkgs to get an up-to-date origin/haskell-updates branch.
+echo "Fetching origin..."
+git fetch origin >/dev/null
+
+# Make sure we are currently on a local haskell-updates branch.
+curr_branch="$(git rev-parse --abbrev-ref HEAD)"
+if [[ "$curr_branch" != "haskell-updates" ]]; then
+    die "Current branch is not called \"haskell-updates\"."
+fi
+
+# Make sure our local haskell-updates branch is on the same commit as
+# origin/haskell-updates.
+curr_branch_commit="$(git rev-parse haskell-updates)"
+origin_haskell_updates_commit="$(git rev-parse origin/haskell-updates)"
+if [[ "$curr_branch_commit" != "$origin_haskell_updates_commit" ]]; then
+    die "Current branch is not at the same commit as origin/haskell-updates"
+fi
+
+# Merge the current open haskell-updates PR.
+echo "Merging https://github.com/NixOS/nixpkgs/pull/${curr_haskell_updates_pr_num}..."
+gh pr merge --repo NixOS/nixpkgs --merge "$curr_haskell_updates_pr_num"
+
+# Update stackage, Hackage hashes, and regenerate Haskell package set
+./maintainers/scripts/haskell/update-package-set.sh
+
+# Push these new commits to the haskell-updates branch
+echo "Pushing commits just created to the remote $push_remote/haskell-updates branch..."
+git push "$push_remote" haskell-updates
+
+# Open new PR
+new_pr_body=$(cat <<EOF
+### This Merge
+
+This PR is the regular merge of the \`haskell-updates\` branch into \`staging\`.
+
+This branch is being continually built and tested by hydra at https://hydra.nixos.org/jobset/nixpkgs/haskell-updates. You may be able to find an up-to-date Hydra build report at [cdepillabout/nix-haskell-updates-status](https://github.com/cdepillabout/nix-haskell-updates-status).
+
+We roughly aim to merge these \`haskell-updates\` PRs at least once every two weeks. See the @NixOS/haskell [team calendar](https://cloud.maralorn.de/apps/calendar/p/H6migHmKX7xHoTFa) for who is currently in charge of this branch.
+
+### haskellPackages Workflow Summary
+
+Our workflow is currently described in [\`pkgs/development/haskell-modules/HACKING.md\`](https://github.com/NixOS/nixpkgs/blob/haskell-updates/pkgs/development/haskell-modules/HACKING.md).
+
+The short version is this:
+* We regularly update the Stackage and Hackage pins on \`haskell-updates\` (normally at the beginning of a merge window).
+* The community fixes builds of Haskell packages on that branch.
+* We aim at at least one merge of \`haskell-updates\` into \`staging\` every two weeks.
+* We only do the merge if the [\`mergeable\`](https://hydra.nixos.org/job/nixpkgs/haskell-updates/mergeable) job is succeeding on hydra.
+* If a [\`maintained\`](https://hydra.nixos.org/job/nixpkgs/haskell-updates/maintained) package is still broken at the time of merge, we will only merge if the maintainer has been pinged 7 days in advance. (If you care about a Haskell package, become a maintainer!)
+
+More information about Haskell packages in nixpkgs can be found [in the nixpkgs manual](https://nixos.org/manual/nixpkgs/unstable/#haskell).
+
+---
+
+This is the follow-up to #${curr_haskell_updates_pr_num}. Come to [#haskell:nixos.org](https://matrix.to/#/#haskell:nixos.org) if you have any questions.
+EOF
+)
+
+echo "Opening a PR for the next haskell-updates merge cycle..."
+gh pr create --repo NixOS/nixpkgs --base staging --head haskell-updates --title "haskellPackages: update stackage and hackage" --body "$new_pr_body"

maintainers/scripts/haskell/regenerate-hackage-packages.sh

@@ -0,0 +1,117 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils haskellPackages.cabal2nix-unstable.bin git -I nixpkgs=.
+
+set -euo pipefail
+
+self=$0
+
+print_help () {
+cat <<END_HELP
+Usage: $self [options]
+
+Options:
+   --do-commit    Commit changes to this file.
+   -f | --fast    Do not update the transitive-broken.yaml file.
+   -h | --help    Show this help.
+
+This script is used to regenerate nixpkgs' Haskell package set, using the
+tool hackage2nix from the nixos/cabal2nix repo. hackage2nix looks at the
+config files in pkgs/development/haskell-modules/configuration-hackage2nix
+and generates a Nix expression for package version specified there, using the
+Cabal files from the Hackage database (available under all-cabal-hashes) and
+its companion tool cabal2nix.
+
+Unless --fast is used, it will then use the generated nix expression by
+running regenerate-transitive-broken-packages.sh which updates the transitive-broken.yaml
+file. Then it re-runs hackage2nix.
+
+Related scripts are update-hackage.sh, for updating the snapshot of the
+Hackage database used by hackage2nix, and update-cabal2nix-unstable.sh,
+for updating the version of hackage2nix used to perform this task.
+
+Note that this script doesn't gcroot anything, so it may be broken by an
+unfortunately timed nix-store --gc.
+
+END_HELP
+}
+
+DO_COMMIT=0
+REGENERATE_TRANSITIVE=1
+
+options=$(getopt -o "fh" -l "help,fast,do-commit" -- "$@")
+
+eval set -- "$options"
+
+while true; do
+   case "$1" in
+      --do-commit)
+         DO_COMMIT=1
+         ;;
+      -f|--fast)
+         REGENERATE_TRANSITIVE=0
+         ;;
+      -h|--help)
+         print_help
+         exit 0
+         ;;
+      --)
+         break;;
+      *)
+         print_help
+         exit 1
+         ;;
+   esac
+   shift
+done
+
+HACKAGE2NIX="${HACKAGE2NIX:-hackage2nix}"
+
+config_dir=pkgs/development/haskell-modules/configuration-hackage2nix
+
+run_hackage2nix() {
+"$HACKAGE2NIX" \
+   --hackage "$unpacked_hackage" \
+   --preferred-versions <(for n in "$unpacked_hackage"/*/preferred-versions; do cat "$n"; echo; done) \
+   --nixpkgs "$PWD" \
+   --config "$compiler_config" \
+   --config "$config_dir/main.yaml" \
+   --config "$config_dir/stackage.yaml" \
+   --config "$config_dir/broken.yaml" \
+   --config "$config_dir/transitive-broken.yaml"
+}
+
+echo "Obtaining Hackage data …"
+extraction_derivation='with import ./. {}; runCommandLocal "unpacked-cabal-hashes" { } "tar xf ${all-cabal-hashes} --strip-components=1 --one-top-level=$out"'
+unpacked_hackage="$(nix-build -E "$extraction_derivation" --no-out-link)"
+
+echo "Generating compiler configuration …"
+compiler_config="$(nix-build -A haskellPackages.cabal2nix-unstable.compilerConfig --no-out-link)"
+
+echo "Running hackage2nix to regenerate pkgs/development/haskell-modules/hackage-packages.nix …"
+run_hackage2nix
+
+if [[ "$REGENERATE_TRANSITIVE" -eq 1 ]]; then
+
+echo "Regenerating transitive-broken.yaml … (pass --fast to $self to skip this step)"
+
+maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh
+
+echo "Running hackage2nix again to reflect changes in transitive-broken.yaml …"
+
+run_hackage2nix
+
+fi
+
+nixfmt pkgs/development/haskell-modules/hackage-packages.nix
+
+if [[ "$DO_COMMIT" -eq 1 ]]; then
+git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+git add pkgs/development/haskell-modules/hackage-packages.nix
+git commit --edit -F - << EOF
+haskellPackages: regenerate package set based on current config
+
+(generated by maintainers/scripts/haskell/regenerate-hackage-packages.sh)
+EOF
+fi
+
+echo "Regeneration of hackage-packages.nix finished."

maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh

@@ -0,0 +1,25 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils jq -I nixpkgs=.
+
+set -euo pipefail
+
+TMP_TEMPLATE=transitive-broken.XXXXXXX
+readonly TMP_TEMPLATE
+
+tmpfile=$(mktemp "$TMP_TEMPLATE")
+
+trap 'rm -f "${tmpfile}"' 0
+
+config_file=pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
+
+cat > $tmpfile << EOF
+# This file is automatically generated by
+# maintainers/scripts/haskell/regenerate-transitive-broken-packages.sh
+# It is supposed to list all haskellPackages that cannot evaluate because they
+# depend on a dependency marked as broken.
+dont-distribute-packages:
+EOF
+
+nix-instantiate --eval --option restrict-eval true -I . --strict --json maintainers/scripts/haskell/transitive-broken-packages.nix | jq -r . | LC_ALL=C.UTF-8 sort --ignore-case >> $tmpfile
+
+mv $tmpfile $config_file

maintainers/scripts/haskell/test-configurations.nix

@@ -0,0 +1,163 @@
+/*
+  Nix expression to test for regressions in the Haskell configuration overlays.
+
+  test-configurations.nix determines all attributes touched by given Haskell
+  configuration overlays (i. e. pkgs/development/haskell-modules/configuration-*.nix)
+  and builds all derivations (or at least a reasonable subset) affected by
+  these overrides.
+
+  By default, it checks `configuration-{common,nix,ghc-9.8.x}.nix`. You can
+  invoke it like this:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix --keep-going
+
+  It is possible to specify other configurations:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg files '[ "configuration-ghc-9.0.x.nix" "configuration-ghc-9.2.x.nix" ]' \
+      --keep-going
+
+  You can also just supply a single string:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --argstr files "configuration-arm.nix" --keep-going
+
+  You can even supply full paths which is handy, as it allows for tab-completing
+  the configurations:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --argstr files pkgs/development/haskell-modules/configuration-arm.nix \
+      --keep-going
+
+  By default, derivation that fail to evaluate are skipped, unless they are
+  “just” marked as broken. You can check for other eval errors like this:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg skipEvalErrors false --keep-going
+
+  You can also disable checking broken packages by passing a nixpkgs config:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg config '{ allowBroken = false; }' --keep-going
+
+  By default the haskell.packages.ghc*Binary sets used for bootstrapping GHC
+  are _not_ tested. You can change this using:
+
+    nix-build maintainers/scripts/haskell/test-configurations.nix \
+      --arg skipBinaryGHCs false --keep-going
+*/
+{
+  files ? [
+    "configuration-common.nix"
+    "configuration-nix.nix"
+    "configuration-ghc-9.8.x.nix"
+  ],
+  nixpkgsPath ? ../../..,
+  config ? {
+    allowBroken = true;
+  },
+  skipEvalErrors ? true,
+  skipBinaryGHCs ? true,
+}:
+
+let
+  pkgs = import nixpkgsPath { inherit config; };
+  inherit (pkgs) lib;
+
+  # see usage explanation for the input format `files` allows
+  files' = map baseNameOf (if !builtins.isList files then [ files ] else files);
+
+  packageSetsWithVersionedHead =
+    pkgs.haskell.packages
+    // (
+      let
+        headSet = pkgs.haskell.packages.ghcHEAD;
+        # Determine the next GHC release version following GHC HEAD.
+        # GHC HEAD always has an uneven, tentative version number, e.g. 9.7.
+        # GHC releases always have even numbers, i.e. GHC 9.8 is branched off from
+        # GHC HEAD 9.7. Since we use the to be release number for GHC HEAD's
+        # configuration file, we need to calculate this here.
+        headVersion = lib.pipe headSet.ghc.version [
+          lib.versions.splitVersion
+          (lib.take 2)
+          lib.concatStrings
+          lib.strings.toInt
+          (builtins.add 1)
+          toString
+        ];
+      in
+      {
+        "ghc${headVersion}" = headSet;
+      }
+    );
+
+  setsForFile =
+    fileName:
+    let
+      # extract the unique part of the config's file name
+      configName = builtins.head (builtins.match "configuration-(.+).nix" fileName);
+      # match the major and minor version of the GHC the config is intended for, if any
+      configVersion = lib.concatStrings (builtins.match "ghc-([0-9]+).([0-9]+).x" configName);
+      # return all package sets under haskell.packages matching the version components
+      setsForVersion = map (name: packageSetsWithVersionedHead.${name}) (
+        builtins.filter (
+          setName:
+          lib.hasPrefix "ghc${configVersion}" setName && (skipBinaryGHCs -> !(lib.hasInfix "Binary" setName))
+        ) (builtins.attrNames packageSetsWithVersionedHead)
+      );
+
+      defaultSets = [ pkgs.haskellPackages ];
+    in
+    {
+      # use plain haskellPackages for the version-agnostic files
+      # TODO(@sternenseemann): also consider currently selected versioned sets
+      "common" = defaultSets;
+      "nix" = defaultSets;
+      "arm" = defaultSets;
+      "darwin" = defaultSets;
+    }
+    .${configName} or setsForVersion;
+
+  # attribute set that has all the attributes of haskellPackages set to null
+  availableHaskellPackages = builtins.listToAttrs (
+    map (attr: lib.nameValuePair attr null) (builtins.attrNames pkgs.haskellPackages)
+  );
+
+  # evaluate a configuration and only return the attributes changed by it,
+  # pass availableHaskellPackages as super in case intersectAttrs is used
+  overriddenAttrs =
+    fileName:
+    builtins.attrNames (
+      lib.fix (
+        self:
+        import (nixpkgsPath + "/pkgs/development/haskell-modules/${fileName}") {
+          haskellLib = pkgs.haskell.lib.compose;
+          inherit pkgs;
+        } self availableHaskellPackages
+      )
+    );
+
+  # list of derivations that are affected by overrides in the given configuration
+  # overlays. For common, nix, darwin etc. only the derivation from the default
+  # package set will be emitted.
+  packages =
+    builtins.filter
+      (
+        v:
+        lib.warnIf (v.meta.broken or false) "${v.pname} is marked as broken" (
+          v != null && (skipEvalErrors -> (builtins.tryEval (v.outPath or v)).success)
+        )
+      )
+      (
+        lib.concatMap (
+          fileName:
+          let
+            sets = setsForFile fileName;
+            attrs = overriddenAttrs fileName;
+          in
+          lib.concatMap (set: map (attr: set.${attr}) attrs) sets
+        ) files'
+      );
+in
+
+packages

maintainers/scripts/haskell/transitive-broken-packages.nix

@@ -0,0 +1,17 @@
+let
+  nixpkgs = import ../../..;
+  inherit (nixpkgs { }) pkgs lib;
+  getEvaluating =
+    x:
+    builtins.attrNames (
+      lib.filterAttrs (
+        _: v: (builtins.tryEval (v.outPath or null)).success && lib.isDerivation v && !v.meta.broken
+      ) x
+    );
+  brokenDeps = lib.subtractLists (getEvaluating pkgs.haskellPackages) (
+    getEvaluating (nixpkgs { config.allowBroken = true; }).haskellPackages
+  );
+in
+''
+  ${lib.concatMapStringsSep "\n" (x: " - ${x}") brokenDeps}
+''

maintainers/scripts/haskell/unbreak.nu

@@ -0,0 +1,31 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i nu -p nushell flock gnused -I nixpkgs=.
+
+# This script tests to build all packages listed in broken.yaml, expecting a build failure.
+# It will remove all packages that build fine from the list.
+
+# Attention: For unknown reasons, the script can't be easily cancelled and needs to be killed manually if it shouldn't run to completion.
+
+use std log
+
+let broken_config = "pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
+
+def is-broken [package : string]: nothing -> bool {
+    let res = with-env { NIXPKGS_ALLOW_BROKEN: "1" } {
+        # rather high timeout of half an hour, just to prevent never-ending builds
+        ^nix-build --no-out-link -j 1 --cores 1 --timeout 1800 -A $"haskellPackages.($package)" | complete
+    }
+    if $res.exit_code == 0 {
+      log warning $"($package) is not broken anymore!"
+      return false
+    } else {
+      log info $"($package) is still broken."
+      log debug $"($package) build log:\n($res.stderr)"
+      return true
+    }
+}
+
+def main [] {
+    $broken_config | open | get broken-packages
+      | par-each {|package| if not (is-broken $package) { ^flock -x $broken_config -c $"sed -i -e '/^  - ($package) /d' ($broken_config)" }}
+}

maintainers/scripts/haskell/update-cabal2nix-unstable.sh

@@ -0,0 +1,27 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p coreutils curl jq gnused haskellPackages.cabal2nix-unstable.bin nix-prefetch-scripts -I nixpkgs=.
+
+# Updates cabal2nix-unstable to the latest master of the nixos/cabal2nix repository.
+# See regenerate-hackage-packages.sh for details on the purpose of this script.
+
+set -euo pipefail
+
+# fetch current master HEAD from Github
+head_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/NixOS/cabal2nix/branches/master)"
+# extract commit hash
+commit="$(jq -r .commit.sha <<< "$head_info")"
+# extract commit timestamp and convert to date
+date="$(date "--date=$(jq -r .commit.commit.committer.date <<< "$head_info")" +%F)"
+
+# generate nix expression from cabal file, replacing the version with the commit date
+function mkPackage() {
+  output=pkgs/development/haskell-modules/cabal2nix-unstable/$1.nix
+  echo "# This file defines $1-unstable, used by maintainers/scripts/haskell/regenerate-hackage-packages.sh." > "$output"
+  cabal2nix --subpath "$1" "https://github.com/NixOS/cabal2nix/archive/$commit.tar.gz" | sed -Ee 's/version = "(.*)"/version = "\1-unstable-'"$date"'"/' >> "$output"
+  nixfmt "$output"
+}
+
+mkPackage "cabal2nix"
+mkPackage "distribution-nixpkgs"
+mkPackage "hackage-db"
+mkPackage "language-nix"

maintainers/scripts/haskell/update-hackage.sh

@@ -0,0 +1,56 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl jq git gnused -I nixpkgs=.
+#
+# SYNOPSIS
+#
+#   Update Hackage index and hashes data exposed via pkgs.all-cabal-hashes.
+#
+# DESCRIPTION
+#
+#   Find latest revision of the commercialhaskell/all-cabal-hashes repository's
+#   hackage branch and update pkgs/data/misc/hackage/pin.json accordingly.
+#
+#   This data is used by hackage2nix to generate hackage-packages.nix. Since
+#   hackage2nix uses the latest version of a package unless an explicit
+#   constraint is configured, running this script indirectly updates packages
+#   (when hackage2nix is executed afterwards).
+#
+#   Prints a version difference to stdout if the pin has been updated, nothing
+#   otherwise.
+#
+# EXIT STATUS
+#
+#   Always exit with zero (even if nothing changed) unless there was an error.
+
+set -euo pipefail
+
+if [[ "${1:-}" == "--do-commit" ]]; then
+  echo "$0: --do-commit is no longer supported. Use update-package-set.sh instead."
+  exit 100
+fi
+
+pin_file=pkgs/data/misc/hackage/pin.json
+current_commit="$(jq -r .commit $pin_file)"
+old_date="$(jq -r .msg $pin_file | sed 's/Update from Hackage at //')"
+git_info="$(curl -H "Accept: application/vnd.github.v3+json" https://api.github.com/repos/commercialhaskell/all-cabal-hashes/branches/hackage)"
+head_commit="$(echo "$git_info" | jq -r .commit.sha)"
+commit_msg="$(echo "$git_info" | jq -r .commit.commit.message)"
+new_date="$(echo "$commit_msg" | sed 's/Update from Hackage at //')"
+
+if [ "$current_commit" != "$head_commit" ]; then
+   echo "Updating all-cabal-hashes from $old_date to $new_date" >&2
+   url="https://github.com/commercialhaskell/all-cabal-hashes/archive/$head_commit.tar.gz"
+   hash="$(nix-prefetch-url "$url")"
+   jq -n \
+     --arg commit "$head_commit" \
+     --arg hash "$hash" \
+     --arg url "$url" \
+     --arg commit_msg "$commit_msg" \
+     '{commit: $commit, url: $url, sha256: $hash, msg: $commit_msg}' \
+     > $pin_file
+else
+   echo "No new all-cabal-hashes version" >&2
+   exit 0
+fi
+
+echo "$old_date -> $new_date"

maintainers/scripts/haskell/update-package-set.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash
+#! nix-shell -p git -I nixpkgs=.
+set -euo pipefail
+
+filesToStage=(
+  'pkgs/data/misc/hackage/pin.json'
+  'pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml'
+  'pkgs/development/haskell-modules/hackage-packages.nix'
+)
+
+if ! git diff --quiet --cached; then
+  echo "Please commit staged changes before running $0" >&2
+  exit 100
+fi
+
+if ! git diff --quiet -- "${filesToStage[@]}"; then
+  echo -n "Please commit your changes to the following files before running $0: " >&2
+  echo    "${filesToStage[@]}" >&2
+  exit 100
+fi
+
+stackage_diff="$(./maintainers/scripts/haskell/update-stackage.sh)"
+hackage_diff="$(./maintainers/scripts/haskell/update-hackage.sh)"
+readonly stackage_diff hackage_diff
+
+# Prefer Stackage version diff in the commit header, fall back to Hackage
+if [[ -n "$stackage_diff" ]]; then
+  commit_message="haskellPackages: stackage $stackage_diff"
+  if [[ -n "$hackage_diff" ]]; then
+    commit_message="$commit_message
+
+all-cabal-hashes: $hackage_diff"
+  fi
+elif [[ -n "$hackage_diff" ]]; then
+  commit_message="haskellPackages: hackage $hackage_diff
+
+all-cabal-hashes: $hackage_diff"
+else
+  echo "Neither Hackage nor Stackage changed. Nothing to do." >&2
+  exit 0
+fi
+
+commit_message="$commit_message
+
+(generated by maintainers/scripts/haskell/update-package-set.sh)"
+
+# Using fast here because after the hackage-update eval errors will likely break the transitive dependencies check.
+./maintainers/scripts/haskell/regenerate-hackage-packages.sh --fast
+
+# A --do-commit flag probably doesn't make much sense
+git add -- "${filesToStage[@]}"
+git commit -m "$commit_message"

maintainers/scripts/haskell/update-stackage.sh

@@ -0,0 +1,109 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl jq git gnused gnugrep -I nixpkgs=.
+# shellcheck shell=bash
+#
+# SYNOPSIS
+#
+#   Update version constraints in hackage2nix config file from Stackage.
+#
+# DESCRIPTION
+#
+#   Fetches the latest snapshot of the configured Stackage solver which is
+#   configured via the SOLVER (either LTS or Nightly) and VERSION variables in
+#   the script.
+#
+#   VERSION is only applicable if SOLVER is LTS. SOLVER=LTS and VERSION=22
+#   will cause update-stackage.sh to fetch the latest LTS-22.XX version.
+#   If empty, the latest version of the solver is used.
+#
+#   If the configuration file has been updated, update-stackage.sh prints a
+#   version difference to stdout, e.g. 23.11 -> 23.13. Otherwise, stdout remains
+#   empty.
+#
+# EXIT STATUS
+#
+#   Always exit with zero (even if nothing changed) unless there was an error.
+
+set -eu -o pipefail
+
+if [[ "${1:-}" == "--do-commit" ]]; then
+  echo "$0: --do-commit is no longer supported. Use update-package-set.sh instead."
+  exit 100
+fi
+
+# Stackage solver to use, LTS or Nightly
+# (should be capitalized like the display name)
+SOLVER=LTS
+# Stackage solver verson, if any. Use latest if empty
+VERSION=
+TMP_TEMPLATE=update-stackage.XXXXXXX
+readonly SOLVER
+readonly VERSION
+readonly TMP_TEMPLATE
+
+toLower() {
+    printf "%s" "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+tmpfile=$(mktemp "$TMP_TEMPLATE")
+tmpfile_new=$(mktemp "$TMP_TEMPLATE")
+
+stackage_config="pkgs/development/haskell-modules/configuration-hackage2nix/stackage.yaml"
+
+trap 'rm "${tmpfile}" "${tmpfile_new}"' 0
+touch "$tmpfile" "$tmpfile_new" # Creating files here so that trap creates no errors.
+
+curl -L -s "https://stackage.org/$(toLower "$SOLVER")${VERSION:+-$VERSION}/cabal.config" >"$tmpfile"
+old_version=$(grep '^# Stackage' $stackage_config | sed -e 's/.\+ \([A-Za-z]\+ [0-9.-]\+\)$/\1/g')
+version="$SOLVER $(sed -rn "s/^--.*http:..(www.)?stackage.org.snapshot.$(toLower "$SOLVER")-//p" "$tmpfile")"
+
+if [[ "$old_version" == "$version" ]]; then
+   echo "No new stackage version" >&2
+   exit 0 # Nothing to do
+fi
+
+echo "Updating Stackage from $old_version to $version." >&2
+
+# Create a simple yaml version of the file.
+sed -r \
+    -e '/^--/d' \
+    -e 's|^constraints:||' \
+    -e 's|^ +|  - |' \
+    -e 's|,$||' \
+    -e '/^with-compiler:/d' \
+    -e '/installed$/d' \
+    -e '/^$/d' \
+    < "${tmpfile}" | LC_ALL=C.UTF-8 sort --ignore-case >"${tmpfile_new}"
+
+cat > $stackage_config << EOF
+# Stackage $version
+# This file is auto-generated by
+# maintainers/scripts/haskell/update-stackage.sh
+default-package-overrides:
+EOF
+
+# Drop restrictions on some tools where we always want the latest version.
+sed -r \
+    -e '/ cabal2nix /d' \
+    -e '/ distribution-nixpkgs /d' \
+    -e '/ jailbreak-cabal /d' \
+    -e '/ language-nix /d' \
+    -e '/ hackage-db /d' \
+    -e '/ cabal-install /d' \
+    -e '/ cabal-install-solver /d' \
+    -e '/ lsp /d' \
+    -e '/ lsp-types /d' \
+    -e '/ lsp-test /d' \
+    -e '/ hie-bios /d' \
+    -e '/ ShellCheck /d' \
+    -e '/ Agda /d' \
+    -e '/ stack /d' \
+    -e '/ git-annex /d' \
+    < "${tmpfile_new}" >> $stackage_config
+# Explanations:
+# cabal2nix, distribution-nixpkgs, jailbreak-cabal, language-nix: These are our packages and we know what we are doing.
+# lsp, lsp-types, lsp-test, hie-bios: These are tightly coupled to hls which is not in stackage. They have no rdeps in stackage.
+# ShellCheck: latest version of command-line dev tool.
+# Agda: The Agda community is fast-moving; we strive to always include the newest versions of Agda and the Agda packages in nixpkgs.
+
+echo "$old_version -> $version"

maintainers/scripts/haskell/upload-nixos-package-list-to-hackage.sh

@@ -0,0 +1,43 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p curl gnused -I nixpkgs=.
+
+# On Hackage every package description shows a category "Distributions" which
+# lists a "NixOS" version.
+# This script uploads a csv to hackage which will update the displayed versions
+# based on the current versions in nixpkgs. This happens with a simple http
+# request.
+
+# For authorization you just need to have any valid hackage account. This
+# script uses the `username` and `password-command` field from your
+# ~/.cabal/config file.
+
+# e.g. username: maralorn
+#      password-command: pass hackage.haskell.org (this can be any command, but not an arbitrary shell expression. Like cabal we only read the first output line and ignore the rest.)
+# Those fields are specified under `upload` on the `cabal` man page.
+
+if test -z "$CABAL_DIR"; then
+  dirs=(
+    "$HOME/.cabal"
+    "${XDG_CONFIG_HOME:-$HOME/.config}/cabal"
+  )
+  missing=true
+
+  for dir in "${dirs[@]}"; do
+    if test -d "$dir"; then
+      export CABAL_DIR="$dir"
+      missing=false
+      break
+    fi
+  done
+
+  if $missing; then
+    echo "Could not find the cabal configuration directory in any of: ${dirs[@]}" >&2
+    exit 101
+  fi
+fi
+
+package_list="$(nix-build -A haskell.package-list)/nixos-hackage-packages.csv"
+username=$(grep "^username:" "$CABAL_DIR/config" | sed "s/^username: //")
+password_command=$(grep "^password-command:" "$CABAL_DIR/config" | sed "s/^password-command: //")
+curl -u "$username:$($password_command | head -n1)" --digest -H "Content-type: text/csv" -T "$package_list" https://hackage.haskell.org/distro/NixOS/packages.csv
+echo