push sheeet

nixos/modules/services/continuous-integration/buildbot/master.nix

@@ -0,0 +1,317 @@
+# NixOS module for Buildbot continuous integration server.
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.buildbot-master;
+  opt = options.services.buildbot-master;
+
+  package = cfg.package.python.pkgs.toPythonModule cfg.package;
+  python = cfg.package.python;
+
+  escapeStr = lib.escape [ "'" ];
+
+  defaultMasterCfg = pkgs.writeText "master.cfg" ''
+    from buildbot.plugins import *
+    ${cfg.extraImports}
+    factory = util.BuildFactory()
+    c = BuildmasterConfig = dict(
+     workers       = [${lib.concatStringsSep "," cfg.workers}],
+     protocols     = { 'pb': {'port': ${toString cfg.pbPort} } },
+     title         = '${escapeStr cfg.title}',
+     titleURL      = '${escapeStr cfg.titleUrl}',
+     buildbotURL   = '${escapeStr cfg.buildbotUrl}',
+     db            = dict(db_url='${escapeStr cfg.dbUrl}'),
+     www           = dict(port=${toString cfg.port}),
+     change_source = [ ${lib.concatStringsSep "," cfg.changeSource} ],
+     schedulers    = [ ${lib.concatStringsSep "," cfg.schedulers} ],
+     builders      = [ ${lib.concatStringsSep "," cfg.builders} ],
+     services      = [ ${lib.concatStringsSep "," cfg.reporters} ],
+     configurators = [ ${lib.concatStringsSep "," cfg.configurators} ],
+    )
+    for step in [ ${lib.concatStringsSep "," cfg.factorySteps} ]:
+      factory.addStep(step)
+
+    ${cfg.extraConfig}
+  '';
+
+  tacFile = pkgs.writeText "buildbot-master.tac" ''
+    import os
+
+    from twisted.application import service
+    from buildbot.master import BuildMaster
+
+    basedir = '${cfg.buildbotDir}'
+
+    configfile = '${cfg.masterCfg}'
+
+    # Default umask for server
+    umask = None
+
+    # note: this line is matched against to check that this is a buildmaster
+    # directory; do not edit it.
+    application = service.Application('buildmaster')
+
+    m = BuildMaster(basedir, configfile, umask)
+    m.setServiceParent(application)
+  '';
+
+in
+{
+  options = {
+    services.buildbot-master = {
+
+      factorySteps = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Factory Steps";
+        default = [ ];
+        example = [
+          "steps.Git(repourl='https://github.com/buildbot/pyflakes.git', mode='incremental')"
+          "steps.ShellCommand(command=['trial', 'pyflakes'])"
+        ];
+      };
+
+      changeSource = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Change Sources.";
+        default = [ ];
+        example = [
+          "changes.GitPoller('https://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
+        ];
+      };
+
+      configurators = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Configurator Steps, see <https://docs.buildbot.net/latest/manual/configuration/configurators.html>";
+        default = [ ];
+        example = [
+          "util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)"
+        ];
+      };
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot continuous integration server.";
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra configuration to append to master.cfg";
+        default = "c['buildbotNetUsageData'] = None";
+      };
+
+      extraImports = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra python imports to prepend to master.cfg";
+        default = "";
+        example = "from buildbot.process.project import Project";
+      };
+
+      masterCfg = lib.mkOption {
+        type = lib.types.path;
+        description = "Optionally pass master.cfg path. Other options in this configuration will be ignored.";
+        default = defaultMasterCfg;
+        defaultText = lib.literalMD ''generated configuration file'';
+        example = "/etc/nixos/buildbot/master.cfg";
+      };
+
+      schedulers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Schedulers.";
+        default = [
+          "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])"
+          "schedulers.ForceScheduler(name='force',builderNames=['runtests'])"
+        ];
+      };
+
+      builders = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Builders.";
+        default = [
+          "util.BuilderConfig(name='runtests',workernames=['example-worker'],factory=factory)"
+        ];
+      };
+
+      workers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Workers.";
+        default = [ "worker.Worker('example-worker', 'pass')" ];
+      };
+
+      reporters = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = "List of reporter objects used to present build status to various users.";
+      };
+
+      user = lib.mkOption {
+        default = "buildbot";
+        type = lib.types.str;
+        description = "User the buildbot server should execute under.";
+      };
+
+      group = lib.mkOption {
+        default = "buildbot";
+        type = lib.types.str;
+        description = "Primary group of buildbot user.";
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "List of extra groups that the buildbot user should be a part of.";
+      };
+
+      home = lib.mkOption {
+        default = "/home/buildbot";
+        type = lib.types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = lib.mkOption {
+        default = "${cfg.home}/master";
+        defaultText = lib.literalExpression ''"''${config.${opt.home}}/master"'';
+        type = lib.types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      pbPort = lib.mkOption {
+        default = 9989;
+        type = lib.types.either lib.types.str lib.types.port;
+        example = "'tcp:9990:interface=127.0.0.1'";
+        description = ''
+          The buildmaster will listen on a TCP port of your choosing
+          for connections from workers.
+          It can also use this port for connections from remote Change Sources,
+          status clients, and debug tools.
+          This port should be visible to the outside world, and you’ll need to tell
+          your worker admins about your choice.
+          If put in (single) quotes, this can also be used as a connection string,
+          as defined in the [ConnectionStrings guide](https://twistedmatrix.com/documents/current/core/howto/endpoints.html).
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        default = "0.0.0.0";
+        type = lib.types.str;
+        description = "Specifies the bind address on which the buildbot HTTP interface listens.";
+      };
+
+      buildbotUrl = lib.mkOption {
+        default = "http://localhost:8010/";
+        type = lib.types.str;
+        description = "Specifies the Buildbot URL.";
+      };
+
+      title = lib.mkOption {
+        default = "Buildbot";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Title.";
+      };
+
+      titleUrl = lib.mkOption {
+        default = "Buildbot";
+        type = lib.types.str;
+        description = "Specifies the Buildbot TitleURL.";
+      };
+
+      dbUrl = lib.mkOption {
+        default = "sqlite:///state.sqlite";
+        type = lib.types.str;
+        description = "Specifies the database connection string.";
+      };
+
+      port = lib.mkOption {
+        default = 8010;
+        type = lib.types.port;
+        description = "Specifies port number on which the buildbot HTTP interface listens.";
+      };
+
+      package = lib.mkPackageOption pkgs "buildbot-full" {
+        example = "buildbot";
+      };
+
+      packages = lib.mkOption {
+        default = [ pkgs.git ];
+        defaultText = lib.literalExpression "[ pkgs.git ]";
+        type = lib.types.listOf lib.types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+
+      pythonPackages = lib.mkOption {
+        type = lib.types.functionTo (lib.types.listOf lib.types.package);
+        default = pythonPackages: [ ];
+        defaultText = lib.literalExpression "pythonPackages: with pythonPackages; [ ]";
+        description = "Packages to add the to the PYTHONPATH of the buildbot process.";
+        example = lib.literalExpression "pythonPackages: with pythonPackages; [ requests ]";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.groups = lib.optionalAttrs (cfg.group == "buildbot") {
+      buildbot = { };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "buildbot") {
+      buildbot = {
+        description = "Buildbot User.";
+        isNormalUser = true;
+        createHome = true;
+        inherit (cfg) home group extraGroups;
+        useDefaultShell = true;
+      };
+    };
+
+    systemd.services.buildbot-master = {
+      description = "Buildbot Continuous Integration Server.";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.packages ++ cfg.pythonPackages python.pkgs;
+      environment.PYTHONPATH = "${
+        python.withPackages (self: cfg.pythonPackages self ++ [ package ])
+      }/${python.sitePackages}";
+
+      preStart = ''
+        mkdir -vp "${cfg.buildbotDir}"
+        # Link the tac file so buildbot command line tools recognize the directory
+        ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac"
+        ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}"
+        rm -f buildbot.tac.new master.cfg.sample
+      '';
+
+      serviceConfig = {
+        Type = "simple";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${cfg.buildbotDir}/buildbot.tac";
+        # To reload on upgrade, set the following in your configuration:
+        # systemd.services.buildbot-master.reloadIfChanged = true;
+        ExecReload = [
+          "${pkgs.coreutils}/bin/ln -sf ${tacFile} ${cfg.buildbotDir}/buildbot.tac"
+          "${pkgs.coreutils}/bin/kill -HUP $MAINPID"
+        ];
+      };
+    };
+  };
+
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "buildbot-master" "bpPort" ]
+      [ "services" "buildbot-master" "pbPort" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "buildbot-master" "status" ] ''
+      Since Buildbot 0.9.0, status targets are deprecated and ignored.
+      Review your configuration and migrate to reporters (available at services.buildbot-master.reporters).
+    '')
+  ];
+
+  meta.maintainers = lib.teams.buildbot.members;
+}

nixos/modules/services/continuous-integration/buildbot/worker.nix

@@ -0,0 +1,201 @@
+# NixOS module for Buildbot Worker.
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.buildbot-worker;
+  opt = options.services.buildbot-worker;
+
+  package = pkgs.python3.pkgs.toPythonModule cfg.package;
+  python = package.pythonModule;
+
+  tacFile = pkgs.writeText "aur-buildbot-worker.tac" ''
+    import os
+    from io import open
+
+    from buildbot_worker.bot import Worker
+    from twisted.application import service
+
+    basedir = '${cfg.buildbotDir}'
+
+    # note: this line is matched against to check that this is a worker
+    # directory; do not edit it.
+    application = service.Application('buildbot-worker')
+
+    master_url_split = '${cfg.masterUrl}'.split(':')
+    buildmaster_host = master_url_split[0]
+    port = int(master_url_split[1])
+    workername = '${cfg.workerUser}'
+
+    with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+        passwd = passwd_file.read().strip('\r\n')
+    keepalive = ${toString cfg.keepalive}
+    umask = None
+    maxdelay = 300
+    numcpus = None
+    allow_shutdown = None
+
+    s = Worker(buildmaster_host, port, workername, passwd, basedir,
+               keepalive, umask=umask, maxdelay=maxdelay,
+               numcpus=numcpus, allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+  '';
+
+in
+{
+  options = {
+    services.buildbot-worker = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot Worker.";
+      };
+
+      user = lib.mkOption {
+        default = "bbworker";
+        type = lib.types.str;
+        description = "User the buildbot Worker should execute under.";
+      };
+
+      group = lib.mkOption {
+        default = "bbworker";
+        type = lib.types.str;
+        description = "Primary group of buildbot Worker user.";
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "List of extra groups that the Buildbot Worker user should be a part of.";
+      };
+
+      home = lib.mkOption {
+        default = "/home/bbworker";
+        type = lib.types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = lib.mkOption {
+        default = "${cfg.home}/worker";
+        defaultText = lib.literalExpression ''"''${config.${opt.home}}/worker"'';
+        type = lib.types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      workerUser = lib.mkOption {
+        default = "example-worker";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker user.";
+      };
+
+      workerPass = lib.mkOption {
+        default = "pass";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker password.";
+      };
+
+      workerPassFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File used to store the Buildbot Worker password";
+      };
+
+      hostMessage = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = "Description of this worker";
+      };
+
+      adminMessage = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = "Name of the administrator of this worker";
+      };
+
+      masterUrl = lib.mkOption {
+        default = "localhost:9989";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker connection string.";
+      };
+
+      keepalive = lib.mkOption {
+        default = 600;
+        type = lib.types.int;
+        description = ''
+          This is a number that indicates how frequently keepalive messages should be sent
+          from the worker to the buildmaster, expressed in seconds.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "buildbot-worker" { };
+
+      packages = lib.mkOption {
+        default = with pkgs; [ git ];
+        defaultText = lib.literalExpression "[ pkgs.git ]";
+        type = lib.types.listOf lib.types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.buildbot-worker.workerPassFile = lib.mkDefault (
+      pkgs.writeText "buildbot-worker-password" cfg.workerPass
+    );
+
+    users.groups = lib.optionalAttrs (cfg.group == "bbworker") {
+      bbworker = { };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "bbworker") {
+      bbworker = {
+        description = "Buildbot Worker User.";
+        isNormalUser = true;
+        createHome = true;
+        home = cfg.home;
+        group = cfg.group;
+        extraGroups = cfg.extraGroups;
+        useDefaultShell = true;
+      };
+    };
+
+    systemd.services.buildbot-worker = {
+      description = "Buildbot Worker.";
+      after = [
+        "network.target"
+        "buildbot-master.service"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.packages;
+      environment.PYTHONPATH = "${python.withPackages (p: [ package ])}/${python.sitePackages}";
+
+      preStart = ''
+        mkdir -vp "${cfg.buildbotDir}/info"
+        ${lib.optionalString (cfg.hostMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host"
+        ''}
+        ${lib.optionalString (cfg.adminMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin"
+        ''}
+      '';
+
+      serviceConfig = {
+        Type = "simple";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+      };
+
+    };
+  };
+
+  meta.maintainers = lib.teams.buildbot.members;
+
+}

nixos/modules/services/continuous-integration/buildkite-agents.nix

@@ -0,0 +1,262 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.buildkite-agents;
+
+  hooksDir =
+    hooks:
+    let
+      mkHookEntry = name: text: ''
+        ln --symbolic ${pkgs.writeShellApplication { inherit name text; }}/bin/${name} $out/${name}
+      '';
+    in
+    pkgs.runCommand "buildkite-agent-hooks"
+      {
+        preferLocalBuild = true;
+      }
+      ''
+        mkdir $out
+        ${lib.concatStringsSep "\n" (lib.mapAttrsToList mkHookEntry hooks)}
+      '';
+
+  buildkiteOptions =
+    {
+      name ? "",
+      config,
+      ...
+    }:
+    {
+      options = {
+        enable = lib.mkOption {
+          default = true;
+          type = lib.types.bool;
+          description = "Whether to enable this buildkite agent";
+        };
+
+        package = lib.mkPackageOption pkgs "buildkite-agent" { };
+
+        dataDir = lib.mkOption {
+          default = "/var/lib/buildkite-agent-${name}";
+          description = "The workdir for the agent";
+          type = lib.types.str;
+        };
+
+        extraGroups = lib.mkOption {
+          default = [ "keys" ];
+          description = "Groups the user for this buildkite agent should belong to";
+          type = lib.types.listOf lib.types.str;
+        };
+
+        runtimePackages = lib.mkOption {
+          default = [
+            pkgs.bash
+            pkgs.gnutar
+            pkgs.gzip
+            pkgs.git
+            pkgs.nix
+          ];
+          defaultText = lib.literalExpression "[ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]";
+          description = "Add programs to the buildkite-agent environment";
+          type = lib.types.listOf lib.types.package;
+        };
+
+        tokenPath = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            The token from your Buildkite "Agents" page.
+
+            A run-time path to the token file, which is supposed to be provisioned
+            outside of Nix store.
+          '';
+        };
+
+        name = lib.mkOption {
+          type = lib.types.str;
+          default = "%hostname-${name}-%n";
+          description = ''
+            The name of the agent as seen in the buildkite dashboard.
+          '';
+        };
+
+        tags = lib.mkOption {
+          type = lib.types.attrsOf (lib.types.either lib.types.str (lib.types.listOf lib.types.str));
+          default = { };
+          example = {
+            queue = "default";
+            docker = "true";
+            ruby2 = "true";
+          };
+          description = ''
+            Tags for the agent.
+          '';
+        };
+
+        extraConfig = lib.mkOption {
+          type = lib.types.lines;
+          default = "";
+          example = "debug=true";
+          description = ''
+            Extra lines to be added verbatim to the configuration file.
+          '';
+        };
+
+        privateSshKeyPath = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          ## maximum care is taken so that secrets (ssh keys and the CI token)
+          ## don't end up in the Nix store.
+          apply = final: if final == null then null else toString final;
+
+          description = ''
+            OpenSSH private key
+
+            A run-time path to the key file, which is supposed to be provisioned
+            outside of Nix store.
+          '';
+        };
+
+        hooks = lib.mkOption {
+          type = lib.types.attrsOf lib.types.lines;
+          default = { };
+          example = lib.literalExpression ''
+            {
+              environment = '''
+                export SECRET_VAR=`head -1 /run/keys/secret`
+              ''';
+            }'';
+          description = ''
+            "Agent" hooks to install.
+            See <https://buildkite.com/docs/agent/v3/hooks> for possible options.
+          '';
+        };
+
+        hooksPath = lib.mkOption {
+          type = lib.types.path;
+          default = hooksDir config.hooks;
+          defaultText = lib.literalMD "generated from {option}`services.buildkite-agents.<name>.hooks`";
+          description = ''
+            Path to the directory storing the hooks.
+            Consider using {option}`services.buildkite-agents.<name>.hooks.<name>`
+            instead.
+          '';
+        };
+
+        shell = lib.mkOption {
+          type = lib.types.str;
+          default = "${pkgs.bash}/bin/bash -e -c";
+          defaultText = lib.literalExpression ''"''${pkgs.bash}/bin/bash -e -c"'';
+          description = ''
+            Command that buildkite-agent 3 will execute when it spawns a shell.
+          '';
+        };
+      };
+    };
+  enabledAgents = lib.filterAttrs (n: v: v.enable) cfg;
+  mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents);
+in
+{
+  options.services.buildkite-agents = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule buildkiteOptions);
+    default = { };
+    description = ''
+      Attribute set of buildkite agents.
+      The attribute key is combined with the hostname and a unique integer to
+      create the final agent name. This can be overridden by setting the `name`
+      attribute.
+    '';
+  };
+
+  config.users.users = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = {
+        name = "buildkite-agent-${name}";
+        home = cfg.dataDir;
+        createHome = true;
+        description = "Buildkite agent user";
+        extraGroups = cfg.extraGroups;
+        isSystemUser = true;
+        group = "buildkite-agent-${name}";
+      };
+    }
+  );
+  config.users.groups = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = { };
+    }
+  );
+
+  config.systemd.services = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = {
+        description = "Buildkite Agent";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = cfg.runtimePackages ++ [
+          cfg.package
+          pkgs.coreutils
+        ];
+        environment = config.networking.proxy.envVars // {
+          HOME = cfg.dataDir;
+          NIX_REMOTE = "daemon";
+        };
+
+        ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
+        ##     don't end up in the Nix store.
+        preStart =
+          let
+            sshDir = "${cfg.dataDir}/.ssh";
+            tagStr =
+              name: value:
+              if lib.isList value then
+                lib.concatStringsSep "," (builtins.map (v: "${name}=${v}") value)
+              else
+                "${name}=${value}";
+            tagsStr = lib.concatStringsSep "," (lib.mapAttrsToList tagStr cfg.tags);
+          in
+          lib.optionalString (cfg.privateSshKeyPath != null) ''
+            mkdir -m 0700 -p "${sshDir}"
+            install -m600 "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa"
+          ''
+          + ''
+            cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF
+            token="$(cat ${toString cfg.tokenPath})"
+            name="${cfg.name}"
+            shell="${cfg.shell}"
+            tags="${tagsStr}"
+            build-path="${cfg.dataDir}/builds"
+            hooks-path="${cfg.hooksPath}"
+            ${cfg.extraConfig}
+            EOF
+          '';
+
+        serviceConfig = {
+          ExecStart = "${cfg.package}/bin/buildkite-agent start --config ${cfg.dataDir}/buildkite-agent.cfg";
+          User = "buildkite-agent-${name}";
+          RestartSec = 5;
+          Restart = "on-failure";
+          TimeoutSec = 10;
+          # set a long timeout to give buildkite-agent a chance to finish current builds
+          TimeoutStopSec = "2 min";
+          KillMode = "mixed";
+        };
+      };
+    }
+  );
+
+  config.assertions = mapAgents (
+    name: cfg: [
+      {
+        assertion = cfg.hooksPath != hooksDir cfg.hooks -> cfg.hooks == { };
+        message = ''
+          Options `services.buildkite-agents.${name}.hooksPath' and
+          `services.buildkite-agents.${name}.hooks.<name>' are mutually exclusive.
+        '';
+      }
+    ]
+  );
+}

nixos/modules/services/continuous-integration/gitea-actions-runner.nix

@@ -0,0 +1,283 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    any
+    attrValues
+    concatStringsSep
+    escapeShellArg
+    hasInfix
+    hasSuffix
+    optionalAttrs
+    optionals
+    literalExpression
+    mapAttrs'
+    mkEnableOption
+    mkOption
+    mkPackageOption
+    mkIf
+    nameValuePair
+    types
+    ;
+
+  inherit (utils)
+    escapeSystemdPath
+    ;
+
+  cfg = config.services.gitea-actions-runner;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+  # Check whether any runner instance label requires a container runtime
+  # Empty label strings result in the upstream defined defaultLabels, which require docker
+  # https://gitea.com/gitea/act_runner/src/tag/v0.1.5/internal/app/cmd/register.go#L93-L98
+  hasDockerScheme =
+    instance: instance.labels == [ ] || any (label: hasInfix ":docker:" label) instance.labels;
+  wantsContainerRuntime = any hasDockerScheme (attrValues cfg.instances);
+
+  hasHostScheme = instance: any (label: hasSuffix ":host" label) instance.labels;
+
+  # provide shorthands for whether container runtimes are enabled
+  hasDocker = config.virtualisation.docker.enable;
+  hasPodman = config.virtualisation.podman.enable;
+
+  tokenXorTokenFile =
+    instance:
+    (instance.token == null && instance.tokenFile != null)
+    || (instance.token != null && instance.tokenFile == null);
+in
+{
+  meta.maintainers = with lib.maintainers; [
+    hexa
+  ];
+
+  options.services.gitea-actions-runner = with types; {
+    package = mkPackageOption pkgs "gitea-actions-runner" { };
+
+    instances = mkOption {
+      default = { };
+      description = ''
+        Gitea Actions Runner instances.
+      '';
+      type = attrsOf (submodule {
+        options = {
+          enable = mkEnableOption "Gitea Actions Runner instance";
+
+          name = mkOption {
+            type = str;
+            example = literalExpression "config.networking.hostName";
+            description = ''
+              The name identifying the runner instance towards the Gitea/Forgejo instance.
+            '';
+          };
+
+          url = mkOption {
+            type = str;
+            example = "https://forge.example.com";
+            description = ''
+              Base URL of your Gitea/Forgejo instance.
+            '';
+          };
+
+          token = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              Plain token to register at the configured Gitea/Forgejo instance.
+            '';
+          };
+
+          tokenFile = mkOption {
+            type = nullOr (either str path);
+            default = null;
+            description = ''
+              Path to an environment file, containing the `TOKEN` environment
+              variable, that holds a token to register at the configured
+              Gitea/Forgejo instance.
+            '';
+          };
+
+          labels = mkOption {
+            type = listOf str;
+            example = literalExpression ''
+              [
+                # provide a debian base with nodejs for actions
+                "debian-latest:docker://node:18-bullseye"
+                # fake the ubuntu name, because node provides no ubuntu builds
+                "ubuntu-latest:docker://node:18-bullseye"
+                # provide native execution on the host
+                #"native:host"
+              ]
+            '';
+            description = ''
+              Labels used to map jobs to their runtime environment. Changing these
+              labels currently requires a new registration token.
+
+              Many common actions require bash, git and nodejs, as well as a filesystem
+              that follows the filesystem hierarchy standard.
+            '';
+          };
+          settings = mkOption {
+            description = ''
+              Configuration for `act_runner daemon`.
+              See <https://gitea.com/gitea/act_runner/src/branch/main/internal/pkg/config/config.example.yaml> for an example configuration
+            '';
+
+            type = types.submodule {
+              freeformType = settingsFormat.type;
+            };
+
+            default = { };
+          };
+
+          hostPackages = mkOption {
+            type = listOf package;
+            default = with pkgs; [
+              bash
+              coreutils
+              curl
+              gawk
+              gitMinimal
+              gnused
+              nodejs
+              wget
+            ];
+            defaultText = literalExpression ''
+              with pkgs; [
+                bash
+                coreutils
+                curl
+                gawk
+                gitMinimal
+                gnused
+                nodejs
+                wget
+              ]
+            '';
+            description = ''
+              List of packages, that are available to actions, when the runner is configured
+              with a host execution label.
+            '';
+          };
+        };
+      });
+    };
+  };
+
+  config = mkIf (cfg.instances != { }) {
+    assertions = [
+      {
+        assertion = any tokenXorTokenFile (attrValues cfg.instances);
+        message = "Instances of gitea-actions-runner can have `token` or `tokenFile`, not both.";
+      }
+      {
+        assertion = wantsContainerRuntime -> hasDocker || hasPodman;
+        message = "Label configuration on gitea-actions-runner instance requires either docker or podman.";
+      }
+    ];
+
+    systemd.services =
+      let
+        mkRunnerService =
+          name: instance:
+          let
+            wantsContainerRuntime = hasDockerScheme instance;
+            wantsHost = hasHostScheme instance;
+            wantsDocker = wantsContainerRuntime && config.virtualisation.docker.enable;
+            wantsPodman = wantsContainerRuntime && config.virtualisation.podman.enable;
+            configFile = settingsFormat.generate "config.yaml" instance.settings;
+          in
+          nameValuePair "gitea-runner-${escapeSystemdPath name}" {
+            inherit (instance) enable;
+            description = "Gitea Actions Runner";
+            wants = [ "network-online.target" ];
+            after = [
+              "network-online.target"
+            ]
+            ++ optionals wantsDocker [
+              "docker.service"
+            ]
+            ++ optionals wantsPodman [
+              "podman.service"
+            ];
+            wantedBy = [
+              "multi-user.target"
+            ];
+            environment =
+              optionalAttrs (instance.token != null) {
+                TOKEN = "${instance.token}";
+              }
+              // optionalAttrs wantsPodman {
+                DOCKER_HOST = "unix:///run/podman/podman.sock";
+              }
+              // {
+                HOME = "/var/lib/gitea-runner/${name}";
+              };
+            path =
+              with pkgs;
+              [
+                coreutils
+              ]
+              ++ lib.optionals wantsHost instance.hostPackages;
+            serviceConfig = {
+              DynamicUser = true;
+              User = "gitea-runner";
+              StateDirectory = "gitea-runner";
+              WorkingDirectory = "-/var/lib/gitea-runner/${name}";
+
+              # gitea-runner might fail when gitea is restarted during upgrade.
+              Restart = "on-failure";
+              RestartSec = 2;
+
+              ExecStartPre = [
+                (pkgs.writeShellScript "gitea-register-runner-${name}" ''
+                  export INSTANCE_DIR="$STATE_DIRECTORY/${name}"
+                  mkdir -vp "$INSTANCE_DIR"
+                  cd "$INSTANCE_DIR"
+
+                  # force reregistration on changed labels
+                  export LABELS_FILE="$INSTANCE_DIR/.labels"
+                  export LABELS_WANTED="$(echo ${escapeShellArg (concatStringsSep "\n" instance.labels)} | sort)"
+                  export LABELS_CURRENT="$(cat $LABELS_FILE 2>/dev/null || echo 0)"
+
+                  if [ ! -e "$INSTANCE_DIR/.runner" ] || [ "$LABELS_WANTED" != "$LABELS_CURRENT" ]; then
+                    # remove existing registration file, so that changing the labels forces a re-registration
+                    rm -v "$INSTANCE_DIR/.runner" || true
+
+                    # perform the registration
+                    ${cfg.package}/bin/act_runner register --no-interactive \
+                      --instance ${escapeShellArg instance.url} \
+                      --token "$TOKEN" \
+                      --name ${escapeShellArg instance.name} \
+                      --labels ${escapeShellArg (concatStringsSep "," instance.labels)} \
+                      --config ${configFile}
+
+                    # and write back the configured labels
+                    echo "$LABELS_WANTED" > "$LABELS_FILE"
+                  fi
+
+                '')
+              ];
+              ExecStart = "${cfg.package}/bin/act_runner daemon --config ${configFile}";
+              SupplementaryGroups =
+                optionals wantsDocker [
+                  "docker"
+                ]
+                ++ optionals wantsPodman [
+                  "podman"
+                ];
+            }
+            // optionalAttrs (instance.tokenFile != null) {
+              EnvironmentFile = instance.tokenFile;
+            };
+          };
+      in
+      mapAttrs' mkRunnerService cfg.instances;
+  };
+}

nixos/modules/services/continuous-integration/github-runner/options.nix

@@ -0,0 +1,282 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.services.github-runners = lib.mkOption {
+    description = ''
+      Multiple GitHub Runners.
+    '';
+    example = {
+      runner1 = {
+        enable = true;
+        url = "https://github.com/owner/repo";
+        name = "runner1";
+        tokenFile = "/secrets/token1";
+      };
+
+      runner2 = {
+        enable = true;
+        url = "https://github.com/owner/repo";
+        name = "runner2";
+        tokenFile = "/secrets/token2";
+      };
+    };
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, config, ... }:
+        {
+          options = {
+            enable = lib.mkOption {
+              default = false;
+              example = true;
+              description = ''
+                Whether to enable GitHub Actions runner.
+
+                Note: GitHub recommends using self-hosted runners with private repositories only. Learn more here:
+                [About self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners).
+              '';
+              type = lib.types.bool;
+            };
+
+            url = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Repository to add the runner to.
+
+                Changing this option triggers a new runner registration.
+
+                IMPORTANT: If your token is org-wide (not per repository), you need to
+                provide a github org link, not a single repository, so do it like this
+                `https://github.com/nixos`, not like this
+                `https://github.com/nixos/nixpkgs`.
+                Otherwise, you are going to get a `404 NotFound`
+                from `POST https://api.github.com/actions/runner-registration`
+                in the configure script.
+              '';
+              example = "https://github.com/nixos/nixpkgs";
+            };
+
+            tokenFile = lib.mkOption {
+              type = lib.types.path;
+              description = ''
+                The full path to a file which contains either
+
+                * a fine-grained personal access token (PAT),
+                * a classic PAT
+                * or a runner registration token
+
+                Changing this option or the `tokenFile`’s content triggers a new runner registration.
+
+                We suggest using the fine-grained PATs. A runner registration token is valid
+                only for 1 hour after creation, so the next time the runner configuration changes
+                this will give you hard-to-debug HTTP 404 errors in the configure step.
+
+                The file should contain exactly one line with the token without any newline.
+                (Use `echo -n '…token…' > …token file…` to make sure no newlines sneak in.)
+
+                If the file contains a PAT, the service creates a new registration token
+                on startup as needed.
+                If a registration token is given, it can be used to re-register a runner of the same
+                name but is time-limited as noted above.
+
+                For fine-grained PATs:
+
+                Give it "Read and Write access to organization/repository self hosted runners",
+                depending on whether it is organization wide or per-repository. You might have to
+                experiment a little, fine-grained PATs are a `beta` Github feature and still subject
+                to change; nonetheless they are the best option at the moment.
+
+                For classic PATs:
+
+                Make sure the PAT has a scope of `admin:org` for organization-wide registrations
+                or a scope of `repo` for a single repository.
+
+                For runner registration tokens:
+
+                Nothing special needs to be done, but updating will break after one hour,
+                so these are not recommended.
+              '';
+              example = "/run/secrets/github-runner/nixos.token";
+            };
+
+            name = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Name of the runner to configure. If null, defaults to the hostname.
+
+                Changing this option triggers a new runner registration.
+              '';
+              example = "nixos";
+              default = name;
+            };
+
+            runnerGroup = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Name of the runner group to add this runner to (defaults to the default runner group).
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = null;
+            };
+
+            extraLabels = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                Extra labels in addition to the default (unless disabled through the `noDefaultLabels` option).
+
+                Changing this option triggers a new runner registration.
+              '';
+              example = lib.literalExpression ''[ "nixos" ]'';
+              default = [ ];
+            };
+
+            noDefaultLabels = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Disables adding the default labels. Also see the `extraLabels` option.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = false;
+            };
+
+            replace = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Replace any existing runner with the same name.
+
+                Without this flag, registering a new runner with the same name fails.
+              '';
+              default = false;
+            };
+
+            extraPackages = lib.mkOption {
+              type = lib.types.listOf lib.types.package;
+              description = ''
+                Extra packages to add to `PATH` of the service to make them available to workflows.
+              '';
+              default = [ ];
+            };
+
+            extraEnvironment = lib.mkOption {
+              type = lib.types.attrs;
+              description = ''
+                Extra environment variables to set for the runner, as an attrset.
+              '';
+              example = {
+                GIT_CONFIG = "/path/to/git/config";
+              };
+              default = { };
+            };
+
+            serviceOverrides = lib.mkOption {
+              type = lib.types.attrs;
+              description = ''
+                Modify the systemd service. Can be used to, e.g., adjust the sandboxing options.
+                See {manpage}`systemd.exec(5)` for more options.
+              '';
+              example = {
+                ProtectHome = false;
+                RestrictAddressFamilies = [ "AF_PACKET" ];
+              };
+              default = { };
+            };
+
+            package = lib.mkPackageOption pkgs "github-runner" { } // {
+              apply =
+                # Support old github-runner versions which don't have the `nodeRuntimes` arg yet.
+                pkg: pkg.override (old: lib.optionalAttrs (old ? nodeRuntimes) { inherit (config) nodeRuntimes; });
+            };
+
+            ephemeral = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                If enabled, causes the following behavior:
+
+                - Passes the `--ephemeral` flag to the runner configuration script
+                - De-registers and stops the runner with GitHub after it has processed one job
+                - On stop, systemd wipes the runtime directory (this always happens, even without using the ephemeral option)
+                - Restarts the service after its successful exit
+                - On start, wipes the state directory and configures a new runner
+
+                You should only enable this option if `tokenFile` points to a file which contains a
+                personal access token (PAT). If you're using the option with a registration token, restarting the
+                service will fail as soon as the registration token expired.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = false;
+            };
+
+            user = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                User under which to run the service.
+
+                If this option and the `group` option is set to `null`,
+                the service runs as a dynamically allocated user.
+
+                Also see the `group` option for an overview on the effects of the `user` and `group` settings.
+              '';
+              default = null;
+              defaultText = lib.literalExpression "username";
+            };
+
+            group = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Group under which to run the service.
+
+                The effect of this option depends on the value of the `user` option:
+
+                - `group == null` and `user == null`:
+                  The service runs with a dynamically allocated user and group.
+                - `group == null` and `user != null`:
+                  The service runs as the given user and its default group.
+                - `group != null` and `user == null`:
+                  This configuration is invalid. In this case, the service would use the given group
+                  but run as root implicitly. If this is really what you want, set `user = "root"` explicitly.
+              '';
+              default = null;
+              defaultText = lib.literalExpression "groupname";
+            };
+
+            workDir = lib.mkOption {
+              type = with lib.types; nullOr str;
+              description = ''
+                Working directory, available as `$GITHUB_WORKSPACE` during workflow runs
+                and used as a default for [repository checkouts](https://github.com/actions/checkout).
+                The service cleans this directory on every service start.
+
+                A value of `null` will default to the systemd `RuntimeDirectory`.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = null;
+            };
+
+            nodeRuntimes = lib.mkOption {
+              type =
+                with lib.types;
+                nonEmptyListOf (enum [
+                  "node20"
+                  "node24"
+                ]);
+              default = [
+                "node20"
+                "node24"
+              ];
+              description = ''
+                List of Node.js runtimes the runner should support.
+              '';
+            };
+          };
+        }
+      )
+    );
+  };
+}

nixos/modules/services/continuous-integration/github-runner/service.nix

@@ -0,0 +1,333 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  config.assertions = lib.flatten (
+    lib.flip lib.mapAttrsToList config.services.github-runners (
+      name: cfg:
+      map (lib.mkIf cfg.enable) [
+        {
+          assertion = !cfg.noDefaultLabels || (cfg.extraLabels != [ ]);
+          message = "`services.github-runners.${name}`: The `extraLabels` option is mandatory if `noDefaultLabels` is set";
+        }
+        {
+          assertion = cfg.group == null || cfg.user != null;
+          message = ''`services.github-runners.${name}`: Setting `group` while leaving `user` unset runs the service as `root`. If this is really what you want, set `user = "root"` explicitly'';
+        }
+      ]
+    )
+  );
+
+  config.systemd.services =
+    let
+      enabledRunners = lib.filterAttrs (_: cfg: cfg.enable) config.services.github-runners;
+    in
+    (lib.flip lib.mapAttrs' enabledRunners (
+      name: cfg:
+      let
+        svcName = "github-runner-${name}";
+        systemdDir = "github-runner/${name}";
+
+        # %t: Runtime directory root (usually /run); see systemd.unit(5)
+        runtimeDir = "%t/${systemdDir}";
+        # %S: State directory root (usually /var/lib); see systemd.unit(5)
+        stateDir = "%S/${systemdDir}";
+        # %L: Log directory root (usually /var/log); see systemd.unit(5)
+        logsDir = "%L/${systemdDir}";
+        # Name of file stored in service state directory
+        currentConfigTokenFilename = ".current-token";
+
+        workDir = if cfg.workDir == null then runtimeDir else cfg.workDir;
+      in
+      lib.nameValuePair svcName {
+        description = "GitHub Actions runner";
+
+        wantedBy = [ "multi-user.target" ];
+        wants = [ "network-online.target" ];
+        after = [
+          "network.target"
+          "network-online.target"
+        ];
+
+        environment = {
+          HOME = workDir;
+          RUNNER_ROOT = stateDir;
+        }
+        // cfg.extraEnvironment;
+
+        path =
+          (with pkgs; [
+            bashInteractive
+            coreutils
+            git
+            gnutar
+            gzip
+          ])
+          ++ [
+            config.nix.package
+          ]
+          ++ cfg.extraPackages;
+
+        serviceConfig = lib.mkMerge [
+          {
+            ExecStart = "${cfg.package}/bin/Runner.Listener run --startuptype service";
+
+            # Does the following, sequentially:
+            # - If the module configuration or the token has changed, purge the state directory,
+            #   and create the current and the new token file with the contents of the configured
+            #   token. While both files have the same content, only the later is accessible by
+            #   the service user.
+            # - Configure the runner using the new token file. When finished, delete it.
+            # - Set up the directory structure by creating the necessary symlinks.
+            ExecStartPre =
+              let
+                # Wrapper script which expects the full path of the state, working and logs
+                # directory as arguments. Overrides the respective systemd variables to provide
+                # unambiguous directory names. This becomes relevant, for example, if the
+                # caller overrides any of the StateDirectory=, RuntimeDirectory= or LogDirectory=
+                # to contain more than one directory. This causes systemd to set the respective
+                # environment variables with the path of all of the given directories, separated
+                # by a colon.
+                writeScript =
+                  name: lines:
+                  pkgs.writeShellScript "${svcName}-${name}.sh" ''
+                    set -euo pipefail
+
+                    STATE_DIRECTORY="$1"
+                    WORK_DIRECTORY="$2"
+                    LOGS_DIRECTORY="$3"
+
+                    ${lines}
+                  '';
+                runnerRegistrationConfig = lib.getAttrs [
+                  "ephemeral"
+                  "extraLabels"
+                  "name"
+                  "noDefaultLabels"
+                  "runnerGroup"
+                  "tokenFile"
+                  "url"
+                  "workDir"
+                ] cfg;
+                newConfigPath = builtins.toFile "${svcName}-config.json" (builtins.toJSON runnerRegistrationConfig);
+                currentConfigPath = "$STATE_DIRECTORY/.nixos-current-config.json";
+                newConfigTokenPath = "$STATE_DIRECTORY/.new-token";
+                currentConfigTokenPath = "$STATE_DIRECTORY/${currentConfigTokenFilename}";
+
+                runnerCredFiles = [
+                  ".credentials"
+                  ".credentials_rsaparams"
+                  ".runner"
+                ];
+                unconfigureRunner = writeScript "unconfigure" ''
+                  copy_tokens() {
+                    # Copy the configured token file to the state dir and allow the service user to read the file
+                    install --mode=666 ${lib.escapeShellArg cfg.tokenFile} "${newConfigTokenPath}"
+                    # Also copy current file to allow for a diff on the next start
+                    install --mode=600 ${lib.escapeShellArg cfg.tokenFile} "${currentConfigTokenPath}"
+                  }
+                  clean_state() {
+                    find "$STATE_DIRECTORY/" -mindepth 1 -delete
+                    copy_tokens
+                  }
+                  diff_config() {
+                    changed=0
+                    # Check for module config changes
+                    [[ -f "${currentConfigPath}" ]] \
+                      && ${pkgs.diffutils}/bin/diff -q '${newConfigPath}' "${currentConfigPath}" >/dev/null 2>&1 \
+                      || changed=1
+                    # Also check the content of the token file
+                    [[ -f "${currentConfigTokenPath}" ]] \
+                      && ${pkgs.diffutils}/bin/diff -q "${currentConfigTokenPath}" ${lib.escapeShellArg cfg.tokenFile} >/dev/null 2>&1 \
+                      || changed=1
+                    # If the config has changed, remove old state and copy tokens
+                    if [[ "$changed" -eq 1 ]]; then
+                      echo "Config has changed, removing old runner state."
+                      echo "The old runner will still appear in the GitHub Actions UI." \
+                           "You have to remove it manually."
+                      clean_state
+                    fi
+                  }
+                  if [[ "${lib.optionalString cfg.ephemeral "1"}" ]]; then
+                    # In ephemeral mode, we always want to start with a clean state
+                    clean_state
+                  elif [[ "$(ls -A "$STATE_DIRECTORY")" ]]; then
+                    # There are state files from a previous run; diff them to decide if we need a new registration
+                    diff_config
+                  else
+                    # The state directory is entirely empty which indicates a first start
+                    copy_tokens
+                  fi
+                  # Always clean workDir
+                  find -H "$WORK_DIRECTORY" -mindepth 1 -delete
+                '';
+                configureRunner =
+                  writeScript "configure" # bash
+                    ''
+                      if [[ -e "${newConfigTokenPath}" ]]; then
+                        echo "Configuring GitHub Actions Runner"
+                        # shellcheck disable=SC2054  # don't complain about commas in --labels
+                        args=(
+                          --unattended
+                          --disableupdate
+                          --work "$WORK_DIRECTORY"
+                          --url ${lib.escapeShellArg cfg.url}
+                          --labels ${lib.escapeShellArg (lib.concatStringsSep "," cfg.extraLabels)}
+                          ${lib.optionalString (cfg.name != null) "--name ${lib.escapeShellArg cfg.name}"}
+                          ${lib.optionalString cfg.replace "--replace"}
+                          ${lib.optionalString (
+                            cfg.runnerGroup != null
+                          ) "--runnergroup ${lib.escapeShellArg cfg.runnerGroup}"}
+                          ${lib.optionalString cfg.ephemeral "--ephemeral"}
+                          ${lib.optionalString cfg.noDefaultLabels "--no-default-labels"}
+                        )
+                        # If the token file contains a PAT (i.e., it starts with "ghp_" or "github_pat_"), we have to use the --pat option,
+                        # if it is not a PAT, we assume it contains a registration token and use the --token option
+                        token=$(<"${newConfigTokenPath}")
+                        if [[ "$token" =~ ^ghp_* ]] || [[ "$token" =~ ^github_pat_* ]]; then
+                          args+=(--pat "$token")
+                        else
+                          args+=(--token "$token")
+                        fi
+                        ${cfg.package}/bin/Runner.Listener configure "''${args[@]}"
+                        # Move the automatically created _diag dir to the logs dir
+                        mkdir -p  "$STATE_DIRECTORY/_diag"
+                        cp    -r  "$STATE_DIRECTORY/_diag/." "$LOGS_DIRECTORY/"
+                        rm    -rf "$STATE_DIRECTORY/_diag/"
+                        # Cleanup token from config
+                        rm "${newConfigTokenPath}"
+                        # Symlink to new config
+                        ln -s '${newConfigPath}' "${currentConfigPath}"
+                      fi
+                    '';
+                setupWorkDir = writeScript "setup-work-dirs" ''
+                  # Link _diag dir
+                  ln -s "$LOGS_DIRECTORY" "$WORK_DIRECTORY/_diag"
+
+                  # Link the runner credentials to the work dir
+                  ln -s "$STATE_DIRECTORY"/{${lib.concatStringsSep "," runnerCredFiles}} "$WORK_DIRECTORY/"
+                '';
+              in
+              map
+                (
+                  x:
+                  "${x} ${
+                    lib.escapeShellArgs [
+                      stateDir
+                      workDir
+                      logsDir
+                    ]
+                  }"
+                )
+                [
+                  "+${unconfigureRunner}" # runs as root
+                  configureRunner
+                  setupWorkDir
+                ];
+
+            # If running in ephemeral mode, restart the service on-exit (i.e., successful de-registration of the runner)
+            # to trigger a fresh registration.
+            Restart = if cfg.ephemeral then "on-success" else "no";
+            # If the runner exits with `ReturnCode.RetryableError = 2`, always restart the service:
+            # https://github.com/actions/runner/blob/40ed7f8/src/Runner.Common/Constants.cs#L146
+            RestartForceExitStatus = [ 2 ];
+
+            # Contains _diag
+            LogsDirectory = [ systemdDir ];
+            # Default RUNNER_ROOT which contains ephemeral Runner data
+            RuntimeDirectory = [ systemdDir ];
+            # Home of persistent runner data, e.g., credentials
+            StateDirectory = [ systemdDir ];
+            StateDirectoryMode = "0700";
+            WorkingDirectory = workDir;
+
+            InaccessiblePaths = [
+              # Token file path given in the configuration, if visible to the service
+              "-${cfg.tokenFile}"
+              # Token file in the state directory
+              "${stateDir}/${currentConfigTokenFilename}"
+            ];
+
+            KillSignal = "SIGINT";
+
+            # Hardening (may overlap with DynamicUser=)
+            # The following options are only for optimizing:
+            # systemd-analyze security github-runner
+            AmbientCapabilities = lib.mkBefore [ "" ];
+            CapabilityBoundingSet = lib.mkBefore [ "" ];
+            # ProtectClock= adds DeviceAllow=char-rtc r
+            DeviceAllow = lib.mkBefore [ "" ];
+            NoNewPrivileges = lib.mkDefault true;
+            PrivateDevices = lib.mkDefault true;
+            PrivateMounts = lib.mkDefault true;
+            PrivateTmp = lib.mkDefault true;
+            PrivateUsers = lib.mkDefault true;
+            ProtectClock = lib.mkDefault true;
+            ProtectControlGroups = lib.mkDefault true;
+            ProtectHome = lib.mkDefault true;
+            ProtectHostname = lib.mkDefault true;
+            ProtectKernelLogs = lib.mkDefault true;
+            ProtectKernelModules = lib.mkDefault true;
+            ProtectKernelTunables = lib.mkDefault true;
+            ProtectSystem = lib.mkDefault "strict";
+            RemoveIPC = lib.mkDefault true;
+            RestrictNamespaces = lib.mkDefault true;
+            RestrictRealtime = lib.mkDefault true;
+            RestrictSUIDSGID = lib.mkDefault true;
+            UMask = lib.mkDefault "0066";
+            ProtectProc = lib.mkDefault "invisible";
+            SystemCallFilter = lib.mkBefore [
+              "~@clock"
+              "~@cpu-emulation"
+              "~@module"
+              "~@mount"
+              "~@obsolete"
+              "~@raw-io"
+              "~@reboot"
+              "~capset"
+              "~setdomainname"
+              "~sethostname"
+            ];
+            RestrictAddressFamilies = lib.mkBefore [
+              "AF_INET"
+              "AF_INET6"
+              "AF_UNIX"
+              "AF_NETLINK"
+            ];
+
+            BindPaths = lib.optionals (cfg.workDir != null) [ cfg.workDir ];
+
+            # Needs network access
+            PrivateNetwork = lib.mkDefault false;
+            # Cannot be true due to Node
+            MemoryDenyWriteExecute = lib.mkDefault false;
+
+            # The more restrictive "pid" option makes `nix` commands in CI emit
+            # "GC Warning: Couldn't read /proc/stat"
+            # You may want to set this to "pid" if not using `nix` commands
+            ProcSubset = lib.mkDefault "all";
+            # Coverage programs for compiled code such as `cargo-tarpaulin` disable
+            # ASLR (address space layout randomization) which requires the
+            # `personality` syscall
+            # You may want to set this to `true` if not using coverage tooling on
+            # compiled code
+            LockPersonality = lib.mkDefault false;
+
+            DynamicUser = lib.mkDefault true;
+          }
+          (lib.mkIf (cfg.user != null) {
+            DynamicUser = false;
+            User = cfg.user;
+          })
+          (lib.mkIf (cfg.group != null) {
+            DynamicUser = false;
+            Group = cfg.group;
+          })
+          cfg.serviceOverrides
+        ];
+      }
+    ));
+}

nixos/modules/services/continuous-integration/github-runners.nix

@@ -0,0 +1,10 @@
+{ lib, ... }:
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "github-runner" ] "Use `services.github-runners.*` instead")
+    ./github-runner/options.nix
+    ./github-runner/service.nix
+  ];
+
+  meta.maintainers = with lib.maintainers; [ veehaitch ];
+}

nixos/modules/services/continuous-integration/gitlab-runner.nix

@@ -0,0 +1,896 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (builtins)
+    hashString
+    map
+    substring
+    toJSON
+    toString
+    unsafeDiscardStringContext
+    ;
+
+  inherit (lib)
+    any
+    assertMsg
+    attrValues
+    concatStringsSep
+    escapeShellArg
+    filterAttrs
+    hasPrefix
+    isStorePath
+    literalExpression
+    mapAttrs'
+    mapAttrsToList
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    mkRemovedOptionModule
+    mkRenamedOptionModule
+    nameValuePair
+    optional
+    optionalAttrs
+    optionals
+    teams
+    toShellVar
+    types
+    ;
+
+  cfg = config.services.gitlab-runner;
+  hasDocker = config.virtualisation.docker.enable;
+  hasPodman = config.virtualisation.podman.enable && config.virtualisation.podman.dockerSocket.enable;
+
+  /*
+    The whole logic of this module is to diff the hashes of the desired vs existing runners
+    The hash is recorded in the runner's name because we can't do better yet
+    See https://gitlab.com/gitlab-org/gitlab-runner/-/issues/29350 for more details
+  */
+  genRunnerName =
+    name: service:
+    let
+      hash = substring 0 12 (hashString "md5" (unsafeDiscardStringContext (toJSON service)));
+    in
+    if service ? description && service.description != null then
+      "${hash} ${service.description}"
+    else
+      "${name}_${config.networking.hostName}_${hash}";
+
+  hashedServices = mapAttrs' (
+    name: service: nameValuePair (genRunnerName name service) service
+  ) cfg.services;
+  configPath = ''"$HOME"/.gitlab-runner/config.toml'';
+  configureScript = pkgs.writeShellApplication {
+    name = "gitlab-runner-configure";
+    runtimeInputs = [
+      cfg.package
+    ]
+    ++ (with pkgs; [
+      bash
+      gawk
+      jq
+      moreutils
+      remarshal
+      util-linux
+      perl
+      python3
+    ]);
+    text =
+      if (cfg.configFile != null) then
+        ''
+          cp ${cfg.configFile} ${configPath}
+          # make config file readable by service
+          chown -R --reference="$HOME" "$(dirname ${configPath})"
+        ''
+      else
+        ''
+          export CONFIG_FILE=${configPath}
+
+          mkdir -p "$(dirname ${configPath})"
+          touch ${configPath}
+
+          # update global options
+          remarshal --if toml --of json --stringify ${configPath} \
+            | jq -cM 'with_entries(select([.key] | inside(["runners"])))' \
+            | jq -scM '.[0] + .[1]' - <(echo ${escapeShellArg (toJSON cfg.settings)}) \
+            | remarshal --if json --of toml \
+            | sponge ${configPath}
+
+          # remove no longer existing services
+          gitlab-runner verify --delete
+
+          ${toShellVar "NEEDED_SERVICES" (lib.mapAttrs (name: value: 1) hashedServices)}
+
+          declare -A REGISTERED_SERVICES
+
+          while IFS="," read -r name token;
+          do
+            REGISTERED_SERVICES["$name"]="$token"
+          done < <(gitlab-runner --log-format json list 2>&1 | grep Token  | jq -r '.msg +"," + .Token')
+
+          echo "NEEDED_SERVICES: " "''${!NEEDED_SERVICES[@]}"
+          echo "REGISTERED_SERVICES:" "''${!REGISTERED_SERVICES[@]}"
+
+          # difference between current and desired state
+          declare -A NEW_SERVICES
+          for name in "''${!NEEDED_SERVICES[@]}"; do
+            if [ ! -v 'REGISTERED_SERVICES[$name]' ]; then
+              NEW_SERVICES[$name]=1
+            fi
+          done
+
+          declare -A OLD_SERVICES
+          # shellcheck disable=SC2034
+          for name in "''${!REGISTERED_SERVICES[@]}"; do
+            if [ ! -v 'NEEDED_SERVICES[$name]' ]; then
+              OLD_SERVICES[$name]=1
+            fi
+          done
+
+          # register new services
+          ${concatStringsSep "\n" (
+            mapAttrsToList (name: service: ''
+              # TODO so here we should mention NEW_SERVICES
+              if [ -v 'NEW_SERVICES["${name}"]' ] ; then
+                bash -c ${
+                  escapeShellArg (
+                    concatStringsSep " \\\n " (
+                      [
+                        "set -a && source ${
+                          if service.registrationConfigFile != null then
+                            service.registrationConfigFile
+                          else
+                            service.authenticationTokenConfigFile
+                        } &&"
+                        "gitlab-runner register"
+                        "--non-interactive"
+                        "--name '${name}'"
+                        "--executor ${service.executor}"
+                        "--limit ${toString service.limit}"
+                        "--request-concurrency ${toString service.requestConcurrency}"
+                      ]
+                      ++ optional (
+                        service.authenticationTokenConfigFile == null
+                      ) "--maximum-timeout ${toString service.maximumTimeout}"
+                      ++ service.registrationFlags
+                      ++ optional (service.buildsDir != null) "--builds-dir ${service.buildsDir}"
+                      ++ optional (service.cloneUrl != null) "--clone-url ${service.cloneUrl}"
+                      ++ optional (
+                        service.preGetSourcesScript != null
+                      ) "--pre-get-sources-script ${service.preGetSourcesScript}"
+                      ++ optional (
+                        service.postGetSourcesScript != null
+                      ) "--post-get-sources-script ${service.postGetSourcesScript}"
+                      ++ optional (service.preBuildScript != null) "--pre-build-script ${service.preBuildScript}"
+                      ++ optional (service.postBuildScript != null) "--post-build-script ${service.postBuildScript}"
+                      ++ optional (
+                        service.authenticationTokenConfigFile == null && service.tagList != [ ]
+                      ) "--tag-list ${concatStringsSep "," service.tagList}"
+                      ++ optional (service.authenticationTokenConfigFile == null && service.runUntagged) "--run-untagged"
+                      ++ optional (
+                        service.authenticationTokenConfigFile == null && service.protected
+                      ) "--access-level ref_protected"
+                      ++ optional service.debugTraceDisabled "--debug-trace-disabled"
+                      ++ map (e: "--env ${escapeShellArg e}") (
+                        mapAttrsToList (name: value: "${name}=${value}") service.environmentVariables
+                      )
+                      ++ optionals (hasPrefix "docker" service.executor) (
+                        assert (
+                          assertMsg (
+                            service.dockerImage != null
+                          ) "dockerImage option is required for ${service.executor} executor (${name})"
+                        );
+                        [ "--docker-image ${service.dockerImage}" ]
+                        ++ optional service.dockerDisableCache "--docker-disable-cache"
+                        ++ optional service.dockerPrivileged "--docker-privileged"
+                        ++ optional (service.dockerPullPolicy != null) "--docker-pull-policy ${service.dockerPullPolicy}"
+                        ++ map (v: "--docker-volumes ${escapeShellArg v}") service.dockerVolumes
+                        ++ map (v: "--docker-extra-hosts ${escapeShellArg v}") service.dockerExtraHosts
+                        ++ map (v: "--docker-allowed-images ${escapeShellArg v}") service.dockerAllowedImages
+                        ++ map (v: "--docker-allowed-services ${escapeShellArg v}") service.dockerAllowedServices
+                      )
+                    )
+                  )
+                } && sleep 1 || exit 1
+              fi
+            '') hashedServices
+          )}
+
+          # check key is in array https://stackoverflow.com/questions/30353951/how-to-check-if-dictionary-contains-a-key-in-bash
+
+          echo "NEW_SERVICES: ''${NEW_SERVICES[*]}"
+          echo "OLD_SERVICES: ''${OLD_SERVICES[*]}"
+          # unregister old services
+          for NAME in "''${!OLD_SERVICES[@]}"
+          do
+            [ -n "$NAME" ] && gitlab-runner unregister \
+              --name "$NAME" && sleep 1
+          done
+
+          # make config file readable by service
+          chown -R --reference="$HOME" "$(dirname ${configPath})"
+        '';
+  };
+  startScript = pkgs.writeShellScriptBin "gitlab-runner-start" ''
+    export CONFIG_FILE=${configPath}
+    exec gitlab-runner run --working-directory $HOME
+  '';
+in
+{
+  options.services.gitlab-runner = {
+    enable = mkEnableOption "Gitlab Runner";
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Configuration file for gitlab-runner.
+
+        {option}`configFile` takes precedence over {option}`services`.
+        {option}`checkInterval` and {option}`concurrent` will be ignored too.
+
+        This option is deprecated, please use {option}`services` instead.
+        You can use {option}`registrationConfigFile` and
+        {option}`registrationFlags`
+        for settings not covered by this module.
+      '';
+    };
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = (pkgs.formats.json { }).type;
+      };
+      default = { };
+      description = ''
+        Global gitlab-runner configuration. See
+        <https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-global-section>
+        for supported values.
+      '';
+    };
+    gracefulTermination = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Finish all remaining jobs before stopping.
+        If not set gitlab-runner will stop immediately without waiting
+        for jobs to finish, which will lead to failed builds.
+      '';
+    };
+    gracefulTimeout = mkOption {
+      type = types.str;
+      default = "infinity";
+      example = "5min 20s";
+      description = ''
+        Time to wait until a graceful shutdown is turned into a forceful one.
+      '';
+    };
+    package = mkPackageOption pkgs "gitlab-runner" {
+      example = "gitlab-runner_1_11";
+    };
+    extraPackages = mkOption {
+      type = types.listOf types.package;
+      default = [ ];
+      description = ''
+        Extra packages to add to PATH for the gitlab-runner process.
+      '';
+    };
+    services = mkOption {
+      description = "GitLab Runner services.";
+      default = { };
+      example = literalExpression ''
+        {
+          # runner for building in docker via host's nix-daemon
+          # nix store will be readable in runner, might be insecure
+          nix = {
+            # File should contain at least these two variables:
+            # - `CI_SERVER_URL`
+            # - `REGISTRATION_TOKEN`
+            #
+            # NOTE: Support for runner registration tokens will be removed in GitLab 18.0.
+            # Please migrate to runner authentication tokens soon. For reference, the example
+            # runners below this one are configured with authentication tokens instead.
+            registrationConfigFile = "/run/secrets/gitlab-runner-registration";
+
+            dockerImage = "alpine";
+            dockerVolumes = [
+              "/nix/store:/nix/store:ro"
+              "/nix/var/nix/db:/nix/var/nix/db:ro"
+              "/nix/var/nix/daemon-socket:/nix/var/nix/daemon-socket:ro"
+            ];
+            dockerDisableCache = true;
+            preBuildScript = pkgs.writeScript "setup-container" '''
+              mkdir -p -m 0755 /nix/var/log/nix/drvs
+              mkdir -p -m 0755 /nix/var/nix/gcroots
+              mkdir -p -m 0755 /nix/var/nix/profiles
+              mkdir -p -m 0755 /nix/var/nix/temproots
+              mkdir -p -m 0755 /nix/var/nix/userpool
+              mkdir -p -m 1777 /nix/var/nix/gcroots/per-user
+              mkdir -p -m 1777 /nix/var/nix/profiles/per-user
+              mkdir -p -m 0755 /nix/var/nix/profiles/per-user/root
+              mkdir -p -m 0700 "$HOME/.nix-defexpr"
+
+              . ''${pkgs.nix}/etc/profile.d/nix.sh
+
+              ''${pkgs.nix}/bin/nix-env -i ''${concatStringsSep " " (with pkgs; [ nix cacert git openssh ])}
+
+              ''${pkgs.nix}/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable
+              ''${pkgs.nix}/bin/nix-channel --update nixpkgs
+            ''';
+            environmentVariables = {
+              ENV = "/etc/profile";
+              USER = "root";
+              NIX_REMOTE = "daemon";
+              PATH = "/nix/var/nix/profiles/default/bin:/nix/var/nix/profiles/default/sbin:/bin:/sbin:/usr/bin:/usr/sbin";
+              NIX_SSL_CERT_FILE = "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt";
+            };
+            tagList = [ "nix" ];
+          };
+          # runner for building docker images
+          docker-images = {
+            # File should contain at least these two variables:
+            # `CI_SERVER_URL`
+            # `CI_SERVER_TOKEN`
+            authenticationTokenConfigFile = "/run/secrets/gitlab-runner-docker-images-token-env";
+
+            dockerImage = "docker:stable";
+            dockerVolumes = [
+              "/var/run/docker.sock:/var/run/docker.sock"
+            ];
+            tagList = [ "docker-images" ];
+          };
+          # runner for executing stuff on host system (very insecure!)
+          # make sure to add required packages (including git!)
+          # to `environment.systemPackages`
+          shell = {
+            # File should contain at least these two variables:
+            # `CI_SERVER_URL`
+            # `CI_SERVER_TOKEN`
+            authenticationTokenConfigFile = "/run/secrets/gitlab-runner-shell-token-env";
+
+            executor = "shell";
+            tagList = [ "shell" ];
+          };
+          # runner for everything else
+          default = {
+            # File should contain at least these two variables:
+            # `CI_SERVER_URL`
+            # `CI_SERVER_TOKEN`
+            authenticationTokenConfigFile = "/run/secrets/gitlab-runner-default-token-env";
+            dockerImage = "debian:stable";
+          };
+        }
+      '';
+      type = types.attrsOf (
+        types.submodule {
+          options = {
+            authenticationTokenConfigFile = mkOption {
+              type = with types; nullOr path;
+              default = null;
+              description = ''
+                Absolute path to a file containing environment variables used for
+                gitlab-runner registrations with *runner authentication tokens*.
+                They replace the deprecated *runner registration tokens*, as
+                outlined in the [GitLab documentation].
+
+                A list of all supported environment variables can be found with
+                `gitlab-runner register --help`.
+
+                The ones you probably want to set are:
+                - `CI_SERVER_URL=<CI server URL>`
+                - `CI_SERVER_TOKEN=<runner authentication token secret>`
+
+                ::: {.warning}
+                Make sure to use a quoted absolute path,
+                or it is going to be copied to Nix Store.
+                :::
+
+                [GitLab documentation]: https://docs.gitlab.com/17.0/ee/ci/runners/new_creation_workflow.html#estimated-time-frame-for-planned-changes
+              '';
+            };
+            registrationConfigFile = mkOption {
+              type = with types; nullOr path;
+              default = null;
+              description = ''
+                Absolute path to a file with environment variables
+                used for gitlab-runner registration with *runner registration
+                tokens*.
+
+                A list of all supported environment variables can be found in
+                `gitlab-runner register --help`.
+
+                The ones you probably want to set are:
+                - `CI_SERVER_URL=<CI server URL>`
+                - `REGISTRATION_TOKEN=<registration secret>`
+
+                Support for *runner registration tokens* is deprecated since
+                GitLab 16.0, has been disabled by default in GitLab 17.0 and
+                will be removed in GitLab 18.0, as outlined in the
+                [GitLab documentation]. Please consider migrating to
+                [runner authentication tokens] and check the documentation on
+                {option}`services.gitlab-runner.services.<name>.authenticationTokenConfigFile`.
+
+                ::: {.warning}
+                Make sure to use a quoted absolute path,
+                or it is going to be copied to Nix Store.
+                :::
+
+                [GitLab documentation]: https://docs.gitlab.com/17.0/ee/ci/runners/new_creation_workflow.html#estimated-time-frame-for-planned-changes
+                [runner authentication tokens]: https://docs.gitlab.com/17.0/ee/ci/runners/new_creation_workflow.html#the-new-runner-registration-workflow
+              '';
+            };
+            registrationFlags = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [ "--docker-helper-image my/gitlab-runner-helper" ];
+              description = ''
+                Extra command-line flags passed to
+                `gitlab-runner register`.
+                Execute `gitlab-runner register --help`
+                for a list of supported flags.
+              '';
+            };
+            environmentVariables = mkOption {
+              type = types.attrsOf types.str;
+              default = { };
+              example = {
+                NAME = "value";
+              };
+              description = ''
+                Custom environment variables injected to build environment.
+                For secrets you can use {option}`registrationConfigFile`
+                with `RUNNER_ENV` variable set.
+              '';
+            };
+            description = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                Name/description of the runner.
+              '';
+            };
+            executor = mkOption {
+              type = types.str;
+              default = "docker";
+              description = ''
+                Select executor, eg. shell, docker, etc.
+                See [runner documentation](https://docs.gitlab.com/runner/executors/README.html) for more information.
+              '';
+            };
+            buildsDir = mkOption {
+              type = types.nullOr types.path;
+              default = null;
+              example = "/var/lib/gitlab-runner/builds";
+              description = ''
+                Absolute path to a directory where builds will be stored
+                in context of selected executor (Locally, Docker, SSH).
+              '';
+            };
+            cloneUrl = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              example = "http://gitlab.example.local";
+              description = ''
+                Overwrite the URL for the GitLab instance. Used if the Runner can’t connect to GitLab on the URL GitLab exposes itself.
+              '';
+            };
+            dockerImage = mkOption {
+              type = types.nullOr types.str;
+              default = null;
+              description = ''
+                Docker image to be used.
+              '';
+            };
+            dockerPullPolicy = mkOption {
+              type = types.nullOr (
+                types.enum [
+                  "always"
+                  "never"
+                  "if-not-present"
+                ]
+              );
+              default = null;
+              description = ''
+                Default pull-policy for Docker images
+              '';
+            };
+            dockerVolumes = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [ "/var/run/docker.sock:/var/run/docker.sock" ];
+              description = ''
+                Bind-mount a volume and create it
+                if it doesn't exist prior to mounting.
+              '';
+            };
+            dockerDisableCache = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Disable all container caching.
+              '';
+            };
+            dockerPrivileged = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Give extended privileges to container.
+              '';
+            };
+            dockerExtraHosts = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [ "other-host:127.0.0.1" ];
+              description = ''
+                Add a custom host-to-IP mapping.
+              '';
+            };
+            dockerAllowedImages = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [
+                "ruby:*"
+                "python:*"
+                "php:*"
+                "my.registry.tld:5000/*:*"
+              ];
+              description = ''
+                Whitelist allowed images.
+              '';
+            };
+            dockerAllowedServices = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              example = [
+                "postgres:9"
+                "redis:*"
+                "mysql:*"
+              ];
+              description = ''
+                Whitelist allowed services.
+              '';
+            };
+            preGetSourcesScript = mkOption {
+              type = types.nullOr (types.either types.str types.path);
+              default = null;
+              description = ''
+                Runner-specific command script executed before code is pulled.
+              '';
+            };
+            postGetSourcesScript = mkOption {
+              type = types.nullOr (types.either types.str types.path);
+              default = null;
+              description = ''
+                Runner-specific command script executed after code is pulled.
+              '';
+            };
+            preBuildScript = mkOption {
+              type = types.nullOr (types.either types.str types.path);
+              default = null;
+              description = ''
+                Runner-specific command script executed after code is pulled,
+                just before build executes.
+              '';
+            };
+            postBuildScript = mkOption {
+              type = types.nullOr (types.either types.str types.path);
+              default = null;
+              description = ''
+                Runner-specific command script executed after code is pulled
+                and just after build executes.
+              '';
+            };
+            tagList = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              description = ''
+                Tag list.
+
+                This option has no effect for runners registered with an runner
+                authentication tokens and will be ignored.
+              '';
+            };
+            runUntagged = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                Register to run untagged builds; defaults to
+                `true` when {option}`tagList` is empty.
+
+                This option has no effect for runners registered with an runner
+                authentication tokens and will be ignored.
+              '';
+            };
+            limit = mkOption {
+              type = types.int;
+              default = 0;
+              description = ''
+                Limit how many jobs can be handled concurrently by this service.
+                0 (default) simply means don't limit.
+              '';
+            };
+            requestConcurrency = mkOption {
+              type = types.int;
+              default = 0;
+              description = ''
+                Limit number of concurrent requests for new jobs from GitLab.
+              '';
+            };
+            maximumTimeout = mkOption {
+              type = types.int;
+              default = 0;
+              description = ''
+                What is the maximum timeout (in seconds) that will be set for
+                job when using this Runner. 0 (default) simply means don't limit.
+
+                This option has no effect for runners registered with an runner
+                authentication tokens and will be ignored.
+              '';
+            };
+            protected = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                When set to true Runner will only run on pipelines
+                triggered on protected branches.
+
+                This option has no effect for runners registered with an runner
+                authentication tokens and will be ignored.
+              '';
+            };
+            debugTraceDisabled = mkOption {
+              type = types.bool;
+              default = false;
+              description = ''
+                When set to true Runner will disable the possibility of
+                using the `CI_DEBUG_TRACE` feature.
+              '';
+            };
+          };
+        }
+      );
+    };
+    clear-docker-cache = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to periodically prune gitlab runner's Docker resources. If
+          enabled, a systemd timer will run {command}`clear-docker-cache` as
+          specified by the `dates` option.
+        '';
+      };
+
+      flags = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        example = [ "prune" ];
+        description = ''
+          Any additional flags passed to {command}`clear-docker-cache`.
+        '';
+      };
+
+      dates = mkOption {
+        default = "weekly";
+        type = types.str;
+        description = ''
+          Specification (in the format described by
+          {manpage}`systemd.time(7)`) of the time at
+          which the prune will occur.
+        '';
+      };
+
+      package = mkOption {
+        default = config.virtualisation.docker.package;
+        defaultText = literalExpression "config.virtualisation.docker.package";
+        example = literalExpression "pkgs.docker";
+        description = "Docker package to use for clearing up docker cache.";
+      };
+    };
+  };
+  config = mkIf cfg.enable {
+    assertions = mapAttrsToList (name: serviceConfig: {
+      assertion =
+        serviceConfig.registrationConfigFile == null || serviceConfig.authenticationTokenConfigFile == null;
+      message = "`services.gitlab-runner.${name}.registrationConfigFile` and `services.gitlab-runner.services.${name}.authenticationTokenConfigFile` are mutually exclusive.";
+    }) cfg.services;
+
+    warnings =
+      mapAttrsToList (
+        name: serviceConfig:
+        "services.gitlab-runner.services.${name}.`registrationConfigFile` points to a file in Nix Store. You should use quoted absolute path to prevent this."
+      ) (filterAttrs (name: serviceConfig: isStorePath serviceConfig.registrationConfigFile) cfg.services)
+      ++
+        mapAttrsToList
+          (
+            name: serviceConfig:
+            "services.gitlab-runner.services.${name}.`authenticationTokenConfigFile` points to a file in Nix Store. You should use quoted absolute path to prevent this."
+          )
+          (
+            filterAttrs (
+              name: serviceConfig: isStorePath serviceConfig.authenticationTokenConfigFile
+            ) cfg.services
+          )
+      ++
+        mapAttrsToList
+          (name: serviceConfig: ''
+            Runner registration tokens have been deprecated and disabled by default in GitLab >= 17.0.
+            Consider migrating to runner authentication tokens by setting `services.gitlab-runner.services.${name}.authenticationTokenConfigFile`.
+            https://docs.gitlab.com/17.0/ee/ci/runners/new_creation_workflow.html'')
+          (
+            filterAttrs (name: serviceConfig: serviceConfig.authenticationTokenConfigFile == null) cfg.services
+          )
+      ++
+        mapAttrsToList
+          (
+            name: serviceConfig:
+            ''`services.gitlab-runner.services.${name}.protected` with runner authentication tokens has no effect and will be ignored. Please remove it from your configuration.''
+          )
+          (
+            filterAttrs (
+              name: serviceConfig:
+              serviceConfig.authenticationTokenConfigFile != null && serviceConfig.protected == true
+            ) cfg.services
+          )
+      ++
+        mapAttrsToList
+          (
+            name: serviceConfig:
+            ''`services.gitlab-runner.services.${name}.runUntagged` with runner authentication tokens has no effect and will be ignored. Please remove it from your configuration.''
+          )
+          (
+            filterAttrs (
+              name: serviceConfig:
+              serviceConfig.authenticationTokenConfigFile != null && serviceConfig.runUntagged == true
+            ) cfg.services
+          )
+      ++
+        mapAttrsToList
+          (
+            name: v:
+            ''`services.gitlab-runner.services.${name}.maximumTimeout` with runner authentication tokens has no effect and will be ignored. Please remove it from your configuration.''
+          )
+          (
+            filterAttrs (
+              name: serviceConfig:
+              serviceConfig.authenticationTokenConfigFile != null && serviceConfig.maximumTimeout != 0
+            ) cfg.services
+          )
+      ++
+        mapAttrsToList
+          (
+            name: v:
+            ''`services.gitlab-runner.services.${name}.tagList` with runner authentication tokens has no effect and will be ignored. Please remove it from your configuration.''
+          )
+          (
+            filterAttrs (
+              serviceName: serviceConfig:
+              serviceConfig.authenticationTokenConfigFile != null && serviceConfig.tagList != [ ]
+            ) cfg.services
+          );
+
+    environment.systemPackages = [ cfg.package ];
+    systemd.services.gitlab-runner = {
+      description = "Gitlab Runner";
+      documentation = [ "https://docs.gitlab.com/runner/" ];
+      after = [
+        "network.target"
+      ]
+      ++ optional hasDocker "docker.service"
+      ++ optional hasPodman "podman.service";
+
+      requires = optional hasDocker "docker.service" ++ optional hasPodman "podman.service";
+      wantedBy = [ "multi-user.target" ];
+      environment = config.networking.proxy.envVars // {
+        HOME = "/var/lib/gitlab-runner";
+      };
+
+      path =
+        (with pkgs; [
+          bash
+          gawk
+          jq
+          moreutils
+          remarshal
+          util-linux
+        ])
+        ++ [ cfg.package ]
+        ++ cfg.extraPackages;
+
+      reloadIfChanged = true;
+      serviceConfig = {
+        # Set `DynamicUser` under `systemd.services.gitlab-runner.serviceConfig`
+        # to `lib.mkForce false` in your configuration to run this service as root.
+        # You can also set `User` and `Group` options to run this service as desired user.
+        # Make sure to restart service or changes won't apply.
+        DynamicUser = true;
+        StateDirectory = "gitlab-runner";
+        SupplementaryGroups = optional hasDocker "docker" ++ optional hasPodman "podman";
+        ExecStartPre = "!${configureScript}/bin/gitlab-runner-configure";
+        ExecStart = "${startScript}/bin/gitlab-runner-start";
+        ExecReload = "!${configureScript}/bin/gitlab-runner-configure";
+      }
+      // optionalAttrs cfg.gracefulTermination {
+        TimeoutStopSec = "${cfg.gracefulTimeout}";
+        KillSignal = "SIGQUIT";
+        KillMode = "process";
+      };
+    };
+    # Enable periodic clear-docker-cache script
+    systemd.services.gitlab-runner-clear-docker-cache =
+      mkIf (cfg.clear-docker-cache.enable && (any (s: s.executor == "docker") (attrValues cfg.services)))
+        {
+          description = "Prune gitlab-runner docker resources";
+          restartIfChanged = false;
+          unitConfig.X-StopOnRemoval = false;
+
+          serviceConfig.Type = "oneshot";
+
+          path = [
+            cfg.clear-docker-cache.package
+            pkgs.gawk
+          ];
+
+          script = ''
+            ${pkgs.gitlab-runner}/bin/clear-docker-cache ${toString cfg.clear-docker-cache.flags}
+          '';
+
+          startAt = cfg.clear-docker-cache.dates;
+        };
+    # Enable docker if `docker` executor is used in any service
+    virtualisation.docker.enable = mkIf (any (s: s.executor == "docker") (attrValues cfg.services)) (
+      mkDefault true
+    );
+  };
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "packages" ]
+      [ "services" "gitlab-runner" "extraPackages" ]
+    )
+    (mkRemovedOptionModule [
+      "services"
+      "gitlab-runner"
+      "configOptions"
+    ] "Use services.gitlab-runner.services option instead")
+    (mkRemovedOptionModule [
+      "services"
+      "gitlab-runner"
+      "workDir"
+    ] "You should move contents of workDir (if any) to /var/lib/gitlab-runner")
+
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "checkInterval" ]
+      [ "services" "gitlab-runner" "settings" "check_interval" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "concurrent" ]
+      [ "services" "gitlab-runner" "settings" "concurrent" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "sentryDSN" ]
+      [ "services" "gitlab-runner" "settings" "sentry_dsn" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "prometheusListenAddress" ]
+      [ "services" "gitlab-runner" "settings" "listen_address" ]
+    )
+
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "sessionServer" "listenAddress" ]
+      [ "services" "gitlab-runner" "settings" "session_server" "listen_address" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "sessionServer" "advertiseAddress" ]
+      [ "services" "gitlab-runner" "settings" "session_server" "advertise_address" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "gitlab-runner" "sessionServer" "sessionTimeout" ]
+      [ "services" "gitlab-runner" "settings" "session_server" "session_timeout" ]
+    )
+  ];
+
+  meta.maintainers = teams.gitlab.members;
+}

nixos/modules/services/continuous-integration/gocd-agent/default.nix

@@ -0,0 +1,231 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.gocd-agent;
+  opt = options.services.gocd-agent;
+in
+{
+  options = {
+    services.gocd-agent = {
+      enable = lib.mkEnableOption "gocd-agent";
+
+      user = lib.mkOption {
+        default = "gocd-agent";
+        type = lib.types.str;
+        description = ''
+          User the Go.CD agent should execute under.
+        '';
+      };
+
+      group = lib.mkOption {
+        default = "gocd-agent";
+        type = lib.types.str;
+        description = ''
+          If the default user "gocd-agent" is configured then this is the primary
+          group of that user.
+        '';
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [
+          "wheel"
+          "docker"
+        ];
+        description = ''
+          List of extra groups that the "gocd-agent" user should be a part of.
+        '';
+      };
+
+      packages = lib.mkOption {
+        default = [
+          pkgs.stdenv
+          pkgs.jre
+          pkgs.git
+          config.programs.ssh.package
+          pkgs.nix
+        ];
+        defaultText = lib.literalExpression "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]";
+        type = lib.types.listOf lib.types.package;
+        description = ''
+          Packages to add to PATH for the Go.CD agent process.
+        '';
+      };
+
+      agentConfig = lib.mkOption {
+        default = "";
+        type = lib.types.str;
+        example = ''
+          agent.auto.register.resources=ant,java
+          agent.auto.register.environments=QA,Performance
+          agent.auto.register.hostname=Agent01
+        '';
+        description = ''
+          Agent registration configuration.
+        '';
+      };
+
+      goServer = lib.mkOption {
+        default = "https://127.0.0.1:8154/go";
+        type = lib.types.str;
+        description = ''
+          URL of the GoCD Server to attach the Go.CD Agent to.
+        '';
+      };
+
+      workDir = lib.mkOption {
+        default = "/var/lib/go-agent";
+        type = lib.types.str;
+        description = ''
+          Specifies the working directory in which the Go.CD agent java archive resides.
+        '';
+      };
+
+      initialJavaHeapSize = lib.mkOption {
+        default = "128m";
+        type = lib.types.str;
+        description = ''
+          Specifies the initial java heap memory size for the Go.CD agent java process.
+        '';
+      };
+
+      maxJavaHeapMemory = lib.mkOption {
+        default = "256m";
+        type = lib.types.str;
+        description = ''
+          Specifies the java maximum heap memory size for the Go.CD agent java process.
+        '';
+      };
+
+      startupOptions = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [
+          "-Xms${cfg.initialJavaHeapSize}"
+          "-Xmx${cfg.maxJavaHeapMemory}"
+          "-Djava.io.tmpdir=/tmp"
+          "-Dcruise.console.publish.interval=10"
+          "-Djava.security.egd=file:/dev/./urandom"
+        ];
+        defaultText = lib.literalExpression ''
+          [
+            "-Xms''${config.${opt.initialJavaHeapSize}}"
+            "-Xmx''${config.${opt.maxJavaHeapMemory}}"
+            "-Djava.io.tmpdir=/tmp"
+            "-Dcruise.console.publish.interval=10"
+            "-Djava.security.egd=file:/dev/./urandom"
+          ]
+        '';
+        description = ''
+          Specifies startup command line arguments to pass to Go.CD agent
+          java process.
+        '';
+      };
+
+      extraOptions = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        example = [
+          "-X debug"
+          "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5006"
+          "-verbose:gc"
+          "-Xloggc:go-agent-gc.log"
+          "-XX:+PrintGCTimeStamps"
+          "-XX:+PrintTenuringDistribution"
+          "-XX:+PrintGCDetails"
+          "-XX:+PrintGC"
+        ];
+        description = ''
+          Specifies additional command line arguments to pass to Go.CD agent
+          java process.  Example contains debug and gcLog arguments.
+        '';
+      };
+
+      environment = lib.mkOption {
+        default = { };
+        type = with lib.types; attrsOf str;
+        description = ''
+          Additional environment variables to be passed to the Go.CD agent process.
+          As a base environment, Go.CD agent receives NIX_PATH from
+          {option}`environment.sessionVariables`, NIX_REMOTE is set to
+          "daemon".
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.groups = lib.optionalAttrs (cfg.group == "gocd-agent") {
+      gocd-agent.gid = config.ids.gids.gocd-agent;
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "gocd-agent") {
+      gocd-agent = {
+        description = "gocd-agent user";
+        createHome = true;
+        home = cfg.workDir;
+        group = cfg.group;
+        extraGroups = cfg.extraGroups;
+        useDefaultShell = true;
+        uid = config.ids.uids.gocd-agent;
+      };
+    };
+
+    systemd.services.gocd-agent = {
+      description = "GoCD Agent";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment =
+        let
+          selectedSessionVars = lib.filterAttrs (
+            n: v: builtins.elem n [ "NIX_PATH" ]
+          ) config.environment.sessionVariables;
+        in
+        selectedSessionVars
+        // {
+          NIX_REMOTE = "daemon";
+          AGENT_WORK_DIR = cfg.workDir;
+          AGENT_STARTUP_ARGS = ''${lib.concatStringsSep " " cfg.startupOptions}'';
+          LOG_DIR = cfg.workDir;
+          LOG_FILE = "${cfg.workDir}/go-agent-start.log";
+        }
+        // cfg.environment;
+
+      path = cfg.packages;
+
+      script = ''
+        MPATH="''${PATH}";
+        source /etc/profile
+        export PATH="''${MPATH}:''${PATH}";
+
+        if ! test -f ~/.nixpkgs/config.nix; then
+          mkdir -p ~/.nixpkgs/
+          echo "{ allowUnfree = true; }" > ~/.nixpkgs/config.nix
+        fi
+
+        mkdir -p config
+        rm -f config/autoregister.properties
+        ln -s "${pkgs.writeText "autoregister.properties" cfg.agentConfig}" config/autoregister.properties
+
+        ${pkgs.git}/bin/git config --global --add http.sslCAinfo ${config.security.pki.caBundle}
+        ${pkgs.jre}/bin/java ${lib.concatStringsSep " " cfg.startupOptions} \
+                        ${lib.concatStringsSep " " cfg.extraOptions} \
+                              -jar ${pkgs.gocd-agent}/go-agent/agent-bootstrapper.jar \
+                              -serverUrl ${cfg.goServer}
+      '';
+
+      serviceConfig = {
+        User = cfg.user;
+        WorkingDirectory = cfg.workDir;
+        RestartSec = 30;
+        Restart = "on-failure";
+      };
+    };
+  };
+}

nixos/modules/services/continuous-integration/gocd-server/default.nix

@@ -0,0 +1,233 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  cfg = config.services.gocd-server;
+  opt = options.services.gocd-server;
+in
+{
+  options = {
+    services.gocd-server = {
+      enable = mkEnableOption "gocd-server";
+
+      user = mkOption {
+        default = "gocd-server";
+        type = types.str;
+        description = ''
+          User the Go.CD server should execute under.
+        '';
+      };
+
+      group = mkOption {
+        default = "gocd-server";
+        type = types.str;
+        description = ''
+          If the default user "gocd-server" is configured then this is the primary group of that user.
+        '';
+      };
+
+      extraGroups = mkOption {
+        default = [ ];
+        type = types.listOf types.str;
+        example = [
+          "wheel"
+          "docker"
+        ];
+        description = ''
+          List of extra groups that the "gocd-server" user should be a part of.
+        '';
+      };
+
+      listenAddress = mkOption {
+        default = "0.0.0.0";
+        example = "localhost";
+        type = types.str;
+        description = ''
+          Specifies the bind address on which the Go.CD server HTTP interface listens.
+        '';
+      };
+
+      port = mkOption {
+        default = 8153;
+        type = types.port;
+        description = ''
+          Specifies port number on which the Go.CD server HTTP interface listens.
+        '';
+      };
+
+      sslPort = mkOption {
+        default = 8154;
+        type = types.port;
+        description = ''
+          Specifies port number on which the Go.CD server HTTPS interface listens.
+        '';
+      };
+
+      workDir = mkOption {
+        default = "/var/lib/go-server";
+        type = types.str;
+        description = ''
+          Specifies the working directory in which the Go.CD server java archive resides.
+        '';
+      };
+
+      packages = mkOption {
+        default = [
+          pkgs.stdenv
+          pkgs.jre
+          pkgs.git
+          config.programs.ssh.package
+          pkgs.nix
+        ];
+        defaultText = literalExpression "[ pkgs.stdenv pkgs.jre pkgs.git config.programs.ssh.package pkgs.nix ]";
+        type = types.listOf types.package;
+        description = ''
+          Packages to add to PATH for the Go.CD server's process.
+        '';
+      };
+
+      initialJavaHeapSize = mkOption {
+        default = "512m";
+        type = types.str;
+        description = ''
+          Specifies the initial java heap memory size for the Go.CD server's java process.
+        '';
+      };
+
+      maxJavaHeapMemory = mkOption {
+        default = "1024m";
+        type = types.str;
+        description = ''
+          Specifies the java maximum heap memory size for the Go.CD server's java process.
+        '';
+      };
+
+      startupOptions = mkOption {
+        type = types.listOf types.str;
+        default = [
+          "-Xms${cfg.initialJavaHeapSize}"
+          "-Xmx${cfg.maxJavaHeapMemory}"
+          "-Dcruise.listen.host=${cfg.listenAddress}"
+          "-Duser.language=en"
+          "-Djruby.rack.request.size.threshold.bytes=30000000"
+          "-Duser.country=US"
+          "-Dcruise.config.dir=${cfg.workDir}/conf"
+          "-Dcruise.config.file=${cfg.workDir}/conf/cruise-config.xml"
+          "-Dcruise.server.port=${toString cfg.port}"
+          "-Dcruise.server.ssl.port=${toString cfg.sslPort}"
+          "--add-opens=java.base/java.lang=ALL-UNNAMED"
+          "--add-opens=java.base/java.util=ALL-UNNAMED"
+        ];
+        defaultText = literalExpression ''
+          [
+            "-Xms''${config.${opt.initialJavaHeapSize}}"
+            "-Xmx''${config.${opt.maxJavaHeapMemory}}"
+            "-Dcruise.listen.host=''${config.${opt.listenAddress}}"
+            "-Duser.language=en"
+            "-Djruby.rack.request.size.threshold.bytes=30000000"
+            "-Duser.country=US"
+            "-Dcruise.config.dir=''${config.${opt.workDir}}/conf"
+            "-Dcruise.config.file=''${config.${opt.workDir}}/conf/cruise-config.xml"
+            "-Dcruise.server.port=''${toString config.${opt.port}}"
+            "-Dcruise.server.ssl.port=''${toString config.${opt.sslPort}}"
+            "--add-opens=java.base/java.lang=ALL-UNNAMED"
+            "--add-opens=java.base/java.util=ALL-UNNAMED"
+          ]
+        '';
+
+        description = ''
+          Specifies startup command line arguments to pass to Go.CD server
+          java process.
+        '';
+      };
+
+      extraOptions = mkOption {
+        default = [ ];
+        type = types.listOf types.str;
+        example = [
+          "-X debug"
+          "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
+          "-verbose:gc"
+          "-Xloggc:go-server-gc.log"
+          "-XX:+PrintGCTimeStamps"
+          "-XX:+PrintTenuringDistribution"
+          "-XX:+PrintGCDetails"
+          "-XX:+PrintGC"
+        ];
+        description = ''
+          Specifies additional command line arguments to pass to Go.CD server's
+          java process.  Example contains debug and gcLog arguments.
+        '';
+      };
+
+      environment = mkOption {
+        default = { };
+        type = with types; attrsOf str;
+        description = ''
+          Additional environment variables to be passed to the gocd-server process.
+          As a base environment, gocd-server receives NIX_PATH from
+          {option}`environment.sessionVariables`, NIX_REMOTE is set to
+          "daemon".
+        '';
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users.groups = optionalAttrs (cfg.group == "gocd-server") {
+      gocd-server.gid = config.ids.gids.gocd-server;
+    };
+
+    users.users = optionalAttrs (cfg.user == "gocd-server") {
+      gocd-server = {
+        description = "gocd-server user";
+        createHome = true;
+        home = cfg.workDir;
+        group = cfg.group;
+        extraGroups = cfg.extraGroups;
+        useDefaultShell = true;
+        uid = config.ids.uids.gocd-server;
+      };
+    };
+
+    systemd.services.gocd-server = {
+      description = "GoCD Server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment =
+        let
+          selectedSessionVars = lib.filterAttrs (
+            n: v: builtins.elem n [ "NIX_PATH" ]
+          ) config.environment.sessionVariables;
+        in
+        selectedSessionVars
+        // {
+          NIX_REMOTE = "daemon";
+        }
+        // cfg.environment;
+
+      path = cfg.packages;
+
+      script = ''
+        ${pkgs.git}/bin/git config --global --add http.sslCAinfo ${config.security.pki.caBundle}
+        ${pkgs.jre}/bin/java -server ${concatStringsSep " " cfg.startupOptions} \
+                               ${concatStringsSep " " cfg.extraOptions}  \
+                              -jar ${pkgs.gocd-server}/go-server/lib/go.jar
+      '';
+
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.workDir;
+      };
+    };
+  };
+}

nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix

@@ -0,0 +1,125 @@
+/*
+  This file is for options that NixOS and nix-darwin have in common.
+
+  Platform-specific code is in the respective default.nix files.
+*/
+
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  inherit (lib)
+    filterAttrs
+    literalExpression
+    mkIf
+    mkOption
+    mkRemovedOptionModule
+    mkRenamedOptionModule
+    types
+    mkPackageOption
+    ;
+
+  cfg = config.services.hercules-ci-agent;
+
+  inherit (import ./settings.nix { inherit pkgs lib; }) format settingsModule;
+
+in
+{
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "hercules-ci-agent" "extraOptions" ]
+      [ "services" "hercules-ci-agent" "settings" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "hercules-ci-agent" "baseDirectory" ]
+      [ "services" "hercules-ci-agent" "settings" "baseDirectory" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "hercules-ci-agent" "concurrentTasks" ]
+      [ "services" "hercules-ci-agent" "settings" "concurrentTasks" ]
+    )
+    (mkRemovedOptionModule [ "services" "hercules-ci-agent" "patchNix" ]
+      "Nix versions packaged in this version of Nixpkgs don't need a patched nix-daemon to work correctly in Hercules CI Agent clusters."
+    )
+  ];
+
+  options.services.hercules-ci-agent = {
+    enable = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable to run Hercules CI Agent as a system service.
+
+        [Hercules CI](https://hercules-ci.com) is a
+        continuous integation service that is centered around Nix.
+
+        Support is available at [help@hercules-ci.com](mailto:help@hercules-ci.com).
+      '';
+    };
+    package = mkPackageOption pkgs "hercules-ci-agent" { };
+    settings = mkOption {
+      description = ''
+        These settings are written to the `agent.toml` file.
+
+        Not all settings are listed as options, can be set nonetheless.
+
+        For the exhaustive list of settings, see <https://docs.hercules-ci.com/hercules-ci/reference/agent-config/>.
+      '';
+      type = types.submoduleWith { modules = [ settingsModule ]; };
+    };
+
+    /*
+      Internal and/or computed values.
+
+      These are written as options instead of let binding to allow sharing with
+      default.nix on both NixOS and nix-darwin.
+    */
+    tomlFile = mkOption {
+      type = types.path;
+      internal = true;
+      defaultText = lib.literalMD "generated `hercules-ci-agent.toml`";
+      description = ''
+        The fully assembled config file.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # Make sure that nix.extraOptions does not override trusted-users
+    assertions = [
+      {
+        assertion =
+          (cfg.settings.nixUserIsTrusted or false)
+          -> builtins.match ".*(^|\n)[ \t]*trusted-users[ \t]*=.*" config.nix.extraOptions == null;
+        message = ''
+          hercules-ci-agent: Please do not set `trusted-users` in `nix.extraOptions`.
+
+          The hercules-ci-agent module by default relies on `nix.settings.trusted-users`
+          to be effectful, but a line like `trusted-users = ...` in `nix.extraOptions`
+          will override the value set in `nix.settings.trusted-users`.
+
+          Instead of setting `trusted-users` in the `nix.extraOptions` string, you should
+          set an option with additive semantics, such as
+           - the NixOS option `nix.settings.trusted-users`, or
+           - the Nix option in the `extraOptions` string, `extra-trusted-users`
+        '';
+      }
+    ];
+    nix.extraOptions = ''
+      # A store path that was missing at first may well have finished building,
+      # even shortly after the previous lookup. This *also* applies to the daemon.
+      narinfo-cache-negative-ttl = 0
+    '';
+    services.hercules-ci-agent = {
+      tomlFile = format.generate "hercules-ci-agent.toml" cfg.settings;
+      settings.config._module.args = {
+        packageOption = options.services.hercules-ci-agent.package;
+        inherit pkgs;
+      };
+    };
+  };
+}

nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix

@@ -0,0 +1,119 @@
+/*
+  This file is for NixOS-specific options and configs.
+
+  Code that is shared with nix-darwin goes in common.nix.
+*/
+
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}:
+let
+  inherit (lib) mkIf mkDefault;
+
+  cfg = config.services.hercules-ci-agent;
+
+  command = "${cfg.package}/bin/hercules-ci-agent --config ${cfg.tomlFile}";
+  testCommand = "${command} --test-configuration";
+
+in
+{
+  imports = [
+    ./common.nix
+    (lib.mkRenamedOptionModule
+      [ "services" "hercules-ci-agent" "user" ]
+      [ "systemd" "services" "hercules-ci-agent" "serviceConfig" "User" ]
+    )
+  ];
+
+  config = mkIf cfg.enable {
+    systemd.services.hercules-ci-agent = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      path = [ config.nix.package ];
+      startLimitBurst = 30 * 1000000; # practically infinite
+      serviceConfig = {
+        User = "hercules-ci-agent";
+        ExecStart = command;
+        ExecStartPre = testCommand;
+        Restart = "on-failure";
+        RestartSec = 120;
+
+        # If a worker goes OOM, don't kill the main process. It needs to
+        # report the failure and it's unlikely to be part of the problem.
+        OOMPolicy = "continue";
+
+        # Work around excessive stack use by libstdc++ regex
+        # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86164
+        # A 256 MiB stack allows between 400 KiB and 1.5 MiB file to be matched by ".*".
+        LimitSTACK = 256 * 1024 * 1024;
+      };
+    };
+
+    # Changes in the secrets do not affect the unit in any way that would cause
+    # a restart, which is currently necessary to reload the secrets.
+    systemd.paths.hercules-ci-agent-restart-files = {
+      wantedBy = [ "hercules-ci-agent.service" ];
+      pathConfig = {
+        Unit = "hercules-ci-agent-restarter.service";
+        PathChanged = [
+          cfg.settings.clusterJoinTokenPath
+          cfg.settings.binaryCachesPath
+        ];
+      };
+    };
+    systemd.services.hercules-ci-agent-restarter = {
+      serviceConfig.Type = "oneshot";
+      script = ''
+        # Wait a bit, with the effect of bundling up file changes into a single
+        # run of this script and hopefully a single restart.
+        sleep 10
+        if systemctl is-active --quiet hercules-ci-agent.service; then
+          if ${testCommand}; then
+            systemctl restart hercules-ci-agent.service
+          else
+            echo 1>&2 "WARNING: Not restarting agent because config is not valid at this time."
+          fi
+        else
+          echo 1>&2 "Not restarting hercules-ci-agent despite config file update, because it is not already active."
+        fi
+      '';
+    };
+
+    # Trusted user allows simplified configuration and better performance
+    # when operating in a cluster.
+    nix.settings.trusted-users = [ config.systemd.services.hercules-ci-agent.serviceConfig.User ];
+    services.hercules-ci-agent = {
+      settings = {
+        nixUserIsTrusted = true;
+        labels =
+          let
+            mkIfNotNull = x: mkIf (x != null) x;
+          in
+          {
+            nixos.configurationRevision = mkIfNotNull config.system.configurationRevision;
+            nixos.release = config.system.nixos.release;
+            nixos.label = mkIfNotNull config.system.nixos.label;
+            nixos.codeName = config.system.nixos.codeName;
+            nixos.tags = config.system.nixos.tags;
+            nixos.systemName = mkIfNotNull config.system.name;
+          };
+      };
+    };
+
+    users.users.hercules-ci-agent = {
+      home = cfg.settings.baseDirectory;
+      createHome = true;
+      group = "hercules-ci-agent";
+      description = "Hercules CI Agent system user";
+      isSystemUser = true;
+    };
+
+    users.groups.hercules-ci-agent = { };
+  };
+
+  meta.maintainers = [ lib.maintainers.roberth ];
+}

nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix

@@ -0,0 +1,161 @@
+# Not a module
+{ pkgs, lib }:
+let
+  inherit (lib)
+    types
+    literalExpression
+    mkOption
+    ;
+
+  format = pkgs.formats.toml { };
+
+  settingsModule =
+    {
+      config,
+      packageOption,
+      pkgs,
+      ...
+    }:
+    {
+      freeformType = format.type;
+      options = {
+        apiBaseUrl = mkOption {
+          description = ''
+            API base URL that the agent will connect to.
+
+            When using Hercules CI Enterprise, set this to the URL where your
+            Hercules CI server is reachable.
+          '';
+          type = types.str;
+          default = "https://hercules-ci.com";
+        };
+        baseDirectory = mkOption {
+          type = types.path;
+          default = "/var/lib/hercules-ci-agent";
+          description = ''
+            State directory (secrets, work directory, etc) for agent
+          '';
+        };
+        concurrentTasks = mkOption {
+          description = ''
+            Number of tasks to perform simultaneously.
+
+            A task is a single derivation build, an evaluation or an effect run.
+            At minimum, you need 2 concurrent tasks for `x86_64-linux`
+            in your cluster, to allow for import from derivation.
+
+            `concurrentTasks` can be around the CPU core count or lower if memory is
+            the bottleneck.
+
+            The optimal value depends on the resource consumption characteristics of your workload,
+            including memory usage and in-task parallelism. This is typically determined empirically.
+
+            When scaling, it is generally better to have a double-size machine than two machines,
+            because each split of resources causes inefficiencies; particularly with regards
+            to build latency because of extra downloads.
+          '';
+          type = types.either types.ints.positive (types.enum [ "auto" ]);
+          default = "auto";
+          defaultText = lib.literalMD ''
+            `"auto"`, meaning equal to the number of CPU cores.
+          '';
+        };
+        labels = mkOption {
+          description = ''
+            A key-value map of user data.
+
+            This data will be available to organization members in the dashboard and API.
+
+            The values can be of any TOML type that corresponds to a JSON type, but arrays
+            can not contain tables/objects due to limitations of the TOML library. Values
+            involving arrays of non-primitive types may not be representable currently.
+          '';
+          type = format.type;
+          defaultText = literalExpression ''
+            {
+              agent.source = "..."; # One of "nixpkgs", "flake", "override"
+              lib.version = "...";
+              pkgs.version = "...";
+            }
+          '';
+        };
+        workDirectory = mkOption {
+          description = ''
+            The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation.
+          '';
+          type = types.path;
+          default = config.baseDirectory + "/work";
+          defaultText = literalExpression ''baseDirectory + "/work"'';
+        };
+        staticSecretsDirectory = mkOption {
+          description = ''
+            This is the default directory to look for statically configured secrets like `cluster-join-token.key`.
+
+            See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration.
+          '';
+          type = types.path;
+          default = config.baseDirectory + "/secrets";
+          defaultText = literalExpression ''baseDirectory + "/secrets"'';
+        };
+        clusterJoinTokenPath = mkOption {
+          description = ''
+            Location of the cluster-join-token.key file.
+
+            You can retrieve the contents of the file when creating a new agent via
+            <https://hercules-ci.com/dashboard>.
+
+            As this value is confidential, it should not be in the store, but
+            installed using other means, such as agenix, NixOps
+            `deployment.keys`, or manual installation.
+
+            The contents of the file are used for authentication between the agent and the API.
+          '';
+          type = types.path;
+          default = config.staticSecretsDirectory + "/cluster-join-token.key";
+          defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"'';
+        };
+        binaryCachesPath = mkOption {
+          description = ''
+            Path to a JSON file containing binary cache secret keys.
+
+            As these values are confidential, they should not be in the store, but
+            copied over using other means, such as agenix, NixOps
+            `deployment.keys`, or manual installation.
+
+            The format is described on <https://docs.hercules-ci.com/hercules-ci-agent/binary-caches-json/>.
+          '';
+          type = types.path;
+          default = config.staticSecretsDirectory + "/binary-caches.json";
+          defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"'';
+        };
+        secretsJsonPath = mkOption {
+          description = ''
+            Path to a JSON file containing secrets for effects.
+
+            As these values are confidential, they should not be in the store, but
+            copied over using other means, such as agenix, NixOps
+            `deployment.keys`, or manual installation.
+
+            The format is described on <https://docs.hercules-ci.com/hercules-ci-agent/secrets-json/>.
+          '';
+          type = types.path;
+          default = config.staticSecretsDirectory + "/secrets.json";
+          defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"'';
+        };
+      };
+      config = {
+        labels = {
+          agent.source =
+            if packageOption.highestPrio == (lib.modules.mkOptionDefault { }).priority then
+              "nixpkgs"
+            else
+              lib.mkOptionDefault "override";
+          pkgs.version = pkgs.lib.version;
+          lib.version = lib.version;
+        };
+      };
+    };
+in
+{
+  inherit format settingsModule;
+}

nixos/modules/services/continuous-integration/hydra/default.nix

@@ -0,0 +1,578 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+
+  cfg = config.services.hydra;
+
+  baseDir = "/var/lib/hydra";
+
+  hydraConf = pkgs.writeScript "hydra.conf" cfg.extraConfig;
+
+  hydraEnv = {
+    HYDRA_DBI = cfg.dbi;
+    HYDRA_CONFIG = "${baseDir}/hydra.conf";
+    HYDRA_DATA = "${baseDir}";
+  };
+
+  env = {
+    NIX_REMOTE = "daemon";
+    PGPASSFILE = "${baseDir}/pgpass";
+    NIX_REMOTE_SYSTEMS = lib.concatStringsSep ":" cfg.buildMachinesFiles;
+  }
+  // lib.optionalAttrs (cfg.smtpHost != null) {
+    EMAIL_SENDER_TRANSPORT = "SMTP";
+    EMAIL_SENDER_TRANSPORT_host = cfg.smtpHost;
+  }
+  // hydraEnv
+  // cfg.extraEnv;
+
+  serverEnv =
+    env
+    // {
+      HYDRA_TRACKER = cfg.tracker;
+      XDG_CACHE_HOME = "${baseDir}/www/.cache";
+      COLUMNS = "80";
+      PGPASSFILE = "${baseDir}/pgpass-www"; # grrr
+    }
+    // (lib.optionalAttrs cfg.debugServer { DBIC_TRACE = "1"; });
+
+  localDB = "dbi:Pg:dbname=hydra;user=hydra;";
+
+  haveLocalDB = cfg.dbi == localDB;
+
+  hydra-package =
+    let
+      makeWrapperArgs = lib.concatStringsSep " " (
+        lib.mapAttrsToList (key: value: "--set-default \"${key}\" \"${value}\"") hydraEnv
+      );
+    in
+    pkgs.buildEnv rec {
+      name = "hydra-env";
+      nativeBuildInputs = [ pkgs.makeWrapper ];
+      paths = [ cfg.package ];
+
+      postBuild = ''
+        if [ -L "$out/bin" ]; then
+            unlink "$out/bin"
+        fi
+        mkdir -p "$out/bin"
+
+        for path in ${lib.concatStringsSep " " paths}; do
+          if [ -d "$path/bin" ]; then
+            cd "$path/bin"
+            for prg in *; do
+              if [ -f "$prg" ]; then
+                rm -f "$out/bin/$prg"
+                if [ -x "$prg" ]; then
+                  makeWrapper "$path/bin/$prg" "$out/bin/$prg" ${makeWrapperArgs}
+                fi
+              fi
+            done
+          fi
+        done
+      '';
+    };
+
+in
+
+{
+  ###### interface
+  options = {
+
+    services.hydra = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to run Hydra services.
+        '';
+      };
+
+      dbi = lib.mkOption {
+        type = lib.types.str;
+        default = localDB;
+        example = "dbi:Pg:dbname=hydra;host=postgres.example.org;user=foo;";
+        description = ''
+          The DBI string for Hydra database connection.
+
+          NOTE: Attempts to set `application_name` will be overridden by
+          `hydra-TYPE` (where TYPE is e.g. `evaluator`, `queue-runner`,
+          etc.) in all hydra services to more easily distinguish where
+          queries are coming from.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "hydra" { };
+
+      hydraURL = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          The base URL for the Hydra webserver instance. Used for links in emails.
+        '';
+      };
+
+      listenHost = lib.mkOption {
+        type = lib.types.str;
+        default = "*";
+        example = "localhost";
+        description = ''
+          The hostname or address to listen on or `*` to listen
+          on all interfaces.
+        '';
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 3000;
+        description = ''
+          TCP port the web server should listen to.
+        '';
+      };
+
+      minimumDiskFree = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = ''
+          Threshold of minimum disk space (GiB) to determine if the queue runner should run or not.
+        '';
+      };
+
+      minimumDiskFreeEvaluator = lib.mkOption {
+        type = lib.types.int;
+        default = 0;
+        description = ''
+          Threshold of minimum disk space (GiB) to determine if the evaluator should run or not.
+        '';
+      };
+
+      notificationSender = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          Sender email address used for email notifications.
+        '';
+      };
+
+      smtpHost = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "localhost";
+        description = ''
+          Hostname of the SMTP server to use to send email.
+        '';
+      };
+
+      tracker = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = ''
+          Piece of HTML that is included on all pages.
+        '';
+      };
+
+      logo = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = ''
+          Path to a file containing the logo of your Hydra instance.
+        '';
+      };
+
+      debugServer = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to run the server in debug mode.";
+      };
+
+      maxServers = lib.mkOption {
+        type = lib.types.int;
+        default = 25;
+        description = "Maximum number of starman workers to spawn.";
+      };
+
+      minSpareServers = lib.mkOption {
+        type = lib.types.int;
+        default = 4;
+        description = "Minimum number of spare starman workers to keep.";
+      };
+
+      maxSpareServers = lib.mkOption {
+        type = lib.types.int;
+        default = 5;
+        description = "Maximum number of spare starman workers to keep.";
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra lines for the Hydra configuration.";
+      };
+
+      extraEnv = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        default = { };
+        description = "Extra environment variables for Hydra.";
+      };
+
+      gcRootsDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/nix/var/nix/gcroots/hydra";
+        description = "Directory that holds Hydra garbage collector roots.";
+      };
+
+      buildMachinesFiles = lib.mkOption {
+        type = lib.types.listOf lib.types.path;
+        default = lib.optional (config.nix.buildMachines != [ ]) "/etc/nix/machines";
+        defaultText = lib.literalExpression ''lib.optional (config.nix.buildMachines != []) "/etc/nix/machines"'';
+        example = [
+          "/etc/nix/machines"
+          "/var/lib/hydra/provisioner/machines"
+        ];
+        description = "List of files containing build machines.";
+      };
+
+      useSubstitutes = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to use binary caches for downloading store paths. Note that
+          binary substitutions trigger (a potentially large number of) additional
+          HTTP requests that slow down the queue monitor thread significantly.
+          Also, this Hydra instance will serve those downloaded store paths to
+          its users with its own signature attached as if it had built them
+          itself, so don't enable this feature unless your active binary caches
+          are absolute trustworthy.
+        '';
+      };
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.maxServers != 0 && cfg.maxSpareServers != 0 && cfg.minSpareServers != 0;
+        message = "services.hydra.{minSpareServers,maxSpareServers,minSpareServers} cannot be 0";
+      }
+      {
+        assertion = cfg.minSpareServers < cfg.maxSpareServers;
+        message = "services.hydra.minSpareServers cannot be bigger than services.hydra.maxSpareServers";
+      }
+    ];
+
+    users.groups.hydra = {
+      gid = config.ids.gids.hydra;
+    };
+
+    users.users.hydra = {
+      description = "Hydra";
+      group = "hydra";
+      # We don't enable `createHome` here because the creation of the home directory is handled by the hydra-init service below.
+      home = baseDir;
+      useDefaultShell = true;
+      uid = config.ids.uids.hydra;
+    };
+
+    users.users.hydra-queue-runner = {
+      description = "Hydra queue runner";
+      group = "hydra";
+      useDefaultShell = true;
+      home = "${baseDir}/queue-runner"; # really only to keep SSH happy
+      uid = config.ids.uids.hydra-queue-runner;
+    };
+
+    users.users.hydra-www = {
+      description = "Hydra web server";
+      group = "hydra";
+      useDefaultShell = true;
+      uid = config.ids.uids.hydra-www;
+    };
+
+    services.hydra.extraConfig = ''
+      using_frontend_proxy = 1
+      base_uri = ${cfg.hydraURL}
+      notification_sender = ${cfg.notificationSender}
+      max_servers = ${toString cfg.maxServers}
+      ${lib.optionalString (cfg.logo != null) ''
+        hydra_logo = ${cfg.logo}
+      ''}
+      gc_roots_dir = ${cfg.gcRootsDir}
+      use-substitutes = ${if cfg.useSubstitutes then "1" else "0"}
+    '';
+
+    environment.systemPackages = [ hydra-package ];
+
+    environment.variables = hydraEnv;
+
+    nix.settings = lib.mkMerge [
+      {
+        keep-outputs = true;
+        keep-derivations = true;
+        trusted-users = [ "hydra-queue-runner" ];
+      }
+
+      (lib.mkIf (lib.versionOlder (lib.getVersion config.nix.package.out) "2.4pre") {
+        # The default (`true') slows Nix down a lot since the build farm
+        # has so many GC roots.
+        gc-check-reachability = false;
+      })
+    ];
+
+    systemd.slices.system-hydra = {
+      description = "Hydra CI Server Slice";
+      documentation = [
+        "file://${cfg.package}/share/doc/hydra/index.html"
+        "https://nixos.org/hydra/manual/"
+      ];
+    };
+
+    systemd.services.hydra-init = {
+      wantedBy = [ "multi-user.target" ];
+      requires = lib.optional haveLocalDB "postgresql.target";
+      after = lib.optional haveLocalDB "postgresql.target";
+      environment = env // {
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-init";
+      };
+      path = [ pkgs.util-linux ];
+      preStart = ''
+        mkdir -p ${baseDir}
+        chown hydra:hydra ${baseDir}
+        chmod 0750 ${baseDir}
+
+        ln -sf ${hydraConf} ${baseDir}/hydra.conf
+
+        mkdir -m 0700 ${baseDir}/www || true
+        chown hydra-www:hydra ${baseDir}/www
+
+        mkdir -m 0700 ${baseDir}/queue-runner || true
+        mkdir -m 0750 ${baseDir}/build-logs || true
+        mkdir -m 0750 ${baseDir}/runcommand-logs || true
+        chown hydra-queue-runner:hydra \
+          ${baseDir}/queue-runner \
+          ${baseDir}/build-logs \
+          ${baseDir}/runcommand-logs
+
+        ${lib.optionalString haveLocalDB ''
+          if ! [ -e ${baseDir}/.db-created ]; then
+            runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createuser hydra
+            runuser -u ${config.services.postgresql.superUser} ${config.services.postgresql.package}/bin/createdb -- -O hydra hydra
+            touch ${baseDir}/.db-created
+          fi
+          echo "create extension if not exists pg_trgm" | runuser -u ${config.services.postgresql.superUser} -- ${config.services.postgresql.package}/bin/psql hydra
+        ''}
+
+        if [ ! -e ${cfg.gcRootsDir} ]; then
+
+          # Move legacy roots directory.
+          if [ -e /nix/var/nix/gcroots/per-user/hydra/hydra-roots ]; then
+            mv /nix/var/nix/gcroots/per-user/hydra/hydra-roots ${cfg.gcRootsDir}
+          fi
+
+          mkdir -p ${cfg.gcRootsDir}
+        fi
+
+        # Move legacy hydra-www roots.
+        if [ -e /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots ]; then
+          find /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots/ -type f -print0 \
+            | xargs -0 -r mv -f -t ${cfg.gcRootsDir}/
+          rmdir /nix/var/nix/gcroots/per-user/hydra-www/hydra-roots
+        fi
+
+        chown hydra:hydra ${cfg.gcRootsDir}
+        chmod 2775 ${cfg.gcRootsDir}
+      '';
+      serviceConfig.ExecStart = "${hydra-package}/bin/hydra-init";
+      serviceConfig.PermissionsStartOnly = true;
+      serviceConfig.User = "hydra";
+      serviceConfig.Type = "oneshot";
+      serviceConfig.RemainAfterExit = true;
+      serviceConfig.Slice = "system-hydra.slice";
+    };
+
+    systemd.services.hydra-server = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "hydra-init.service" ];
+      after = [ "hydra-init.service" ];
+      environment = serverEnv // {
+        HYDRA_DBI = "${serverEnv.HYDRA_DBI};application_name=hydra-server";
+      };
+      restartTriggers = [ hydraConf ];
+      serviceConfig = {
+        ExecStart =
+          "@${hydra-package}/bin/hydra-server hydra-server -f -h '${cfg.listenHost}' "
+          + "-p ${toString cfg.port} --min_spare_servers ${toString cfg.minSpareServers} --max_spare_servers ${toString cfg.maxSpareServers} "
+          + "--max_servers ${toString cfg.maxServers} --max_requests 100 ${lib.optionalString cfg.debugServer "-d"}";
+        User = "hydra-www";
+        PermissionsStartOnly = true;
+        Restart = "always";
+        Slice = "system-hydra.slice";
+      };
+    };
+
+    systemd.services.hydra-queue-runner = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "hydra-init.service" ];
+      after = [
+        "hydra-init.service"
+        "network.target"
+      ];
+      path = [
+        config.nix.package
+        hydra-package
+        pkgs.bzip2
+        pkgs.hostname-debian
+        pkgs.openssh
+      ];
+      restartTriggers = [ hydraConf ];
+      environment = env // {
+        PGPASSFILE = "${baseDir}/pgpass-queue-runner"; # grrr
+        IN_SYSTEMD = "1"; # to get log severity levels
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-queue-runner";
+      };
+      serviceConfig = {
+        ExecStart = "@${hydra-package}/bin/hydra-queue-runner hydra-queue-runner -v";
+        ExecStopPost = "${hydra-package}/bin/hydra-queue-runner --unlock";
+        User = "hydra-queue-runner";
+        Restart = "always";
+        Slice = "system-hydra.slice";
+
+        # Ensure we can get core dumps.
+        LimitCORE = "infinity";
+        WorkingDirectory = "${baseDir}/queue-runner";
+      };
+    };
+
+    systemd.services.hydra-evaluator = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "hydra-init.service" ];
+      wants = [ "network-online.target" ];
+      after = [
+        "hydra-init.service"
+        "network.target"
+        "network-online.target"
+      ];
+      path = with pkgs; [
+        hostname-debian
+        hydra-package
+        jq
+      ];
+      restartTriggers = [ hydraConf ];
+      environment = env // {
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-evaluator";
+      };
+      serviceConfig = {
+        ExecStart = "@${hydra-package}/bin/hydra-evaluator hydra-evaluator";
+        User = "hydra";
+        Restart = "always";
+        WorkingDirectory = baseDir;
+        Slice = "system-hydra.slice";
+      };
+    };
+
+    systemd.services.hydra-update-gc-roots = {
+      requires = [ "hydra-init.service" ];
+      after = [ "hydra-init.service" ];
+      environment = env // {
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-update-gc-roots";
+      };
+      serviceConfig = {
+        ExecStart = "@${hydra-package}/bin/hydra-update-gc-roots hydra-update-gc-roots";
+        User = "hydra";
+        Slice = "system-hydra.slice";
+      };
+      startAt = "2,14:15";
+    };
+
+    systemd.services.hydra-send-stats = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "hydra-init.service" ];
+      environment = env // {
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-send-stats";
+      };
+      serviceConfig = {
+        ExecStart = "@${hydra-package}/bin/hydra-send-stats hydra-send-stats";
+        User = "hydra";
+        Slice = "system-hydra.slice";
+      };
+    };
+
+    systemd.services.hydra-notify = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "hydra-init.service" ];
+      after = [ "hydra-init.service" ];
+      restartTriggers = [ hydraConf ];
+      path = [ pkgs.zstd ];
+      environment = env // {
+        PGPASSFILE = "${baseDir}/pgpass-queue-runner";
+        HYDRA_DBI = "${env.HYDRA_DBI};application_name=hydra-notify";
+      };
+      serviceConfig = {
+        ExecStart = "@${hydra-package}/bin/hydra-notify hydra-notify";
+        # FIXME: run this under a less privileged user?
+        User = "hydra-queue-runner";
+        Restart = "always";
+        RestartSec = 5;
+        Slice = "system-hydra.slice";
+      };
+    };
+
+    # If there is less than a certain amount of free disk space, stop
+    # the queue/evaluator to prevent builds from failing or aborting.
+    systemd.services.hydra-check-space = {
+      script = ''
+        if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFree} * 1024**3)) ]; then
+            echo "stopping Hydra queue runner due to lack of free space..."
+            systemctl stop hydra-queue-runner
+        fi
+        if [ $(($(stat -f -c '%a' /nix/store) * $(stat -f -c '%S' /nix/store))) -lt $((${toString cfg.minimumDiskFreeEvaluator} * 1024**3)) ]; then
+            echo "stopping Hydra evaluator due to lack of free space..."
+            systemctl stop hydra-evaluator
+        fi
+      '';
+      startAt = "*:0/5";
+      serviceConfig.Slice = "system-hydra.slice";
+    };
+
+    # Periodically compress build logs. The queue runner compresses
+    # logs automatically after a step finishes, but this doesn't work
+    # if the queue runner is stopped prematurely.
+    systemd.services.hydra-compress-logs = {
+      path = [
+        pkgs.bzip2
+        pkgs.zstd
+      ];
+      script = ''
+        set -eou pipefail
+        compression=$(sed -nr 's/compress_build_logs_compression = ()/\1/p' ${baseDir}/hydra.conf)
+        if [[ $compression == "" || $compression == bzip2 ]]; then
+          compressionCmd=(bzip2)
+        elif [[ $compression == zstd ]]; then
+          compressionCmd=(zstd --rm)
+        fi
+        find ${baseDir}/build-logs -ignore_readdir_race -type f -name "*.drv" -mtime +3 -size +0c -print0 | xargs -0 -r "''${compressionCmd[@]}" --force --quiet
+      '';
+      startAt = "Sun 01:45";
+      serviceConfig.Slice = "system-hydra.slice";
+    };
+
+    services.postgresql.enable = lib.mkIf haveLocalDB true;
+
+    services.postgresql.identMap = lib.optionalString haveLocalDB ''
+      hydra hydra hydra
+      hydra hydra-queue-runner hydra
+      hydra hydra-www hydra
+      hydra root hydra
+    '';
+
+    services.postgresql.authentication = lib.optionalString haveLocalDB ''
+      local all hydra peer map=hydra
+    '';
+
+  };
+
+}

nixos/modules/services/continuous-integration/jenkins/default.nix

@@ -0,0 +1,287 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.jenkins;
+  jenkinsUrl = "http://${cfg.listenAddress}:${toString cfg.port}${cfg.prefix}";
+in
+{
+  options = {
+    services.jenkins = {
+      enable = lib.mkEnableOption "Jenkins, a continuous integration server";
+
+      user = lib.mkOption {
+        default = "jenkins";
+        type = lib.types.str;
+        description = ''
+          User the jenkins server should execute under.
+        '';
+      };
+
+      group = lib.mkOption {
+        default = "jenkins";
+        type = lib.types.str;
+        description = ''
+          If the default user "jenkins" is configured then this is the primary
+          group of that user.
+        '';
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [
+          "wheel"
+          "dialout"
+        ];
+        description = ''
+          List of extra groups that the "jenkins" user should be a part of.
+        '';
+      };
+
+      home = lib.mkOption {
+        default = "/var/lib/jenkins";
+        type = lib.types.path;
+        description = ''
+          The path to use as JENKINS_HOME. If the default user "jenkins" is configured then
+          this is the home of the "jenkins" user.
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        default = "0.0.0.0";
+        example = "localhost";
+        type = lib.types.str;
+        description = ''
+          Specifies the bind address on which the jenkins HTTP interface listens.
+          The default is the wildcard address.
+        '';
+      };
+
+      port = lib.mkOption {
+        default = 8080;
+        type = lib.types.port;
+        description = ''
+          Specifies port number on which the jenkins HTTP interface listens.
+          The default is 8080.
+        '';
+      };
+
+      prefix = lib.mkOption {
+        default = "";
+        example = "/jenkins";
+        type = lib.types.str;
+        description = ''
+          Specifies a urlPrefix to use with jenkins.
+          If the example /jenkins is given, the jenkins server will be
+          accessible using localhost:8080/jenkins.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "jenkins" { };
+
+      javaPackage = lib.mkPackageOption pkgs "jdk21" { };
+
+      packages = lib.mkOption {
+        default = [
+          pkgs.stdenv
+          pkgs.git
+          pkgs.jdk21
+          config.programs.ssh.package
+          pkgs.nix
+        ];
+        defaultText = lib.literalExpression "[ pkgs.stdenv pkgs.git pkgs.jdk17 config.programs.ssh.package pkgs.nix ]";
+        type = lib.types.listOf lib.types.package;
+        description = ''
+          Packages to add to PATH for the jenkins process.
+        '';
+      };
+
+      environment = lib.mkOption {
+        default = { };
+        type = with lib.types; attrsOf str;
+        description = ''
+          Additional environment variables to be passed to the jenkins process.
+          As a base environment, jenkins receives NIX_PATH from
+          {option}`environment.sessionVariables`, NIX_REMOTE is set to
+          "daemon" and JENKINS_HOME is set to the value of
+          {option}`services.jenkins.home`.
+          This option has precedence and can be used to override those
+          mentioned variables.
+        '';
+      };
+
+      plugins = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr (lib.types.attrsOf lib.types.package);
+        description = ''
+          A set of plugins to activate. Note that this will completely
+          remove and replace any previously installed plugins. If you
+          have manually-installed plugins that you want to keep while
+          using this module, set this option to
+          `null`. You can generate this set with a
+          tool such as `jenkinsPlugins2nix`.
+        '';
+        example = lib.literalExpression ''
+          import path/to/jenkinsPlugins2nix-generated-plugins.nix { inherit (pkgs) fetchurl stdenv; }
+        '';
+      };
+
+      extraOptions = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [ "--debug=9" ];
+        description = ''
+          Additional command line arguments to pass to Jenkins.
+        '';
+      };
+
+      extraJavaOptions = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [ "-Xmx80m" ];
+        description = ''
+          Additional command line arguments to pass to the Java run time (as opposed to Jenkins).
+        '';
+      };
+
+      withCLI = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to make the CLI available.
+
+          More info about the CLI available at
+          [
+          https://www.jenkins.io/doc/book/managing/cli](https://www.jenkins.io/doc/book/managing/cli) .
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment = {
+      # server references the dejavu fonts
+      systemPackages = [
+        pkgs.dejavu_fonts
+      ]
+      ++ lib.optional cfg.withCLI cfg.package;
+
+      variables =
+        { }
+        // lib.optionalAttrs cfg.withCLI {
+          # Make it more convenient to use the `jenkins-cli`.
+          JENKINS_URL = jenkinsUrl;
+        };
+    };
+
+    users.groups = lib.optionalAttrs (cfg.group == "jenkins") {
+      jenkins.gid = config.ids.gids.jenkins;
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "jenkins") {
+      jenkins = {
+        description = "jenkins user";
+        createHome = true;
+        home = cfg.home;
+        group = cfg.group;
+        extraGroups = cfg.extraGroups;
+        useDefaultShell = true;
+        uid = config.ids.uids.jenkins;
+      };
+    };
+
+    systemd.services.jenkins = {
+      description = "Jenkins Continuous Integration Server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment =
+        let
+          selectedSessionVars = lib.filterAttrs (
+            n: v: builtins.elem n [ "NIX_PATH" ]
+          ) config.environment.sessionVariables;
+        in
+        selectedSessionVars
+        // {
+          JENKINS_HOME = cfg.home;
+          NIX_REMOTE = "daemon";
+        }
+        // cfg.environment;
+
+      path = cfg.packages;
+
+      # Force .war (re)extraction, or else we might run stale Jenkins.
+
+      preStart =
+        let
+          replacePlugins = lib.optionalString (cfg.plugins != null) (
+            let
+              pluginCmds = lib.mapAttrsToList (n: v: "cp ${v} ${cfg.home}/plugins/${n}.jpi") cfg.plugins;
+            in
+            ''
+              rm -r ${cfg.home}/plugins || true
+              mkdir -p ${cfg.home}/plugins
+              ${lib.concatStringsSep "\n" pluginCmds}
+            ''
+          );
+        in
+        ''
+          rm -rf ${cfg.home}/war
+          ${replacePlugins}
+        '';
+
+      # For reference: https://wiki.jenkins.io/display/JENKINS/JenkinsLinuxStartupScript
+      script = ''
+        ${cfg.javaPackage}/bin/java ${lib.concatStringsSep " " cfg.extraJavaOptions} -jar ${cfg.package}/webapps/jenkins.war --httpListenAddress=${cfg.listenAddress} \
+                                                  --httpPort=${toString cfg.port} \
+                                                  --prefix=${cfg.prefix} \
+                                                  -Djava.awt.headless=true \
+                                                  ${lib.concatStringsSep " " cfg.extraOptions}
+      '';
+
+      postStart = ''
+        until [[ $(${pkgs.curl.bin}/bin/curl -L -s --head -w '\n%{http_code}' ${jenkinsUrl} | tail -n1) =~ ^(200|403)$ ]]; do
+          sleep 1
+        done
+      '';
+
+      serviceConfig = {
+        User = cfg.user;
+        StateDirectory = lib.mkIf (lib.hasPrefix "/var/lib/jenkins" cfg.home) "jenkins";
+        # For (possible) socket use
+        RuntimeDirectory = "jenkins";
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = "";
+        LockPersonality = true;
+        # MemoryDenyWriteExecute = false;   Breaks execution;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectSystem = "full";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_UNIX"
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        UMask = 27;
+      };
+    };
+  };
+}

nixos/modules/services/continuous-integration/jenkins/job-builder.nix

@@ -0,0 +1,260 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  jenkinsCfg = config.services.jenkins;
+  cfg = config.services.jenkins.jobBuilder;
+
+in
+{
+  options = {
+    services.jenkins.jobBuilder = {
+      enable = lib.mkEnableOption ''
+        the Jenkins Job Builder (JJB) service. It
+        allows defining jobs for Jenkins in a declarative manner.
+
+        Jobs managed through the Jenkins WebUI (or by other means) are left
+        unchanged.
+
+        Note that it really is declarative configuration; if you remove a
+        previously defined job, the corresponding job directory will be
+        deleted.
+
+        Please see the Jenkins Job Builder documentation for more info:
+        <https://jenkins-job-builder.readthedocs.io/>
+      '';
+
+      accessUser = lib.mkOption {
+        default = "admin";
+        type = lib.types.str;
+        description = ''
+          User id in Jenkins used to reload config.
+        '';
+      };
+
+      accessToken = lib.mkOption {
+        default = "";
+        type = lib.types.str;
+        description = ''
+          User token in Jenkins used to reload config.
+          WARNING: This token will be world readable in the Nix store. To keep
+          it secret, use the {option}`accessTokenFile` option instead.
+        '';
+      };
+
+      accessTokenFile = lib.mkOption {
+        default = "${config.services.jenkins.home}/secrets/initialAdminPassword";
+        defaultText = lib.literalExpression ''"''${config.services.jenkins.home}/secrets/initialAdminPassword"'';
+        type = lib.types.str;
+        example = "/run/keys/jenkins-job-builder-access-token";
+        description = ''
+          File containing the API token for the {option}`accessUser`
+          user.
+        '';
+      };
+
+      yamlJobs = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        example = ''
+          - job:
+              name: jenkins-job-test-1
+              builders:
+                - shell: echo 'Hello world!'
+        '';
+        description = ''
+          Job descriptions for Jenkins Job Builder in YAML format.
+        '';
+      };
+
+      jsonJobs = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        example = lib.literalExpression ''
+          [
+            '''
+              [ { "job":
+                  { "name": "jenkins-job-test-2",
+                    "builders": [ "shell": "echo 'Hello world!'" ]
+                  }
+                }
+              ]
+            '''
+          ]
+        '';
+        description = ''
+          Job descriptions for Jenkins Job Builder in JSON format.
+        '';
+      };
+
+      nixJobs = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.attrs;
+        example = lib.literalExpression ''
+          [ { job =
+              { name = "jenkins-job-test-3";
+                builders = [
+                  { shell = "echo 'Hello world!'"; }
+                ];
+              };
+            }
+          ]
+        '';
+        description = ''
+          Job descriptions for Jenkins Job Builder in Nix format.
+
+          This is a trivial wrapper around jsonJobs, using builtins.toJSON
+          behind the scene.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf (jenkinsCfg.enable && cfg.enable) {
+    assertions = [
+      {
+        assertion =
+          if cfg.accessUser != "" then
+            (cfg.accessToken != "" && cfg.accessTokenFile == "")
+            || (cfg.accessToken == "" && cfg.accessTokenFile != "")
+          else
+            true;
+        message = ''
+          One of accessToken and accessTokenFile options must be non-empty
+          strings, but not both. Current values:
+            services.jenkins.jobBuilder.accessToken = "${cfg.accessToken}"
+            services.jenkins.jobBuilder.accessTokenFile = "${cfg.accessTokenFile}"
+        '';
+      }
+    ];
+
+    systemd.services.jenkins-job-builder = {
+      description = "Jenkins Job Builder Service";
+      # JJB can run either before or after jenkins. We chose after, so we can
+      # always use curl to notify (running) jenkins to reload its config.
+      after = [ "jenkins.service" ];
+      wantedBy = [ "multi-user.target" ];
+
+      path = with pkgs; [
+        jenkins-job-builder
+        curl
+      ];
+
+      # Q: Why manipulate files directly instead of using "jenkins-jobs upload [...]"?
+      # A: Because this module is for administering a local jenkins install,
+      #    and using local file copy allows us to not worry about
+      #    authentication.
+      script =
+        let
+          yamlJobsFile = builtins.toFile "jobs.yaml" cfg.yamlJobs;
+          jsonJobsFiles = map (x: (builtins.toFile "jobs.json" x)) (
+            cfg.jsonJobs ++ [ (builtins.toJSON cfg.nixJobs) ]
+          );
+          jobBuilderOutputDir = "/run/jenkins-job-builder/output";
+          # Stamp file is placed in $JENKINS_HOME/jobs/$JOB_NAME/ to indicate
+          # ownership. Enables tracking and removal of stale jobs.
+          ownerStamp = ".config-xml-managed-by-nixos-jenkins-job-builder";
+          reloadScript = ''
+            echo "Asking Jenkins to reload config"
+            curl_opts="--silent --fail --show-error"
+            access_token_file=${
+              if cfg.accessTokenFile != "" then
+                cfg.accessTokenFile
+              else
+                "$RUNTIME_DIRECTORY/jenkins_access_token.txt"
+            }
+            if [ "${cfg.accessToken}" != "" ]; then
+               (umask 0077; printf "${cfg.accessToken}" >"$access_token_file")
+            fi
+            jenkins_url="http://${jenkinsCfg.listenAddress}:${toString jenkinsCfg.port}${jenkinsCfg.prefix}"
+            auth_file="$RUNTIME_DIRECTORY/jenkins_auth_file.txt"
+            trap 'rm -f "$auth_file"' EXIT
+            (umask 0077; printf "${cfg.accessUser}:@password_placeholder@" >"$auth_file")
+            "${pkgs.replace-secret}/bin/replace-secret" "@password_placeholder@" "$access_token_file" "$auth_file"
+
+            if ! "${pkgs.jenkins}/bin/jenkins-cli" -s "$jenkins_url" -auth "@$auth_file" reload-configuration; then
+                echo "error: failed to reload configuration"
+                exit 1
+            fi
+          '';
+        in
+        ''
+          joinByString()
+          {
+              local separator="$1"
+              shift
+              local first="$1"
+              shift
+              printf "%s" "$first" "''${@/#/$separator}"
+          }
+
+          # Map a relative directory path in the output from
+          # jenkins-job-builder (jobname) to the layout expected by jenkins:
+          # each directory level gets prepended "jobs/".
+          getJenkinsJobDir()
+          {
+              IFS='/' read -ra input_dirs <<< "$1"
+              printf "jobs/"
+              joinByString "/jobs/" "''${input_dirs[@]}"
+          }
+
+          # The inverse of getJenkinsJobDir (remove the "jobs/" prefixes)
+          getJobname()
+          {
+              IFS='/' read -ra input_dirs <<< "$1"
+              local i=0
+              local nelem=''${#input_dirs[@]}
+              for e in "''${input_dirs[@]}"; do
+                  if [ $((i % 2)) -eq 1 ]; then
+                      printf "$e"
+                      if [ $i -lt $(( nelem - 1 )) ]; then
+                          printf "/"
+                      fi
+                  fi
+                  i=$((i + 1))
+              done
+          }
+
+          rm -rf ${jobBuilderOutputDir}
+          cur_decl_jobs=/run/jenkins-job-builder/declarative-jobs
+          rm -f "$cur_decl_jobs"
+
+          # Create / update jobs
+          mkdir -p ${jobBuilderOutputDir}
+          for inputFile in ${yamlJobsFile} ${lib.concatStringsSep " " jsonJobsFiles}; do
+              HOME="${jenkinsCfg.home}" "${pkgs.jenkins-job-builder}/bin/jenkins-jobs" --ignore-cache test --config-xml -o "${jobBuilderOutputDir}" "$inputFile"
+          done
+
+          find "${jobBuilderOutputDir}" -type f -name config.xml | while read -r f; do echo "$(dirname "$f")"; done | sort | while read -r dir; do
+              jobname="$(realpath --relative-to="${jobBuilderOutputDir}" "$dir")"
+              jenkinsjobname=$(getJenkinsJobDir "$jobname")
+              jenkinsjobdir="${jenkinsCfg.home}/$jenkinsjobname"
+              echo "Creating / updating job \"$jobname\""
+              mkdir -p "$jenkinsjobdir"
+              touch "$jenkinsjobdir/${ownerStamp}"
+              cp "$dir"/config.xml "$jenkinsjobdir/config.xml"
+              echo "$jenkinsjobname" >> "$cur_decl_jobs"
+          done
+
+          # Remove stale jobs
+          find "${jenkinsCfg.home}" -type f -name "${ownerStamp}" | while read -r f; do echo "$(dirname "$f")"; done | sort --reverse | while read -r dir; do
+              jenkinsjobname="$(realpath --relative-to="${jenkinsCfg.home}" "$dir")"
+              grep --quiet --line-regexp "$jenkinsjobname" "$cur_decl_jobs" 2>/dev/null && continue
+              jobname=$(getJobname "$jenkinsjobname")
+              echo "Deleting stale job \"$jobname\""
+              jobdir="${jenkinsCfg.home}/$jenkinsjobname"
+              rm -rf "$jobdir"
+          done
+        ''
+        + (lib.optionalString (cfg.accessUser != "") reloadScript);
+      serviceConfig = {
+        Type = "oneshot";
+        User = jenkinsCfg.user;
+        RuntimeDirectory = "jenkins-job-builder";
+      };
+    };
+  };
+}

nixos/modules/services/continuous-integration/jenkins/slave.nix

@@ -0,0 +1,82 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib) mkIf mkOption types;
+  cfg = config.services.jenkinsSlave;
+  masterCfg = config.services.jenkins;
+in
+{
+  options = {
+    services.jenkinsSlave = {
+      # todo:
+      # * assure the profile of the jenkins user has a JRE and any specified packages. This would
+      # enable ssh slaves.
+      # * Optionally configure the node as a jenkins ad-hoc slave. This would imply configuration
+      # properties for the master node.
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          If true the system will be configured to work as a jenkins slave.
+          If the system is also configured to work as a jenkins master then this has no effect.
+          In progress: Currently only assures the jenkins user is configured.
+        '';
+      };
+
+      user = mkOption {
+        default = "jenkins";
+        type = types.str;
+        description = ''
+          User the jenkins slave agent should execute under.
+        '';
+      };
+
+      group = mkOption {
+        default = "jenkins";
+        type = types.str;
+        description = ''
+          If the default slave agent user "jenkins" is configured then this is
+          the primary group of that user.
+        '';
+      };
+
+      home = mkOption {
+        default = "/var/lib/jenkins";
+        type = types.path;
+        description = ''
+          The path to use as JENKINS_HOME. If the default user "jenkins" is configured then
+          this is the home of the "jenkins" user.
+        '';
+      };
+
+      javaPackage = lib.mkPackageOption pkgs "jdk" { };
+    };
+  };
+
+  config = mkIf (cfg.enable && !masterCfg.enable) {
+    users.groups = lib.optionalAttrs (cfg.group == "jenkins") {
+      jenkins.gid = config.ids.gids.jenkins;
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "jenkins") {
+      jenkins = {
+        description = "jenkins user";
+        createHome = true;
+        home = cfg.home;
+        group = cfg.group;
+        useDefaultShell = true;
+        uid = config.ids.uids.jenkins;
+      };
+    };
+
+    programs.java = {
+      enable = true;
+      package = cfg.javaPackage;
+    };
+  };
+}

nixos/modules/services/continuous-integration/woodpecker/agents.nix

@@ -0,0 +1,168 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.woodpecker-agents;
+
+  agentModule = lib.types.submodule {
+    options = {
+      enable = lib.mkEnableOption "this Woodpecker-Agent. Agents execute tasks generated by a Server, every install will need one server and at least one agent";
+
+      package = lib.mkPackageOption pkgs "woodpecker-agent" { };
+
+      environment = lib.mkOption {
+        default = { };
+        type = lib.types.attrsOf lib.types.str;
+        example = lib.literalExpression ''
+          {
+            WOODPECKER_SERVER = "localhost:9000";
+            WOODPECKER_BACKEND = "docker";
+            DOCKER_HOST = "unix:///run/podman/podman.sock";
+          }
+        '';
+        description = "woodpecker-agent config environment variables, for other options read the [documentation](https://woodpecker-ci.org/docs/administration/configuration/agent)";
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [ "podman" ];
+        description = ''
+          Additional groups for the systemd service.
+        '';
+      };
+
+      path = lib.mkOption {
+        type = lib.types.listOf lib.types.package;
+        default = [ ];
+        example = [ "" ];
+        description = ''
+          Additional packages that should be added to the agent's `PATH`.
+          Mostly useful for the `local` backend.
+        '';
+      };
+
+      environmentFile = lib.mkOption {
+        type = lib.types.listOf lib.types.path;
+        default = [ ];
+        example = [ "/var/secrets/woodpecker-agent.env" ];
+        description = ''
+          File to load environment variables
+          from. This is helpful for specifying secrets.
+          Example content of environmentFile:
+          ```
+          WOODPECKER_AGENT_SECRET=your-shared-secret-goes-here
+          ```
+        '';
+      };
+    };
+  };
+
+  mkAgentService = name: agentCfg: {
+    name = "woodpecker-agent-${name}";
+    value = {
+      description = "Woodpecker-Agent Service - ${name}";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig = {
+        DynamicUser = true;
+        SupplementaryGroups = agentCfg.extraGroups;
+        EnvironmentFile = agentCfg.environmentFile;
+        ExecStart = lib.getExe agentCfg.package;
+        Restart = "on-failure";
+        RestartSec = 15;
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        ProtectSystem = "strict";
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateUsers = true;
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        RestrictAddressFamilies = [ "AF_UNIX AF_INET AF_INET6" ];
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        PrivateMounts = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = "~@clock @privileged @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @reboot @setuid @swap";
+        BindReadOnlyPaths = [
+          "-/etc/resolv.conf"
+          "-/etc/nsswitch.conf"
+          "-/etc/ssl/certs"
+          "-/etc/static/ssl/certs"
+          "-/etc/hosts"
+          "-/etc/localtime"
+        ];
+      };
+      inherit (agentCfg) environment path;
+    };
+  };
+in
+{
+  meta.maintainers = with lib.maintainers; [ ambroisie ];
+
+  options = {
+    services.woodpecker-agents = {
+      agents = lib.mkOption {
+        default = { };
+        type = lib.types.attrsOf agentModule;
+        example = lib.literalExpression ''
+          {
+            podman = {
+              environment = {
+                WOODPECKER_SERVER = "localhost:9000";
+                WOODPECKER_BACKEND = "docker";
+                DOCKER_HOST = "unix:///run/podman/podman.sock";
+              };
+
+              extraGroups = [ "podman" ];
+
+              environmentFile = [ "/run/secrets/woodpecker/agent-secret.txt" ];
+            };
+
+            exec = {
+              environment = {
+                WOODPECKER_SERVER = "localhost:9000";
+                WOODPECKER_BACKEND = "local";
+              };
+
+              environmentFile = [ "/run/secrets/woodpecker/agent-secret.txt" ];
+
+              path = [
+                # Needed to clone repos
+                git
+                git-lfs
+                woodpecker-plugin-git
+                # Used by the runner as the default shell
+                bash
+                # Most likely to be used in pipeline definitions
+                coreutils
+              ];
+            };
+          }
+        '';
+        description = "woodpecker-agents configurations";
+      };
+    };
+  };
+
+  config = {
+    systemd.services =
+      let
+        mkServices = lib.mapAttrs' mkAgentService;
+        enabledAgents = lib.filterAttrs (_: agent: agent.enable) cfg.agents;
+      in
+      mkServices enabledAgents;
+  };
+}

nixos/modules/services/continuous-integration/woodpecker/server.nix

@@ -0,0 +1,96 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.woodpecker-server;
+in
+{
+  meta.maintainers = with lib.maintainers; [ ambroisie ];
+
+  options = {
+    services.woodpecker-server = {
+      enable = lib.mkEnableOption "the Woodpecker-Server, a CI/CD application for automatic builds, deployments and tests";
+      package = lib.mkPackageOption pkgs "woodpecker-server" { };
+      environment = lib.mkOption {
+        default = { };
+        type = lib.types.attrsOf lib.types.str;
+        example = lib.literalExpression ''
+          {
+            WOODPECKER_HOST = "https://woodpecker.example.com";
+            WOODPECKER_OPEN = "true";
+            WOODPECKER_GITEA = "true";
+            WOODPECKER_GITEA_CLIENT = "ffffffff-ffff-ffff-ffff-ffffffffffff";
+            WOODPECKER_GITEA_URL = "https://git.example.com";
+          }
+        '';
+        description = "woodpecker-server config environment variables, for other options read the [documentation](https://woodpecker-ci.org/docs/administration/configuration/server)";
+      };
+      environmentFile = lib.mkOption {
+        type = with lib.types; coercedTo path (f: [ f ]) (listOf path);
+        default = [ ];
+        example = [ "/root/woodpecker-server.env" ];
+        description = ''
+          File to load environment variables
+          from. This is helpful for specifying secrets.
+          Example content of environmentFile:
+          ```
+          WOODPECKER_AGENT_SECRET=your-shared-secret-goes-here
+          WOODPECKER_GITEA_SECRET=gto_**************************************
+          ```
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services = {
+      woodpecker-server = {
+        description = "Woodpecker-Server Service";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network-online.target" ];
+        wants = [ "network-online.target" ];
+        serviceConfig = {
+          DynamicUser = true;
+          WorkingDirectory = "%S/woodpecker-server";
+          StateDirectory = "woodpecker-server";
+          StateDirectoryMode = "0700";
+          UMask = "0007";
+          ConfigurationDirectory = "woodpecker-server";
+          EnvironmentFile = cfg.environmentFile;
+          ExecStart = "${cfg.package}/bin/woodpecker-server";
+          Restart = "on-failure";
+          RestartSec = 15;
+          CapabilityBoundingSet = "";
+          # Security
+          NoNewPrivileges = true;
+          # Sandboxing
+          ProtectSystem = "strict";
+          ProtectHome = true;
+          PrivateTmp = true;
+          PrivateDevices = true;
+          PrivateUsers = true;
+          ProtectHostname = true;
+          ProtectClock = true;
+          ProtectKernelTunables = true;
+          ProtectKernelModules = true;
+          ProtectKernelLogs = true;
+          ProtectControlGroups = true;
+          RestrictAddressFamilies = [ "AF_UNIX AF_INET AF_INET6" ];
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          PrivateMounts = true;
+          # System Call Filtering
+          SystemCallArchitectures = "native";
+          SystemCallFilter = "~@clock @privileged @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @reboot @setuid @swap";
+        };
+        inherit (cfg) environment;
+      };
+    };
+  };
+}