push sheeet

nixos/modules/services/accessibility/orca.nix

@@ -0,0 +1,28 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.orca;
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkPackageOption
+    ;
+in
+{
+  options.services.orca = {
+    enable = mkEnableOption "Orca screen reader";
+    package = mkPackageOption pkgs "orca" { };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+    systemd.services.display-manager = lib.mkIf config.services.displayManager.enable {
+      path = [ cfg.package ];
+    };
+    services.speechd.enable = true;
+  };
+}

nixos/modules/services/accessibility/speechd.nix

@@ -0,0 +1,31 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.speechd;
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkPackageOption
+    ;
+in
+{
+  options.services.speechd = {
+    # FIXME: figure out how to deprecate this EXTREMELY CAREFULLY
+    # default guessed conservatively in ../misc/graphical-desktop.nix
+    enable = mkEnableOption "speech-dispatcher speech synthesizer daemon";
+    package = mkPackageOption pkgs "speechd" { };
+  };
+
+  config = mkIf cfg.enable {
+    environment = {
+      systemPackages = [ cfg.package ];
+    };
+    systemd.packages = [ cfg.package ];
+    # have to set `wantedBy` since `systemd.packages` ignores `[Install]`
+    systemd.user.sockets.speech-dispatcher.wantedBy = [ "sockets.target" ];
+  };
+}

nixos/modules/services/admin/docuum.nix

@@ -0,0 +1,93 @@
+{
+  config,
+  pkgs,
+  lib,
+  utils,
+  ...
+}:
+
+let
+  cfg = config.services.docuum;
+  inherit (lib)
+    mkIf
+    mkEnableOption
+    mkOption
+    getExe
+    types
+    optionals
+    concatMap
+    ;
+in
+{
+  options.services.docuum = {
+    enable = mkEnableOption "docuum daemon";
+
+    threshold = mkOption {
+      description = "Threshold for deletion in bytes, like `10 GB`, `10 GiB`, `10GB` or percentage-based thresholds like `50%`";
+      type = types.str;
+      default = "10 GB";
+      example = "50%";
+    };
+
+    minAge = mkOption {
+      description = "Sets the minimum age of images to be considered for deletion.";
+      type = types.nullOr types.str;
+      default = null;
+      example = "1d";
+    };
+
+    keep = mkOption {
+      description = "Prevents deletion of images for which repository:tag matches the specified regex.";
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "^my-image" ];
+    };
+
+    deletionChunkSize = mkOption {
+      description = "Removes specified quantity of images at a time.";
+      type = types.int;
+      default = 1;
+      example = 10;
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.virtualisation.docker.enable;
+        message = "docuum requires docker on the host";
+      }
+    ];
+
+    systemd.services.docuum = {
+      after = [ "docker.socket" ];
+      requires = [ "docker.socket" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ config.virtualisation.docker.package ];
+      environment.HOME = "/var/lib/docuum";
+
+      serviceConfig = {
+        DynamicUser = true;
+        StateDirectory = "docuum";
+        SupplementaryGroups = [ "docker" ];
+        ExecStart = utils.escapeSystemdExecArgs (
+          [
+            (getExe pkgs.docuum)
+            "--threshold"
+            cfg.threshold
+            "--deletion-chunk-size"
+            cfg.deletionChunkSize
+          ]
+          ++ (concatMap (keep: [
+            "--keep"
+            keep
+          ]) cfg.keep)
+          ++ (optionals (cfg.minAge != null) [
+            "--min-age"
+            cfg.minAge
+          ])
+        );
+      };
+    };
+  };
+}

nixos/modules/services/admin/meshcentral.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.meshcentral;
+  configFormat = pkgs.formats.json { };
+  configFile = configFormat.generate "meshcentral-config.json" cfg.settings;
+in
+with lib;
+{
+  options.services.meshcentral = with types; {
+    enable = mkEnableOption "MeshCentral computer management server";
+    package = mkPackageOption pkgs "meshcentral" { };
+    settings = mkOption {
+      description = ''
+        Settings for MeshCentral. Refer to upstream documentation for details:
+
+        - [JSON Schema definition](https://github.com/Ylianst/MeshCentral/blob/master/meshcentral-config-schema.json)
+        - [simple sample configuration](https://github.com/Ylianst/MeshCentral/blob/master/sample-config.json)
+        - [complex sample configuration](https://github.com/Ylianst/MeshCentral/blob/master/sample-config-advanced.json)
+        - [Old homepage with documentation link](https://www.meshcommander.com/meshcentral2)
+      '';
+      type = types.submodule {
+        freeformType = attrsOf configFormat.type;
+      };
+      example = {
+        settings = {
+          WANonly = true;
+          Cert = "meshcentral.example.com";
+          TlsOffload = "10.0.0.2,fd42::2";
+          Port = 4430;
+        };
+        domains."".certUrl = "https://meshcentral.example.com/";
+      };
+    };
+  };
+  config = mkIf cfg.enable {
+    services.meshcentral.settings.settings.autoBackup.backupPath =
+      lib.mkDefault "/var/lib/meshcentral/backups";
+    systemd.services.meshcentral = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/meshcentral --datapath /var/lib/meshcentral --configfile ${configFile}";
+        DynamicUser = true;
+        StateDirectory = "meshcentral";
+        CacheDirectory = "meshcentral";
+      };
+    };
+  };
+  meta.maintainers = [ ];
+}

nixos/modules/services/admin/oxidized.nix

@@ -0,0 +1,163 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.oxidized;
+in
+{
+  options.services.oxidized = {
+    enable = lib.mkEnableOption "the oxidized configuration backup service";
+
+    package = lib.mkPackageOption pkgs "oxidized" { };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "oxidized";
+      description = ''
+        User under which the oxidized service runs.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "oxidized";
+      description = ''
+        Group under which the oxidized service runs.
+      '';
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/oxidized";
+      description = "State directory for the oxidized service.";
+    };
+
+    configFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      example = lib.literalExpression ''
+        pkgs.writeText "oxidized-config.yml" '''
+          ---
+          debug: true
+          use_syslog: true
+          input:
+            default: ssh
+            ssh:
+              secure: true
+          interval: 3600
+          model_map:
+            dell: powerconnect
+            hp: procurve
+          source:
+            default: csv
+            csv:
+              delimiter: !ruby/regexp /:/
+              file: "/var/lib/oxidized/.config/oxidized/router.db"
+              map:
+                name: 0
+                model: 1
+                username: 2
+                password: 3
+          pid: "/var/lib/oxidized/.config/oxidized/pid"
+          rest: 127.0.0.1:8888
+          retries: 3
+          # ... additional config
+        ''';
+      '';
+      description = ''
+        Path to the oxidized configuration file.
+      '';
+    };
+
+    routerDB = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      example = lib.literalExpression ''
+        pkgs.writeText "oxidized-router.db" '''
+          hostname-sw1:powerconnect:username1:password2
+          hostname-sw2:procurve:username2:password2
+          # ... additional hosts
+        '''
+      '';
+      description = ''
+        Path to the file/database which contains the targets for oxidized.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.groups.${cfg.group} = { };
+    users.users.${cfg.user} = {
+      description = "Oxidized service user";
+      group = cfg.group;
+      home = cfg.dataDir;
+      createHome = true;
+      isSystemUser = true;
+    };
+
+    systemd.tmpfiles.settings."10-oxidized" = {
+      "${cfg.dataDir}" = {
+        d = {
+          mode = "0750";
+          user = cfg.user;
+          group = cfg.group;
+        };
+      };
+
+      "${cfg.dataDir}/.config" = {
+        d = {
+          mode = "0750";
+          user = cfg.user;
+          group = cfg.group;
+        };
+      };
+
+      "${cfg.dataDir}/.config/oxidized" = {
+        d = {
+          mode = "0750";
+          user = cfg.user;
+          group = cfg.group;
+        };
+      };
+
+    }
+    // lib.optionalAttrs (cfg.configFile != null) {
+      "${cfg.dataDir}/.config/oxidized/config" = {
+        "L+" = {
+          argument = "${cfg.configFile}";
+          user = cfg.user;
+          group = cfg.group;
+        };
+      };
+
+    }
+    // lib.optionalAttrs (cfg.routerDB != null) {
+      "${cfg.dataDir}/.config/oxidized/router.db" = {
+        "L+" = {
+          argument = "${cfg.routerDB}";
+          user = cfg.user;
+          group = cfg.group;
+        };
+      };
+    };
+
+    systemd.services.oxidized = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      serviceConfig = {
+        ExecStart = lib.getExe cfg.package;
+        User = cfg.user;
+        Group = cfg.group;
+        UMask = "0077";
+        NoNewPrivileges = true;
+        Restart = "always";
+        WorkingDirectory = cfg.dataDir;
+        KillSignal = "SIGKILL";
+        PIDFile = "${cfg.dataDir}/.config/oxidized/pid";
+      };
+    };
+  };
+}

nixos/modules/services/admin/pgadmin.nix

@@ -0,0 +1,282 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.pgadmin;
+
+  _base = with lib.types; [
+    int
+    bool
+    str
+  ];
+  base =
+    with lib.types;
+    oneOf (
+      [
+        (listOf (oneOf _base))
+        (attrsOf (oneOf _base))
+      ]
+      ++ _base
+    );
+
+  formatAttrset =
+    attr:
+    "{${
+      lib.concatStringsSep "\n" (
+        lib.mapAttrsToList (key: value: "${builtins.toJSON key}: ${formatPyValue value},") attr
+      )
+    }}";
+
+  formatPyValue =
+    value:
+    if builtins.isString value then
+      builtins.toJSON value
+    else if value ? _expr then
+      value._expr
+    else if builtins.isInt value then
+      toString value
+    else if builtins.isBool value then
+      (if value then "True" else "False")
+    else if builtins.isAttrs value then
+      (formatAttrset value)
+    else if builtins.isList value then
+      "[${lib.concatStringsSep "\n" (map (v: "${formatPyValue v},") value)}]"
+    else
+      throw "Unrecognized type";
+
+  formatPy =
+    attrs:
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (key: value: "${key} = ${formatPyValue value}") attrs
+    );
+
+  pyType =
+    with lib.types;
+    attrsOf (oneOf [
+      (attrsOf base)
+      (listOf base)
+      base
+    ]);
+in
+{
+  options.services.pgadmin = {
+    enable = lib.mkEnableOption "PostgreSQL Admin 4";
+
+    port = lib.mkOption {
+      description = "Port for pgadmin4 to run on";
+      type = lib.types.port;
+      default = 5050;
+    };
+
+    package = lib.mkPackageOption pkgs "pgadmin4" { };
+
+    initialEmail = lib.mkOption {
+      description = "Initial email for the pgAdmin account";
+      type = lib.types.str;
+    };
+
+    initialPasswordFile = lib.mkOption {
+      description = ''
+        Initial password file for the pgAdmin account. Minimum length by default is 6.
+        Please see `services.pgadmin.minimumPasswordLength`.
+        NOTE: Should be string not a store path, to prevent the password from being world readable
+      '';
+      type = lib.types.path;
+    };
+
+    minimumPasswordLength = lib.mkOption {
+      description = "Minimum length of the password";
+      type = lib.types.int;
+      default = 6;
+    };
+
+    emailServer = {
+      enable = lib.mkOption {
+        description = ''
+          Enable SMTP email server. This is necessary, if you want to use password recovery or change your own password
+        '';
+        type = lib.types.bool;
+        default = false;
+      };
+      address = lib.mkOption {
+        description = "SMTP server for email delivery";
+        type = lib.types.str;
+        default = "localhost";
+      };
+      port = lib.mkOption {
+        description = "SMTP server port for email delivery";
+        type = lib.types.port;
+        default = 25;
+      };
+      useSSL = lib.mkOption {
+        description = "SMTP server should use SSL";
+        type = lib.types.bool;
+        default = false;
+      };
+      useTLS = lib.mkOption {
+        description = "SMTP server should use TLS";
+        type = lib.types.bool;
+        default = false;
+      };
+      username = lib.mkOption {
+        description = "SMTP server username for email delivery";
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+      };
+      sender = lib.mkOption {
+        description = ''
+          SMTP server sender email for email delivery. Some servers require this to be a valid email address from that server
+        '';
+        type = lib.types.str;
+        example = "noreply@example.com";
+      };
+      passwordFile = lib.mkOption {
+        description = ''
+          Password for SMTP email account.
+          NOTE: Should be string not a store path, to prevent the password from being world readable
+        '';
+        type = lib.types.path;
+      };
+    };
+
+    openFirewall = lib.mkEnableOption "firewall passthrough for pgadmin4";
+
+    settings = lib.mkOption {
+      description = ''
+        Settings for pgadmin4.
+        [Documentation](https://www.pgadmin.org/docs/pgadmin4/development/config_py.html)
+      '';
+      type = pyType;
+      default = { };
+    };
+  };
+
+  config = lib.mkIf (cfg.enable) {
+    networking.firewall.allowedTCPPorts = lib.mkIf (cfg.openFirewall) [ cfg.port ];
+
+    services.pgadmin.settings = {
+      DEFAULT_SERVER_PORT = cfg.port;
+      PASSWORD_LENGTH_MIN = cfg.minimumPasswordLength;
+      SERVER_MODE = true;
+      UPGRADE_CHECK_ENABLED = false;
+    }
+    // (lib.optionalAttrs cfg.openFirewall {
+      DEFAULT_SERVER = lib.mkDefault "::";
+    })
+    // (lib.optionalAttrs cfg.emailServer.enable {
+      MAIL_SERVER = cfg.emailServer.address;
+      MAIL_PORT = cfg.emailServer.port;
+      MAIL_USE_SSL = cfg.emailServer.useSSL;
+      MAIL_USE_TLS = cfg.emailServer.useTLS;
+      MAIL_USERNAME = cfg.emailServer.username;
+      SECURITY_EMAIL_SENDER = cfg.emailServer.sender;
+    });
+
+    systemd.services.pgadmin = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      requires = [ "network.target" ];
+      # we're adding this optionally so just in case there's any race it'll be caught
+      # in case postgres doesn't start, pgadmin will just start normally
+      wants = [ "postgresql.target" ];
+
+      path = [
+        config.services.postgresql.package
+        pkgs.coreutils
+        pkgs.bash
+      ];
+
+      preStart = ''
+        # NOTE: this is idempotent (aka running it twice has no effect)
+        # Check here for password length to prevent pgadmin from starting
+        # and presenting a hard to find error message
+        # see https://github.com/NixOS/nixpkgs/issues/270624
+        PW_FILE="$CREDENTIALS_DIRECTORY/initial_password"
+        PW_LENGTH=$(wc -m < "$PW_FILE")
+        if [ $PW_LENGTH -lt ${toString cfg.minimumPasswordLength} ]; then
+            echo "Password must be at least ${toString cfg.minimumPasswordLength} characters long"
+            exit 1
+        fi
+        (
+          # Email address:
+          echo ${lib.escapeShellArg cfg.initialEmail}
+
+          # file might not contain newline. echo hack fixes that.
+          PW=$(cat "$PW_FILE")
+
+          # Password:
+          echo "$PW"
+          # Retype password:
+          echo "$PW"
+        ) | ${cfg.package}/bin/pgadmin4-cli setup-db
+      '';
+
+      restartTriggers = [
+        "/etc/pgadmin/config_system.py"
+      ];
+
+      serviceConfig = {
+        User = "pgadmin";
+        DynamicUser = true;
+        LogsDirectory = "pgadmin";
+        StateDirectory = "pgadmin";
+        ExecStart = "${cfg.package}/bin/pgadmin4";
+        LoadCredential = [
+          "initial_password:${cfg.initialPasswordFile}"
+        ]
+        ++ lib.optional cfg.emailServer.enable "email_password:${cfg.emailServer.passwordFile}";
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = "";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectSystem = "full";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_UNIX"
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        UMask = 27;
+      };
+    };
+
+    users.users.pgadmin = {
+      isSystemUser = true;
+      group = "pgadmin";
+    };
+
+    users.groups.pgadmin = { };
+
+    environment.etc."pgadmin/config_system.py" = {
+      text =
+        lib.optionalString cfg.emailServer.enable ''
+          import os
+          with open(os.path.join(os.environ['CREDENTIALS_DIRECTORY'], 'email_password')) as f:
+            pw = f.read()
+          MAIL_PASSWORD = pw
+        ''
+        + formatPy cfg.settings;
+      mode = "0600";
+      user = "pgadmin";
+      group = "pgadmin";
+    };
+  };
+}

nixos/modules/services/admin/salt/master.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+
+  cfg = config.services.salt.master;
+
+  fullConfig = lib.recursiveUpdate {
+    # Provide defaults for some directories to allow an immutable config dir
+
+    # Default is equivalent to /etc/salt/master.d/*.conf
+    default_include = "/var/lib/salt/master.d/*.conf";
+    # Default is in /etc/salt/pki/master
+    pki_dir = "/var/lib/salt/pki/master";
+  } cfg.configuration;
+
+in
+
+{
+  options = {
+    services.salt.master = {
+      enable = lib.mkEnableOption "Salt configuration management system master service";
+      configuration = lib.mkOption {
+        type = lib.types.attrs;
+        default = { };
+        description = "Salt master configuration as Nix attribute set.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment = {
+      # Set this up in /etc/salt/master so `salt`, `salt-key`, etc. work.
+      # The alternatives are
+      # - passing --config-dir to all salt commands, not just the master unit,
+      # - setting a global environment variable,
+      etc."salt/master".source = pkgs.writeText "master" (builtins.toJSON fullConfig);
+      systemPackages = with pkgs; [ salt ];
+    };
+    systemd.services.salt-master = {
+      description = "Salt Master";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = with pkgs; [
+        util-linux # for dmesg
+      ];
+      serviceConfig = {
+        ExecStart = "${pkgs.salt}/bin/salt-master";
+        LimitNOFILE = 16384;
+        Type = "notify";
+        NotifyAccess = "all";
+      };
+      restartTriggers = [
+        config.environment.etc."salt/master".source
+      ];
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ Flakebi ];
+}

nixos/modules/services/admin/salt/minion.nix

@@ -0,0 +1,66 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+
+  cfg = config.services.salt.minion;
+
+  fullConfig = lib.recursiveUpdate {
+    # Provide defaults for some directories to allow an immutable config dir
+    # NOTE: the config dir being immutable prevents `minion_id` caching
+
+    # Default is equivalent to /etc/salt/minion.d/*.conf
+    default_include = "/var/lib/salt/minion.d/*.conf";
+    # Default is in /etc/salt/pki/minion
+    pki_dir = "/var/lib/salt/pki/minion";
+  } cfg.configuration;
+
+in
+
+{
+  options = {
+    services.salt.minion = {
+      enable = lib.mkEnableOption "Salt configuration management system minion service";
+      configuration = lib.mkOption {
+        type = lib.types.attrs;
+        default = { };
+        description = ''
+          Salt minion configuration as Nix attribute set.
+          See <https://docs.saltstack.com/en/latest/ref/configuration/minion.html>
+          for details.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment = {
+      # Set this up in /etc/salt/minion so `salt-call`, etc. work.
+      # The alternatives are
+      # - passing --config-dir to all salt commands, not just the minion unit,
+      # - setting aglobal environment variable.
+      etc."salt/minion".source = pkgs.writeText "minion" (builtins.toJSON fullConfig);
+      systemPackages = with pkgs; [ salt ];
+    };
+    systemd.services.salt-minion = {
+      description = "Salt Minion";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = with pkgs; [
+        util-linux
+      ];
+      serviceConfig = {
+        ExecStart = "${pkgs.salt}/bin/salt-minion";
+        LimitNOFILE = 8192;
+        Type = "notify";
+        NotifyAccess = "all";
+      };
+      restartTriggers = [
+        config.environment.etc."salt/minion".source
+      ];
+    };
+  };
+}

nixos/modules/services/amqp/activemq/ActiveMQBroker.java

@@ -0,0 +1,19 @@
+import org.apache.activemq.broker.BrokerService;
+import org.apache.activemq.broker.BrokerFactory;
+import java.net.URI;
+
+public class ActiveMQBroker {
+
+  public static void main(String[] args) throws Throwable {
+    URI uri = new URI((args.length > 0) ? args[0] : "xbean:activemq.xml");
+    BrokerService broker = BrokerFactory.createBroker(uri);
+    broker.start();
+    if (broker.waitUntilStarted()) {
+      broker.waitUntilStopped();
+    } else {
+      System.out.println("Failed starting broker");
+      System.exit(-1);
+    };
+  }
+
+}

nixos/modules/services/amqp/activemq/default.nix

@@ -0,0 +1,144 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.activemq;
+
+  activemqBroker =
+    pkgs.runCommand "activemq-broker"
+      {
+        nativeBuildInputs = [ pkgs.jdk ];
+      }
+      ''
+        mkdir -p $out/lib
+        source ${pkgs.activemq}/lib/classpath.env
+        export CLASSPATH
+        ln -s "${./ActiveMQBroker.java}" ActiveMQBroker.java
+        javac -d $out/lib ActiveMQBroker.java
+      '';
+
+in
+{
+
+  options = {
+    services.activemq = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Enable the Apache ActiveMQ message broker service.
+        '';
+      };
+      configurationDir = lib.mkOption {
+        default = "${pkgs.activemq}/conf";
+        defaultText = lib.literalExpression ''"''${pkgs.activemq}/conf"'';
+        type = lib.types.str;
+        description = ''
+          The base directory for ActiveMQ's configuration.
+          By default, this directory is searched for a file named activemq.xml,
+          which should contain the configuration for the broker service.
+        '';
+      };
+      configurationURI = lib.mkOption {
+        type = lib.types.str;
+        default = "xbean:activemq.xml";
+        description = ''
+          The URI that is passed along to the BrokerFactory to
+          set up the configuration of the ActiveMQ broker service.
+          You should not need to change this. For custom configuration,
+          set the `configurationDir` instead, and create
+          an activemq.xml configuration file in it.
+        '';
+      };
+      baseDir = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/activemq";
+        description = ''
+          The base directory where ActiveMQ stores its persistent data and logs.
+          This will be overridden if you set "activemq.base" and "activemq.data"
+          in the `javaProperties` option. You can also override
+          this in activemq.xml.
+        '';
+      };
+      javaProperties = lib.mkOption {
+        type = lib.types.attrs;
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "java.net.preferIPv4Stack" = "true";
+          }
+        '';
+        apply =
+          attrs:
+          {
+            "activemq.base" = "${cfg.baseDir}";
+            "activemq.data" = "${cfg.baseDir}/data";
+            "activemq.conf" = "${cfg.configurationDir}";
+            "activemq.home" = "${pkgs.activemq}";
+          }
+          // attrs;
+        description = ''
+          Specifies Java properties that are sent to the ActiveMQ
+          broker service with the "-D" option. You can set properties
+          here to change the behaviour and configuration of the broker.
+          All essential properties that are not set here are automatically
+          given reasonable defaults.
+        '';
+      };
+      extraJavaOptions = lib.mkOption {
+        type = lib.types.separatedString " ";
+        default = "";
+        example = "-Xmx2G -Xms2G -XX:MaxPermSize=512M";
+        description = ''
+          Add extra options here that you want to be sent to the
+          Java runtime when the broker service is started.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.activemq = {
+      description = "ActiveMQ server user";
+      group = "activemq";
+      uid = config.ids.uids.activemq;
+    };
+
+    users.groups.activemq.gid = config.ids.gids.activemq;
+
+    systemd.services.activemq_init = {
+      wantedBy = [ "activemq.service" ];
+      partOf = [ "activemq.service" ];
+      before = [ "activemq.service" ];
+      serviceConfig.Type = "oneshot";
+      script = ''
+        mkdir -p "${cfg.javaProperties."activemq.data"}"
+        chown -R activemq "${cfg.javaProperties."activemq.data"}"
+      '';
+    };
+
+    systemd.services.activemq = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      path = [ pkgs.jre ];
+      serviceConfig.User = "activemq";
+      script = ''
+        source ${pkgs.activemq}/lib/classpath.env
+        export CLASSPATH=${activemqBroker}/lib:${cfg.configurationDir}:$CLASSPATH
+        exec java \
+          ${
+            lib.concatStringsSep " \\\n" (
+              lib.mapAttrsToList (name: value: "-D${name}=${value}") cfg.javaProperties
+            )
+          } \
+          ${cfg.extraJavaOptions} ActiveMQBroker "${cfg.configurationURI}"
+      '';
+    };
+
+  };
+
+}

nixos/modules/services/amqp/rabbitmq.nix

@@ -0,0 +1,243 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.rabbitmq;
+
+  inherit (builtins) concatStringsSep;
+
+  config_file_content = lib.generators.toKeyValue { } cfg.configItems;
+  config_file = pkgs.writeText "rabbitmq.conf" config_file_content;
+
+  advanced_config_file = pkgs.writeText "advanced.config" cfg.config;
+
+in
+{
+
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "rabbitmq" "cookie" ] ''
+      This option wrote the Erlang cookie to the store, while it should be kept secret.
+      Please remove it from your NixOS configuration and deploy a cookie securely instead.
+      The renamed `unsafeCookie` must ONLY be used in isolated non-production environments such as NixOS VM tests.
+    '')
+  ];
+
+  ###### interface
+  options = {
+    services.rabbitmq = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable the RabbitMQ server, an Advanced Message
+          Queuing Protocol (AMQP) broker.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "rabbitmq-server" { };
+
+      listenAddress = lib.mkOption {
+        default = "127.0.0.1";
+        example = "";
+        description = ''
+          IP address on which RabbitMQ will listen for AMQP
+          connections.  Set to the empty string to listen on all
+          interfaces.  Note that RabbitMQ creates a user named
+          `guest` with password
+          `guest` by default, so you should delete
+          this user if you intend to allow external access.
+
+          Together with 'port' setting it's mostly an alias for
+          configItems."listeners.tcp.1" and it's left for backwards
+          compatibility with previous version of this module.
+        '';
+        type = lib.types.str;
+      };
+
+      port = lib.mkOption {
+        default = 5672;
+        description = ''
+          Port on which RabbitMQ will listen for AMQP connections.
+        '';
+        type = lib.types.port;
+      };
+
+      dataDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/rabbitmq";
+        description = ''
+          Data directory for rabbitmq.
+        '';
+      };
+
+      unsafeCookie = lib.mkOption {
+        default = "";
+        type = lib.types.str;
+        description = ''
+          Erlang cookie is a string of arbitrary length which must
+          be the same for several nodes to be allowed to communicate.
+          Leave empty to generate automatically.
+
+          Setting the cookie via this option exposes the cookie to the store, which
+          is not recommended for security reasons.
+          Only use this option in an isolated non-production environment such as
+          NixOS VM tests.
+        '';
+      };
+
+      configItems = lib.mkOption {
+        default = { };
+        type = lib.types.attrsOf lib.types.str;
+        example = lib.literalExpression ''
+          {
+            "auth_backends.1.authn" = "rabbit_auth_backend_ldap";
+            "auth_backends.1.authz" = "rabbit_auth_backend_internal";
+          }
+        '';
+        description = ''
+          Configuration options in RabbitMQ's new config file format,
+          which is a simple key-value format that can not express nested
+          data structures. This is known as the `rabbitmq.conf` file,
+          although outside NixOS that filename may have Erlang syntax, particularly
+          prior to RabbitMQ 3.7.0.
+
+          If you do need to express nested data structures, you can use
+          `config` option. Configuration from `config`
+          will be merged into these options by RabbitMQ at runtime to
+          form the final configuration.
+
+          See <https://www.rabbitmq.com/configure.html#config-items>
+          For the distinct formats, see <https://www.rabbitmq.com/configure.html#config-file-formats>
+        '';
+      };
+
+      config = lib.mkOption {
+        default = "";
+        type = lib.types.str;
+        description = ''
+          Verbatim advanced configuration file contents using the Erlang syntax.
+          This is also known as the `advanced.config` file or the old config format.
+
+          `configItems` is preferred whenever possible. However, nested
+          data structures can only be expressed properly using the `config` option.
+
+          The contents of this option will be merged into the `configItems`
+          by RabbitMQ at runtime to form the final configuration.
+
+          See the second table on <https://www.rabbitmq.com/configure.html#config-items>
+          For the distinct formats, see <https://www.rabbitmq.com/configure.html#config-file-formats>
+        '';
+      };
+
+      plugins = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = "The names of plugins to enable";
+      };
+
+      pluginDirs = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.path;
+        description = "The list of directories containing external plugins";
+      };
+
+      managementPlugin = {
+        enable = lib.mkEnableOption "the management plugin";
+        port = lib.mkOption {
+          default = 15672;
+          type = lib.types.port;
+          description = ''
+            On which port to run the management plugin
+          '';
+        };
+      };
+    };
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+
+    # This is needed so we will have 'rabbitmqctl' in our PATH
+    environment.systemPackages = [ cfg.package ];
+
+    services.epmd.enable = true;
+
+    users.users.rabbitmq = {
+      description = "RabbitMQ server user";
+      home = "${cfg.dataDir}";
+      createHome = true;
+      group = "rabbitmq";
+      uid = config.ids.uids.rabbitmq;
+    };
+
+    users.groups.rabbitmq.gid = config.ids.gids.rabbitmq;
+
+    services.rabbitmq.configItems = {
+      "listeners.tcp.1" = lib.mkDefault "${cfg.listenAddress}:${toString cfg.port}";
+    }
+    // lib.optionalAttrs cfg.managementPlugin.enable {
+      "management.tcp.port" = toString cfg.managementPlugin.port;
+      "management.tcp.ip" = cfg.listenAddress;
+    };
+
+    services.rabbitmq.plugins = lib.optional cfg.managementPlugin.enable "rabbitmq_management";
+
+    systemd.services.rabbitmq = {
+      description = "RabbitMQ Server";
+
+      wantedBy = [ "multi-user.target" ];
+      after = [
+        "network.target"
+        "epmd.socket"
+      ];
+      wants = [
+        "network.target"
+        "epmd.socket"
+      ];
+
+      path = [
+        cfg.package
+        pkgs.coreutils # mkdir/chown/chmod for preStart
+      ];
+
+      environment = {
+        RABBITMQ_MNESIA_BASE = "${cfg.dataDir}/mnesia";
+        RABBITMQ_LOGS = "-";
+        SYS_PREFIX = "";
+        RABBITMQ_CONFIG_FILE = config_file;
+        RABBITMQ_PLUGINS_DIR = lib.concatStringsSep ":" cfg.pluginDirs;
+        RABBITMQ_ENABLED_PLUGINS_FILE = pkgs.writeText "enabled_plugins" ''
+          [ ${lib.concatStringsSep "," cfg.plugins} ].
+        '';
+      }
+      // lib.optionalAttrs (cfg.config != "") { RABBITMQ_ADVANCED_CONFIG_FILE = advanced_config_file; };
+
+      serviceConfig = {
+        ExecStart = "${cfg.package}/sbin/rabbitmq-server";
+        ExecStop = "${cfg.package}/sbin/rabbitmqctl shutdown";
+        User = "rabbitmq";
+        Group = "rabbitmq";
+        LogsDirectory = "rabbitmq";
+        WorkingDirectory = cfg.dataDir;
+        Type = "notify";
+        NotifyAccess = "all";
+        UMask = "0027";
+        LimitNOFILE = "100000";
+        Restart = "on-failure";
+        RestartSec = "10";
+        TimeoutStartSec = "3600";
+      };
+
+      preStart = ''
+        ${lib.optionalString (cfg.unsafeCookie != "") ''
+          install -m 600 <(echo -n ${cfg.unsafeCookie}) ${cfg.dataDir}/.erlang.cookie
+        ''}
+      '';
+    };
+
+  };
+
+}

nixos/modules/services/audio/alsa.nix

@@ -0,0 +1,459 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.hardware.alsa;
+
+  quote = x: ''"${lib.escape [ "\"" ] x}"'';
+
+  alsactl = lib.getExe' pkgs.alsa-utils "alsactl";
+
+  # Creates a volume control
+  mkControl = name: opts: ''
+    pcm.${name} {
+      type softvol
+      slave.pcm ${quote opts.device}
+      control.name ${quote (if opts.name != null then opts.name else name)}
+      control.card ${quote opts.card}
+      max_dB ${toString opts.maxVolume}
+    }
+  '';
+
+  # modprobe.conf for naming sound cards
+  cardsConfig =
+    let
+      # Reverse the mapping from card name→driver to card driver→name
+      drivers = lib.unique (lib.mapAttrsToList (n: v: v.driver) cfg.cardAliases);
+      options = lib.forEach drivers (
+        drv:
+        let
+          byDriver = lib.filterAttrs (n: v: v.driver == drv);
+          ids = lib.mapAttrs (n: v: v.id) (byDriver cfg.cardAliases);
+        in
+        {
+          driver = drv;
+          names = lib.attrNames ids;
+          ids = lib.attrValues ids;
+        }
+      );
+      toList = x: lib.concatStringsSep "," (map toString x);
+    in
+    lib.forEach options (i: "options ${i.driver} index=${toList i.ids} id=${toList i.names}");
+
+  defaultDeviceVars = {
+    "ALSA_AUDIO_OUT" = cfg.defaultDevice.playback;
+    "ALSA_AUDIO_IN" = cfg.defaultDevice.capture;
+  };
+
+in
+
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "sound" "enable" ] ''
+      The option was heavily overloaded and can be removed from most configurations.
+      To specifically configure the user space part of ALSA, see `hardware.alsa`.
+    '')
+    (lib.mkRemovedOptionModule [ "sound" "mediaKeys" ] ''
+      The media keys can be configured with any hotkey daemon (that better
+      integrates with your desktop setup). To continue using the actkbd daemon
+      (which was used up to NixOS 24.05), add these lines to your configuration:
+
+        services.actkbd.enable = true;
+        services.actkbd.bindings = [
+          # Mute
+          { keys = [ 113 ]; events = [ "key" ];
+            command = "''${pkgs.alsa-utils}/bin/amixer -q set Master toggle";
+          }
+          # Volume down
+          { keys = [ 114 ]; events = [ "key" "rep" ];
+            command = "''${pkgs.alsa-utils}/bin/amixer -q set Master 1- unmute";
+          }
+          # Volume up
+          { keys = [ 115 ]; events = [ "key" "rep" ];
+            command = "''${pkgs.alsa-utils}/bin/amixer -q set Master 1+ unmute";
+          }
+          # Mic Mute
+          { keys = [ 190 ]; events = [ "key" ];
+            command = "''${pkgs.alsa-utils}/bin/amixer -q set Capture toggle";
+          }
+        ];
+    '')
+    (lib.mkRenamedOptionModule
+      [ "sound" "enableOSSEmulation" ]
+      [ "hardware" "alsa" "enableOSSEmulation" ]
+    )
+    (lib.mkRenamedOptionModule [ "sound" "extraConfig" ] [ "hardware" "alsa" "config" ])
+  ];
+
+  options.hardware.alsa = {
+
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to set up the user space part of the Advanced Linux Sound Architecture (ALSA)
+
+        ::: {.warning}
+        Enable this option only if you want to use ALSA as your main sound system,
+        not if you're using a sound server (e.g. PulseAudio or Pipewire).
+        :::
+      '';
+    };
+
+    enableOSSEmulation = lib.mkEnableOption "the OSS emulation";
+
+    enableRecorder = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to set up a loopback device that continuously records and
+        allows to play back audio from the computer.
+
+        The loopback device is named `pcm.recorder`, audio can be saved
+        by capturing from this device as with any microphone.
+
+        ::: {.note}
+        By default the output is duplicated to the recorder assuming stereo
+        audio, for a more complex layout you have to override the pcm.splitter
+        device using `hardware.alsa.config`.
+        See the generated /etc/asound.conf for its definition.
+        :::
+      '';
+    };
+
+    defaultDevice.playback = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "dmix:CARD=1,DEV=0";
+      description = ''
+        The default playback device.
+        Leave empty to let ALSA pick the default automatically.
+
+        ::: {.note}
+        The device can be changed at runtime by setting the ALSA_AUDIO_OUT
+        environment variables (but only before starting a program).
+        :::
+      '';
+    };
+
+    defaultDevice.capture = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "dsnoop:CARD=0,DEV=2";
+      description = ''
+        The default capture device (i.e. microphone).
+        Leave empty to let ALSA pick the default automatically.
+
+        ::: {.note}
+        The device can be changed at runtime by setting the ALSA_AUDIO_IN
+        environment variables (but only before starting a program).
+        :::
+      '';
+    };
+
+    controls = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule {
+          options.name = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            default = null;
+            description = ''
+              Name of the control, as it appears in `alsamixer`.
+              If null it will be the same as the softvol device name.
+            '';
+          };
+          options.device = lib.mkOption {
+            type = lib.types.str;
+            default = "default";
+            description = ''
+              Name of the PCM device to control (slave).
+            '';
+          };
+          options.card = lib.mkOption {
+            type = lib.types.str;
+            default = "default";
+            description = ''
+              Name of the PCM card to control (slave).
+            '';
+          };
+          options.maxVolume = lib.mkOption {
+            type = lib.types.float;
+            default = 0.0;
+            description = ''
+              The maximum volume in dB.
+            '';
+          };
+        }
+      );
+      default = { };
+      example = lib.literalExpression ''
+        {
+          firefox = { device = "front"; maxVolume = -25.0; };
+          mpv     = { device = "front"; maxVolume = -25.0; };
+          # and run programs with `env ALSA_AUDIO_OUT=<name>`
+        }
+      '';
+      description = ''
+        Virtual volume controls (softvols) to add to a sound card.
+        These can be used to control the volume of specific applications
+        or a digital output device (HDMI video card).
+      '';
+    };
+
+    cardAliases = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule {
+          options.driver = lib.mkOption {
+            type = lib.types.str;
+            description = ''
+              Name of the kernel module that provides the card.
+            '';
+          };
+          options.id = lib.mkOption {
+            type = lib.types.int;
+            default = "default";
+            description = ''
+              The ID of the sound card
+            '';
+          };
+        }
+      );
+      default = { };
+      example = lib.literalExpression ''
+        {
+          soundchip = { driver = "snd_intel_hda"; id = 0; };
+          videocard = { driver = "snd_intel_hda"; id = 1; };
+          usb       = { driver = "snd_usb_audio"; id = 2; };
+        }
+      '';
+      description = ''
+        Assign custom names and reorder the sound cards.
+
+        ::: {.note}
+        You can find the card ids by looking at `/proc/asound/cards`.
+        :::
+      '';
+    };
+
+    deviceAliases = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          hdmi1 = "hw:CARD=videocard,DEV=5";
+          hdmi2 = "hw:CARD=videocard,DEV=6";
+        }
+      '';
+      description = ''
+        Assign custom names to sound cards.
+      '';
+    };
+
+    config = lib.mkOption {
+      type = lib.types.lines;
+      default = "";
+      example = lib.literalExpression ''
+        # Send audio to a remote host via SSH
+        pcm.remote {
+          @args [ HOSTNAME ]
+          @args.HOSTNAME { type string }
+          type file
+          format raw
+          slave.pcm pcm.null
+          file {
+            @func concat
+            strings [
+              "| ''${lib.getExec pkgs.openssh} -C "
+              $HOSTNAME
+              " aplay -f %f -c %c -r %r -"
+            ]
+          }
+        }
+      '';
+      description = ''
+        The content of the system-wide ALSA configuration (/etc/asound.conf).
+
+        Documentation of the configuration language and examples can be found
+        in the unofficial ALSA wiki: <https://alsa.opensrc.org/Asoundrc>
+      '';
+    };
+
+  };
+
+  options.hardware.alsa.enablePersistence = lib.mkOption {
+    type = lib.types.bool;
+    defaultText = lib.literalExpression "config.hardware.alsa.enable";
+    default = config.hardware.alsa.enable;
+    description = ''
+      Whether to enable ALSA sound card state saving on shutdown.
+      This is generally not necessary if you're using an external sound server.
+    '';
+  };
+
+  config = lib.mkMerge [
+
+    (lib.mkIf cfg.enable {
+      # Disable sound servers enabled by default and,
+      # if the user enabled one manually, cause a conflict.
+      services.pipewire.enable = false;
+      services.pulseaudio.enable = false;
+
+      hardware.alsa.config =
+        let
+          conf = [
+            ''
+              pcm.!default fromenv
+
+              # Read the capture and playback device from
+              # the ALSA_AUDIO_IN, ALSA_AUDIO_OUT variables
+              pcm.fromenv {
+                type asym
+                playback.pcm {
+                  type plug
+                  slave.pcm {
+                    @func getenv
+                    vars [ ALSA_AUDIO_OUT ]
+                    default pcm.sysdefault
+                  }
+                }
+                capture.pcm {
+                  type plug
+                  slave.pcm {
+                    @func getenv
+                    vars [ ALSA_AUDIO_IN ]
+                    default pcm.sysdefault
+                  }
+                }
+              }
+            ''
+            (lib.optional cfg.enableRecorder ''
+              pcm.!default "splitter:fromenv,recorder"
+
+              # Send audio to two stereo devices
+              pcm.splitter {
+                @args [ A B ]
+                @args.A.type string
+                @args.B.type string
+                type asym
+                playback.pcm {
+                  type plug
+                  route_policy "duplicate"
+                  slave.pcm {
+                    type multi
+                    slaves.a.pcm $A
+                    slaves.b.pcm $B
+                    slaves.a.channels 2
+                    slaves.b.channels 2
+                    bindings [
+                     { slave a channel 0 }
+                     { slave a channel 1 }
+                     { slave b channel 0 }
+                     { slave b channel 1 }
+                    ]
+                  }
+                }
+                capture.pcm $A
+              }
+
+              # Device which records and plays back audio
+              pcm.recorder {
+                type asym
+                capture.pcm {
+                  type dsnoop
+                  ipc_key 9165218
+                  ipc_perm 0666
+                  slave.pcm "hw:loopback,1,0"
+                  slave.period_size 1024
+                  slave.buffer_size 8192
+                }
+                playback.pcm {
+                  type dmix
+                  ipc_key 6181923
+                  ipc_perm 0666
+                  slave.pcm "hw:loopback,0,0"
+                  slave.period_size 1024
+                  slave.buffer_size 8192
+                }
+              }
+            '')
+            (lib.mapAttrsToList mkControl cfg.controls)
+            (lib.mapAttrsToList (n: v: "pcm.${n} ${quote v}") cfg.deviceAliases)
+          ];
+        in
+        lib.mkBefore (lib.concatStringsSep "\n" (lib.flatten conf));
+
+      hardware.alsa.cardAliases = lib.mkIf cfg.enableRecorder {
+        loopback.driver = "snd_aloop";
+        loopback.id = 2;
+      };
+
+      # Set default PCM devices
+      environment.sessionVariables = defaultDeviceVars;
+      systemd.globalEnvironment = defaultDeviceVars;
+
+      environment.etc."asound.conf".text = cfg.config;
+
+      boot.kernelModules =
+        [ ]
+        ++ lib.optionals cfg.enableOSSEmulation [
+          "snd_pcm_oss"
+          "snd_mixer_oss"
+        ]
+        ++ lib.optionals cfg.enableRecorder [ "snd_aloop" ];
+
+      # Assign names to the sound cards
+      boot.extraModprobeConfig = lib.concatStringsSep "\n" cardsConfig;
+
+      # Provide alsamixer, aplay, arecord, etc.
+      environment.systemPackages = [ pkgs.alsa-utils ];
+    })
+
+    (lib.mkIf config.hardware.alsa.enablePersistence {
+
+      # Install udev rules for restoring card settings on boot
+      services.udev.extraRules = ''
+        ACTION=="add", SUBSYSTEM=="sound", KERNEL=="controlC*", KERNELS!="card*", GOTO="alsa_restore_go"
+        GOTO="alsa_restore_end"
+
+        LABEL="alsa_restore_go"
+        TEST!="/etc/alsa/state-daemon.conf", RUN+="${alsactl} restore -gU $attr{device/number}"
+        TEST=="/etc/alsa/state-daemon.conf", RUN+="${alsactl} nrestore -gU $attr{device/number}"
+        LABEL="alsa_restore_end"
+      '';
+
+      # Service to store/restore the sound card settings
+      systemd.services.alsa-store = {
+        description = "Store Sound Card State";
+        wantedBy = [ "multi-user.target" ];
+        restartIfChanged = false;
+        unitConfig = {
+          RequiresMountsFor = "/var/lib/alsa";
+          ConditionVirtualization = "!systemd-nspawn";
+        };
+        serviceConfig = {
+          Type = "oneshot";
+          RemainAfterExit = true;
+          StateDirectory = "alsa";
+          # Note: the service should never be restated, otherwise any
+          # setting changed between the last `store` and now will be lost.
+          # To prevent NixOS from starting it in case it has failed we
+          # expand the exit codes considered successful
+          SuccessExitStatus = [
+            0
+            99
+          ];
+          ExecStart = "${alsactl} restore -gU";
+          ExecStop = "${alsactl} store -gU";
+        };
+      };
+    })
+
+  ];
+
+  meta.maintainers = with lib.maintainers; [ rnhmjoj ];
+
+}

nixos/modules/services/audio/botamusique.nix

@@ -0,0 +1,113 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.botamusique;
+
+  format = pkgs.formats.ini { };
+  configFile = format.generate "botamusique.ini" cfg.settings;
+in
+{
+  meta.maintainers = with lib.maintainers; [ hexa ];
+
+  options.services.botamusique = {
+    enable = lib.mkEnableOption "botamusique, a bot to play audio streams on mumble";
+
+    package = lib.mkPackageOption pkgs "botamusique" { };
+
+    settings = lib.mkOption {
+      type =
+        with lib.types;
+        submodule {
+          freeformType = format.type;
+          options = {
+            server.host = lib.mkOption {
+              type = types.str;
+              default = "localhost";
+              example = "mumble.example.com";
+              description = "Hostname of the mumble server to connect to.";
+            };
+
+            server.port = lib.mkOption {
+              type = types.port;
+              default = 64738;
+              description = "Port of the mumble server to connect to.";
+            };
+
+            bot.username = lib.mkOption {
+              type = types.str;
+              default = "botamusique";
+              description = "Name the bot should appear with.";
+            };
+
+            bot.comment = lib.mkOption {
+              type = types.str;
+              default = "Hi, I'm here to play radio, local music or youtube/soundcloud music. Have fun!";
+              description = "Comment displayed for the bot.";
+            };
+          };
+        };
+      default = { };
+      description = ''
+        Your {file}`configuration.ini` as a Nix attribute set. Look up
+        possible options in the [configuration.example.ini](https://github.com/azlux/botamusique/blob/master/configuration.example.ini).
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.botamusique = {
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      unitConfig.Documentation = "https://github.com/azlux/botamusique/wiki";
+
+      environment.HOME = "/var/lib/botamusique";
+
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/botamusique --config ${configFile}";
+        Restart = "always"; # the bot exits when the server connection is lost
+
+        # Hardening
+        CapabilityBoundingSet = [ "" ];
+        DynamicUser = true;
+        IPAddressDeny = [
+          "link-local"
+          "multicast"
+        ];
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        ProcSubset = "pid";
+        PrivateDevices = true;
+        PrivateUsers = true;
+        PrivateTmp = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        StateDirectory = "botamusique";
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service @resources"
+          "~@privileged"
+        ];
+        UMask = "0077";
+        WorkingDirectory = "/var/lib/botamusique";
+      };
+    };
+  };
+}

nixos/modules/services/audio/gmediarender.nix

@@ -0,0 +1,132 @@
+{
+  pkgs,
+  lib,
+  config,
+  utils,
+  ...
+}:
+let
+  cfg = config.services.gmediarender;
+in
+{
+  options.services.gmediarender = {
+    enable = lib.mkEnableOption "the gmediarender DLNA renderer";
+
+    audioDevice = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        The audio device to use.
+      '';
+    };
+
+    audioSink = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        The audio sink to use.
+      '';
+    };
+
+    friendlyName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        A "friendly name" for identifying the endpoint.
+      '';
+    };
+
+    initialVolume = lib.mkOption {
+      type = lib.types.nullOr lib.types.int;
+      default = 0;
+      description = ''
+        A default volume attenuation (in dB) for the endpoint.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "gmediarender" {
+      default = "gmrender-resurrect";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.nullOr lib.types.port;
+      default = null;
+      description = "Port that will be used to accept client connections.";
+    };
+
+    uuid = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        A UUID for uniquely identifying the endpoint.  If you have
+        multiple renderers on your network, you MUST set this.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd = {
+      services.gmediarender = {
+        wants = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+        wantedBy = [ "multi-user.target" ];
+        description = "gmediarender server daemon";
+        environment = {
+          XDG_CACHE_HOME = "%t/gmediarender";
+        };
+        serviceConfig = {
+          DynamicUser = true;
+          User = "gmediarender";
+          Group = "gmediarender";
+          SupplementaryGroups = [ "audio" ];
+          ExecStart =
+            "${cfg.package}/bin/gmediarender "
+            + lib.optionalString (
+              cfg.audioDevice != null
+            ) "--gstout-audiodevice=${utils.escapeSystemdExecArg cfg.audioDevice} "
+
+            + lib.optionalString (
+              cfg.audioSink != null
+            ) "--gstout-audiosink=${utils.escapeSystemdExecArg cfg.audioSink} "
+
+            + lib.optionalString (
+              cfg.friendlyName != null
+            ) "--friendly-name=${utils.escapeSystemdExecArg cfg.friendlyName} "
+
+            + lib.optionalString (cfg.initialVolume != 0) "--initial-volume=${toString cfg.initialVolume} "
+            + lib.optionalString (cfg.port != null) "--port=${toString cfg.port} "
+            + lib.optionalString (cfg.uuid != null) "--uuid=${utils.escapeSystemdExecArg cfg.uuid} ";
+          Restart = "always";
+          RuntimeDirectory = "gmediarender";
+
+          # Security options:
+          CapabilityBoundingSet = "";
+          LockPersonality = true;
+          MemoryDenyWriteExecute = true;
+          NoNewPrivileges = true;
+          # PrivateDevices = true;
+          PrivateTmp = true;
+          PrivateUsers = true;
+          ProcSubset = "pid";
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectProc = "invisible";
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+          SystemCallArchitectures = "native";
+          SystemCallFilter = [
+            "@system-service"
+            "~@privileged"
+          ];
+          UMask = 66;
+        };
+      };
+    };
+  };
+}

nixos/modules/services/audio/gonic.nix

@@ -0,0 +1,105 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.gonic;
+  settingsFormat = pkgs.formats.keyValue {
+    mkKeyValue = lib.generators.mkKeyValueDefault { } " ";
+    listsAsDuplicateKeys = true;
+  };
+in
+{
+  options = {
+    services.gonic = {
+
+      enable = lib.mkEnableOption "Gonic music server";
+
+      settings = lib.mkOption rec {
+        type = settingsFormat.type;
+        apply = lib.recursiveUpdate default;
+        default = {
+          listen-addr = "127.0.0.1:4747";
+          cache-path = "/var/cache/gonic";
+          tls-cert = null;
+          tls-key = null;
+        };
+        example = {
+          music-path = [ "/mnt/music" ];
+          podcast-path = "/mnt/podcasts";
+        };
+        description = ''
+          Configuration for Gonic, see <https://github.com/sentriz/gonic#configuration-options> for supported values.
+        '';
+      };
+
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.gonic = {
+      description = "Gonic Media Server";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart =
+          let
+            # these values are null by default but should not appear in the final config
+            filteredSettings = lib.filterAttrs (
+              n: v: !((n == "tls-cert" || n == "tls-key") && v == null)
+            ) cfg.settings;
+          in
+          "${pkgs.gonic}/bin/gonic -config-path ${settingsFormat.generate "gonic" filteredSettings}";
+        DynamicUser = true;
+        StateDirectory = "gonic";
+        CacheDirectory = "gonic";
+        WorkingDirectory = "/var/lib/gonic";
+        RuntimeDirectory = "gonic";
+        RootDirectory = "/run/gonic";
+        ReadWritePaths = "";
+        BindPaths = [
+          cfg.settings.playlists-path
+          cfg.settings.podcast-path
+        ];
+        BindReadOnlyPaths = [
+          # gonic can access scrobbling services
+          "-/etc/resolv.conf"
+          "${config.security.pki.caBundle}:/etc/ssl/certs/ca-certificates.crt"
+          builtins.storeDir
+        ]
+        ++ cfg.settings.music-path
+        ++ lib.optional (cfg.settings.tls-cert != null) cfg.settings.tls-cert
+        ++ lib.optional (cfg.settings.tls-key != null) cfg.settings.tls-key;
+        CapabilityBoundingSet = "";
+        RestrictAddressFamilies = [
+          "AF_UNIX"
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        PrivateDevices = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+        RestrictRealtime = true;
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        UMask = "0066";
+        ProtectHostname = true;
+      };
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.autrimpo ];
+}

nixos/modules/services/audio/goxlr-utility.nix

@@ -0,0 +1,60 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.goxlr-utility;
+in
+{
+
+  options = {
+    services.goxlr-utility = {
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Whether to enable goxlr-utility for controlling your TC-Helicon GoXLR or GoXLR Mini
+        '';
+      };
+      package = lib.mkPackageOption pkgs "goxlr-utility" { };
+      autoStart.xdg = lib.mkOption {
+        default = true;
+        type = with lib.types; bool;
+        description = ''
+          Start the daemon automatically using XDG autostart.
+          Sets `xdg.autostart.enable = true` if not already enabled.
+        '';
+      };
+    };
+  };
+
+  config =
+    let
+      goxlr-autostart = pkgs.stdenv.mkDerivation {
+        name = "autostart-goxlr-daemon";
+        priority = 5;
+
+        buildCommand = ''
+          mkdir -p $out/etc/xdg/autostart
+          cp ${cfg.package}/share/applications/goxlr-utility.desktop $out/etc/xdg/autostart/goxlr-daemon.desktop
+          chmod +w $out/etc/xdg/autostart/goxlr-daemon.desktop
+          echo "X-KDE-autostart-phase=2" >> $out/etc/xdg/autostart/goxlr-daemon.desktop
+          substituteInPlace $out/etc/xdg/autostart/goxlr-daemon.desktop \
+            --replace-fail goxlr-launcher goxlr-daemon
+        '';
+      };
+    in
+    lib.mkIf config.services.goxlr-utility.enable {
+      services.udev.packages = [ cfg.package ];
+
+      xdg.autostart.enable = lib.mkIf cfg.autoStart.xdg true;
+      environment.systemPackages = lib.mkIf cfg.autoStart.xdg [
+        cfg.package
+        goxlr-autostart
+      ];
+    };
+
+  meta.maintainers = with lib.maintainers; [ errnoh ];
+}

nixos/modules/services/audio/hqplayerd.nix

@@ -0,0 +1,156 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hqplayerd;
+  pkg = pkgs.hqplayerd;
+  # XXX: This is hard-coded in the distributed binary, don't try to change it.
+  stateDir = "/var/lib/hqplayer";
+  configDir = "/etc/hqplayer";
+in
+{
+  options = {
+    services.hqplayerd = {
+      enable = lib.mkEnableOption "HQPlayer Embedded";
+
+      auth = {
+        username = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            Username used for HQPlayer's WebUI.
+
+            Without this you will need to manually create the credentials after
+            first start by going to http://your.ip/8088/auth
+          '';
+        };
+
+        password = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          default = null;
+          description = ''
+            Password used for HQPlayer's WebUI.
+
+            Without this you will need to manually create the credentials after
+            first start by going to http://your.ip/8088/auth
+          '';
+        };
+      };
+
+      licenseFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = ''
+          Path to the HQPlayer license key file.
+
+          Without this, the service will run in trial mode and restart every 30
+          minutes.
+        '';
+      };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Opens ports needed for the WebUI and controller API.
+        '';
+      };
+
+      config = lib.mkOption {
+        type = lib.types.nullOr lib.types.lines;
+        default = null;
+        description = ''
+          HQplayer daemon configuration, written to /etc/hqplayer/hqplayerd.xml.
+
+          Refer to share/doc/hqplayerd/readme.txt in the hqplayerd derivation for possible values.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion =
+          (cfg.auth.username != null -> cfg.auth.password != null)
+          && (cfg.auth.password != null -> cfg.auth.username != null);
+        message = "You must set either both services.hqplayer.auth.username and password, or neither.";
+      }
+    ];
+
+    environment = {
+      etc = {
+        "hqplayer/hqplayerd.xml" = lib.mkIf (cfg.config != null) {
+          source = pkgs.writeText "hqplayerd.xml" cfg.config;
+        };
+        "hqplayer/hqplayerd4-key.xml" = lib.mkIf (cfg.licenseFile != null) { source = cfg.licenseFile; };
+      };
+      systemPackages = [ pkg ];
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPorts = [
+        8088
+        4321
+      ];
+    };
+
+    systemd = {
+      tmpfiles.rules = [
+        "d ${configDir}      0755 hqplayer hqplayer - -"
+        "d ${stateDir}       0755 hqplayer hqplayer - -"
+        "d ${stateDir}/home  0755 hqplayer hqplayer - -"
+      ];
+
+      packages = [ pkg ];
+
+      services.hqplayerd = {
+        wantedBy = [ "multi-user.target" ];
+        after = [ "systemd-tmpfiles-setup.service" ];
+
+        environment.HOME = "${stateDir}/home";
+
+        unitConfig.ConditionPathExists = [
+          configDir
+          stateDir
+        ];
+
+        restartTriggers = lib.optionals (cfg.config != null) [
+          config.environment.etc."hqplayer/hqplayerd.xml".source
+        ];
+
+        preStart = ''
+          cp -r "${pkg}/var/lib/hqplayer/web" "${stateDir}"
+          chmod -R u+wX "${stateDir}/web"
+
+          if [ ! -f "${configDir}/hqplayerd.xml" ]; then
+            echo "creating initial config file"
+            install -m 0644 "${pkg}/etc/hqplayer/hqplayerd.xml" "${configDir}/hqplayerd.xml"
+          fi
+        ''
+        + lib.optionalString (cfg.auth.username != null && cfg.auth.password != null) ''
+          ${pkg}/bin/hqplayerd -s ${cfg.auth.username} ${cfg.auth.password}
+        '';
+      };
+    };
+
+    users.groups = {
+      hqplayer.gid = config.ids.gids.hqplayer;
+    };
+
+    users.users = {
+      hqplayer = {
+        description = "hqplayer daemon user";
+        extraGroups = [
+          "audio"
+          "video"
+        ];
+        group = "hqplayer";
+        uid = config.ids.uids.hqplayer;
+      };
+    };
+  };
+}

nixos/modules/services/audio/icecast.nix

@@ -0,0 +1,133 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.icecast;
+  configFile = pkgs.writeText "icecast.xml" ''
+    <icecast>
+      <hostname>${cfg.hostname}</hostname>
+
+      <authentication>
+        <admin-user>${cfg.admin.user}</admin-user>
+        <admin-password>${cfg.admin.password}</admin-password>
+      </authentication>
+
+      <paths>
+        <logdir>${cfg.logDir}</logdir>
+        <adminroot>${pkgs.icecast}/share/icecast/admin</adminroot>
+        <webroot>${pkgs.icecast}/share/icecast/web</webroot>
+        <alias source="/" dest="/status.xsl"/>
+      </paths>
+
+      <listen-socket>
+        <port>${toString cfg.listen.port}</port>
+        <bind-address>${cfg.listen.address}</bind-address>
+      </listen-socket>
+
+      <security>
+        <chroot>0</chroot>
+        <changeowner>
+            <user>${cfg.user}</user>
+            <group>${cfg.group}</group>
+        </changeowner>
+      </security>
+
+      ${cfg.extraConf}
+    </icecast>
+  '';
+in
+{
+
+  ###### interface
+
+  options = {
+
+    services.icecast = {
+
+      enable = lib.mkEnableOption "Icecast server";
+
+      hostname = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        description = "DNS name or IP address that will be used for the stream directory lookups or possibly the playlist generation if a Host header is not provided.";
+        default = config.networking.domain;
+        defaultText = lib.literalExpression "config.networking.domain";
+      };
+
+      admin = {
+        user = lib.mkOption {
+          type = lib.types.str;
+          description = "Username used for all administration functions.";
+          default = "admin";
+        };
+
+        password = lib.mkOption {
+          type = lib.types.str;
+          description = "Password used for all administration functions.";
+        };
+      };
+
+      logDir = lib.mkOption {
+        type = lib.types.path;
+        description = "Base directory used for logging.";
+        default = "/var/log/icecast";
+      };
+
+      listen = {
+        port = lib.mkOption {
+          type = lib.types.port;
+          description = "TCP port that will be used to accept client connections.";
+          default = 8000;
+        };
+
+        address = lib.mkOption {
+          type = lib.types.str;
+          description = "Address Icecast will listen on.";
+          default = "::";
+        };
+      };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        description = "User privileges for the server.";
+        default = "nobody";
+      };
+
+      group = lib.mkOption {
+        type = lib.types.str;
+        description = "Group privileges for the server.";
+        default = "nogroup";
+      };
+
+      extraConf = lib.mkOption {
+        type = lib.types.lines;
+        description = "icecast.xml content.";
+        default = "";
+      };
+
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.services.icecast = {
+      after = [ "network.target" ];
+      description = "Icecast Network Audio Streaming Server";
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = "mkdir -p ${cfg.logDir} && chown ${cfg.user}:${cfg.group} ${cfg.logDir}";
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${pkgs.icecast}/bin/icecast -c ${configFile}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      };
+    };
+
+  };
+
+}

nixos/modules/services/audio/jack.nix

@@ -0,0 +1,305 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.jack;
+
+  pcmPlugin = cfg.jackd.enable && cfg.alsa.enable;
+  loopback = cfg.jackd.enable && cfg.loopback.enable;
+
+  enable32BitAlsaPlugins =
+    cfg.alsa.support32Bit && pkgs.stdenv.hostPlatform.isx86_64 && pkgs.pkgsi686Linux.alsa-lib != null;
+
+  umaskNeeded = lib.versionOlder cfg.jackd.package.version "1.9.12";
+  bridgeNeeded = lib.versionAtLeast cfg.jackd.package.version "1.9.12";
+in
+{
+  options = {
+    services.jack = {
+      jackd = {
+        enable = lib.mkEnableOption ''
+          JACK Audio Connection Kit. You need to add yourself to the "jackaudio" group
+        '';
+
+        package =
+          lib.mkPackageOption pkgs "jack2" {
+            example = "jack1";
+          }
+          // {
+            # until jack1 promiscuous mode is fixed
+            internal = true;
+          };
+
+        extraOptions = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          default = [
+            "-dalsa"
+          ];
+          example = lib.literalExpression ''
+            [ "-dalsa" "--device" "hw:1" ];
+          '';
+          description = ''
+            Specifies startup command line arguments to pass to JACK server.
+          '';
+        };
+
+        session = lib.mkOption {
+          type = lib.types.lines;
+          description = ''
+            Commands to run after JACK is started.
+          '';
+        };
+
+      };
+
+      alsa = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Route audio to/from generic ALSA-using applications using ALSA JACK PCM plugin.
+          '';
+        };
+
+        support32Bit = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = ''
+            Whether to support sound for 32-bit ALSA applications on 64-bit system.
+          '';
+        };
+      };
+
+      loopback = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = ''
+            Create ALSA loopback device, instead of using PCM plugin. Has broader
+            application support (things like Steam will work), but may need fine-tuning
+            for concrete hardware.
+          '';
+        };
+
+        index = lib.mkOption {
+          type = lib.types.int;
+          default = 10;
+          description = ''
+            Index of an ALSA loopback device.
+          '';
+        };
+
+        config = lib.mkOption {
+          type = lib.types.lines;
+          description = ''
+            ALSA config for loopback device.
+          '';
+        };
+
+        dmixConfig = lib.mkOption {
+          type = lib.types.lines;
+          default = "";
+          example = ''
+            period_size 2048
+            periods 2
+          '';
+          description = ''
+            For music production software that still doesn't support JACK natively you
+            would like to put buffer/period adjustments here
+            to decrease dmix device latency.
+          '';
+        };
+
+        session = lib.mkOption {
+          type = lib.types.lines;
+          description = ''
+            Additional commands to run to setup loopback device.
+          '';
+        };
+      };
+
+    };
+
+  };
+
+  config = lib.mkMerge [
+
+    (lib.mkIf pcmPlugin {
+      environment.etc."alsa/conf.d/98-jack.conf".text = ''
+        pcm_type.jack {
+          libs.native = ${pkgs.alsa-plugins}/lib/alsa-lib/libasound_module_pcm_jack.so ;
+          ${lib.optionalString enable32BitAlsaPlugins "libs.32Bit = ${pkgs.pkgsi686Linux.alsa-plugins}/lib/alsa-lib/libasound_module_pcm_jack.so ;"}
+        }
+        pcm.!default {
+          @func getenv
+          vars [ PCM ]
+          default "plug:jack"
+        }
+      '';
+    })
+
+    (lib.mkIf loopback {
+      boot.kernelModules = [ "snd-aloop" ];
+      boot.kernelParams = [ "snd-aloop.index=${toString cfg.loopback.index}" ];
+      environment.etc."alsa/conf.d/99-jack-loopback.conf".text = cfg.loopback.config;
+    })
+
+    (lib.mkIf cfg.jackd.enable {
+      services.jack.jackd.session = ''
+        ${lib.optionalString bridgeNeeded "${pkgs.a2jmidid}/bin/a2jmidid -e &"}
+      '';
+      # https://alsa.opensrc.org/Jack_and_Loopback_device_as_Alsa-to-Jack_bridge#id06
+      services.jack.loopback.config = ''
+        pcm.loophw00 {
+          type hw
+          card ${toString cfg.loopback.index}
+          device 0
+          subdevice 0
+        }
+        pcm.amix {
+          type dmix
+          ipc_key 219345
+          slave {
+            pcm loophw00
+            ${cfg.loopback.dmixConfig}
+          }
+        }
+        pcm.asoftvol {
+          type softvol
+          slave.pcm "amix"
+          control { name Master }
+        }
+        pcm.cloop {
+          type hw
+          card ${toString cfg.loopback.index}
+          device 1
+          subdevice 0
+          format S32_LE
+        }
+        pcm.loophw01 {
+          type hw
+          card ${toString cfg.loopback.index}
+          device 0
+          subdevice 1
+        }
+        pcm.ploop {
+          type hw
+          card ${toString cfg.loopback.index}
+          device 1
+          subdevice 1
+          format S32_LE
+        }
+        pcm.aduplex {
+          type asym
+          playback.pcm "asoftvol"
+          capture.pcm "loophw01"
+        }
+        pcm.!default {
+          type plug
+          slave.pcm aduplex
+        }
+      '';
+      services.jack.loopback.session = ''
+        alsa_in -j cloop -dcloop &
+        alsa_out -j ploop -dploop &
+        while [ "$(jack_lsp cloop)" == "" ] || [ "$(jack_lsp ploop)" == "" ]; do sleep 1; done
+        jack_connect cloop:capture_1 system:playback_1
+        jack_connect cloop:capture_2 system:playback_2
+        jack_connect system:capture_1 ploop:playback_1
+        jack_connect system:capture_2 ploop:playback_2
+      '';
+
+      assertions = [
+        {
+          assertion = !(cfg.alsa.enable && cfg.loopback.enable);
+          message = "For JACK both alsa and loopback options shouldn't be used at the same time.";
+        }
+      ];
+
+      users.users.jackaudio = {
+        group = "jackaudio";
+        extraGroups = [ "audio" ];
+        description = "JACK Audio system service user";
+        isSystemUser = true;
+      };
+      # https://jackaudio.org/faq/linux_rt_config.html
+      security.pam.loginLimits = [
+        {
+          domain = "@jackaudio";
+          type = "-";
+          item = "rtprio";
+          value = "99";
+        }
+        {
+          domain = "@jackaudio";
+          type = "-";
+          item = "memlock";
+          value = "unlimited";
+        }
+      ];
+      users.groups.jackaudio = { };
+
+      environment = {
+        systemPackages = [ cfg.jackd.package ];
+        etc."alsa/conf.d/50-jack.conf".source = "${pkgs.alsa-plugins}/etc/alsa/conf.d/50-jack.conf";
+        variables.JACK_PROMISCUOUS_SERVER = "jackaudio";
+      };
+
+      services.udev.extraRules = ''
+        ACTION=="add", SUBSYSTEM=="sound", ATTRS{id}!="Loopback", TAG+="systemd", ENV{SYSTEMD_WANTS}="jack.service"
+      '';
+
+      systemd.services.jack = {
+        description = "JACK Audio Connection Kit";
+        serviceConfig = {
+          User = "jackaudio";
+          SupplementaryGroups = lib.optional (
+            config.services.pulseaudio.enable && !config.services.pulseaudio.systemWide
+          ) "users";
+          ExecStart = "${cfg.jackd.package}/bin/jackd ${lib.escapeShellArgs cfg.jackd.extraOptions}";
+          LimitRTPRIO = 99;
+          LimitMEMLOCK = "infinity";
+        }
+        // lib.optionalAttrs umaskNeeded {
+          UMask = "007";
+        };
+        path = [ cfg.jackd.package ];
+        environment = {
+          JACK_PROMISCUOUS_SERVER = "jackaudio";
+          JACK_NO_AUDIO_RESERVATION = "1";
+        };
+        restartIfChanged = false;
+      };
+      systemd.services.jack-session = {
+        description = "JACK session";
+        script = ''
+          ${pkgs.jack-example-tools}/bin/jack_wait -w
+          ${cfg.jackd.session}
+          ${lib.optionalString cfg.loopback.enable cfg.loopback.session}
+        '';
+        serviceConfig = {
+          RemainAfterExit = true;
+          User = "jackaudio";
+          StateDirectory = "jack";
+          LimitRTPRIO = 99;
+          LimitMEMLOCK = "infinity";
+        };
+        path = [ cfg.jackd.package ];
+        environment = {
+          JACK_PROMISCUOUS_SERVER = "jackaudio";
+          HOME = "/var/lib/jack";
+        };
+        wantedBy = [ "jack.service" ];
+        partOf = [ "jack.service" ];
+        after = [ "jack.service" ];
+        restartIfChanged = false;
+      };
+    })
+
+  ];
+
+  meta.maintainers = [ ];
+}

nixos/modules/services/audio/jmusicbot.nix

@@ -0,0 +1,49 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.jmusicbot;
+in
+{
+  options = {
+    services.jmusicbot = {
+      enable = lib.mkEnableOption "jmusicbot, a Discord music bot that's easy to set up and run yourself";
+
+      package = lib.mkPackageOption pkgs "jmusicbot" { };
+
+      stateDir = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          The directory where config.txt and serversettings.json is saved.
+          If left as the default value this directory will automatically be created before JMusicBot starts, otherwise the sysadmin is responsible for ensuring the directory exists with appropriate ownership and permissions.
+          Untouched by the value of this option config.txt needs to be placed manually into this directory.
+        '';
+        default = "/var/lib/jmusicbot/";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.jmusicbot = {
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      description = "Discord music bot that's easy to set up and run yourself!";
+      serviceConfig = lib.mkMerge [
+        {
+          ExecStart = "${cfg.package}/bin/JMusicBot";
+          WorkingDirectory = cfg.stateDir;
+          Restart = "always";
+          RestartSec = 20;
+          DynamicUser = true;
+        }
+        (lib.mkIf (cfg.stateDir == "/var/lib/jmusicbot") { StateDirectory = "jmusicbot"; })
+      ];
+    };
+  };
+
+  meta.maintainers = [ ];
+}

nixos/modules/services/audio/lavalink.nix

@@ -0,0 +1,335 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkOption
+    mkEnableOption
+    mkIf
+    types
+    ;
+
+  cfg = config.services.lavalink;
+
+  format = pkgs.formats.yaml { };
+in
+
+{
+  options.services.lavalink = {
+    enable = mkEnableOption "Lavalink";
+
+    package = lib.mkPackageOption pkgs "lavalink" { };
+
+    password = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      example = "s3cRe!p4SsW0rD";
+      description = ''
+        The password for Lavalink's authentication in plain text.
+      '';
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 2333;
+      example = 4567;
+      description = ''
+        The port that Lavalink will use.
+      '';
+    };
+
+    address = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      example = "127.0.0.1";
+      description = ''
+        The network address to bind to.
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      example = true;
+      description = ''
+        Whether to expose the port to the network.
+      '';
+    };
+
+    user = mkOption {
+      type = types.str;
+      default = "lavalink";
+      example = "root";
+      description = ''
+        The user of the service.
+      '';
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "lavalink";
+      example = "medias";
+      description = ''
+        The group of the service.
+      '';
+    };
+
+    home = mkOption {
+      type = types.str;
+      default = "/var/lib/lavalink";
+      example = "/home/lavalink";
+      description = ''
+        The home directory for lavalink.
+      '';
+    };
+
+    enableHttp2 = mkEnableOption "HTTP/2 support";
+
+    jvmArgs = mkOption {
+      type = types.str;
+      default = "-Xmx4G";
+      example = "-Djava.io.tmpdir=/var/lib/lavalink/tmp -Xmx6G";
+      description = ''
+        Set custom JVM arguments.
+      '';
+    };
+
+    environmentFile = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      example = "/run/secrets/lavalink/passwordEnvFile";
+      description = ''
+        Add custom environment variables from a file.
+        See <https://lavalink.dev/configuration/index.html#example-environment-variables> for the full documentation.
+      '';
+    };
+
+    plugins = mkOption {
+      type = types.listOf (
+        types.submodule {
+          options = {
+            dependency = mkOption {
+              type = types.str;
+              example = "dev.lavalink.youtube:youtube-plugin:1.8.0";
+              description = ''
+                The coordinates of the plugin.
+              '';
+            };
+
+            repository = mkOption {
+              type = types.str;
+              example = "https://maven.example.com/releases";
+              default = "https://maven.lavalink.dev/releases";
+              description = ''
+                The plugin repository. Defaults to the lavalink releases repository.
+
+                To use the snapshots repository, use <https://maven.lavalink.dev/snapshots> instead
+              '';
+            };
+
+            hash = mkOption {
+              type = types.str;
+              example = lib.fakeHash;
+              description = ''
+                The hash of the plugin.
+              '';
+            };
+
+            configName = mkOption {
+              type = types.nullOr types.str;
+              example = "youtube";
+              default = null;
+              description = ''
+                The name of the plugin to use as the key for the plugin configuration.
+              '';
+            };
+
+            extraConfig = mkOption {
+              type = types.submodule { freeformType = format.type; };
+              default = { };
+              description = ''
+                The configuration for the plugin.
+
+                The {option}`services.lavalink.plugins.*.configName` option must be set.
+              '';
+            };
+          };
+        }
+      );
+      default = [ ];
+
+      example = lib.literalExpression ''
+        [
+          {
+            dependency = "dev.lavalink.youtube:youtube-plugin:1.8.0";
+            repository = "https://maven.lavalink.dev/snapshots";
+            hash = lib.fakeHash;
+            configName = "youtube";
+            extraConfig = {
+              enabled = true;
+              allowSearch = true;
+              allowDirectVideoIds = true;
+              allowDirectPlaylistIds = true;
+            };
+          }
+        ]
+      '';
+
+      description = ''
+        A list of plugins for lavalink.
+      '';
+    };
+
+    extraConfig = mkOption {
+      type = types.submodule { freeformType = format.type; };
+
+      description = ''
+        Configuration to write to {file}`application.yml`.
+        See <https://lavalink.dev/configuration/#example-applicationyml> for the full documentation.
+
+        Individual configuration parameters can be overwritten using environment variables.
+        See <https://lavalink.dev/configuration/#example-environment-variables> for more information.
+      '';
+
+      default = { };
+
+      example = lib.literalExpression ''
+        {
+          lavalink.server = {
+            sources.twitch = true;
+
+            filters.volume = true;
+          };
+
+          logging.file.path = "./logs/";
+        }
+      '';
+    };
+  };
+
+  config =
+    let
+      pluginSymlinks = lib.concatStringsSep "\n" (
+        map (
+          pluginCfg:
+          let
+            pluginParts = lib.match ''^(.*?:(.*?):)([0-9]+\.[0-9]+\.[0-9]+)$'' pluginCfg.dependency;
+
+            pluginWebPath = lib.replaceStrings [ "." ":" ] [ "/" "/" ] (lib.elemAt pluginParts 0);
+
+            pluginFileName = lib.elemAt pluginParts 1;
+            pluginVersion = lib.elemAt pluginParts 2;
+
+            pluginFile = "${pluginFileName}-${pluginVersion}.jar";
+            pluginUrl = "${pluginCfg.repository}/${pluginWebPath}${pluginVersion}/${pluginFile}";
+
+            plugin = pkgs.fetchurl {
+              url = pluginUrl;
+              inherit (pluginCfg) hash;
+            };
+          in
+          "ln -sf ${plugin} ${cfg.home}/plugins/${pluginFile}"
+        ) cfg.plugins
+      );
+
+      pluginExtraConfigs = builtins.listToAttrs (
+        builtins.map (
+          pluginConfig: lib.attrsets.nameValuePair pluginConfig.configName pluginConfig.extraConfig
+        ) (lib.lists.filter (pluginCfg: pluginCfg.configName != null) cfg.plugins)
+      );
+
+      config = lib.attrsets.recursiveUpdate cfg.extraConfig {
+        server = {
+          inherit (cfg) port address;
+          http2.enabled = cfg.enableHttp2;
+        };
+
+        plugins = pluginExtraConfigs;
+        lavalink.plugins = (
+          builtins.map (
+            pluginConfig:
+            builtins.removeAttrs pluginConfig [
+              "name"
+              "extraConfig"
+              "hash"
+            ]
+          ) cfg.plugins
+        );
+      };
+
+      configWithPassword = lib.attrsets.recursiveUpdate config (
+        lib.attrsets.optionalAttrs (cfg.password != null) { lavalink.server.password = cfg.password; }
+      );
+
+      configFile = format.generate "application.yml" configWithPassword;
+    in
+    mkIf cfg.enable {
+      assertions = [
+        {
+          assertion =
+            !(lib.lists.any (
+              pluginCfg: pluginCfg.extraConfig != { } && pluginCfg.configName == null
+            ) cfg.plugins);
+          message = "Plugins with extra configuration need to have the `configName` attribute defined";
+        }
+      ];
+
+      networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
+
+      users.groups = mkIf (cfg.group == "lavalink") { lavalink = { }; };
+      users.users = mkIf (cfg.user == "lavalink") {
+        lavalink = {
+          inherit (cfg) home;
+          group = "lavalink";
+          description = "The user for the Lavalink server";
+          isSystemUser = true;
+        };
+      };
+
+      systemd.tmpfiles.settings."10-lavalink" =
+        let
+          dirConfig = {
+            inherit (cfg) user group;
+            mode = "0700";
+          };
+        in
+        {
+          "${cfg.home}/plugins".d = mkIf (cfg.plugins != [ ]) dirConfig;
+          ${cfg.home}.d = dirConfig;
+        };
+
+      systemd.services.lavalink = {
+        description = "Lavalink Service";
+
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "syslog.target"
+          "network.target"
+        ];
+
+        script = ''
+          ${pluginSymlinks}
+
+          ln -sf ${configFile} ${cfg.home}/application.yml
+          export _JAVA_OPTIONS="${cfg.jvmArgs}"
+
+          ${lib.getExe cfg.package}
+        '';
+
+        serviceConfig = {
+          User = cfg.user;
+          Group = cfg.group;
+
+          Type = "simple";
+          Restart = "on-failure";
+
+          EnvironmentFile = cfg.environmentFile;
+          WorkingDirectory = cfg.home;
+        };
+      };
+    };
+}

nixos/modules/services/audio/liquidsoap.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  streams = builtins.attrNames config.services.liquidsoap.streams;
+
+  streamService =
+    name:
+    let
+      stream = builtins.getAttr name config.services.liquidsoap.streams;
+    in
+    {
+      inherit name;
+      value = {
+        after = [
+          "network-online.target"
+          "sound.target"
+        ];
+        description = "${name} liquidsoap stream";
+        wantedBy = [ "multi-user.target" ];
+        path = [ pkgs.wget ];
+        serviceConfig = {
+          ExecStart = "${pkgs.liquidsoap}/bin/liquidsoap ${stream}";
+          User = "liquidsoap";
+          LogsDirectory = "liquidsoap";
+          Restart = "always";
+        };
+      };
+    };
+in
+{
+
+  ##### interface
+
+  options = {
+
+    services.liquidsoap.streams = lib.mkOption {
+
+      description = ''
+        Set of Liquidsoap streams to start,
+        one systemd service per stream.
+      '';
+
+      default = { };
+
+      example = lib.literalExpression ''
+        {
+          myStream1 = "/etc/liquidsoap/myStream1.liq";
+          myStream2 = ./myStream2.liq;
+          myStream3 = "out(playlist(\"/srv/music/\"))";
+        }
+      '';
+
+      type = lib.types.attrsOf (lib.types.either lib.types.path lib.types.str);
+    };
+
+  };
+  ##### implementation
+
+  config = lib.mkIf (builtins.length streams != 0) {
+
+    users.users.liquidsoap = {
+      uid = config.ids.uids.liquidsoap;
+      group = "liquidsoap";
+      extraGroups = [ "audio" ];
+      description = "Liquidsoap streaming user";
+      home = "/var/lib/liquidsoap";
+      createHome = true;
+    };
+
+    users.groups.liquidsoap.gid = config.ids.gids.liquidsoap;
+
+    systemd.services = builtins.listToAttrs (map streamService streams);
+  };
+
+}

nixos/modules/services/audio/marytts.nix

@@ -0,0 +1,184 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.marytts;
+  format = pkgs.formats.javaProperties { };
+in
+{
+  options.services.marytts = {
+    enable = lib.mkEnableOption "MaryTTS";
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = format.type;
+      };
+      default = { };
+      description = ''
+        Settings for MaryTTS.
+
+        See the [default settings](https://github.com/marytts/marytts/blob/master/marytts-runtime/conf/marybase.config)
+        for a list of possible keys.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "marytts" { };
+
+    basePath = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/marytts";
+      description = ''
+        The base path in which MaryTTS runs.
+      '';
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 59125;
+      description = ''
+        Port to bind the MaryTTS server to.
+      '';
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      example = true;
+      description = ''
+        Whether to open the port in the firewall for MaryTTS.
+      '';
+    };
+
+    voices = lib.mkOption {
+      type = lib.types.listOf lib.types.path;
+      default = [ ];
+      example = lib.literalExpression ''
+        [
+          (pkgs.fetchzip {
+            url = "https://github.com/marytts/voice-bits1-hsmm/releases/download/v5.2/voice-bits1-hsmm-5.2.zip";
+            hash = "sha256-1nK+qZxjumMev7z5lgKr660NCKH5FDwvZ9sw/YYYeaA=";
+          })
+        ]
+      '';
+      description = ''
+        Paths to the JAR files that contain additional voices for MaryTTS.
+
+        Voices are automatically detected by MaryTTS, so there is no need to alter
+        your config to make use of new voices.
+      '';
+    };
+
+    userDictionaries = lib.mkOption {
+      type = lib.types.listOf lib.types.path;
+      default = [ ];
+      example = lib.literalExpression ''
+        [
+          (pkgs.writeTextFile {
+            name = "userdict-en_US";
+            destination = "/userdict-en_US.txt";
+            text = '''
+              Nixpkgs | n I k s - ' p { - k @ - dZ @ s
+            ''';
+          })
+        ]
+      '';
+      description = ''
+        Paths to the user dictionary files for MaryTTS.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.marytts.settings = {
+      "mary.base" = lib.mkDefault cfg.basePath;
+      "socket.port" = lib.mkDefault cfg.port;
+    };
+
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services.marytts = {
+      description = "MaryTTS server instance";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      # FIXME: MaryTTS's config loading mechanism appears to be horrendously broken
+      # and it doesn't seem to actually read config files outside of precompiled JAR files.
+      # Using system properties directly works for now, but this is really ugly.
+      script = ''
+        ${lib.getExe pkgs.marytts} -classpath "${cfg.basePath}/lib/*:${cfg.package}/lib/*" ${
+          lib.concatStringsSep " " (lib.mapAttrsToList (n: v: ''-D${n}="${v}"'') cfg.settings)
+        }
+      '';
+
+      restartTriggers = cfg.voices ++ cfg.userDictionaries;
+
+      serviceConfig = {
+        DynamicUser = true;
+        User = "marytts";
+        RuntimeDirectory = "marytts";
+        StateDirectory = "marytts";
+        Restart = "on-failure";
+        RestartSec = 5;
+        TimeoutSec = 20;
+
+        # Hardening
+        ProtectClock = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        ProtectKernelModules = true;
+        ProtectHostname = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectHome = true;
+        ProcSubset = "pid";
+
+        PrivateTmp = true;
+        PrivateNetwork = false;
+        PrivateUsers = cfg.port >= 1024;
+        PrivateDevices = true;
+
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+
+        MemoryDenyWriteExecute = false; # Java does not like w^x :(
+        LockPersonality = true;
+        AmbientCapabilities = lib.optional (cfg.port < 1024) "CAP_NET_BIND_SERVICE";
+        CapabilityBoundingSet = "";
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@resources"
+          "~@privileged"
+        ];
+        UMask = "0027";
+      };
+    };
+
+    systemd.tmpfiles.settings."10-marytts" = {
+      "${cfg.basePath}/lib"."L+".argument = "${pkgs.symlinkJoin {
+        name = "marytts-lib";
+
+        # Put user paths before default ones so that user ones have priority
+        paths = cfg.voices ++ [ "${cfg.package}/lib" ];
+      }}";
+
+      "${cfg.basePath}/user-dictionaries"."L+".argument = "${pkgs.symlinkJoin {
+        name = "marytts-user-dictionaries";
+
+        # Put user paths before default ones so that user ones have priority
+        paths = cfg.userDictionaries ++ [ "${cfg.package}/user-dictionaries" ];
+      }}";
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
+  };
+}

nixos/modules/services/audio/mopidy.nix

@@ -0,0 +1,124 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  uid = config.ids.uids.mopidy;
+  gid = config.ids.gids.mopidy;
+  cfg = config.services.mopidy;
+  settingsFormat = pkgs.formats.ini { };
+
+  mopidyConf = settingsFormat.generate "mopidy.conf" cfg.settings;
+
+  mopidyEnv = pkgs.buildEnv {
+    name = "mopidy-with-extensions-${pkgs.mopidy.version}";
+    ignoreCollisions = true;
+    paths = lib.closePropagation cfg.extensionPackages;
+    pathsToLink = [ "/${pkgs.mopidyPackages.python.sitePackages}" ];
+    nativeBuildInputs = [ pkgs.makeWrapper ];
+    postBuild = ''
+      makeWrapper ${lib.getExe pkgs.mopidy} $out/bin/mopidy \
+        --prefix PYTHONPATH : $out/${pkgs.mopidyPackages.python.sitePackages}
+    '';
+  };
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "mopidy" "configuration" ] ''
+      Use RFC42-style services.mopidy.settings instead.
+    '')
+  ];
+
+  options = {
+    services.mopidy = {
+      enable = lib.mkEnableOption "Mopidy, a music player daemon";
+
+      dataDir = lib.mkOption {
+        default = "/var/lib/mopidy";
+        type = lib.types.str;
+        description = ''
+          The directory where Mopidy stores its state.
+        '';
+      };
+
+      extensionPackages = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.package;
+        example = lib.literalExpression "[ pkgs.mopidy-spotify ]";
+        description = ''
+          Mopidy extensions that should be loaded by the service.
+        '';
+      };
+
+      settings = lib.mkOption {
+        inherit (settingsFormat) type;
+        example.mpd = {
+          enabled = true;
+          hostname = "::";
+        };
+        description = ''
+          The configuration that Mopidy should use.
+          See the upstream documentation <https://docs.mopidy.com/stable/config/> for details.
+        '';
+      };
+
+      extraConfigFiles = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = ''
+          Extra config file read by Mopidy when the service starts.
+          Later files in the list overrides earlier configuration.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.tmpfiles.settings."10-mopidy".${cfg.dataDir}.d = {
+      user = "mopidy";
+      group = "mopidy";
+    };
+
+    systemd.services.mopidy = {
+      wantedBy = [ "multi-user.target" ];
+      after = [
+        "network-online.target"
+        "sound.target"
+      ];
+      wants = [ "network-online.target" ];
+      description = "mopidy music player daemon";
+      serviceConfig = {
+        ExecStart = "${mopidyEnv}/bin/mopidy --config ${
+          lib.concatStringsSep ":" ([ mopidyConf ] ++ cfg.extraConfigFiles)
+        }";
+        Restart = "on-failure";
+        User = "mopidy";
+      };
+    };
+
+    systemd.services.mopidy-scan = {
+      description = "mopidy local files scanner";
+      serviceConfig = {
+        ExecStart = "${mopidyEnv}/bin/mopidy --config ${
+          lib.concatStringsSep ":" ([ mopidyConf ] ++ cfg.extraConfigFiles)
+        } local scan";
+        User = "mopidy";
+        Type = "oneshot";
+      };
+    };
+
+    users.users.mopidy = {
+      inherit uid;
+      group = "mopidy";
+      extraGroups = [ "audio" ];
+      description = "Mopidy daemon user";
+      home = cfg.dataDir;
+    };
+
+    users.groups.mopidy.gid = gid;
+  };
+}

nixos/modules/services/audio/mpd.nix

@@ -0,0 +1,311 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  name = "mpd";
+
+  uid = config.ids.uids.mpd;
+  gid = config.ids.gids.mpd;
+  cfg = config.services.mpd;
+
+  credentialsPlaceholder = (
+    creds:
+    let
+      placeholders = (
+        lib.imap0 (
+          i: c: ''password "{{password-${toString i}}}@${lib.concatStringsSep "," c.permissions}"''
+        ) creds
+      );
+    in
+    lib.concatStringsSep "\n" placeholders
+  );
+
+  mpdConf = pkgs.writeText "mpd.conf" ''
+    # This file was automatically generated by NixOS. Edit mpd's configuration
+    # via NixOS' configuration.nix, as this file will be rewritten upon mpd's
+    # restart.
+
+    music_directory     "${cfg.musicDirectory}"
+    playlist_directory  "${cfg.playlistDirectory}"
+    ${lib.optionalString (cfg.dbFile != null) ''
+      db_file             "${cfg.dbFile}"
+    ''}
+    state_file          "${cfg.dataDir}/state"
+    sticker_file        "${cfg.dataDir}/sticker.sql"
+
+    ${lib.optionalString (
+      cfg.network.listenAddress != "any"
+    ) ''bind_to_address "${cfg.network.listenAddress}"''}
+    ${lib.optionalString (cfg.network.port != 6600) ''port "${toString cfg.network.port}"''}
+    ${lib.optionalString (cfg.fluidsynth) ''
+      decoder {
+              plugin "fluidsynth"
+              soundfont "${pkgs.soundfont-fluid}/share/soundfonts/FluidR3_GM2-2.sf2"
+      }
+    ''}
+
+    ${lib.optionalString (cfg.credentials != [ ]) (credentialsPlaceholder cfg.credentials)}
+
+    ${cfg.extraConfig}
+  '';
+
+in
+{
+
+  ###### interface
+
+  options = {
+
+    services.mpd = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable MPD, the music player daemon.
+        '';
+      };
+
+      startWhenNeeded = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If set, {command}`mpd` is socket-activated; that
+          is, instead of having it permanently running as a daemon,
+          systemd will start it on the first incoming connection.
+        '';
+      };
+
+      musicDirectory = lib.mkOption {
+        type = with lib.types; either path (strMatching "(http|https|nfs|smb)://.+");
+        default = "${cfg.dataDir}/music";
+        defaultText = lib.literalExpression ''"''${dataDir}/music"'';
+        description = ''
+          The directory or NFS/SMB network share where MPD reads music from. If left
+          as the default value this directory will automatically be created before
+          the MPD server starts, otherwise the sysadmin is responsible for ensuring
+          the directory exists with appropriate ownership and permissions.
+        '';
+      };
+
+      playlistDirectory = lib.mkOption {
+        type = lib.types.path;
+        default = "${cfg.dataDir}/playlists";
+        defaultText = lib.literalExpression ''"''${dataDir}/playlists"'';
+        description = ''
+          The directory where MPD stores playlists. If left as the default value
+          this directory will automatically be created before the MPD server starts,
+          otherwise the sysadmin is responsible for ensuring the directory exists
+          with appropriate ownership and permissions.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = ''
+          Extra directives added to to the end of MPD's configuration file,
+          mpd.conf. Basic configuration like file location and uid/gid
+          is added automatically to the beginning of the file. For available
+          options see {manpage}`mpd.conf(5)`.
+        '';
+      };
+
+      dataDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/${name}";
+        description = ''
+          The directory where MPD stores its state, tag cache, playlists etc. If
+          left as the default value this directory will automatically be created
+          before the MPD server starts, otherwise the sysadmin is responsible for
+          ensuring the directory exists with appropriate ownership and permissions.
+        '';
+      };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = name;
+        description = "User account under which MPD runs.";
+      };
+
+      group = lib.mkOption {
+        type = lib.types.str;
+        default = name;
+        description = "Group account under which MPD runs.";
+      };
+
+      network = {
+
+        listenAddress = lib.mkOption {
+          type = lib.types.str;
+          default = "127.0.0.1";
+          example = "any";
+          description = ''
+            The address for the daemon to listen on.
+            Use `any` to listen on all addresses.
+          '';
+        };
+
+        port = lib.mkOption {
+          type = lib.types.port;
+          default = 6600;
+          description = ''
+            This setting is the TCP port that is desired for the daemon to get assigned
+            to.
+          '';
+        };
+
+      };
+
+      dbFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = "${cfg.dataDir}/tag_cache";
+        defaultText = lib.literalExpression ''"''${dataDir}/tag_cache"'';
+        description = ''
+          The path to MPD's database. If set to `null` the
+          parameter is omitted from the configuration.
+        '';
+      };
+
+      credentials = lib.mkOption {
+        type = lib.types.listOf (
+          lib.types.submodule {
+            options = {
+              passwordFile = lib.mkOption {
+                type = lib.types.path;
+                description = ''
+                  Path to file containing the password.
+                '';
+              };
+              permissions =
+                let
+                  perms = [
+                    "read"
+                    "add"
+                    "control"
+                    "admin"
+                  ];
+                in
+                lib.mkOption {
+                  type = lib.types.listOf (lib.types.enum perms);
+                  default = [ "read" ];
+                  description = ''
+                    List of permissions that are granted with this password.
+                    Permissions can be "${lib.concatStringsSep "\", \"" perms}".
+                  '';
+                };
+            };
+          }
+        );
+        description = ''
+          Credentials and permissions for accessing the mpd server.
+        '';
+        default = [ ];
+        example = [
+          {
+            passwordFile = "/var/lib/secrets/mpd_readonly_password";
+            permissions = [ "read" ];
+          }
+          {
+            passwordFile = "/var/lib/secrets/mpd_admin_password";
+            permissions = [
+              "read"
+              "add"
+              "control"
+              "admin"
+            ];
+          }
+        ];
+      };
+
+      fluidsynth = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If set, add fluidsynth soundfont and configure the plugin.
+        '';
+      };
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    # install mpd units
+    systemd.packages = [ pkgs.mpd ];
+
+    systemd.sockets.mpd = lib.mkIf cfg.startWhenNeeded {
+      wantedBy = [ "sockets.target" ];
+      listenStreams = [
+        "" # Note: this is needed to override the upstream unit
+        (
+          if pkgs.lib.hasPrefix "/" cfg.network.listenAddress then
+            cfg.network.listenAddress
+          else
+            "${
+              lib.optionalString (cfg.network.listenAddress != "any") "${cfg.network.listenAddress}:"
+            }${toString cfg.network.port}"
+        )
+      ];
+    };
+
+    systemd.services.mpd = {
+      wantedBy = lib.optional (!cfg.startWhenNeeded) "multi-user.target";
+
+      preStart = ''
+        set -euo pipefail
+        install -m 600 ${mpdConf} /run/mpd/mpd.conf
+      ''
+      + lib.optionalString (cfg.credentials != [ ]) (
+        lib.concatStringsSep "\n" (
+          lib.imap0 (
+            i: c:
+            ''${pkgs.replace-secret}/bin/replace-secret '{{password-${toString i}}}' '${c.passwordFile}' /run/mpd/mpd.conf''
+          ) cfg.credentials
+        )
+      );
+
+      serviceConfig = {
+        User = "${cfg.user}";
+        # Note: the first "" overrides the ExecStart from the upstream unit
+        ExecStart = [
+          ""
+          "${pkgs.mpd}/bin/mpd --systemd /run/mpd/mpd.conf"
+        ];
+        RuntimeDirectory = "mpd";
+        StateDirectory =
+          [ ]
+          ++ lib.optionals (cfg.dataDir == "/var/lib/${name}") [ name ]
+          ++ lib.optionals (cfg.playlistDirectory == "/var/lib/${name}/playlists") [
+            name
+            "${name}/playlists"
+          ]
+          ++ lib.optionals (cfg.musicDirectory == "/var/lib/${name}/music") [
+            name
+            "${name}/music"
+          ];
+      };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == name) {
+      ${name} = {
+        inherit uid;
+        group = cfg.group;
+        extraGroups = [ "audio" ];
+        description = "Music Player Daemon user";
+        home = "${cfg.dataDir}";
+      };
+    };
+
+    users.groups = lib.optionalAttrs (cfg.group == name) {
+      ${name}.gid = gid;
+    };
+  };
+
+}

nixos/modules/services/audio/mpdscribble.nix

@@ -0,0 +1,220 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.mpdscribble;
+  mpdCfg = config.services.mpd;
+  mpdOpt = options.services.mpd;
+
+  endpointUrls = {
+    "last.fm" = "http://post.audioscrobbler.com";
+    "libre.fm" = "http://turtle.libre.fm";
+    "jamendo" = "http://postaudioscrobbler.jamendo.com";
+    "listenbrainz" = "http://proxy.listenbrainz.org";
+  };
+
+  mkSection = secname: secCfg: ''
+    [${secname}]
+    url      = ${secCfg.url}
+    username = ${secCfg.username}
+    password = {{${secname}_PASSWORD}}
+    journal  = /var/lib/mpdscribble/${secname}.journal
+  '';
+
+  endpoints = lib.concatStringsSep "\n" (lib.mapAttrsToList mkSection cfg.endpoints);
+  cfgTemplate = pkgs.writeText "mpdscribble.conf" ''
+    ## This file was automatically genenrated by NixOS and will be overwritten.
+    ## Do not edit. Edit your NixOS configuration instead.
+
+    ## mpdscribble - an audioscrobbler for the Music Player Daemon.
+    ## http://mpd.wikia.com/wiki/Client:mpdscribble
+
+    # HTTP proxy URL.
+    ${lib.optionalString (cfg.proxy != null) "proxy = ${cfg.proxy}"}
+
+    # The location of the mpdscribble log file.  The special value
+    # "syslog" makes mpdscribble use the local syslog daemon.  On most
+    # systems, log messages will appear in /var/log/daemon.log then.
+    # "-" means log to stderr (the current terminal).
+    log = -
+
+    # How verbose mpdscribble's logging should be.  Default is 1.
+    verbose = ${toString cfg.verbose}
+
+    # How often should mpdscribble save the journal file? [seconds]
+    journal_interval = ${toString cfg.journalInterval}
+
+    # The host running MPD, possibly protected by a password
+    # ([PASSWORD@]HOSTNAME).
+    host = ${(lib.optionalString (cfg.passwordFile != null) "{{MPD_PASSWORD}}@") + cfg.host}
+
+    # The port that the MPD listens on and mpdscribble should try to
+    # connect to.
+    port = ${toString cfg.port}
+
+    ${endpoints}
+  '';
+
+  cfgFile = "/run/mpdscribble/mpdscribble.conf";
+
+  replaceSecret =
+    secretFile: placeholder: targetFile:
+    lib.optionalString (
+      secretFile != null
+    ) ''${pkgs.replace-secret}/bin/replace-secret '${placeholder}' '${secretFile}' '${targetFile}' '';
+
+  preStart = pkgs.writeShellScript "mpdscribble-pre-start" ''
+    cp -f "${cfgTemplate}" "${cfgFile}"
+    ${replaceSecret cfg.passwordFile "{{MPD_PASSWORD}}" cfgFile}
+    ${lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (
+        secname: cfg: replaceSecret cfg.passwordFile "{{${secname}_PASSWORD}}" cfgFile
+      ) cfg.endpoints
+    )}
+  '';
+
+  localMpd = (cfg.host == "localhost" || cfg.host == "127.0.0.1");
+
+in
+{
+  ###### interface
+
+  options.services.mpdscribble = {
+
+    enable = lib.mkEnableOption "mpdscribble, an MPD client which submits info about tracks being played to Last.fm (formerly AudioScrobbler)";
+
+    proxy = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        HTTP proxy URL.
+      '';
+    };
+
+    verbose = lib.mkOption {
+      default = 1;
+      type = lib.types.int;
+      description = ''
+        Log level for the mpdscribble daemon.
+      '';
+    };
+
+    journalInterval = lib.mkOption {
+      default = 600;
+      example = 60;
+      type = lib.types.int;
+      description = ''
+        How often should mpdscribble save the journal file? [seconds]
+      '';
+    };
+
+    host = lib.mkOption {
+      default = (
+        if mpdCfg.network.listenAddress != "any" then mpdCfg.network.listenAddress else "localhost"
+      );
+      defaultText = lib.literalExpression ''
+        if config.${mpdOpt.network.listenAddress} != "any"
+        then config.${mpdOpt.network.listenAddress}
+        else "localhost"
+      '';
+      type = lib.types.str;
+      description = ''
+        Host for the mpdscribble daemon to search for a mpd daemon on.
+      '';
+    };
+
+    passwordFile = lib.mkOption {
+      default =
+        if localMpd then
+          (lib.findFirst (c: lib.any (x: x == "read") c.permissions) {
+            passwordFile = null;
+          } mpdCfg.credentials).passwordFile
+        else
+          null;
+      defaultText = lib.literalMD ''
+        The first password file with read access configured for MPD when using a local instance,
+        otherwise `null`.
+      '';
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        File containing the password for the mpd daemon.
+        If there is a local mpd configured using {option}`services.mpd.credentials`
+        the default is automatically set to a matching passwordFile of the local mpd.
+      '';
+    };
+
+    port = lib.mkOption {
+      default = mpdCfg.network.port;
+      defaultText = lib.literalExpression "config.${mpdOpt.network.port}";
+      type = lib.types.port;
+      description = ''
+        Port for the mpdscribble daemon to search for a mpd daemon on.
+      '';
+    };
+
+    endpoints = lib.mkOption {
+      type = (
+        let
+          endpoint =
+            { name, ... }:
+            {
+              options = {
+                url = lib.mkOption {
+                  type = lib.types.str;
+                  default = endpointUrls.${name} or "";
+                  description = "The url endpoint where the scrobble API is listening.";
+                };
+                username = lib.mkOption {
+                  type = lib.types.str;
+                  description = ''
+                    Username for the scrobble service.
+                  '';
+                };
+                passwordFile = lib.mkOption {
+                  type = lib.types.nullOr lib.types.str;
+                  description = "File containing the password, either as MD5SUM or cleartext.";
+                };
+              };
+            };
+        in
+        lib.types.attrsOf (lib.types.submodule endpoint)
+      );
+      default = { };
+      example = {
+        "last.fm" = {
+          username = "foo";
+          passwordFile = "/run/secrets/lastfm_password";
+        };
+      };
+      description = ''
+        Endpoints to scrobble to.
+        If the endpoint is one of "${lib.concatStringsSep "\", \"" (lib.attrNames endpointUrls)}" the url is set automatically.
+      '';
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.mpdscribble = {
+      after = [ "network.target" ] ++ (lib.optional localMpd "mpd.service");
+      description = "mpdscribble mpd scrobble client";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        DynamicUser = true;
+        StateDirectory = "mpdscribble";
+        RuntimeDirectory = "mpdscribble";
+        RuntimeDirectoryMode = "700";
+        # TODO use LoadCredential= instead of running preStart with full privileges?
+        ExecStartPre = "+${preStart}";
+        ExecStart = "${pkgs.mpdscribble}/bin/mpdscribble --no-daemon --conf ${cfgFile}";
+      };
+    };
+  };
+
+}

nixos/modules/services/audio/music-assistant.nix

@@ -0,0 +1,134 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkIf
+    mkEnableOption
+    mkOption
+    mkPackageOption
+    types
+    ;
+
+  inherit (types)
+    listOf
+    enum
+    str
+    ;
+
+  cfg = config.services.music-assistant;
+
+  finalPackage = cfg.package.override {
+    inherit (cfg) providers;
+  };
+in
+
+{
+  meta.buildDocsInSandbox = false;
+
+  options.services.music-assistant = {
+    enable = mkEnableOption "Music Assistant";
+
+    package = mkPackageOption pkgs "music-assistant" { };
+
+    extraOptions = mkOption {
+      type = listOf str;
+      default = [
+        "--config"
+        "/var/lib/music-assistant"
+      ];
+      example = [
+        "--log-level"
+        "DEBUG"
+      ];
+      description = ''
+        List of extra options to pass to the music-assistant executable.
+      '';
+    };
+
+    providers = mkOption {
+      type = listOf (enum cfg.package.providerNames);
+      default = [ ];
+      example = [
+        "opensubsonic"
+        "snapcast"
+      ];
+      description = ''
+        List of provider names for which dependencies will be installed.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.music-assistant = {
+      description = "Music Assistant";
+      documentation = [ "https://music-assistant.io" ];
+
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+
+      wantedBy = [ "multi-user.target" ];
+
+      environment = {
+        HOME = "/var/lib/music-assistant";
+        PYTHONPATH = finalPackage.pythonPath;
+      };
+
+      path =
+        with pkgs;
+        [
+          lsof
+        ]
+        ++ lib.optionals (lib.elem "spotify" cfg.providers) [
+          librespot-ma
+        ]
+        ++ lib.optionals (lib.elem "snapcast" cfg.providers) [
+          snapcast
+        ];
+
+      serviceConfig = {
+        ExecStart = utils.escapeSystemdExecArgs (
+          [
+            (lib.getExe cfg.package)
+          ]
+          ++ cfg.extraOptions
+        );
+        DynamicUser = true;
+        StateDirectory = "music-assistant";
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = [ "" ];
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_NETLINK"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged @resources"
+        ];
+        RestrictSUIDSGID = true;
+        UMask = "0077";
+      };
+    };
+  };
+}

nixos/modules/services/audio/mympd.nix

@@ -0,0 +1,145 @@
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}:
+
+let
+  cfg = config.services.mympd;
+in
+{
+  options = {
+
+    services.mympd = {
+
+      enable = lib.mkEnableOption "MyMPD server";
+
+      package = lib.mkPackageOption pkgs "mympd" { };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open ports needed for the functionality of the program.
+        '';
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = [ "music" ];
+        description = ''
+          Additional groups for the systemd service.
+        '';
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType =
+            with lib.types;
+            attrsOf (
+              nullOr (oneOf [
+                str
+                bool
+                int
+              ])
+            );
+          options = {
+            http_port = lib.mkOption {
+              type = lib.types.port;
+              description = ''
+                The HTTP port where mympd's web interface will be available.
+
+                The HTTPS/SSL port can be configured via {option}`config`.
+              '';
+              example = "8080";
+            };
+
+            ssl = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Whether to enable listening on the SSL port.
+
+                Refer to <https://jcorporation.github.io/myMPD/020-configuration/configuration-files#ssl-options>
+                for more information.
+              '';
+              default = false;
+            };
+          };
+        };
+        description = ''
+          Manages the configuration files declaratively. For all the configuration
+          options, see <https://jcorporation.github.io/myMPD/020-configuration/configuration-files>.
+
+          Each key represents the "File" column from the upstream configuration table, and the
+          value is the content of that file.
+        '';
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.mympd = {
+      # upstream service config: https://github.com/jcorporation/myMPD/blob/master/contrib/initscripts/mympd.service.in
+      after = [ "mpd.service" ];
+      wantedBy = [ "multi-user.target" ];
+      preStart = with lib; ''
+        config_dir="/var/lib/mympd/config"
+        mkdir -p "$config_dir"
+
+        ${pipe cfg.settings [
+          (mapAttrsToList (
+            name: value: ''
+              echo -n "${if isBool value then boolToString value else toString value}" > "$config_dir/${name}"
+            ''
+          ))
+          (concatStringsSep "\n")
+        ]}
+      '';
+      unitConfig = {
+        Description = "myMPD server daemon";
+        Documentation = "man:mympd(1)";
+      };
+      serviceConfig = {
+        AmbientCapabilities = "CAP_NET_BIND_SERVICE";
+        CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
+        DynamicUser = true;
+        ExecStart = lib.getExe cfg.package;
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        PrivateDevices = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        RestrictRealtime = true;
+        StateDirectory = "mympd";
+        CacheDirectory = "mympd";
+        RestrictAddressFamilies = "AF_INET AF_INET6 AF_NETLINK AF_UNIX";
+        RestrictNamespaces = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = "@system-service";
+        SupplementaryGroups = cfg.extraGroups;
+      };
+    };
+
+    networking.firewall = lib.mkMerge [
+      (lib.mkIf cfg.openFirewall {
+        allowedTCPPorts = [ cfg.settings.http_port ];
+      })
+      (lib.mkIf (cfg.openFirewall && cfg.settings.ssl && cfg.settings.ssl_port != null) {
+        allowedTCPPorts = [ cfg.settings.ssl_port ];
+      })
+    ];
+
+  };
+
+  meta.maintainers = [ lib.maintainers.eliandoran ];
+
+}

nixos/modules/services/audio/navidrome.nix

@@ -0,0 +1,182 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkEnableOption
+    mkPackageOption
+    mkOption
+    maintainers
+    ;
+  inherit (lib.types)
+    bool
+    port
+    str
+    submodule
+    ;
+  cfg = config.services.navidrome;
+  settingsFormat = pkgs.formats.json { };
+in
+{
+  options = {
+    services.navidrome = {
+
+      enable = mkEnableOption "Navidrome music server";
+
+      package = mkPackageOption pkgs "navidrome" { };
+
+      settings = mkOption {
+        type = submodule {
+          freeformType = settingsFormat.type;
+
+          options = {
+            Address = mkOption {
+              default = "127.0.0.1";
+              description = "Address to run Navidrome on.";
+              type = str;
+            };
+
+            Port = mkOption {
+              default = 4533;
+              description = "Port to run Navidrome on.";
+              type = port;
+            };
+
+            EnableInsightsCollector = mkOption {
+              default = false;
+              description = "Enable anonymous usage data collection, see <https://www.navidrome.org/docs/getting-started/insights/> for details.";
+              type = bool;
+            };
+          };
+        };
+        default = { };
+        example = {
+          MusicFolder = "/mnt/music";
+        };
+        description = "Configuration for Navidrome, see <https://www.navidrome.org/docs/usage/configuration-options/> for supported values.";
+      };
+
+      user = mkOption {
+        type = str;
+        default = "navidrome";
+        description = "User under which Navidrome runs.";
+      };
+
+      group = mkOption {
+        type = str;
+        default = "navidrome";
+        description = "Group under which Navidrome runs.";
+      };
+
+      openFirewall = mkOption {
+        type = bool;
+        default = false;
+        description = "Whether to open the TCP port in the firewall";
+      };
+
+      environmentFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = "Environment file, used to set any secret ND_* environment variables.";
+      };
+    };
+  };
+
+  config =
+    let
+      inherit (lib) mkIf optional getExe;
+      WorkingDirectory = "/var/lib/navidrome";
+    in
+    mkIf cfg.enable {
+      systemd = {
+        tmpfiles.settings.navidromeDirs = {
+          "${cfg.settings.DataFolder or WorkingDirectory}"."d" = {
+            mode = "700";
+            inherit (cfg) user group;
+          };
+          "${cfg.settings.CacheFolder or (WorkingDirectory + "/cache")}"."d" = {
+            mode = "700";
+            inherit (cfg) user group;
+          };
+          "${cfg.settings.MusicFolder or (WorkingDirectory + "/music")}"."d" = {
+            mode = ":700";
+            user = ":${cfg.user}";
+            group = ":${cfg.group}";
+          };
+        };
+        services.navidrome = {
+          description = "Navidrome Media Server";
+          after = [ "network.target" ];
+          wantedBy = [ "multi-user.target" ];
+          serviceConfig = {
+            ExecStart = ''
+              ${getExe cfg.package} --configfile ${settingsFormat.generate "navidrome.json" cfg.settings}
+            '';
+            EnvironmentFile = lib.mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+            User = cfg.user;
+            Group = cfg.group;
+            StateDirectory = "navidrome";
+            inherit WorkingDirectory;
+            RuntimeDirectory = "navidrome";
+            RootDirectory = "/run/navidrome";
+            ReadWritePaths = "";
+            BindPaths =
+              optional (cfg.settings ? DataFolder) cfg.settings.DataFolder
+              ++ optional (cfg.settings ? CacheFolder) cfg.settings.CacheFolder;
+            BindReadOnlyPaths = [
+              # navidrome uses online services to download additional album metadata / covers
+              "${config.security.pki.caBundle}:/etc/ssl/certs/ca-certificates.crt"
+              builtins.storeDir
+              "/etc"
+            ]
+            ++ optional (cfg.settings ? MusicFolder) cfg.settings.MusicFolder
+            ++ lib.optionals config.services.resolved.enable [
+              "/run/systemd/resolve/stub-resolv.conf"
+              "/run/systemd/resolve/resolv.conf"
+            ];
+            CapabilityBoundingSet = "";
+            RestrictAddressFamilies = [
+              "AF_UNIX"
+              "AF_INET"
+              "AF_INET6"
+            ];
+            RestrictNamespaces = true;
+            PrivateDevices = true;
+            PrivateUsers = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHome = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            SystemCallArchitectures = "native";
+            SystemCallFilter = [
+              "@system-service"
+              "~@privileged"
+            ];
+            RestrictRealtime = true;
+            LockPersonality = true;
+            MemoryDenyWriteExecute = true;
+            UMask = "0066";
+            ProtectHostname = true;
+          };
+        };
+      };
+
+      users.users = mkIf (cfg.user == "navidrome") {
+        navidrome = {
+          inherit (cfg) group;
+          isSystemUser = true;
+        };
+      };
+
+      users.groups = mkIf (cfg.group == "navidrome") { navidrome = { }; };
+
+      networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.settings.Port ];
+    };
+  meta.maintainers = with maintainers; [ fsnkty ];
+}

nixos/modules/services/audio/networkaudiod.nix

@@ -0,0 +1,22 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  name = "networkaudiod";
+  cfg = config.services.networkaudiod;
+in
+{
+  options = {
+    services.networkaudiod = {
+      enable = lib.mkEnableOption "Networkaudiod (NAA)";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.packages = [ pkgs.networkaudiod ];
+    systemd.services.networkaudiod.wantedBy = [ "multi-user.target" ];
+  };
+}

nixos/modules/services/audio/pulseaudio.nix

@@ -0,0 +1,341 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.pulseaudio;
+
+  hasZeroconf =
+    let
+      z = cfg.zeroconf;
+    in
+    z.publish.enable || z.discovery.enable;
+
+  overriddenPackage = cfg.package.override (
+    lib.optionalAttrs hasZeroconf { zeroconfSupport = true; }
+  );
+  binary = "${lib.getBin overriddenPackage}/bin/pulseaudio";
+  binaryNoDaemon = "${binary} --daemonize=no";
+
+  # Forces 32bit pulseaudio and alsa-plugins to be built/supported for apps
+  # using 32bit alsa on 64bit linux.
+  enable32BitAlsaPlugins =
+    cfg.support32Bit
+    && pkgs.stdenv.hostPlatform.isx86_64
+    && (pkgs.pkgsi686Linux.alsa-lib != null && pkgs.pkgsi686Linux.libpulseaudio != null);
+
+  myConfigFile =
+    let
+      addModuleIf = cond: mod: lib.optionalString cond "load-module ${mod}";
+      allAnon = lib.optional cfg.tcp.anonymousClients.allowAll "auth-anonymous=1";
+      ipAnon =
+        let
+          a = cfg.tcp.anonymousClients.allowedIpRanges;
+        in
+        lib.optional (a != [ ]) ''auth-ip-acl=${lib.concatStringsSep ";" a}'';
+    in
+    pkgs.writeTextFile {
+      name = "default.pa";
+      text = ''
+        .include ${cfg.configFile}
+        ${addModuleIf cfg.zeroconf.publish.enable "module-zeroconf-publish"}
+        ${addModuleIf cfg.zeroconf.discovery.enable "module-zeroconf-discover"}
+        ${addModuleIf cfg.tcp.enable (
+          lib.concatStringsSep " " ([ "module-native-protocol-tcp" ] ++ allAnon ++ ipAnon)
+        )}
+        ${addModuleIf config.services.jack.jackd.enable "module-jack-sink"}
+        ${addModuleIf config.services.jack.jackd.enable "module-jack-source"}
+        ${cfg.extraConfig}
+      '';
+    };
+
+  ids = config.ids;
+
+  uid = ids.uids.pulseaudio;
+  gid = ids.gids.pulseaudio;
+
+  stateDir = "/run/pulse";
+
+  # Create pulse/client.conf even if PulseAudio is disabled so
+  # that we can disable the autospawn feature in programs that
+  # are built with PulseAudio support (like KDE).
+  clientConf = pkgs.writeText "client.conf" ''
+    autospawn=no
+    ${cfg.extraClientConf}
+  '';
+
+  # Write an /etc/asound.conf that causes all ALSA applications to
+  # be re-routed to the PulseAudio server through ALSA's Pulse
+  # plugin.
+  alsaConf = ''
+    pcm_type.pulse {
+      libs.native = ${pkgs.alsa-plugins}/lib/alsa-lib/libasound_module_pcm_pulse.so ;
+      ${lib.optionalString enable32BitAlsaPlugins "libs.32Bit = ${pkgs.pkgsi686Linux.alsa-plugins}/lib/alsa-lib/libasound_module_pcm_pulse.so ;"}
+    }
+    pcm.!default {
+      type pulse
+      hint.description "Default Audio Device (via PulseAudio)"
+    }
+    ctl_type.pulse {
+      libs.native = ${pkgs.alsa-plugins}/lib/alsa-lib/libasound_module_ctl_pulse.so ;
+      ${lib.optionalString enable32BitAlsaPlugins "libs.32Bit = ${pkgs.pkgsi686Linux.alsa-plugins}/lib/alsa-lib/libasound_module_ctl_pulse.so ;"}
+    }
+    ctl.!default {
+      type pulse
+    }
+  '';
+
+in
+{
+  imports = [
+    (lib.mkRenamedOptionModule [ "hardware" "pulseaudio" ] [ "services" "pulseaudio" ])
+  ];
+
+  options = {
+
+    services.pulseaudio = {
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable the PulseAudio sound server.
+        '';
+      };
+
+      systemWide = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If false, a PulseAudio server is launched automatically for
+          each user that tries to use the sound system. The server runs
+          with user privileges. If true, one system-wide PulseAudio
+          server is launched on boot, running as the user "pulse", and
+          only users in the "pulse-access" group will have access to the server.
+          Please read the PulseAudio documentation for more details.
+
+          Don't enable this option unless you know what you are doing.
+        '';
+      };
+
+      support32Bit = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to include the 32-bit pulseaudio libraries in the system or not.
+          This is only useful on 64-bit systems and currently limited to x86_64-linux.
+        '';
+      };
+
+      configFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        description = ''
+          The path to the default configuration options the PulseAudio server
+          should use. By default, the "default.pa" configuration
+          from the PulseAudio distribution is used.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = ''
+          Literal string to append to `configFile`
+          and the config file generated by the pulseaudio module.
+        '';
+      };
+
+      extraClientConf = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = ''
+          Extra configuration appended to pulse/client.conf file.
+        '';
+      };
+
+      package = lib.mkOption {
+        type = lib.types.package;
+        default = if config.services.jack.jackd.enable then pkgs.pulseaudioFull else pkgs.pulseaudio;
+        defaultText = lib.literalExpression "pkgs.pulseaudio";
+        example = lib.literalExpression "pkgs.pulseaudioFull";
+        description = ''
+          The PulseAudio derivation to use.  This can be used to enable
+          features (such as JACK support, Bluetooth) via the
+          `pulseaudioFull` package.
+        '';
+      };
+
+      extraModules = lib.mkOption {
+        type = lib.types.listOf lib.types.package;
+        default = [ ];
+        example = lib.literalExpression "[ pkgs.pulseaudio-modules-bt ]";
+        description = ''
+          Extra pulseaudio modules to use. This is intended for out-of-tree
+          pulseaudio modules like extra bluetooth codecs.
+
+          Extra modules take precedence over built-in pulseaudio modules.
+        '';
+      };
+
+      daemon = {
+        logLevel = lib.mkOption {
+          type = lib.types.str;
+          default = "notice";
+          description = ''
+            The log level that the system-wide pulseaudio daemon should use,
+            if activated.
+          '';
+        };
+
+        config = lib.mkOption {
+          type = lib.types.attrsOf lib.types.unspecified;
+          default = { };
+          description = "Config of the pulse daemon. See `man pulse-daemon.conf`.";
+          example = lib.literalExpression ''{ realtime-scheduling = "yes"; }'';
+        };
+      };
+
+      zeroconf = {
+        discovery.enable = lib.mkEnableOption "discovery of pulseaudio sinks in the local network";
+        publish.enable = lib.mkEnableOption "publishing the pulseaudio sink in the local network";
+      };
+
+      # TODO: enable by default?
+      tcp = {
+        enable = lib.mkEnableOption "tcp streaming support";
+
+        anonymousClients = {
+          allowAll = lib.mkEnableOption "all anonymous clients to stream to the server";
+          allowedIpRanges = lib.mkOption {
+            type = lib.types.listOf lib.types.str;
+            default = [ ];
+            example = lib.literalExpression ''[ "127.0.0.1" "192.168.1.0/24" ]'';
+            description = ''
+              A list of IP subnets that are allowed to stream to the server.
+            '';
+          };
+        };
+      };
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      {
+        environment.etc."pulse/client.conf".source = clientConf;
+
+        environment.systemPackages = [ overriddenPackage ];
+
+        environment.etc = {
+          "alsa/conf.d/99-pulseaudio.conf".text = alsaConf;
+
+          "pulse/daemon.conf".source = pkgs.writeText "daemon.conf" (
+            lib.generators.toKeyValue { } cfg.daemon.config
+          );
+
+          "openal/alsoft.conf".source = pkgs.writeText "alsoft.conf" "drivers=pulse";
+
+          "libao.conf".source = pkgs.writeText "libao.conf" "default_driver=pulse";
+        };
+
+        services.pulseaudio.configFile = lib.mkDefault "${lib.getBin overriddenPackage}/etc/pulse/default.pa";
+
+        # Disable flat volumes to enable relative ones
+        services.pulseaudio.daemon.config.flat-volumes = lib.mkDefault "no";
+
+        # Upstream defaults to speex-float-1 which results in audible artifacts
+        services.pulseaudio.daemon.config.resample-method = lib.mkDefault "speex-float-5";
+
+        # Allow PulseAudio to get realtime priority using rtkit.
+        security.rtkit.enable = true;
+
+        systemd.packages = [ overriddenPackage ];
+
+        # PulseAudio is packaged with udev rules to handle various audio device quirks
+        services.udev.packages = [ overriddenPackage ];
+      }
+
+      (lib.mkIf (cfg.extraModules != [ ]) {
+        services.pulseaudio.daemon.config.dl-search-path =
+          let
+            overriddenModules = builtins.map (
+              drv: drv.override { pulseaudio = overriddenPackage; }
+            ) cfg.extraModules;
+            modulePaths =
+              builtins.map (drv: "${drv}/lib/pulseaudio/modules")
+                # User-provided extra modules take precedence
+                (overriddenModules ++ [ overriddenPackage ]);
+          in
+          lib.concatStringsSep ":" modulePaths;
+      })
+
+      (lib.mkIf hasZeroconf {
+        services.avahi.enable = true;
+      })
+      (lib.mkIf cfg.zeroconf.publish.enable {
+        services.avahi.publish.enable = true;
+        services.avahi.publish.userServices = true;
+      })
+
+      (lib.mkIf (!cfg.systemWide) {
+        environment.etc = {
+          "pulse/default.pa".source = myConfigFile;
+        };
+        systemd.user = {
+          services.pulseaudio = {
+            restartIfChanged = true;
+            serviceConfig = {
+              RestartSec = "500ms";
+              PassEnvironment = "DISPLAY";
+            };
+          }
+          // lib.optionalAttrs config.services.jack.jackd.enable {
+            environment.JACK_PROMISCUOUS_SERVER = "jackaudio";
+          };
+          sockets.pulseaudio = {
+            wantedBy = [ "sockets.target" ];
+          };
+        };
+      })
+
+      (lib.mkIf cfg.systemWide {
+        users.users.pulse = {
+          # For some reason, PulseAudio wants UID == GID.
+          uid =
+            assert uid == gid;
+            uid;
+          group = "pulse";
+          extraGroups = [ "audio" ];
+          description = "PulseAudio system service user";
+          home = stateDir;
+          homeMode = "755";
+          createHome = true;
+          isSystemUser = true;
+        };
+
+        users.groups.pulse.gid = gid;
+        users.groups.pulse-access = { };
+
+        systemd.services.pulseaudio = {
+          description = "PulseAudio System-Wide Server";
+          wantedBy = [ "sound.target" ];
+          before = [ "sound.target" ];
+          environment.PULSE_RUNTIME_PATH = stateDir;
+          serviceConfig = {
+            Type = "notify";
+            ExecStart = "${binaryNoDaemon} --log-level=${cfg.daemon.logLevel} --system -n --file=${myConfigFile}";
+            Restart = "on-failure";
+            RestartSec = "500ms";
+          };
+        };
+
+        environment.variables.PULSE_COOKIE = "${stateDir}/.config/pulse/cookie";
+      })
+    ]
+  );
+
+}

nixos/modules/services/audio/roon-bridge.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  name = "roon-bridge";
+  cfg = config.services.roon-bridge;
+in
+{
+  options = {
+    services.roon-bridge = {
+      enable = lib.mkEnableOption "Roon Bridge";
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open ports in the firewall for the bridge.
+        '';
+      };
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = "roon-bridge";
+        description = ''
+          User to run the Roon bridge as.
+        '';
+      };
+      group = lib.mkOption {
+        type = lib.types.str;
+        default = "roon-bridge";
+        description = ''
+          Group to run the Roon Bridge as.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.roon-bridge = {
+      after = [ "network.target" ];
+      description = "Roon Bridge";
+      wantedBy = [ "multi-user.target" ];
+
+      environment.ROON_DATAROOT = "/var/lib/${name}";
+
+      serviceConfig = {
+        ExecStart = "${pkgs.roon-bridge}/bin/RoonBridge";
+        LimitNOFILE = 8192;
+        User = cfg.user;
+        Group = cfg.group;
+        StateDirectory = name;
+      };
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPortRanges = [
+        {
+          from = 9100;
+          to = 9200;
+        }
+      ];
+      allowedUDPPorts = [ 9003 ];
+      extraCommands = lib.optionalString (!config.networking.nftables.enable) ''
+        iptables -A INPUT -s 224.0.0.0/4 -j ACCEPT
+        iptables -A INPUT -d 224.0.0.0/4 -j ACCEPT
+        iptables -A INPUT -s 240.0.0.0/5 -j ACCEPT
+        iptables -A INPUT -m pkttype --pkt-type multicast -j ACCEPT
+        iptables -A INPUT -m pkttype --pkt-type broadcast -j ACCEPT
+      '';
+      extraInputRules = lib.optionalString config.networking.nftables.enable ''
+        ip saddr { 224.0.0.0/4, 240.0.0.0/5 } accept
+        ip daddr 224.0.0.0/4 accept
+        pkttype { multicast, broadcast } accept
+      '';
+    };
+
+    users.groups.${cfg.group} = { };
+    users.users.${cfg.user} = lib.optionalAttrs (cfg.user == "roon-bridge") {
+      isSystemUser = true;
+      description = "Roon Bridge user";
+      group = cfg.group;
+      extraGroups = [ "audio" ];
+    };
+  };
+}

nixos/modules/services/audio/roon-server.nix

@@ -0,0 +1,97 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  name = "roon-server";
+  cfg = config.services.roon-server;
+in
+{
+  options = {
+    services.roon-server = {
+      enable = lib.mkEnableOption "Roon Server";
+      package = lib.mkPackageOption pkgs "roon-server" { };
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open ports in the firewall for the server.
+        '';
+      };
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = "roon-server";
+        description = ''
+          User to run the Roon Server as.
+        '';
+      };
+      group = lib.mkOption {
+        type = lib.types.str;
+        default = "roon-server";
+        description = ''
+          Group to run the Roon Server as.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.roon-server = {
+      after = [ "network.target" ];
+      description = "Roon Server";
+      wantedBy = [ "multi-user.target" ];
+
+      environment.ROON_DATAROOT = "/var/lib/${name}";
+      environment.ROON_ID_DIR = "/var/lib/${name}";
+
+      serviceConfig = {
+        ExecStart = "${lib.getExe cfg.package}";
+        LimitNOFILE = 8192;
+        User = cfg.user;
+        Group = cfg.group;
+        StateDirectory = name;
+      };
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPortRanges = [
+        {
+          from = 9100;
+          to = 9200;
+        }
+        {
+          from = 9330;
+          to = 9339;
+        }
+        {
+          from = 30000;
+          to = 30010;
+        }
+      ];
+      allowedUDPPorts = [ 9003 ];
+      extraCommands = lib.optionalString (!config.networking.nftables.enable) ''
+        ## IGMP / Broadcast ##
+        iptables -A INPUT -s 224.0.0.0/4 -j ACCEPT
+        iptables -A INPUT -d 224.0.0.0/4 -j ACCEPT
+        iptables -A INPUT -s 240.0.0.0/5 -j ACCEPT
+        iptables -A INPUT -m pkttype --pkt-type multicast -j ACCEPT
+        iptables -A INPUT -m pkttype --pkt-type broadcast -j ACCEPT
+      '';
+      extraInputRules = lib.optionalString config.networking.nftables.enable ''
+        ip saddr { 224.0.0.0/4, 240.0.0.0/5 } accept
+        ip daddr 224.0.0.0/4 accept
+        pkttype { multicast, broadcast } accept
+      '';
+    };
+
+    users.groups.${cfg.group} = { };
+    users.users.${cfg.user} = lib.optionalAttrs (cfg.user == "roon-server") {
+      isSystemUser = true;
+      description = "Roon Server user";
+      group = cfg.group;
+      extraGroups = [ "audio" ];
+    };
+  };
+}

nixos/modules/services/audio/slimserver.nix

@@ -0,0 +1,69 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.slimserver;
+
+in
+{
+  options = {
+
+    services.slimserver = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable slimserver.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "slimserver" { };
+
+      dataDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/slimserver";
+        description = ''
+          The directory where slimserver stores its state, tag cache,
+          playlists etc.
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - slimserver slimserver - -"
+    ];
+
+    systemd.services.slimserver = {
+      after = [ "network.target" ];
+      description = "Slim Server for Logitech Squeezebox Players";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        User = "slimserver";
+        # Issue 40589: Disable broken image/video support (audio still works!)
+        ExecStart = "${lib.getExe cfg.package} --logdir ${cfg.dataDir}/logs --prefsdir ${cfg.dataDir}/prefs --cachedir ${cfg.dataDir}/cache --noimage --novideo";
+      };
+    };
+
+    users = {
+      users.slimserver = {
+        description = "Slimserver daemon user";
+        home = cfg.dataDir;
+        group = "slimserver";
+        isSystemUser = true;
+      };
+      groups.slimserver = { };
+    };
+  };
+
+}

nixos/modules/services/audio/snapserver.nix

@@ -0,0 +1,257 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  name = "snapserver";
+
+  inherit (lib)
+    literalExpression
+    mkEnableOption
+    mkOption
+    mkPackageOption
+    mkRemovedOptionModule
+    mkRenamedOptionModule
+    types
+    ;
+
+  cfg = config.services.snapserver;
+
+  format = pkgs.formats.ini {
+    listsAsDuplicateKeys = true;
+  };
+
+  configFile = format.generate "snapserver.conf" cfg.settings;
+
+in
+{
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "controlPort" ]
+      [ "services" "snapserver" "tcp" "port" ]
+    )
+
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "listenAddress" ]
+      [ "services" "snapserver" "settings" "stream" "bind_to_address" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "port" ]
+      [ "services" "snapserver" "settings" "stream" "port" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "sampleFormat" ]
+      [ "services" "snapserver" "settings" "stream" "sampleformat" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "codec" ]
+      [ "services" "snapserver" "settings" "stream" "codec" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "streamBuffer" ]
+      [ "services" "snapserver" "settings" "stream" "chunk_ms" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "buffer" ]
+      [ "services" "snapserver" "settings" "stream" "buffer" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "send" ]
+      [ "services" "snapserver" "settings" "stream" "chunk_ms" ]
+    )
+
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "tcp" "enable" ]
+      [ "services" "snapserver" "settings" "tcp" "enabled" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "tcp" "listenAddress" ]
+      [ "services" "snapserver" "settings" "tcp" "bind_to_address" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "tcp" "port" ]
+      [ "services" "snapserver" "settings" "tcp" "port" ]
+    )
+
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "http" "enable" ]
+      [ "services" "snapserver" "settings" "http" "enabled" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "http" "listenAddress" ]
+      [ "services" "snapserver" "settings" "http" "bind_to_address" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "http" "port" ]
+      [ "services" "snapserver" "settings" "http" "port" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "snapserver" "http" "docRoot" ]
+      [ "services" "snapserver" "settings" "http" "doc_root" ]
+    )
+
+    (mkRemovedOptionModule [
+      "services"
+      "snapserver"
+      "streams"
+    ] "Configure `services.snapserver.settings.stream.source` instead")
+  ];
+
+  ###### interface
+
+  options = {
+
+    services.snapserver = {
+
+      enable = mkEnableOption "snapserver";
+
+      package = mkPackageOption pkgs "snapcast" { };
+
+      settings = mkOption {
+        default = { };
+        description = ''
+          Snapserver configuration.
+
+          Refer to the [example configuration](https://github.com/badaix/snapcast/blob/develop/server/etc/snapserver.conf) for possible options.
+        '';
+        type = types.submodule {
+          freeformType = format.type;
+          options = {
+            stream = {
+              bind_to_address = mkOption {
+                default = "::";
+                description = ''
+                  Address to listen on for snapclient connections.
+                '';
+              };
+
+              port = mkOption {
+                type = types.port;
+                default = 1704;
+                description = ''
+                  Port to listen on for snapclient connections.
+                '';
+              };
+
+              source = mkOption {
+                type = with types; either str (listOf str);
+                example = "pipe:///tmp/snapfifo?name=default";
+                description = ''
+                  One or multiple URIs to PCM inpuit streams.
+                '';
+              };
+            };
+
+            tcp = {
+              enabled = mkEnableOption "the TCP JSON-RPC";
+
+              bind_to_address = mkOption {
+                default = "::";
+                description = ''
+                  Address to listen on for snapclient connections.
+                '';
+              };
+
+              port = mkOption {
+                type = types.port;
+                default = 1705;
+                description = ''
+                  Port to listen on for snapclient connections.
+                '';
+              };
+            };
+
+            http = {
+              enabled = mkEnableOption "the HTTP JSON-RPC";
+
+              bind_to_address = mkOption {
+                default = "::";
+                description = ''
+                  Address to listen on for snapclient connections.
+                '';
+              };
+
+              port = mkOption {
+                type = types.port;
+                default = 1780;
+                description = ''
+                  Port to listen on for snapclient connections.
+                '';
+              };
+
+              doc_root = lib.mkOption {
+                type = with lib.types; nullOr path;
+                default = pkgs.snapweb;
+                defaultText = literalExpression "pkgs.snapweb";
+                description = ''
+                  Path to serve from the HTTP servers root.
+                '';
+              };
+            };
+          };
+        };
+      };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to automatically open the specified ports in the firewall.
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."snapserver.conf".source = configFile;
+
+    systemd.services.snapserver = {
+      after = [
+        "network.target"
+        "nss-lookup.target"
+      ];
+      description = "Snapserver";
+      wantedBy = [ "multi-user.target" ];
+      before = [
+        "mpd.service"
+        "mopidy.service"
+      ];
+      restartTriggers = [ configFile ];
+      serviceConfig = {
+        DynamicUser = true;
+        ExecStart = toString [
+          (lib.getExe' cfg.package "snapserver")
+          "--daemon"
+        ];
+        Type = "forking";
+        LimitRTPRIO = 50;
+        LimitRTTIME = "infinity";
+        NoNewPrivileges = true;
+        PIDFile = "/run/${name}/pid";
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectKernelModules = true;
+        Restart = "on-failure";
+        RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX AF_NETLINK";
+        RestrictNamespaces = true;
+        RuntimeDirectory = name;
+        StateDirectory = name;
+      };
+    };
+
+    networking.firewall.allowedTCPPorts =
+      lib.optionals cfg.openFirewall [ cfg.settings.stream.port ]
+      ++ lib.optional (cfg.openFirewall && cfg.settings.tcp.enabled) cfg.settings.tcp.port
+      ++ lib.optional (cfg.openFirewall && cfg.settings.http.enabled) cfg.settings.http.port;
+  };
+
+  meta = {
+    maintainers = with lib.maintainers; [ tobim ];
+  };
+
+}

nixos/modules/services/audio/spotifyd.nix

@@ -0,0 +1,78 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.spotifyd;
+  toml = pkgs.formats.toml { };
+  warnConfig =
+    if cfg.config != "" then
+      lib.trace "Using the stringly typed .config attribute is discouraged. Use the TOML typed .settings attribute instead."
+    else
+      lib.id;
+  spotifydConf =
+    if cfg.settings != { } then
+      toml.generate "spotify.conf" cfg.settings
+    else
+      warnConfig (pkgs.writeText "spotifyd.conf" cfg.config);
+in
+{
+  options = {
+    services.spotifyd = {
+      enable = lib.mkEnableOption "spotifyd, a Spotify playing daemon";
+
+      config = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          (Deprecated) Configuration for Spotifyd. For syntax and directives, see
+          <https://docs.spotifyd.rs/configuration/index.html#config-file>.
+        '';
+      };
+
+      settings = lib.mkOption {
+        default = { };
+        type = toml.type;
+        example = {
+          global.bitrate = 320;
+        };
+        description = ''
+          Configuration for Spotifyd. For syntax and directives, see
+          <https://docs.spotifyd.rs/configuration/index.html#config-file>.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.config == "" || cfg.settings == { };
+        message = "At most one of the .config attribute and the .settings attribute may be set";
+      }
+    ];
+
+    systemd.services.spotifyd = {
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "sound.target"
+      ];
+      description = "spotifyd, a Spotify playing daemon";
+      environment.SHELL = "/bin/sh";
+      serviceConfig = {
+        ExecStart = "${pkgs.spotifyd}/bin/spotifyd --no-daemon --cache-path /var/cache/spotifyd --config-path ${spotifydConf}";
+        Restart = "always";
+        RestartSec = 12;
+        DynamicUser = true;
+        CacheDirectory = "spotifyd";
+        SupplementaryGroups = [ "audio" ];
+      };
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.anderslundstedt ];
+}

nixos/modules/services/audio/squeezelite.nix

@@ -0,0 +1,59 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkOption
+    optionalString
+    types
+    ;
+
+  dataDir = "/var/lib/squeezelite";
+  cfg = config.services.squeezelite;
+  pkg = if cfg.pulseAudio then pkgs.squeezelite-pulse else pkgs.squeezelite;
+  bin = "${pkg}/bin/${pkg.pname}";
+
+in
+{
+
+  ###### interface
+
+  options.services.squeezelite = {
+    enable = mkEnableOption "Squeezelite, a software Squeezebox emulator";
+
+    pulseAudio = mkEnableOption "pulseaudio support";
+
+    extraArguments = mkOption {
+      default = "";
+      type = types.str;
+      description = ''
+        Additional command line arguments to pass to Squeezelite.
+      '';
+    };
+  };
+
+  ###### implementation
+
+  config = mkIf cfg.enable {
+    systemd.services.squeezelite = {
+      wantedBy = [ "multi-user.target" ];
+      after = [
+        "network.target"
+        "sound.target"
+      ];
+      description = "Software Squeezebox emulator";
+      serviceConfig = {
+        DynamicUser = true;
+        ExecStart = "${bin} -N ${dataDir}/player-name ${cfg.extraArguments}";
+        StateDirectory = builtins.baseNameOf dataDir;
+        SupplementaryGroups = "audio";
+      };
+    };
+  };
+}

nixos/modules/services/audio/tts.nix

@@ -0,0 +1,179 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.tts;
+in
+
+{
+  options.services.tts =
+    let
+      inherit (lib)
+        literalExpression
+        mkOption
+        mkEnableOption
+        types
+        ;
+    in
+    {
+      servers = mkOption {
+        type = types.attrsOf (
+          types.submodule (
+            { ... }:
+            {
+              options = {
+                enable = mkEnableOption "Coqui TTS server";
+
+                port = mkOption {
+                  type = types.port;
+                  example = 5000;
+                  description = ''
+                    Port to bind the TTS server to.
+                  '';
+                };
+
+                model = mkOption {
+                  type = types.nullOr types.str;
+                  default = "tts_models/en/ljspeech/tacotron2-DDC";
+                  example = null;
+                  description = ''
+                    Name of the model to download and use for speech synthesis.
+
+                    Check `tts-server --list_models` for possible values.
+
+                    Set to `null` to use a custom model.
+                  '';
+                };
+
+                useCuda = mkOption {
+                  type = types.bool;
+                  default = false;
+                  example = true;
+                  description = ''
+                    Whether to offload computation onto a CUDA compatible GPU.
+                  '';
+                };
+
+                extraArgs = mkOption {
+                  type = types.listOf types.str;
+                  default = [ ];
+                  description = ''
+                    Extra arguments to pass to the server commandline.
+                  '';
+                };
+              };
+            }
+          )
+        );
+        default = { };
+        example = literalExpression ''
+          {
+            english = {
+              port = 5300;
+              model = "tts_models/en/ljspeech/tacotron2-DDC";
+            };
+            german = {
+              port = 5301;
+              model = "tts_models/de/thorsten/tacotron2-DDC";
+            };
+            dutch = {
+              port = 5302;
+              model = "tts_models/nl/mai/tacotron2-DDC";
+            };
+          }
+        '';
+        description = ''
+          TTS server instances.
+        '';
+      };
+    };
+
+  config =
+    let
+      inherit (lib)
+        mkIf
+        mapAttrs'
+        nameValuePair
+        optionalString
+        concatMapStringsSep
+        escapeShellArgs
+        ;
+    in
+    mkIf (cfg.servers != { }) {
+      systemd.services = mapAttrs' (
+        server: options:
+        nameValuePair "tts-${server}" {
+          description = "Coqui TTS server instance ${server}";
+          after = [
+            "network-online.target"
+          ];
+          wantedBy = [
+            "multi-user.target"
+          ];
+          path = with pkgs; [
+            espeak-ng
+          ];
+          environment.HOME = "/var/lib/tts";
+          serviceConfig = {
+            DynamicUser = true;
+            User = "tts";
+            StateDirectory = "tts";
+            ExecStart =
+              "${pkgs.tts}/bin/tts-server --port ${toString options.port} "
+              + optionalString (options.model != null) "--model_name ${options.model} "
+              + optionalString (options.useCuda) "--use_cuda "
+              + (escapeShellArgs options.extraArgs);
+            CapabilityBoundingSet = "";
+            DeviceAllow =
+              if options.useCuda then
+                [
+                  # https://docs.nvidia.com/dgx/pdf/dgx-os-5-user-guide.pdf
+                  "/dev/nvidia1"
+                  "/dev/nvidia2"
+                  "/dev/nvidia3"
+                  "/dev/nvidia4"
+                  "/dev/nvidia-caps/nvidia-cap1"
+                  "/dev/nvidia-caps/nvidia-cap2"
+                  "/dev/nvidiactl"
+                  "/dev/nvidia-modeset"
+                  "/dev/nvidia-uvm"
+                  "/dev/nvidia-uvm-tools"
+                ]
+              else
+                "";
+            DevicePolicy = "closed";
+            LockPersonality = true;
+            # jit via numba->llvmpipe
+            MemoryDenyWriteExecute = false;
+            PrivateDevices = true;
+            PrivateUsers = true;
+            ProtectHome = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            ProtectControlGroups = true;
+            ProtectProc = "invisible";
+            ProcSubset = "pid";
+            RestrictAddressFamilies = [
+              "AF_UNIX"
+              "AF_INET"
+              "AF_INET6"
+            ];
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            SystemCallArchitectures = "native";
+            SystemCallFilter = [
+              "@system-service"
+              "~@privileged"
+            ];
+            UMask = "0077";
+          };
+        }
+      ) cfg.servers;
+    };
+}

nixos/modules/services/audio/ympd.nix

@@ -0,0 +1,101 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.ympd;
+in
+{
+
+  ###### interface
+
+  options = {
+
+    services.ympd = {
+
+      enable = lib.mkEnableOption "ympd, the MPD Web GUI";
+
+      webPort = lib.mkOption {
+        type = lib.types.either lib.types.str lib.types.port; # string for backwards compat
+        default = "8080";
+        description = "The port where ympd's web interface will be available.";
+        example = "ssl://8080:/path/to/ssl-private-key.pem";
+      };
+
+      mpd = {
+        host = lib.mkOption {
+          type = lib.types.str;
+          default = "localhost";
+          description = "The host where MPD is listening.";
+        };
+
+        port = lib.mkOption {
+          type = lib.types.port;
+          default = config.services.mpd.network.port;
+          defaultText = lib.literalExpression "config.services.mpd.network.port";
+          description = "The port where MPD is listening.";
+          example = 6600;
+        };
+      };
+
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.services.ympd = {
+      description = "Standalone MPD Web GUI written in C";
+
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+
+      serviceConfig = {
+        ExecStart = ''
+          ${pkgs.ympd}/bin/ympd \
+            --host ${cfg.mpd.host} \
+            --port ${toString cfg.mpd.port} \
+            --webport ${toString cfg.webPort}
+        '';
+
+        DynamicUser = true;
+        NoNewPrivileges = true;
+
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateIPC = true;
+
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+
+        SystemCallFilter = [
+          "@system-service"
+          "~@process"
+          "~@setuid"
+        ];
+      };
+    };
+
+  };
+
+}

nixos/modules/services/autotierfs.nix

@@ -0,0 +1,95 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.autotierfs;
+  ini = pkgs.formats.ini { };
+  format = lib.types.attrsOf ini.type;
+  stateDir = "/var/lib/autotier";
+
+  generateConfigName =
+    name: builtins.replaceStrings [ "/" ] [ "-" ] (lib.strings.removePrefix "/" name);
+  configFiles = builtins.mapAttrs (
+    name: val: ini.generate "${generateConfigName name}.conf" val
+  ) cfg.settings;
+
+  getMountDeps =
+    settings: builtins.concatStringsSep " " (builtins.catAttrs "Path" (builtins.attrValues settings));
+
+  mountPaths = builtins.attrNames cfg.settings;
+in
+{
+  options.services.autotierfs = {
+    enable = lib.mkEnableOption "the autotier passthrough tiering filesystem";
+    package = lib.mkPackageOption pkgs "autotier" { };
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = format;
+      };
+      default = { };
+      description = ''
+        The contents of the configuration file for autotier.
+        See the [autotier repo](https://github.com/45Drives/autotier#configuration) for supported values.
+      '';
+      example = lib.literalExpression ''
+        {
+          "/mnt/autotier" = {
+            Global = {
+              "Log Level" = 1;
+              "Tier Period" = 1000;
+              "Copy Buffer Size" = "1 MiB";
+            };
+            "Tier 1" = {
+              Path = "/mnt/tier1";
+              Quota = "30GiB";
+            };
+            "Tier 2" = {
+              Path = "/mnt/tier2";
+              Quota = "200GiB";
+            };
+          };
+        }
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.settings != { };
+        message = "`services.autotierfs.settings` must be configured.";
+      }
+    ];
+
+    system.fsPackages = [ cfg.package ];
+
+    # Not necessary for module to work but makes it easier to pass config into cli
+    environment.etc = lib.attrsets.mapAttrs' (
+      name: value:
+      lib.attrsets.nameValuePair "autotier/${(generateConfigName name)}.conf" { source = value; }
+    ) configFiles;
+
+    systemd.tmpfiles.rules = (map (path: "d ${path} 0770 - autotier - -") mountPaths) ++ [
+      "d ${stateDir} 0774 - autotier - -"
+    ];
+
+    users.groups.autotier = { };
+
+    systemd.services = lib.attrsets.mapAttrs' (
+      path: values:
+      lib.attrsets.nameValuePair (generateConfigName path) {
+        description = "Mount autotierfs virtual path ${path}";
+        unitConfig.RequiresMountsFor = getMountDeps values;
+        wantedBy = [ "local-fs.target" ];
+        serviceConfig = {
+          Type = "forking";
+          ExecStart = "${lib.getExe' cfg.package "autotierfs"} -c /etc/autotier/${generateConfigName path}.conf ${path} -o allow_other,default_permissions";
+          ExecStop = "umount ${path}";
+        };
+      }
+    ) cfg.settings;
+  };
+}

nixos/modules/services/backup/automysqlbackup.nix

@@ -0,0 +1,168 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+
+  inherit (lib)
+    concatMapStringsSep
+    concatStringsSep
+    isInt
+    isList
+    literalExpression
+    ;
+  inherit (lib)
+    mapAttrs
+    mapAttrsToList
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkOption
+    mkRenamedOptionModule
+    optional
+    types
+    ;
+
+  cfg = config.services.automysqlbackup;
+  pkg = pkgs.automysqlbackup;
+  user = "automysqlbackup";
+  group = "automysqlbackup";
+
+  toStr =
+    val:
+    if isList val then
+      "( ${concatMapStringsSep " " (val: "'${val}'") val} )"
+    else if isInt val then
+      toString val
+    else if true == val then
+      "'yes'"
+    else if false == val then
+      "'no'"
+    else
+      "'${toString val}'";
+
+  configFile = pkgs.writeText "automysqlbackup.conf" ''
+    #version=${pkg.version}
+    # DONT'T REMOVE THE PREVIOUS VERSION LINE!
+    #
+    ${concatStringsSep "\n" (mapAttrsToList (name: value: "CONFIG_${name}=${toStr value}") cfg.config)}
+  '';
+
+in
+{
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "automysqlbackup" "config" ]
+      [ "services" "automysqlbackup" "settings" ]
+    )
+  ];
+
+  # interface
+  options = {
+    services.automysqlbackup = {
+
+      enable = mkEnableOption "AutoMySQLBackup";
+
+      calendar = mkOption {
+        type = types.str;
+        default = "01:15:00";
+        description = ''
+          Configured when to run the backup service systemd unit (DayOfWeek Year-Month-Day Hour:Minute:Second).
+        '';
+      };
+
+      settings = mkOption {
+        type =
+          with types;
+          attrsOf (oneOf [
+            str
+            int
+            bool
+            (listOf str)
+          ]);
+        default = { };
+        description = ''
+          automysqlbackup configuration. Refer to
+          {file}`''${pkgs.automysqlbackup}/etc/automysqlbackup.conf`
+          for details on supported values.
+        '';
+        example = literalExpression ''
+          {
+            db_names = [ "nextcloud" "matomo" ];
+            table_exclude = [ "nextcloud.oc_users" "nextcloud.oc_whats_new" ];
+            mailcontent = "log";
+            mail_address = "admin@example.org";
+          }
+        '';
+      };
+
+    };
+  };
+
+  # implementation
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion = !config.services.mysqlBackup.enable;
+        message = "Please choose one of services.mysqlBackup or services.automysqlbackup.";
+      }
+    ];
+
+    services.automysqlbackup.config = mapAttrs (name: mkDefault) {
+      mysql_dump_username = user;
+      mysql_dump_host = "localhost";
+      mysql_dump_socket = "/run/mysqld/mysqld.sock";
+      backup_dir = "/var/backup/mysql";
+      db_exclude = [
+        "information_schema"
+        "performance_schema"
+      ];
+      mailcontent = "stdout";
+      mysql_dump_single_transaction = true;
+    };
+
+    systemd.timers.automysqlbackup = {
+      description = "automysqlbackup timer";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = cfg.calendar;
+        AccuracySec = "5m";
+      };
+    };
+
+    systemd.services.automysqlbackup = {
+      description = "automysqlbackup service";
+      serviceConfig = {
+        User = user;
+        Group = group;
+        ExecStart = "${pkg}/bin/automysqlbackup ${configFile}";
+      };
+    };
+
+    environment.systemPackages = [ pkg ];
+
+    users.users.${user} = {
+      group = group;
+      isSystemUser = true;
+    };
+    users.groups.${group} = { };
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.config.backup_dir}' 0750 ${user} ${group} - -"
+    ];
+
+    services.mysql.ensureUsers =
+      optional (config.services.mysql.enable && cfg.config.mysql_dump_host == "localhost")
+        {
+          name = user;
+          ensurePermissions = {
+            "*.*" = "SELECT, SHOW VIEW, TRIGGER, LOCK TABLES, EVENT";
+          };
+        };
+
+  };
+}

nixos/modules/services/backup/bacula.nix

@@ -0,0 +1,764 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+# TODO: test configuration when building nixexpr (use -t parameter)
+# TODO: support sqlite3 (it's deprecate?) and mysql
+
+let
+  inherit (lib)
+    concatStringsSep
+    literalExpression
+    mapAttrsToList
+    mkIf
+    mkOption
+    optional
+    optionalString
+    types
+    ;
+  libDir = "/var/lib/bacula";
+
+  yes_no = bool: if bool then "yes" else "no";
+  tls_conf =
+    tls_cfg:
+    optionalString tls_cfg.enable (
+      concatStringsSep "\n" (
+        [ "TLS Enable = yes;" ]
+        ++ optional (tls_cfg.require != null) "TLS Require = ${yes_no tls_cfg.require};"
+        ++ optional (tls_cfg.certificate != null) ''TLS Certificate = "${tls_cfg.certificate}";''
+        ++ [ ''TLS Key = "${tls_cfg.key}";'' ]
+        ++ optional (tls_cfg.verifyPeer != null) "TLS Verify Peer = ${yes_no tls_cfg.verifyPeer};"
+        ++ optional (
+          tls_cfg.allowedCN != [ ]
+        ) "TLS Allowed CN = ${concatStringsSep " " (tls_cfg.allowedCN)};"
+        ++ optional (
+          tls_cfg.caCertificateFile != null
+        ) ''TLS CA Certificate File = "${tls_cfg.caCertificateFile}";''
+      )
+    );
+
+  fd_cfg = config.services.bacula-fd;
+  fd_conf = pkgs.writeText "bacula-fd.conf" ''
+    Client {
+      Name = "${fd_cfg.name}";
+      FDPort = ${toString fd_cfg.port};
+      WorkingDirectory = ${libDir};
+      Pid Directory = /run;
+      ${fd_cfg.extraClientConfig}
+      ${tls_conf fd_cfg.tls}
+    }
+
+    ${concatStringsSep "\n" (
+      mapAttrsToList (name: value: ''
+        Director {
+          Name = "${name}";
+          Password = ${value.password};
+          Monitor = ${value.monitor};
+          ${tls_conf value.tls}
+        }
+      '') fd_cfg.director
+    )}
+
+    Messages {
+      Name = Standard;
+      syslog = all, !skipped, !restored
+      ${fd_cfg.extraMessagesConfig}
+    }
+  '';
+
+  sd_cfg = config.services.bacula-sd;
+  sd_conf = pkgs.writeText "bacula-sd.conf" ''
+    Storage {
+      Name = "${sd_cfg.name}";
+      SDPort = ${toString sd_cfg.port};
+      WorkingDirectory = ${libDir};
+      Pid Directory = /run;
+      ${sd_cfg.extraStorageConfig}
+      ${tls_conf sd_cfg.tls}
+    }
+
+    ${concatStringsSep "\n" (
+      mapAttrsToList (name: value: ''
+        Autochanger {
+          Name = "${name}";
+          Device = ${concatStringsSep ", " (map (a: "\"${a}\"") value.devices)};
+          Changer Device =  ${value.changerDevice};
+          Changer Command = ${value.changerCommand};
+          ${value.extraAutochangerConfig}
+        }
+      '') sd_cfg.autochanger
+    )}
+
+    ${concatStringsSep "\n" (
+      mapAttrsToList (name: value: ''
+        Device {
+          Name = "${name}";
+          Archive Device = ${value.archiveDevice};
+          Media Type = ${value.mediaType};
+          ${value.extraDeviceConfig}
+        }
+      '') sd_cfg.device
+    )}
+
+    ${concatStringsSep "\n" (
+      mapAttrsToList (name: value: ''
+        Director {
+          Name = "${name}";
+          Password = ${value.password};
+          Monitor = ${value.monitor};
+          ${tls_conf value.tls}
+        }
+      '') sd_cfg.director
+    )}
+
+    Messages {
+      Name = Standard;
+      syslog = all, !skipped, !restored
+      ${sd_cfg.extraMessagesConfig}
+    }
+  '';
+
+  dir_cfg = config.services.bacula-dir;
+  dir_conf = pkgs.writeText "bacula-dir.conf" ''
+    Director {
+      Name = "${dir_cfg.name}";
+      Password = ${dir_cfg.password};
+      DirPort = ${toString dir_cfg.port};
+      Working Directory = ${libDir};
+      Pid Directory = /run/;
+      QueryFile = ${pkgs.bacula}/etc/query.sql;
+      ${tls_conf dir_cfg.tls}
+      ${dir_cfg.extraDirectorConfig}
+    }
+
+    Catalog {
+      Name = PostgreSQL;
+      dbname = bacula;
+      user = bacula;
+    }
+
+    Messages {
+      Name = Standard;
+      syslog = all, !skipped, !restored
+      ${dir_cfg.extraMessagesConfig}
+    }
+
+    ${dir_cfg.extraConfig}
+  '';
+
+  linkOption =
+    name: destination: "[${name}](#opt-${builtins.replaceStrings [ "<" ">" ] [ "_" "_" ] destination})";
+  tlsLink =
+    destination: submodulePath:
+    linkOption "${submodulePath}.${destination}" "${submodulePath}.${destination}";
+
+  tlsOptions =
+    submodulePath:
+    { ... }:
+    {
+      options = {
+        enable = mkOption {
+          type = types.bool;
+          default = false;
+          description = ''
+            Specifies if TLS should be enabled.
+            If this set to `false` TLS will be completely disabled, even if ${tlsLink "tls.require" submodulePath} is true.
+          '';
+        };
+        require = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Require TLS or TLS-PSK encryption.
+            This directive is ignored unless one of ${tlsLink "tls.enable" submodulePath} is true or TLS PSK Enable is set to `yes`.
+            If TLS is not required while TLS or TLS-PSK are enabled, then the Bacula component
+            will connect with other components either with or without TLS or TLS-PSK
+
+            If ${tlsLink "tls.enable" submodulePath} or TLS-PSK is enabled and TLS is required, then the Bacula
+            component will refuse any connection request that does not use TLS.
+          '';
+        };
+        certificate = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          description = ''
+            The full path to the PEM encoded TLS certificate.
+            It will be used as either a client or server certificate,
+            depending on the connection direction.
+            This directive is required in a server context, but it may
+            not be specified in a client context if ${tlsLink "tls.verifyPeer" submodulePath} is
+            `false` in the corresponding server context.
+          '';
+        };
+        key = mkOption {
+          type = types.path;
+          description = ''
+            The path of a PEM encoded TLS private key.
+            It must correspond to the TLS certificate.
+          '';
+        };
+        verifyPeer = mkOption {
+          type = types.nullOr types.bool;
+          default = null;
+          description = ''
+            Verify peer certificate.
+            Instructs server to request and verify the client's X.509 certificate.
+            Any client certificate signed by a known-CA will be accepted.
+            Additionally, the client's X509 certificate Common Name must meet the value of the Address directive.
+            If ${tlsLink "tls.allowedCN" submodulePath} is used,
+            the client's x509 certificate Common Name must also correspond to
+            one of the CN specified in the ${tlsLink "tls.allowedCN" submodulePath} directive.
+            This directive is valid only for a server and not in client context.
+
+            Standard from Bacula is `true`.
+          '';
+        };
+        allowedCN = mkOption {
+          type = types.listOf types.str;
+          default = [ ];
+          description = ''
+            Common name attribute of allowed peer certificates.
+            This directive is valid for a server and in a client context.
+            If this directive is specified, the peer certificate will be verified against this list.
+            In the case this directive is configured on a server side, the allowed
+            CN list will not be checked if ${tlsLink "tls.verifyPeer" submodulePath} is false.
+          '';
+        };
+        caCertificateFile = mkOption {
+          type = types.nullOr types.path;
+          default = null;
+          description = ''
+            The path specifying a PEM encoded TLS CA certificate(s).
+            Multiple certificates are permitted in the file.
+            One of TLS CA Certificate File or TLS CA Certificate Dir are required in a server context, unless
+            ${tlsLink "tls.verifyPeer" submodulePath} is false, and are always required in a client context.
+          '';
+        };
+      };
+    };
+
+  directorOptions =
+    submodulePath:
+    { ... }:
+    {
+      options = {
+        password = mkOption {
+          type = types.str;
+          # TODO: required?
+          description = ''
+            Specifies the password that must be supplied for the default Bacula
+            Console to be authorized. The same password must appear in the
+            Director resource of the Console configuration file. For added
+            security, the password is never passed across the network but instead
+            a challenge response hash code created with the password. This
+            directive is required. If you have either /dev/random or bc on your
+            machine, Bacula will generate a random password during the
+            configuration process, otherwise it will be left blank and you must
+            manually supply it.
+
+            The password is plain text. It is not generated through any special
+            process but as noted above, it is better to use random text for
+            security reasons.
+          '';
+        };
+
+        monitor = mkOption {
+          type = types.enum [
+            "no"
+            "yes"
+          ];
+          default = "no";
+          example = "yes";
+          description = ''
+            If Monitor is set to `no`, this director will have
+            full access to this Storage daemon. If Monitor is set to
+            `yes`, this director will only be able to fetch the
+            current status of this Storage daemon.
+
+            Please note that if this director is being used by a Monitor, we
+            highly recommend to set this directive to yes to avoid serious
+            security problems.
+          '';
+        };
+
+        tls = mkOption {
+          type = types.submodule (tlsOptions "${submodulePath}.director.<name>");
+          description = ''
+            TLS Options for the Director in this Configuration.
+          '';
+        };
+      };
+    };
+
+  autochangerOptions =
+    { ... }:
+    {
+      options = {
+        changerDevice = mkOption {
+          type = types.str;
+          description = ''
+            The specified name-string must be the generic SCSI device name of the
+            autochanger that corresponds to the normal read/write Archive Device
+            specified in the Device resource. This generic SCSI device name
+            should be specified if you have an autochanger or if you have a
+            standard tape drive and want to use the Alert Command (see below).
+            For example, on Linux systems, for an Archive Device name of
+            `/dev/nst0`, you would specify
+            `/dev/sg0` for the Changer Device name.  Depending
+            on your exact configuration, and the number of autochangers or the
+            type of autochanger, what you specify here can vary. This directive
+            is optional. See the Using AutochangersAutochangersChapter chapter of
+            this manual for more details of using this and the following
+            autochanger directives.
+          '';
+        };
+
+        changerCommand = mkOption {
+          type = types.str;
+          description = ''
+            The name-string specifies an external program to be called that will
+            automatically change volumes as required by Bacula. Normally, this
+            directive will be specified only in the AutoChanger resource, which
+            is then used for all devices. However, you may also specify the
+            different Changer Command in each Device resource. Most frequently,
+            you will specify the Bacula supplied mtx-changer script as follows:
+
+            `"/path/mtx-changer %c %o %S %a %d"`
+
+            and you will install the mtx on your system (found in the depkgs
+            release). An example of this command is in the default bacula-sd.conf
+            file. For more details on the substitution characters that may be
+            specified to configure your autochanger please see the
+            AutochangersAutochangersChapter chapter of this manual. For FreeBSD
+            users, you might want to see one of the several chio scripts in
+            examples/autochangers.
+          '';
+          default = "/etc/bacula/mtx-changer %c %o %S %a %d";
+        };
+
+        devices = mkOption {
+          description = "";
+          type = types.listOf types.str;
+        };
+
+        extraAutochangerConfig = mkOption {
+          default = "";
+          type = types.lines;
+          description = ''
+            Extra configuration to be passed in Autochanger directive.
+          '';
+          example = ''
+
+          '';
+        };
+      };
+    };
+
+  deviceOptions =
+    { ... }:
+    {
+      options = {
+        archiveDevice = mkOption {
+          # TODO: required?
+          type = types.str;
+          description = ''
+            The specified name-string gives the system file name of the storage
+            device managed by this storage daemon. This will usually be the
+            device file name of a removable storage device (tape drive), for
+            example `/dev/nst0` or
+            `/dev/rmt/0mbn`. For a DVD-writer, it will be for
+            example `/dev/hdc`. It may also be a directory name
+            if you are archiving to disk storage. In this case, you must supply
+            the full absolute path to the directory. When specifying a tape
+            device, it is preferable that the "non-rewind" variant of the device
+            file name be given.
+          '';
+        };
+
+        mediaType = mkOption {
+          # TODO: required?
+          type = types.str;
+          description = ''
+            The specified name-string names the type of media supported by this
+            device, for example, `DLT7000`. Media type names are
+            arbitrary in that you set them to anything you want, but they must be
+            known to the volume database to keep track of which storage daemons
+            can read which volumes. In general, each different storage type
+            should have a unique Media Type associated with it. The same
+            name-string must appear in the appropriate Storage resource
+            definition in the Director's configuration file.
+
+            Even though the names you assign are arbitrary (i.e. you choose the
+            name you want), you should take care in specifying them because the
+            Media Type is used to determine which storage device Bacula will
+            select during restore. Thus you should probably use the same Media
+            Type specification for all drives where the Media can be freely
+            interchanged. This is not generally an issue if you have a single
+            Storage daemon, but it is with multiple Storage daemons, especially
+            if they have incompatible media.
+
+            For example, if you specify a Media Type of `DDS-4`
+            then during the restore, Bacula will be able to choose any Storage
+            Daemon that handles `DDS-4`. If you have an
+            autochanger, you might want to name the Media Type in a way that is
+            unique to the autochanger, unless you wish to possibly use the
+            Volumes in other drives. You should also ensure to have unique Media
+            Type names if the Media is not compatible between drives. This
+            specification is required for all devices.
+
+            In addition, if you are using disk storage, each Device resource will
+            generally have a different mount point or directory. In order for
+            Bacula to select the correct Device resource, each one must have a
+            unique Media Type.
+          '';
+        };
+
+        extraDeviceConfig = mkOption {
+          default = "";
+          type = types.lines;
+          description = ''
+            Extra configuration to be passed in Device directive.
+          '';
+          example = ''
+            LabelMedia = yes
+            Random Access = no
+            AutomaticMount = no
+            RemovableMedia = no
+            MaximumOpenWait = 60
+            AlwaysOpen = no
+          '';
+        };
+      };
+    };
+
+in
+{
+  options = {
+    services.bacula-fd = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to enable the Bacula File Daemon.
+        '';
+      };
+
+      name = mkOption {
+        default = "${config.networking.hostName}-fd";
+        defaultText = literalExpression ''"''${config.networking.hostName}-fd"'';
+        type = types.str;
+        description = ''
+          The client name that must be used by the Director when connecting.
+          Generally, it is a good idea to use a name related to the machine so
+          that error messages can be easily identified if you have multiple
+          Clients. This directive is required.
+        '';
+      };
+
+      port = mkOption {
+        default = 9102;
+        type = types.port;
+        description = ''
+          This specifies the port number on which the Client listens for
+          Director connections. It must agree with the FDPort specified in
+          the Client resource of the Director's configuration file.
+        '';
+      };
+
+      director = mkOption {
+        default = { };
+        description = ''
+          This option defines director resources in Bacula File Daemon.
+        '';
+        type = types.attrsOf (types.submodule (directorOptions "services.bacula-fd"));
+      };
+
+      tls = mkOption {
+        type = types.submodule (tlsOptions "services.bacula-fd");
+        default = { };
+        description = ''
+          TLS Options for the File Daemon.
+          Important notice: The backup won't be encrypted.
+        '';
+      };
+
+      extraClientConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Client directive.
+        '';
+        example = ''
+          Maximum Concurrent Jobs = 20;
+          Heartbeat Interval = 30;
+        '';
+      };
+
+      extraMessagesConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Messages directive.
+        '';
+        example = ''
+          console = all
+        '';
+      };
+    };
+
+    services.bacula-sd = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to enable Bacula Storage Daemon.
+        '';
+      };
+
+      name = mkOption {
+        default = "${config.networking.hostName}-sd";
+        defaultText = literalExpression ''"''${config.networking.hostName}-sd"'';
+        type = types.str;
+        description = ''
+          Specifies the Name of the Storage daemon.
+        '';
+      };
+
+      port = mkOption {
+        default = 9103;
+        type = types.port;
+        description = ''
+          Specifies port number on which the Storage daemon listens for
+          Director connections.
+        '';
+      };
+
+      director = mkOption {
+        default = { };
+        description = ''
+          This option defines Director resources in Bacula Storage Daemon.
+        '';
+        type = types.attrsOf (types.submodule (directorOptions "services.bacula-sd"));
+      };
+
+      device = mkOption {
+        default = { };
+        description = ''
+          This option defines Device resources in Bacula Storage Daemon.
+        '';
+        type = types.attrsOf (types.submodule deviceOptions);
+      };
+
+      autochanger = mkOption {
+        default = { };
+        description = ''
+          This option defines Autochanger resources in Bacula Storage Daemon.
+        '';
+        type = types.attrsOf (types.submodule autochangerOptions);
+      };
+
+      extraStorageConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Storage directive.
+        '';
+        example = ''
+          Maximum Concurrent Jobs = 20;
+          Heartbeat Interval = 30;
+        '';
+      };
+
+      extraMessagesConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Messages directive.
+        '';
+        example = ''
+          console = all
+        '';
+      };
+      tls = mkOption {
+        type = types.submodule (tlsOptions "services.bacula-sd");
+        default = { };
+        description = ''
+          TLS Options for the Storage Daemon.
+          Important notice: The backup won't be encrypted.
+        '';
+      };
+
+    };
+
+    services.bacula-dir = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to enable Bacula Director Daemon.
+        '';
+      };
+
+      name = mkOption {
+        default = "${config.networking.hostName}-dir";
+        defaultText = literalExpression ''"''${config.networking.hostName}-dir"'';
+        type = types.str;
+        description = ''
+          The director name used by the system administrator. This directive is
+          required.
+        '';
+      };
+
+      port = mkOption {
+        default = 9101;
+        type = types.port;
+        description = ''
+          Specify the port (a positive integer) on which the Director daemon
+          will listen for Bacula Console connections. This same port number
+          must be specified in the Director resource of the Console
+          configuration file. The default is 9101, so normally this directive
+          need not be specified. This directive should not be used if you
+          specify DirAddresses (N.B plural) directive.
+        '';
+      };
+
+      password = mkOption {
+        # TODO: required?
+        type = types.str;
+        description = ''
+          Specifies the password that must be supplied for a Director.
+        '';
+      };
+
+      extraMessagesConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Messages directive.
+        '';
+        example = ''
+          console = all
+        '';
+      };
+
+      extraDirectorConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration to be passed in Director directive.
+        '';
+        example = ''
+          Maximum Concurrent Jobs = 20;
+          Heartbeat Interval = 30;
+        '';
+      };
+
+      extraConfig = mkOption {
+        default = "";
+        type = types.lines;
+        description = ''
+          Extra configuration for Bacula Director Daemon.
+        '';
+        example = ''
+          TODO
+        '';
+      };
+
+      tls = mkOption {
+        type = types.submodule (tlsOptions "services.bacula-dir");
+        default = { };
+        description = ''
+          TLS Options for the Director.
+          Important notice: The backup won't be encrypted.
+        '';
+      };
+    };
+  };
+
+  config = mkIf (fd_cfg.enable || sd_cfg.enable || dir_cfg.enable) {
+    systemd.slices.system-bacula = {
+      description = "Bacula Backup System Slice";
+      documentation = [
+        "man:bacula(8)"
+        "https://www.bacula.org/"
+      ];
+    };
+
+    systemd.services.bacula-fd = mkIf fd_cfg.enable {
+      after = [ "network.target" ];
+      description = "Bacula File Daemon";
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.bacula ];
+      serviceConfig = {
+        ExecStart = "${pkgs.bacula}/sbin/bacula-fd -f -u root -g bacula -c ${fd_conf}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        LogsDirectory = "bacula";
+        StateDirectory = "bacula";
+        Slice = "system-bacula.slice";
+      };
+    };
+
+    systemd.services.bacula-sd = mkIf sd_cfg.enable {
+      after = [ "network.target" ];
+      description = "Bacula Storage Daemon";
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.bacula ];
+      serviceConfig = {
+        ExecStart = "${pkgs.bacula}/sbin/bacula-sd -f -u bacula -g bacula -c ${sd_conf}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        LogsDirectory = "bacula";
+        StateDirectory = "bacula";
+        Slice = "system-bacula.slice";
+      };
+    };
+
+    services.postgresql.enable = lib.mkIf dir_cfg.enable true;
+
+    systemd.services.bacula-dir = mkIf dir_cfg.enable {
+      after = [
+        "network.target"
+        "postgresql.target"
+      ];
+      description = "Bacula Director Daemon";
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.bacula ];
+      serviceConfig = {
+        ExecStart = "${pkgs.bacula}/sbin/bacula-dir -f -u bacula -g bacula -c ${dir_conf}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        LogsDirectory = "bacula";
+        StateDirectory = "bacula";
+        Slice = "system-bacula.slice";
+      };
+      preStart = ''
+        if ! test -e "${libDir}/db-created"; then
+            ${pkgs.postgresql}/bin/createuser --no-superuser --no-createdb --no-createrole bacula
+            #${pkgs.postgresql}/bin/createdb --owner bacula bacula
+
+            # populate DB
+            ${pkgs.bacula}/etc/create_bacula_database postgresql
+            ${pkgs.bacula}/etc/make_bacula_tables postgresql
+            ${pkgs.bacula}/etc/grant_bacula_privileges postgresql
+            touch "${libDir}/db-created"
+        else
+            ${pkgs.bacula}/etc/update_bacula_tables postgresql || true
+        fi
+      '';
+    };
+
+    environment.systemPackages = [ pkgs.bacula ];
+
+    users.users.bacula = {
+      group = "bacula";
+      uid = config.ids.uids.bacula;
+      home = "${libDir}";
+      createHome = true;
+      description = "Bacula Daemons user";
+      shell = "${pkgs.bash}/bin/bash";
+    };
+
+    users.groups.bacula.gid = config.ids.gids.bacula;
+  };
+}

nixos/modules/services/backup/borgbackup.md

@@ -0,0 +1,167 @@
+# BorgBackup {#module-borgbase}
+
+*Source:* {file}`modules/services/backup/borgbackup.nix`
+
+*Upstream documentation:* <https://borgbackup.readthedocs.io/>
+
+[BorgBackup](https://www.borgbackup.org/) (short: Borg)
+is a deduplicating backup program. Optionally, it supports compression and
+authenticated encryption.
+
+The main goal of Borg is to provide an efficient and secure way to backup
+data. The data deduplication technique used makes Borg suitable for daily
+backups since only changes are stored. The authenticated encryption technique
+makes it suitable for backups to not fully trusted targets.
+
+## Configuring {#module-services-backup-borgbackup-configuring}
+
+A complete list of options for the Borgbase module may be found
+[here](#opt-services.borgbackup.jobs).
+
+## Basic usage for a local backup {#opt-services-backup-borgbackup-local-directory}
+
+A very basic configuration for backing up to a locally accessible directory is:
+```nix
+{
+  services.borgbackup.jobs = {
+    rootBackup = {
+      paths = "/";
+      exclude = [
+        "/nix"
+        "/path/to/local/repo"
+      ];
+      repo = "/path/to/local/repo";
+      doInit = true;
+      encryption = {
+        mode = "repokey";
+        passphrase = "secret";
+      };
+      compression = "auto,lzma";
+      startAt = "weekly";
+    };
+  };
+}
+```
+
+::: {.warning}
+If you do not want the passphrase to be stored in the world-readable
+Nix store, use passCommand. You find an example below.
+:::
+
+## Create a borg backup server {#opt-services-backup-create-server}
+
+You should use a different SSH key for each repository you write to,
+because the specified keys are restricted to running borg serve and can only
+access this single repository. You need the output of the generate pub file.
+
+```ShellSession
+# sudo ssh-keygen -N '' -t ed25519 -f /run/keys/id_ed25519_my_borg_repo
+# cat /run/keys/id_ed25519_my_borg_repo
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID78zmOyA+5uPG4Ot0hfAy+sLDPU1L4AiIoRYEIVbbQ/ root@nixos
+```
+
+Add the following snippet to your NixOS configuration:
+```nix
+{
+  services.borgbackup.repos = {
+    my_borg_repo = {
+      authorizedKeys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID78zmOyA+5uPG4Ot0hfAy+sLDPU1L4AiIoRYEIVbbQ/ root@nixos"
+      ];
+      path = "/var/lib/my_borg_repo";
+    };
+  };
+}
+```
+
+## Backup to the borg repository server {#opt-services-backup-borgbackup-remote-server}
+
+The following NixOS snippet creates an hourly backup to the service
+(on the host nixos) as created in the section above. We assume
+that you have stored a secret passphrasse in the file
+{file}`/run/keys/borgbackup_passphrase`, which should be only
+accessible by root
+
+```nix
+{
+  services.borgbackup.jobs = {
+    backupToLocalServer = {
+      paths = [ "/etc/nixos" ];
+      doInit = true;
+      repo = "borg@nixos:.";
+      encryption = {
+        mode = "repokey-blake2";
+        passCommand = "cat /run/keys/borgbackup_passphrase";
+      };
+      environment = {
+        BORG_RSH = "ssh -i /run/keys/id_ed25519_my_borg_repo";
+      };
+      compression = "auto,lzma";
+      startAt = "hourly";
+    };
+  };
+}
+```
+
+The following few commands (run as root) let you test your backup.
+```
+> nixos-rebuild switch
+...restarting the following units: polkit.service
+> systemctl restart borgbackup-job-backupToLocalServer
+> sleep 10
+> systemctl restart borgbackup-job-backupToLocalServer
+> export BORG_PASSPHRASE=topSecret
+> borg list --rsh='ssh -i /run/keys/id_ed25519_my_borg_repo' borg@nixos:.
+nixos-backupToLocalServer-2020-03-30T21:46:17 Mon, 2020-03-30 21:46:19 [84feb97710954931ca384182f5f3cb90665f35cef214760abd7350fb064786ac]
+nixos-backupToLocalServer-2020-03-30T21:46:30 Mon, 2020-03-30 21:46:32 [e77321694ecd160ca2228611747c6ad1be177d6e0d894538898de7a2621b6e68]
+```
+
+## Backup to a hosting service {#opt-services-backup-borgbackup-borgbase}
+
+Several companies offer [(paid) hosting services](https://www.borgbackup.org/support/commercial.html)
+for Borg repositories.
+
+To backup your home directory to borgbase you have to:
+
+  - Generate a SSH key without a password, to access the remote server. E.g.
+
+        sudo ssh-keygen -N '' -t ed25519 -f /run/keys/id_ed25519_borgbase
+
+  - Create the repository on the server by following the instructions for your
+    hosting server.
+  - Initialize the repository on the server. Eg.
+
+        sudo borg init --encryption=repokey-blake2  \
+            --rsh "ssh -i /run/keys/id_ed25519_borgbase" \
+            zzz2aaaaa@zzz2aaaaa.repo.borgbase.com:repo
+
+  - Add it to your NixOS configuration, e.g.
+
+        {
+            services.borgbackup.jobs = {
+            my_Remote_Backup = {
+                paths = [ "/" ];
+                exclude = [ "/nix" "'**/.cache'" ];
+                repo =  "zzz2aaaaa@zzz2aaaaa.repo.borgbase.com:repo";
+                  encryption = {
+                  mode = "repokey-blake2";
+                  passCommand = "cat /run/keys/borgbackup_passphrase";
+                };
+                environment = { BORG_RSH = "ssh -i /run/keys/id_ed25519_borgbase"; };
+                compression = "auto,lzma";
+                startAt = "daily";
+            };
+          };
+        }}
+
+## Vorta backup client for the desktop {#opt-services-backup-borgbackup-vorta}
+
+Vorta is a backup client for macOS and Linux desktops. It integrates the
+mighty BorgBackup with your desktop environment to protect your data from
+disk failure, ransomware and theft.
+
+It can be installed in NixOS e.g. by adding `pkgs.vorta`
+to [](#opt-environment.systemPackages).
+
+Details about using Vorta can be found under
+[https://vorta.borgbase.com](https://vorta.borgbase.com/usage) .

nixos/modules/services/backup/borgbackup.nix

@@ -0,0 +1,913 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  isLocalPath =
+    x:
+    builtins.substring 0 1 x == "/" # absolute path
+    || builtins.substring 0 1 x == "." # relative path
+    || builtins.match "[.*:.*]" == null; # not machine:path
+
+  mkExcludeFile =
+    cfg:
+    # Write each exclude pattern to a new line
+    pkgs.writeText "excludefile" (lib.concatMapStrings (s: s + "\n") cfg.exclude);
+
+  mkPatternsFile =
+    cfg:
+    # Write each pattern to a new line
+    pkgs.writeText "patternsfile" (lib.concatMapStrings (s: s + "\n") cfg.patterns);
+
+  mkKeepArgs =
+    cfg:
+    # If cfg.prune.keep e.g. has a yearly attribute,
+    # its content is passed on as --keep-yearly
+    lib.concatStringsSep " " (lib.mapAttrsToList (x: y: "--keep-${x}=${toString y}") cfg.prune.keep);
+
+  mkExtraArgs =
+    cfg:
+    # Create BASH arrays of extra args
+    lib.concatLines (
+      lib.mapAttrsToList
+        (name: values: ''
+          ${name}=(${values})
+        '')
+        {
+          inherit (cfg)
+            extraArgs
+            extraInitArgs
+            extraCreateArgs
+            extraPruneArgs
+            extraCompactArgs
+            ;
+        }
+    );
+
+  mkBackupScript =
+    name: cfg:
+    pkgs.writeShellScript "${name}-script" (
+      ''
+        set -e
+
+        ${mkExtraArgs cfg}
+
+        on_exit()
+        {
+          exitStatus=$?
+          ${cfg.postHook}
+          exit $exitStatus
+        }
+        trap on_exit EXIT
+
+        borgWrapper () {
+          local result
+          borg "$@" && result=$? || result=$?
+          if [[ -z "${toString cfg.failOnWarnings}" ]] && [[ "$result" == 1 || ("$result" -ge 100 && "$result" -le 127) ]]; then
+            echo "ignoring warning return value $result"
+            return 0
+          else
+            return "$result"
+          fi
+        }
+
+        archiveName="${
+          lib.optionalString (cfg.archiveBaseName != null) (cfg.archiveBaseName + "-")
+        }$(date ${cfg.dateFormat})"
+        archiveSuffix="${lib.optionalString cfg.appendFailedSuffix ".failed"}"
+        ${cfg.preHook}
+      ''
+      + lib.optionalString cfg.doInit ''
+        # Run borg init if the repo doesn't exist yet
+        if ! borgWrapper list "''${extraArgs[@]}" > /dev/null; then
+          borgWrapper init "''${extraArgs[@]}" \
+            --encryption ${cfg.encryption.mode} \
+            "''${extraInitArgs[@]}"
+          ${cfg.postInit}
+        fi
+      ''
+      + ''
+        (
+          set -o pipefail
+          ${lib.optionalString (cfg.dumpCommand != null) ''${lib.escapeShellArg cfg.dumpCommand} | \''}
+          borgWrapper create "''${extraArgs[@]}" \
+            --compression ${cfg.compression} \
+            --exclude-from ${mkExcludeFile cfg} \
+            --patterns-from ${mkPatternsFile cfg} \
+            "''${extraCreateArgs[@]}" \
+            "::$archiveName$archiveSuffix" \
+            ${if cfg.paths == null then "-" else lib.escapeShellArgs cfg.paths}
+        )
+      ''
+      + lib.optionalString cfg.appendFailedSuffix ''
+        borgWrapper rename "''${extraArgs[@]}" \
+          "::$archiveName$archiveSuffix" "$archiveName"
+      ''
+      + ''
+        ${cfg.postCreate}
+      ''
+      + lib.optionalString (cfg.prune.keep != { }) ''
+        borgWrapper prune "''${extraArgs[@]}" \
+          ${mkKeepArgs cfg} \
+          ${
+            lib.optionalString (
+              cfg.prune.prefix != null
+            ) "--glob-archives ${lib.escapeShellArg "${cfg.prune.prefix}*"}"
+          } \
+          "''${extraPruneArgs[@]}"
+        borgWrapper compact "''${extraArgs[@]}" "''${extraCompactArgs[@]}"
+        ${cfg.postPrune}
+      ''
+    );
+
+  mkPassEnv =
+    cfg:
+    with cfg.encryption;
+    if passCommand != null then
+      { BORG_PASSCOMMAND = passCommand; }
+    else if passphrase != null then
+      { BORG_PASSPHRASE = passphrase; }
+    else
+      { };
+
+  mkBackupService =
+    name: cfg:
+    let
+      userHome = config.users.users.${cfg.user}.home;
+      backupJobName = "borgbackup-job-${name}";
+      backupScript = mkBackupScript backupJobName cfg;
+    in
+    lib.nameValuePair backupJobName {
+      description = "BorgBackup job ${name}";
+      path = [
+        config.services.borgbackup.package
+        pkgs.openssh
+      ];
+      script =
+        "exec "
+        + lib.optionalString cfg.inhibitsSleep ''
+          ${pkgs.systemd}/bin/systemd-inhibit \
+              --who="borgbackup" \
+              --what="sleep" \
+              --why="Scheduled backup" \
+        ''
+        + backupScript;
+      unitConfig = lib.optionalAttrs (isLocalPath cfg.repo) {
+        RequiresMountsFor = [ cfg.repo ];
+      };
+      serviceConfig = {
+        User = cfg.user;
+        Group = cfg.group;
+        # Only run when no other process is using CPU or disk
+        CPUSchedulingPolicy = "idle";
+        IOSchedulingClass = "idle";
+        ProtectSystem = "strict";
+        ReadWritePaths = [
+          "${userHome}/.config/borg"
+          "${userHome}/.cache/borg"
+        ]
+        ++ cfg.readWritePaths
+        # Borg needs write access to repo if it is not remote
+        ++ lib.optional (isLocalPath cfg.repo) cfg.repo;
+        PrivateTmp = cfg.privateTmp;
+      };
+      environment = {
+        BORG_REPO = cfg.repo;
+      }
+      // (mkPassEnv cfg)
+      // cfg.environment;
+    };
+
+  mkBackupTimers =
+    name: cfg:
+    lib.nameValuePair "borgbackup-job-${name}" {
+      description = "BorgBackup job ${name} timer";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        Persistent = cfg.persistentTimer;
+        OnCalendar = cfg.startAt;
+      };
+      # if remote-backup wait for network
+      after = lib.optional (cfg.persistentTimer && !isLocalPath cfg.repo) "network-online.target";
+      wants = lib.optional (cfg.persistentTimer && !isLocalPath cfg.repo) "network-online.target";
+    };
+
+  # utility function around makeWrapper
+  mkWrapperDrv =
+    {
+      original,
+      name,
+      set ? { },
+    }:
+    pkgs.runCommand "${name}-wrapper"
+      {
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+      }
+
+      ''
+        makeWrapper "${original}" "$out/bin/${name}" \
+          ${lib.concatStringsSep " \\\n " (
+            lib.mapAttrsToList (name: value: ''--set ${name} "${value}"'') set
+          )}
+      '';
+
+  # Returns a singleton list, due to usage of lib.optional
+  mkBorgWrapper =
+    name: cfg:
+    lib.optional (cfg.wrapper != "" && cfg.wrapper != null) (mkWrapperDrv {
+      original = lib.getExe config.services.borgbackup.package;
+      name = cfg.wrapper;
+      set = {
+        BORG_REPO = cfg.repo;
+      }
+      // (mkPassEnv cfg)
+      // cfg.environment;
+    });
+
+  # Paths listed in ReadWritePaths must exist before service is started
+  mkTmpfiles =
+    name: cfg:
+    let
+      settings = { inherit (cfg) user group; };
+    in
+    lib.nameValuePair "borgbackup-job-${name}" (
+      {
+        # Create parent dirs separately, to ensure correct ownership.
+        "${config.users.users."${cfg.user}".home}/.config".d = settings;
+        "${config.users.users."${cfg.user}".home}/.cache".d = settings;
+        "${config.users.users."${cfg.user}".home}/.config/borg".d = settings;
+        "${config.users.users."${cfg.user}".home}/.cache/borg".d = settings;
+      }
+      // lib.optionalAttrs (isLocalPath cfg.repo && !cfg.removableDevice) {
+        "${cfg.repo}".d = settings;
+      }
+    );
+
+  mkPassAssertion = name: cfg: {
+    assertion = with cfg.encryption; mode != "none" -> passCommand != null || passphrase != null;
+    message =
+      "passCommand or passphrase has to be specified because"
+      + " borgbackup.jobs.${name}.encryption != \"none\"";
+  };
+
+  mkRepoService =
+    name: cfg:
+    lib.nameValuePair "borgbackup-repo-${name}" {
+      description = "Create BorgBackup repository ${name} directory";
+      script = ''
+        mkdir -p ${lib.escapeShellArg cfg.path}
+        chown ${cfg.user}:${cfg.group} ${lib.escapeShellArg cfg.path}
+      '';
+      serviceConfig = {
+        # The service's only task is to ensure that the specified path exists
+        Type = "oneshot";
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+
+  mkAuthorizedKey =
+    cfg: appendOnly: key:
+    let
+      # Because of the following line, clients do not need to specify an absolute repo path
+      cdCommand = "cd ${lib.escapeShellArg cfg.path}";
+      restrictedArg = "--restrict-to-${if cfg.allowSubRepos then "path" else "repository"} .";
+      appendOnlyArg = lib.optionalString appendOnly "--append-only";
+      quotaArg = lib.optionalString (cfg.quota != null) "--storage-quota ${cfg.quota}";
+      serveCommand = "borg serve ${restrictedArg} ${appendOnlyArg} ${quotaArg}";
+    in
+    ''command="${cdCommand} && ${serveCommand}",restrict ${key}'';
+
+  mkUsersConfig = name: cfg: {
+    users.${cfg.user} = {
+      openssh.authorizedKeys.keys = (
+        map (mkAuthorizedKey cfg false) cfg.authorizedKeys
+        ++ map (mkAuthorizedKey cfg true) cfg.authorizedKeysAppendOnly
+      );
+      useDefaultShell = true;
+      group = cfg.group;
+      isSystemUser = true;
+    };
+    groups.${cfg.group} = { };
+  };
+
+  mkKeysAssertion = name: cfg: {
+    assertion = cfg.authorizedKeys != [ ] || cfg.authorizedKeysAppendOnly != [ ];
+    message = "borgbackup.repos.${name} does not make sense" + " without at least one public key";
+  };
+
+  mkSourceAssertions = name: cfg: {
+    assertion =
+      lib.count isNull [
+        cfg.dumpCommand
+        cfg.paths
+      ] == 1;
+    message = ''
+      Exactly one of borgbackup.jobs.${name}.paths or borgbackup.jobs.${name}.dumpCommand
+      must be set.
+    '';
+  };
+
+  mkRemovableDeviceAssertions = name: cfg: {
+    assertion = !(isLocalPath cfg.repo) -> !cfg.removableDevice;
+    message = ''
+      borgbackup.repos.${name}: repo isn't a local path, thus it can't be a removable device!
+    '';
+  };
+
+in
+{
+  meta.maintainers = with lib.maintainers; [
+    dotlambda
+    Scrumplex
+  ];
+  meta.doc = ./borgbackup.md;
+
+  ###### interface
+
+  options.services.borgbackup.package = lib.mkPackageOption pkgs "borgbackup" { };
+
+  options.services.borgbackup.jobs = lib.mkOption {
+    description = ''
+      Deduplicating backups using BorgBackup.
+      Adding a job will cause a borg-job-NAME wrapper to be added
+      to your system path, so that you can perform maintenance easily.
+      See also the chapter about BorgBackup in the NixOS manual.
+    '';
+    default = { };
+    example = lib.literalExpression ''
+        { # for a local backup
+          rootBackup = {
+            paths = "/";
+            exclude = [ "/nix" ];
+            repo = "/path/to/local/repo";
+            encryption = {
+              mode = "repokey";
+              passphrase = "secret";
+            };
+            compression = "auto,lzma";
+            startAt = "weekly";
+          };
+        }
+        { # Root backing each day up to a remote backup server. We assume that you have
+          #   * created a password less key: ssh-keygen -N "" -t ed25519 -f /path/to/ssh_key
+          #     best practices are: use -t ed25519, /path/to = /run/keys
+          #   * the passphrase is in the file /run/keys/borgbackup_passphrase
+          #   * you have initialized the repository manually
+          paths = [ "/etc" "/home" ];
+          exclude = [ "/nix" "'**/.cache'" ];
+          doInit = false;
+          repo =  "user3@arep.repo.borgbase.com:repo";
+          encryption = {
+            mode = "repokey-blake2";
+            passCommand = "cat /path/to/passphrase";
+          };
+          environment = { BORG_RSH = "ssh -i /path/to/ssh_key"; };
+          compression = "auto,lzma";
+          startAt = "daily";
+      };
+    '';
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        let
+          globalConfig = config;
+        in
+        { name, config, ... }:
+        {
+          options = {
+
+            paths = lib.mkOption {
+              type = with lib.types; nullOr (coercedTo str lib.singleton (listOf str));
+              default = null;
+              description = ''
+                Path(s) to back up.
+                Mutually exclusive with {option}`dumpCommand`.
+              '';
+              example = "/home/user";
+            };
+
+            dumpCommand = lib.mkOption {
+              type = with lib.types; nullOr path;
+              default = null;
+              description = ''
+                Backup the stdout of this program instead of filesystem paths.
+                Mutually exclusive with {option}`paths`.
+              '';
+              example = "/path/to/createZFSsend.sh";
+            };
+
+            repo = lib.mkOption {
+              type = lib.types.str;
+              description = "Remote or local repository to back up to.";
+              example = "user@machine:/path/to/repo";
+            };
+
+            removableDevice = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = "Whether the repo (which must be local) is a removable device.";
+            };
+
+            archiveBaseName = lib.mkOption {
+              type = lib.types.nullOr (lib.types.strMatching "[^/{}]+");
+              default = "${globalConfig.networking.hostName}-${name}";
+              defaultText = lib.literalExpression ''"''${config.networking.hostName}-<name>"'';
+              description = ''
+                How to name the created archives. A timestamp, whose format is
+                determined by {option}`dateFormat`, will be appended. The full
+                name can be modified at runtime (`$archiveName`).
+                Placeholders like `{hostname}` must not be used.
+                Use `null` for no base name.
+              '';
+            };
+
+            dateFormat = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Arguments passed to {command}`date`
+                to create a timestamp suffix for the archive name.
+              '';
+              default = "+%Y-%m-%dT%H:%M:%S";
+              example = "-u +%s";
+            };
+
+            startAt = lib.mkOption {
+              type = with lib.types; either str (listOf str);
+              default = "daily";
+              description = ''
+                When or how often the backup should run.
+                Must be in the format described in
+                {manpage}`systemd.time(7)`.
+                If you do not want the backup to start
+                automatically, use `[ ]`.
+                It will generate a systemd service borgbackup-job-NAME.
+                You may trigger it manually via systemctl restart borgbackup-job-NAME.
+              '';
+            };
+
+            persistentTimer = lib.mkOption {
+              default = false;
+              type = lib.types.bool;
+              example = true;
+              description = ''
+                Set the `Persistent` option for the
+                {manpage}`systemd.timer(5)`
+                which triggers the backup immediately if the last trigger
+                was missed (e.g. if the system was powered down).
+              '';
+            };
+
+            inhibitsSleep = lib.mkOption {
+              default = false;
+              type = lib.types.bool;
+              example = true;
+              description = ''
+                Prevents the system from sleeping while backing up.
+              '';
+            };
+
+            user = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The user {command}`borg` is run as.
+                User or group need read permission
+                for the specified {option}`paths`.
+              '';
+              default = "root";
+            };
+
+            group = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The group borg is run as. User or group needs read permission
+                for the specified {option}`paths`.
+              '';
+              default = "root";
+            };
+
+            wrapper = lib.mkOption {
+              type = with lib.types; nullOr str;
+              description = ''
+                Name of the wrapper that is installed into {env}`PATH`.
+                Set to `null` or `""` to disable it altogether.
+              '';
+              default = "borg-job-${name}";
+              defaultText = "borg-job-<name>";
+            };
+
+            encryption.mode = lib.mkOption {
+              type = lib.types.enum [
+                "repokey"
+                "keyfile"
+                "repokey-blake2"
+                "keyfile-blake2"
+                "authenticated"
+                "authenticated-blake2"
+                "none"
+              ];
+              description = ''
+                Encryption mode to use. Setting a mode
+                other than `"none"` requires
+                you to specify a {option}`passCommand`
+                or a {option}`passphrase`.
+              '';
+              example = "repokey-blake2";
+            };
+
+            encryption.passCommand = lib.mkOption {
+              type = with lib.types; nullOr str;
+              description = ''
+                A command which prints the passphrase to stdout.
+                Mutually exclusive with {option}`passphrase`.
+              '';
+              default = null;
+              example = "cat /path/to/passphrase_file";
+            };
+
+            encryption.passphrase = lib.mkOption {
+              type = with lib.types; nullOr str;
+              description = ''
+                The passphrase the backups are encrypted with.
+                Mutually exclusive with {option}`passCommand`.
+                If you do not want the passphrase to be stored in the
+                world-readable Nix store, use {option}`passCommand`.
+              '';
+              default = null;
+            };
+
+            compression = lib.mkOption {
+              # "auto" is optional,
+              # compression mode must be given,
+              # compression level is optional
+              type = lib.types.strMatching "none|(auto,)?(lz4|zstd|zlib|lzma)(,[[:digit:]]{1,2})?";
+              description = ''
+                Compression method to use. Refer to
+                {command}`borg help compression`
+                for all available options.
+              '';
+              default = "lz4";
+              example = "auto,lzma";
+            };
+
+            exclude = lib.mkOption {
+              type = with lib.types; listOf str;
+              description = ''
+                Exclude paths matching any of the given patterns. See
+                {command}`borg help patterns` for pattern syntax.
+              '';
+              default = [ ];
+              example = [
+                "/home/*/.cache"
+                "/nix"
+              ];
+            };
+
+            patterns = lib.mkOption {
+              type = with lib.types; listOf str;
+              description = ''
+                Include/exclude paths matching the given patterns. The first
+                matching patterns is used, so if an include pattern (prefix `+`)
+                matches before an exclude pattern (prefix `-`), the file is
+                backed up. See [{command}`borg help patterns`](https://borgbackup.readthedocs.io/en/stable/usage/help.html#borg-patterns) for pattern syntax.
+              '';
+              default = [ ];
+              example = [
+                "+ /home/susan"
+                "- /home/*"
+              ];
+            };
+
+            readWritePaths = lib.mkOption {
+              type = with lib.types; listOf path;
+              description = ''
+                By default, borg cannot write anywhere on the system but
+                `$HOME/.config/borg` and `$HOME/.cache/borg`.
+                If, for example, your preHook script needs to dump files
+                somewhere, put those directories here.
+              '';
+              default = [ ];
+              example = [
+                "/var/backup/mysqldump"
+              ];
+            };
+
+            privateTmp = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Set the `PrivateTmp` option for
+                the systemd-service. Set to false if you need sockets
+                or other files from global /tmp.
+              '';
+              default = true;
+            };
+
+            failOnWarnings = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Fail the whole backup job if any borg command returns a warning
+                (exit code 1), for example because a file changed during backup.
+              '';
+              default = true;
+            };
+
+            doInit = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Run {command}`borg init` if the
+                specified {option}`repo` does not exist.
+                You should set this to `false`
+                if the repository is located on an external drive
+                that might not always be mounted.
+              '';
+              default = true;
+            };
+
+            appendFailedSuffix = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Append a `.failed` suffix
+                to the archive name, which is only removed if
+                {command}`borg create` has a zero exit status.
+              '';
+              default = true;
+            };
+
+            prune.keep = lib.mkOption {
+              # Specifying e.g. `prune.keep.yearly = -1`
+              # means there is no limit of yearly archives to keep
+              # The regex is for use with e.g. --keep-within 1y
+              type = with lib.types; attrsOf (either int (strMatching "[[:digit:]]+[Hdwmy]"));
+              description = ''
+                Prune a repository by deleting all archives not matching any of the
+                specified retention options. See {command}`borg help prune`
+                for the available options.
+              '';
+              default = { };
+              example = lib.literalExpression ''
+                {
+                  within = "1d"; # Keep all archives from the last day
+                  daily = 7;
+                  weekly = 4;
+                  monthly = -1;  # Keep at least one archive for each month
+                }
+              '';
+            };
+
+            prune.prefix = lib.mkOption {
+              type = lib.types.nullOr (lib.types.str);
+              description = ''
+                Only consider archive names starting with this prefix for pruning.
+                By default, only archives created by this job are considered.
+                Use `""` or `null` to consider all archives.
+              '';
+              default = config.archiveBaseName;
+              defaultText = lib.literalExpression "archiveBaseName";
+            };
+
+            environment = lib.mkOption {
+              type = with lib.types; attrsOf str;
+              description = ''
+                Environment variables passed to the backup script.
+                You can for example specify which SSH key to use.
+              '';
+              default = { };
+              example = {
+                BORG_RSH = "ssh -i /path/to/key";
+              };
+            };
+
+            preHook = lib.mkOption {
+              type = lib.types.lines;
+              description = ''
+                Shell commands to run before the backup.
+                This can for example be used to mount file systems.
+              '';
+              default = "";
+              example = ''
+                # To add excluded paths at runtime
+                extraCreateArgs+=("--exclude" "/some/path")
+              '';
+            };
+
+            postInit = lib.mkOption {
+              type = lib.types.lines;
+              description = ''
+                Shell commands to run after {command}`borg init`.
+              '';
+              default = "";
+            };
+
+            postCreate = lib.mkOption {
+              type = lib.types.lines;
+              description = ''
+                Shell commands to run after {command}`borg create`. The name
+                of the created archive is stored in `$archiveName`.
+              '';
+              default = "";
+            };
+
+            postPrune = lib.mkOption {
+              type = lib.types.lines;
+              description = ''
+                Shell commands to run after {command}`borg prune`.
+              '';
+              default = "";
+            };
+
+            postHook = lib.mkOption {
+              type = lib.types.lines;
+              description = ''
+                Shell commands to run just before exit. They are executed
+                even if a previous command exits with a non-zero exit code.
+                The latter is available as `$exitStatus`.
+              '';
+              default = "";
+            };
+
+            extraArgs = lib.mkOption {
+              type = with lib.types; coercedTo (listOf str) lib.escapeShellArgs str;
+              description = ''
+                Additional arguments for all {command}`borg` calls the
+                service has. Handle with care.
+              '';
+              default = [ ];
+              example = [ "--remote-path=/path/to/borg" ];
+            };
+
+            extraInitArgs = lib.mkOption {
+              type = with lib.types; coercedTo (listOf str) lib.escapeShellArgs str;
+              description = ''
+                Additional arguments for {command}`borg init`.
+                Can also be set at runtime using `$extraInitArgs`.
+              '';
+              default = [ ];
+              example = [ "--append-only" ];
+            };
+
+            extraCreateArgs = lib.mkOption {
+              type = with lib.types; coercedTo (listOf str) lib.escapeShellArgs str;
+              description = ''
+                Additional arguments for {command}`borg create`.
+                Can also be set at runtime using `$extraCreateArgs`.
+              '';
+              default = [ ];
+              example = [
+                "--stats"
+                "--checkpoint-interval 600"
+              ];
+            };
+
+            extraPruneArgs = lib.mkOption {
+              type = with lib.types; coercedTo (listOf str) lib.escapeShellArgs str;
+              description = ''
+                Additional arguments for {command}`borg prune`.
+                Can also be set at runtime using `$extraPruneArgs`.
+              '';
+              default = [ ];
+              example = [ "--save-space" ];
+            };
+
+            extraCompactArgs = lib.mkOption {
+              type = with lib.types; coercedTo (listOf str) lib.escapeShellArgs str;
+              description = ''
+                Additional arguments for {command}`borg compact`.
+                Can also be set at runtime using `$extraCompactArgs`.
+              '';
+              default = [ ];
+              example = [ "--cleanup-commits" ];
+            };
+          };
+        }
+      )
+    );
+  };
+
+  options.services.borgbackup.repos = lib.mkOption {
+    description = ''
+      Serve BorgBackup repositories to given public SSH keys,
+      restricting their access to the repository only.
+      See also the chapter about BorgBackup in the NixOS manual.
+      Also, clients do not need to specify the absolute path when accessing the repository,
+      i.e. `user@machine:.` is enough. (Note colon and dot.)
+    '';
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { ... }:
+        {
+          options = {
+            path = lib.mkOption {
+              type = lib.types.path;
+              description = ''
+                Where to store the backups. Note that the directory
+                is created automatically, with correct permissions.
+              '';
+              default = "/var/lib/borgbackup";
+            };
+
+            user = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The user {command}`borg serve` is run as.
+                User or group needs write permission
+                for the specified {option}`path`.
+              '';
+              default = "borg";
+            };
+
+            group = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                The group {command}`borg serve` is run as.
+                User or group needs write permission
+                for the specified {option}`path`.
+              '';
+              default = "borg";
+            };
+
+            authorizedKeys = lib.mkOption {
+              type = with lib.types; listOf str;
+              description = ''
+                Public SSH keys that are given full write access to this repository.
+                You should use a different SSH key for each repository you write to, because
+                the specified keys are restricted to running {command}`borg serve`
+                and can only access this single repository.
+              '';
+              default = [ ];
+            };
+
+            authorizedKeysAppendOnly = lib.mkOption {
+              type = with lib.types; listOf str;
+              description = ''
+                Public SSH keys that can only be used to append new data (archives) to the repository.
+                Note that archives can still be marked as deleted and are subsequently removed from disk
+                upon accessing the repo with full write access, e.g. when pruning.
+              '';
+              default = [ ];
+            };
+
+            allowSubRepos = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Allow clients to create repositories in subdirectories of the
+                specified {option}`path`. These can be accessed using
+                `user@machine:path/to/subrepo`. Note that a
+                {option}`quota` applies to repositories independently.
+                Therefore, if this is enabled, clients can create multiple
+                repositories and upload an arbitrary amount of data.
+              '';
+              default = false;
+            };
+
+            quota = lib.mkOption {
+              # See the definition of parse_file_size() in src/borg/helpers/parseformat.py
+              type = with lib.types; nullOr (strMatching "[[:digit:].]+[KMGTP]?");
+              description = ''
+                Storage quota for the repository. This quota is ensured for all
+                sub-repositories if {option}`allowSubRepos` is enabled
+                but not for the overall storage space used.
+              '';
+              default = null;
+              example = "100G";
+            };
+
+          };
+        }
+      )
+    );
+  };
+
+  ###### implementation
+
+  config = lib.mkIf (with config.services.borgbackup; jobs != { } || repos != { }) (
+    with config.services.borgbackup;
+    {
+      assertions =
+        lib.mapAttrsToList mkPassAssertion jobs
+        ++ lib.mapAttrsToList mkKeysAssertion repos
+        ++ lib.mapAttrsToList mkSourceAssertions jobs
+        ++ lib.mapAttrsToList mkRemovableDeviceAssertions jobs;
+
+      systemd.tmpfiles.settings = lib.mapAttrs' mkTmpfiles jobs;
+
+      systemd.services =
+        # A job named "foo" is mapped to systemd.services.borgbackup-job-foo
+        lib.mapAttrs' mkBackupService jobs
+        # A repo named "foo" is mapped to systemd.services.borgbackup-repo-foo
+        // lib.mapAttrs' mkRepoService repos;
+
+      # A job named "foo" is mapped to systemd.timers.borgbackup-job-foo
+      # only generate the timer if interval (startAt) is set
+      systemd.timers = lib.mapAttrs' mkBackupTimers (lib.filterAttrs (_: cfg: cfg.startAt != [ ]) jobs);
+
+      users = lib.mkMerge (lib.mapAttrsToList mkUsersConfig repos);
+
+      environment.systemPackages = [
+        config.services.borgbackup.package
+      ]
+      ++ (lib.flatten (lib.mapAttrsToList mkBorgWrapper jobs));
+    }
+  );
+}

nixos/modules/services/backup/borgmatic.nix

@@ -0,0 +1,199 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.borgmatic;
+  settingsFormat = pkgs.formats.yaml { };
+
+  postgresql = config.services.postgresql.package;
+  mysql = config.services.mysql.package;
+  requireSudo =
+    s:
+    s ? postgresql_databases
+    && lib.any (d: d ? username && !(d ? password) && !(d ? pg_dump_command)) s.postgresql_databases;
+  addRequiredBinaries =
+    s:
+    s
+    // (lib.optionalAttrs (s ? postgresql_databases && s.postgresql_databases != [ ]) {
+      postgresql_databases = map (
+        d:
+        let
+          as_user = if d ? username && !(d ? password) then "${pkgs.sudo}/bin/sudo -u ${d.username} " else "";
+        in
+        {
+          pg_dump_command =
+            if d.name == "all" && (!(d ? format) || isNull d.format) then
+              "${as_user}${postgresql}/bin/pg_dumpall"
+            else
+              "${as_user}${postgresql}/bin/pg_dump";
+          pg_restore_command = "${as_user}${postgresql}/bin/pg_restore";
+          psql_command = "${as_user}${postgresql}/bin/psql";
+        }
+        // d
+      ) s.postgresql_databases;
+    })
+    // (lib.optionalAttrs (s ? mariadb_databases && s.mariadb_databases != [ ]) {
+      mariadb_databases = map (
+        d:
+        {
+          mariadb_dump_command = "${mysql}/bin/mariadb-dump";
+          mariadb_command = "${mysql}/bin/mariadb";
+        }
+        // d
+      ) s.mariadb_databases;
+    })
+    // (lib.optionalAttrs (s ? mysql_databases && s.mysql_databases != [ ]) {
+      mysql_databases = map (
+        d:
+        {
+          mysql_dump_command = "${mysql}/bin/mysqldump";
+          mysql_command = "${mysql}/bin/mysql";
+        }
+        // d
+      ) s.mysql_databases;
+    });
+
+  repository =
+    with lib.types;
+    submodule {
+      options = {
+        path = lib.mkOption {
+          type = str;
+          description = ''
+            Path to the repository
+          '';
+        };
+        label = lib.mkOption {
+          type = str;
+          description = ''
+            Label to the repository
+          '';
+        };
+      };
+    };
+  cfgType =
+    with lib.types;
+    submodule {
+      freeformType = settingsFormat.type;
+      options = {
+        source_directories = lib.mkOption {
+          type = listOf str;
+          default = [ ];
+          description = ''
+            List of source directories and files to backup. Globs and tildes are
+            expanded. Do not backslash spaces in path names.
+          '';
+          example = [
+            "/home"
+            "/etc"
+            "/var/log/syslog*"
+            "/home/user/path with spaces"
+          ];
+        };
+        repositories = lib.mkOption {
+          type = listOf repository;
+          default = [ ];
+          description = ''
+            A required list of local or remote repositories with paths and
+            optional labels (which can be used with the --repository flag to
+            select a repository). Tildes are expanded. Multiple repositories are
+            backed up to in sequence. Borg placeholders can be used. See the
+            output of "borg help placeholders" for details. See ssh_command for
+            SSH options like identity file or port. If systemd service is used,
+            then add local repository paths in the systemd service file to the
+            ReadWritePaths list.
+          '';
+          example = [
+            {
+              path = "ssh://user@backupserver/./sourcehostname.borg";
+              label = "backupserver";
+            }
+            {
+              path = "/mnt/backup";
+              label = "local";
+            }
+          ];
+        };
+      };
+    };
+
+  cfgfile = settingsFormat.generate "config.yaml" (addRequiredBinaries cfg.settings);
+
+  anycfgRequiresSudo =
+    requireSudo cfg.settings || lib.any requireSudo (lib.attrValues cfg.configurations);
+in
+{
+  options.services.borgmatic = {
+    enable = lib.mkEnableOption "borgmatic";
+
+    settings = lib.mkOption {
+      description = ''
+        See <https://torsion.org/borgmatic/docs/reference/configuration/>
+      '';
+      default = null;
+      type = lib.types.nullOr cfgType;
+    };
+
+    configurations = lib.mkOption {
+      description = ''
+        Set of borgmatic configurations, see <https://torsion.org/borgmatic/docs/reference/configuration/>
+      '';
+      default = { };
+      type = lib.types.attrsOf cfgType;
+    };
+
+    enableConfigCheck = lib.mkEnableOption "checking all configurations during build time" // {
+      default = true;
+    };
+  };
+
+  config =
+    let
+      configFiles =
+        (lib.optionalAttrs (cfg.settings != null) {
+          "borgmatic/config.yaml".source = cfgfile;
+        })
+        // lib.mapAttrs' (
+          name: value:
+          lib.nameValuePair "borgmatic.d/${name}.yaml" {
+            source = settingsFormat.generate "${name}.yaml" (addRequiredBinaries value);
+          }
+        ) cfg.configurations;
+      borgmaticCheck =
+        name: f:
+        pkgs.runCommandCC "${name} validation" { } ''
+          ${pkgs.borgmatic}/bin/borgmatic -c ${f.source} config validate
+          touch $out
+        '';
+    in
+    lib.mkIf cfg.enable {
+
+      warnings =
+        [ ]
+        ++
+          lib.optional (cfg.settings != null && cfg.settings ? location)
+            "`services.borgmatic.settings.location` is deprecated, please move your options out of sections to the global scope"
+        ++
+          lib.optional (lib.catAttrs "location" (lib.attrValues cfg.configurations) != [ ])
+            "`services.borgmatic.configurations.<name>.location` is deprecated, please move your options out of sections to the global scope";
+
+      environment.systemPackages = [ pkgs.borgmatic ];
+
+      environment.etc = configFiles;
+
+      systemd.packages = [ pkgs.borgmatic ];
+      systemd.services.borgmatic.path = [ pkgs.coreutils ];
+      systemd.services.borgmatic.serviceConfig = lib.optionalAttrs anycfgRequiresSudo {
+        NoNewPrivileges = false;
+        CapabilityBoundingSet = "CAP_DAC_READ_SEARCH CAP_NET_RAW CAP_SETUID CAP_SETGID";
+      };
+
+      # Workaround: https://github.com/NixOS/nixpkgs/issues/81138
+      systemd.timers.borgmatic.wantedBy = [ "timers.target" ];
+
+      system.checks = lib.mkIf cfg.enableConfigCheck (lib.mapAttrsToList borgmaticCheck configFiles);
+    };
+}

nixos/modules/services/backup/btrbk.nix

@@ -0,0 +1,392 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  inherit (lib)
+    concatLists
+    concatMap
+    concatMapStringsSep
+    concatStringsSep
+    filterAttrs
+    flatten
+    getAttr
+    isAttrs
+    literalExpression
+    mapAttrs'
+    mapAttrsToList
+    mkIf
+    mkOption
+    optional
+    optionalString
+    sortOn
+    types
+    ;
+
+  # The priority of an option or section.
+  # The configurations format are order-sensitive. Pairs are added as children of
+  # the last sections if possible, otherwise, they start a new section.
+  # We sort them in topological order:
+  # 1. Leaf pairs.
+  # 2. Sections that may contain (1).
+  # 3. Sections that may contain (1) or (2).
+  # 4. Etc.
+  prioOf =
+    { name, value }:
+    if !isAttrs value then
+      0 # Leaf options.
+    else
+      {
+        target = 1; # Contains: options.
+        subvolume = 2; # Contains: options, target.
+        volume = 3; # Contains: options, target, subvolume.
+      }
+      .${name} or (throw "Unknow section '${name}'");
+
+  genConfig' = set: concatStringsSep "\n" (genConfig set);
+  genConfig =
+    set:
+    let
+      pairs = mapAttrsToList (name: value: { inherit name value; }) set;
+      sortedPairs = sortOn prioOf pairs;
+    in
+    concatMap genPair sortedPairs;
+  genSection =
+    sec: secName: value:
+    [ "${sec} ${secName}" ] ++ map (x: " " + x) (genConfig value);
+  genPair =
+    { name, value }:
+    if !isAttrs value then
+      [ "${name} ${value}" ]
+    else
+      concatLists (mapAttrsToList (genSection name) value);
+
+  sudoRule = {
+    users = [ "btrbk" ];
+    commands = [
+      {
+        command = "${pkgs.btrfs-progs}/bin/btrfs";
+        options = [ "NOPASSWD" ];
+      }
+      {
+        command = "${pkgs.coreutils}/bin/mkdir";
+        options = [ "NOPASSWD" ];
+      }
+      {
+        command = "${pkgs.coreutils}/bin/readlink";
+        options = [ "NOPASSWD" ];
+      }
+      # for ssh, they are not the same than the one hard coded in ${pkgs.btrbk}
+      {
+        command = "/run/current-system/sw/bin/btrfs";
+        options = [ "NOPASSWD" ];
+      }
+      {
+        command = "/run/current-system/sw/bin/mkdir";
+        options = [ "NOPASSWD" ];
+      }
+      {
+        command = "/run/current-system/sw/bin/readlink";
+        options = [ "NOPASSWD" ];
+      }
+    ];
+  };
+
+  sudo_doas =
+    if config.security.sudo.enable || config.security.sudo-rs.enable then
+      "sudo"
+    else if config.security.doas.enable then
+      "doas"
+    else
+      throw "The btrbk nixos module needs either sudo or doas enabled in the configuration";
+
+  addDefaults = settings: { backend = "btrfs-progs-${sudo_doas}"; } // settings;
+
+  mkConfigFile =
+    name: settings:
+    pkgs.writeTextFile {
+      name = "btrbk-${name}.conf";
+      text = genConfig' (addDefaults settings);
+      checkPhase = ''
+        set +e
+        ${pkgs.btrbk}/bin/btrbk -c $out dryrun
+        # According to btrbk(1), exit status 2 means parse error
+        # for CLI options or the config file.
+        if [[ $? == 2 ]]; then
+          echo "Btrbk configuration is invalid:"
+          cat $out
+          exit 1
+        fi
+        set -e
+      '';
+    };
+
+  streamCompressMap = {
+    gzip = pkgs.gzip;
+    pigz = pkgs.pigz;
+    bzip2 = pkgs.bzip2;
+    pbzip2 = pkgs.pbzip2;
+    bzip3 = pkgs.bzip3;
+    xz = pkgs.xz;
+    lzo = pkgs.lzo;
+    lz4 = pkgs.lz4;
+    zstd = pkgs.zstd;
+  };
+
+  cfg = config.services.btrbk;
+  sshEnabled = cfg.sshAccess != [ ];
+  serviceEnabled = cfg.instances != { };
+in
+{
+  meta.maintainers = with lib.maintainers; [ oxalica ];
+
+  options = {
+    services.btrbk = {
+      extraPackages = mkOption {
+        description = ''
+          Extra packages for btrbk, like compression utilities for `stream_compress`.
+
+          **Note**: This option will get deprecated in future releases.
+          Required compression programs will get automatically provided to btrbk
+          depending on configured compression method in
+          `services.btrbk.instances.<name>.settings` option.
+        '';
+        type = types.listOf types.package;
+        default = [ ];
+        example = literalExpression "[ pkgs.xz ]";
+      };
+      niceness = mkOption {
+        description = "Niceness for local instances of btrbk. Also applies to remote ones connecting via ssh when positive.";
+        type = types.ints.between (-20) 19;
+        default = 10;
+      };
+      ioSchedulingClass = mkOption {
+        description = "IO scheduling class for btrbk (see {manpage}`ionice(1)` for a quick description). Applies to local instances, and remote ones connecting by ssh if set to idle.";
+        type = types.enum [
+          "idle"
+          "best-effort"
+          "realtime"
+        ];
+        default = "best-effort";
+      };
+      instances = mkOption {
+        description = "Set of btrbk instances. The instance named `btrbk` is the default one.";
+        type =
+          with types;
+          attrsOf (submodule {
+            options = {
+              onCalendar = mkOption {
+                type = types.nullOr types.str;
+                default = "daily";
+                description = ''
+                  How often this btrbk instance is started. See {manpage}`systemd.time(7)` for more information about the format.
+                  Setting it to null disables the timer, thus this instance can only be started manually.
+                '';
+              };
+              snapshotOnly = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Whether to run in snapshot only mode. This skips backup creation and deletion steps.
+                  Useful when you want to manually backup to an external drive that might not always be connected.
+                  Use `btrbk -c /path/to/conf resume` to trigger manual backups.
+                  More examples [here](https://github.com/digint/btrbk#example-backups-to-usb-disk).
+                  See also `snapshot` subcommand in {manpage}`btrbk(1)`.
+                '';
+              };
+              settings = mkOption {
+                type = types.submodule {
+                  freeformType =
+                    let
+                      t = types.attrsOf (
+                        types.either types.str (t // { description = "instances of this type recursively"; })
+                      );
+                    in
+                    t;
+                  options = {
+                    stream_compress = mkOption {
+                      description = ''
+                        Compress the btrfs send stream before transferring it from/to remote locations using a
+                        compression command.
+                      '';
+                      type = types.enum [
+                        "gzip"
+                        "pigz"
+                        "bzip2"
+                        "pbzip2"
+                        "bzip3"
+                        "xz"
+                        "lzo"
+                        "lz4"
+                        "zstd"
+                        "no"
+                      ];
+                      default = "no";
+                    };
+                  };
+                };
+                default = { };
+                example = {
+                  snapshot_preserve_min = "2d";
+                  snapshot_preserve = "14d";
+                  volume = {
+                    "/mnt/btr_pool" = {
+                      target = "/mnt/btr_backup/mylaptop";
+                      subvolume = {
+                        "rootfs" = { };
+                        "home" = {
+                          snapshot_create = "always";
+                        };
+                      };
+                    };
+                  };
+                };
+                description = "configuration options for btrbk. Nested attrsets translate to subsections.";
+              };
+            };
+          });
+        default = { };
+      };
+      sshAccess = mkOption {
+        description = "SSH keys that should be able to make or push snapshots on this system remotely with btrbk";
+        type =
+          with types;
+          listOf (submodule {
+            options = {
+              key = mkOption {
+                type = str;
+                description = "SSH public key allowed to login as user `btrbk` to run remote backups.";
+              };
+              roles = mkOption {
+                type = listOf (enum [
+                  "info"
+                  "source"
+                  "target"
+                  "delete"
+                  "snapshot"
+                  "send"
+                  "receive"
+                ]);
+                example = [
+                  "source"
+                  "info"
+                  "send"
+                ];
+                description = "What actions can be performed with this SSH key. See ssh_filter_btrbk(1) for details";
+              };
+            };
+          });
+        default = [ ];
+      };
+    };
+
+  };
+  config = mkIf (sshEnabled || serviceEnabled) {
+
+    environment.systemPackages = [ pkgs.btrbk ] ++ cfg.extraPackages;
+
+    security.sudo.extraRules = mkIf (sudo_doas == "sudo") [ sudoRule ];
+    security.sudo-rs.extraRules = mkIf (sudo_doas == "sudo") [ sudoRule ];
+
+    security.doas = mkIf (sudo_doas == "doas") {
+      extraRules =
+        let
+          doasCmdNoPass = cmd: {
+            users = [ "btrbk" ];
+            cmd = cmd;
+            noPass = true;
+          };
+        in
+        [
+          (doasCmdNoPass "${pkgs.btrfs-progs}/bin/btrfs")
+          (doasCmdNoPass "${pkgs.coreutils}/bin/mkdir")
+          (doasCmdNoPass "${pkgs.coreutils}/bin/readlink")
+          # for ssh, they are not the same than the one hard coded in ${pkgs.btrbk}
+          (doasCmdNoPass "/run/current-system/sw/bin/btrfs")
+          (doasCmdNoPass "/run/current-system/sw/bin/mkdir")
+          (doasCmdNoPass "/run/current-system/sw/bin/readlink")
+
+          # doas matches command, not binary
+          (doasCmdNoPass "btrfs")
+          (doasCmdNoPass "mkdir")
+          (doasCmdNoPass "readlink")
+        ];
+    };
+    users.users.btrbk = {
+      isSystemUser = true;
+      # ssh needs a home directory
+      home = "/var/lib/btrbk";
+      createHome = true;
+      shell = "${pkgs.bash}/bin/bash";
+      group = "btrbk";
+      openssh.authorizedKeys.keys = map (
+        v:
+        let
+          options = concatMapStringsSep " " (x: "--" + x) v.roles;
+          ioniceClass =
+            {
+              "idle" = 3;
+              "best-effort" = 2;
+              "realtime" = 1;
+            }
+            .${cfg.ioSchedulingClass};
+          sudo_doas_flag = "--${sudo_doas}";
+        in
+        ''command="${pkgs.util-linux}/bin/ionice -t -c ${toString ioniceClass} ${
+          optionalString (cfg.niceness >= 1) "${pkgs.coreutils}/bin/nice -n ${toString cfg.niceness}"
+        } ${pkgs.btrbk}/share/btrbk/scripts/ssh_filter_btrbk.sh ${sudo_doas_flag} ${options}" ${v.key}''
+      ) cfg.sshAccess;
+    };
+    users.groups.btrbk = { };
+    systemd.tmpfiles.rules = [
+      "d /var/lib/btrbk 0750 btrbk btrbk"
+      "d /var/lib/btrbk/.ssh 0700 btrbk btrbk"
+      "f /var/lib/btrbk/.ssh/config 0700 btrbk btrbk - StrictHostKeyChecking=accept-new"
+    ];
+    environment.etc = mapAttrs' (name: instance: {
+      name = "btrbk/${name}.conf";
+      value.source = mkConfigFile name instance.settings;
+    }) cfg.instances;
+    systemd.services = mapAttrs' (name: instance: {
+      name = "btrbk-${name}";
+      value = {
+        description = "Takes BTRFS snapshots and maintains retention policies.";
+        unitConfig.Documentation = "man:btrbk(1)";
+        path = [
+          "/run/wrappers"
+        ]
+        ++ cfg.extraPackages
+        ++ optional (instance.settings.stream_compress != "no") (
+          getAttr instance.settings.stream_compress streamCompressMap
+        );
+        serviceConfig = {
+          User = "btrbk";
+          Group = "btrbk";
+          Type = "oneshot";
+          ExecStart = "${pkgs.btrbk}/bin/btrbk -c /etc/btrbk/${name}.conf ${
+            if instance.snapshotOnly then "snapshot" else "run"
+          }";
+          Nice = cfg.niceness;
+          IOSchedulingClass = cfg.ioSchedulingClass;
+          StateDirectory = "btrbk";
+        };
+      };
+    }) cfg.instances;
+
+    systemd.timers = mapAttrs' (name: instance: {
+      name = "btrbk-${name}";
+      value = {
+        description = "Timer to take BTRFS snapshots and maintain retention policies.";
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnCalendar = instance.onCalendar;
+          AccuracySec = "10min";
+          Persistent = true;
+        };
+      };
+    }) (filterAttrs (name: instance: instance.onCalendar != null) cfg.instances);
+  };
+
+}

nixos/modules/services/backup/duplicati.nix

@@ -0,0 +1,131 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.duplicati;
+
+  parametersFile =
+    if cfg.parametersFile != null then
+      cfg.parametersFile
+    else
+      pkgs.writeText "duplicati-parameters" cfg.parameters;
+in
+{
+  options = {
+    services.duplicati = {
+      enable = lib.mkEnableOption "Duplicati";
+
+      package = lib.mkPackageOption pkgs "duplicati" { };
+
+      port = lib.mkOption {
+        default = 8200;
+        type = lib.types.port;
+        description = ''
+          Port serving the web interface
+        '';
+      };
+
+      dataDir = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/duplicati";
+        description = ''
+          The directory where Duplicati stores its data files.
+
+          ::: {.note}
+          If left as the default value this directory will automatically be created
+          before the Duplicati server starts, otherwise you are responsible for ensuring
+          the directory exists with appropriate ownership and permissions.
+          :::
+        '';
+      };
+
+      interface = lib.mkOption {
+        default = "127.0.0.1";
+        type = lib.types.str;
+        description = ''
+          Listening interface for the web UI
+          Set it to "any" to listen on all available interfaces
+        '';
+      };
+
+      user = lib.mkOption {
+        default = "duplicati";
+        type = lib.types.str;
+        description = ''
+          Duplicati runs as it's own user. It will only be able to backup world-readable files.
+          Run as root with special care.
+        '';
+      };
+
+      parameters = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        example = ''
+          --webservice-allowedhostnames=*
+        '';
+        description = ''
+          This option can be used to store some or all of the options given to the
+          commandline client.
+          Each line in this option should be of the format --option=value.
+          The options in this file take precedence over the options provided
+          through command line arguments.
+          <link xlink:href="https://duplicati.readthedocs.io/en/latest/06-advanced-options/#parameters-file">Duplicati docs: parameters-file</link>
+        '';
+      };
+
+      parametersFile = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.path;
+        description = ''
+          This file can be used to store some or all of the options given to the
+          commandline client.
+          Each line in the file option should be of the format --option=value.
+          The options in this file take precedence over the options provided
+          through command line arguments.
+          <link xlink:href="https://duplicati.readthedocs.io/en/latest/06-advanced-options/#parameters-file">Duplicati docs: parameters-file</link>
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    assertions = [
+      {
+        assertion = !(cfg.parametersFile != null && cfg.parameters != "");
+        message = "cannot set both services.duplicati.parameters and services.duplicati.parametersFile at the same time";
+      }
+    ];
+
+    systemd.services.duplicati = {
+      description = "Duplicati backup";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = lib.mkMerge [
+        {
+          User = cfg.user;
+          Group = "duplicati";
+          ExecStart = "${cfg.package}/bin/duplicati-server --webservice-interface=${cfg.interface} --webservice-port=${toString cfg.port} --server-datafolder=${cfg.dataDir} --parameters-file=${parametersFile}";
+          Restart = "on-failure";
+        }
+        (lib.mkIf (cfg.dataDir == "/var/lib/duplicati") {
+          StateDirectory = "duplicati";
+        })
+      ];
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "duplicati") {
+      duplicati = {
+        uid = config.ids.uids.duplicati;
+        home = cfg.dataDir;
+        group = "duplicati";
+      };
+    };
+    users.groups.duplicati.gid = config.ids.gids.duplicati;
+
+  };
+}

nixos/modules/services/backup/duplicity.nix

@@ -0,0 +1,254 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.duplicity;
+
+  stateDirectory = "/var/lib/duplicity";
+
+  localTarget =
+    if lib.hasPrefix "file://" cfg.targetUrl then lib.removePrefix "file://" cfg.targetUrl else null;
+
+in
+{
+  options.services.duplicity = {
+    enable = lib.mkEnableOption "backups with duplicity";
+
+    root = lib.mkOption {
+      type = lib.types.path;
+      default = "/";
+      description = ''
+        Root directory to backup.
+      '';
+    };
+
+    include = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [ "/home" ];
+      description = ''
+        List of paths to include into the backups. See the FILE SELECTION
+        section in {manpage}`duplicity(1)` for details on the syntax.
+      '';
+    };
+
+    exclude = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        List of paths to exclude from backups. See the FILE SELECTION section in
+        {manpage}`duplicity(1)` for details on the syntax.
+      '';
+    };
+
+    includeFileList = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      example = /path/to/fileList.txt;
+      description = ''
+        File containing newline-separated list of paths to include into the
+        backups. See the FILE SELECTION section in {manpage}`duplicity(1)` for
+        details on the syntax.
+      '';
+    };
+
+    excludeFileList = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      example = /path/to/fileList.txt;
+      description = ''
+        File containing newline-separated list of paths to exclude into the
+        backups. See the FILE SELECTION section in {manpage}`duplicity(1)` for
+        details on the syntax.
+      '';
+    };
+
+    targetUrl = lib.mkOption {
+      type = lib.types.str;
+      example = "s3://host:port/prefix";
+      description = ''
+        Target url to backup to. See the URL FORMAT section in
+        {manpage}`duplicity(1)` for supported urls.
+      '';
+    };
+
+    secretFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        Path of a file containing secrets (gpg passphrase, access key...) in
+        the format of EnvironmentFile as described by
+        {manpage}`systemd.exec(5)`. For example:
+        ```
+        PASSPHRASE=«...»
+        AWS_ACCESS_KEY_ID=«...»
+        AWS_SECRET_ACCESS_KEY=«...»
+        ```
+      '';
+    };
+
+    frequency = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = "daily";
+      description = ''
+        Run duplicity with the given frequency (see
+        {manpage}`systemd.time(7)` for the format).
+        If null, do not run automatically.
+      '';
+    };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [
+        "--backend-retry-delay"
+        "100"
+      ];
+      description = ''
+        Extra command-line flags passed to duplicity. See
+        {manpage}`duplicity(1)`.
+      '';
+    };
+
+    fullIfOlderThan = lib.mkOption {
+      type = lib.types.str;
+      default = "never";
+      example = "1M";
+      description = ''
+        If `"never"` (the default) always do incremental
+        backups (the first backup will be a full backup, of course).  If
+        `"always"` always do full backups.  Otherwise, this
+        must be a string representing a duration. Full backups will be made
+        when the latest full backup is older than this duration. If this is not
+        the case, an incremental backup is performed.
+      '';
+    };
+
+    cleanup = {
+      maxAge = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "6M";
+        description = ''
+          If non-null, delete all backup sets older than the given time.  Old backup sets
+          will not be deleted if backup sets newer than time depend on them.
+        '';
+      };
+      maxFull = lib.mkOption {
+        type = lib.types.nullOr lib.types.int;
+        default = null;
+        example = 2;
+        description = ''
+          If non-null, delete all backups sets that are older than the count:th last full
+          backup (in other words, keep the last count full backups and
+          associated incremental sets).
+        '';
+      };
+      maxIncr = lib.mkOption {
+        type = lib.types.nullOr lib.types.int;
+        default = null;
+        example = 1;
+        description = ''
+          If non-null, delete incremental sets of all backups sets that are
+          older than the count:th last full backup (in other words, keep only
+          old full backups and not their increments).
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd = {
+      services.duplicity = {
+        description = "backup files with duplicity";
+
+        environment.HOME = stateDirectory;
+
+        script =
+          let
+            target = lib.escapeShellArg cfg.targetUrl;
+            extra = lib.escapeShellArgs (
+              [
+                "--archive-dir"
+                stateDirectory
+              ]
+              ++ cfg.extraFlags
+            );
+            dup = "${pkgs.duplicity}/bin/duplicity";
+          in
+          ''
+            set -x
+            ${dup} cleanup ${target} --force ${extra}
+            ${lib.optionalString (
+              cfg.cleanup.maxAge != null
+            ) "${dup} remove-older-than ${lib.escapeShellArg cfg.cleanup.maxAge} ${target} --force ${extra}"}
+            ${lib.optionalString (
+              cfg.cleanup.maxFull != null
+            ) "${dup} remove-all-but-n-full ${toString cfg.cleanup.maxFull} ${target} --force ${extra}"}
+            ${lib.optionalString (
+              cfg.cleanup.maxIncr != null
+            ) "${dup} remove-all-inc-of-but-n-full ${toString cfg.cleanup.maxIncr} ${target} --force ${extra}"}
+            exec ${dup} ${if cfg.fullIfOlderThan == "always" then "full" else "incr"} ${
+              lib.escapeShellArgs (
+                [
+                  cfg.root
+                  cfg.targetUrl
+                ]
+                ++ lib.optionals (cfg.includeFileList != null) [
+                  "--include-filelist"
+                  cfg.includeFileList
+                ]
+                ++ lib.optionals (cfg.excludeFileList != null) [
+                  "--exclude-filelist"
+                  cfg.excludeFileList
+                ]
+                ++ lib.concatMap (p: [
+                  "--include"
+                  p
+                ]) cfg.include
+                ++ lib.concatMap (p: [
+                  "--exclude"
+                  p
+                ]) cfg.exclude
+                ++ (lib.optionals (cfg.fullIfOlderThan != "never" && cfg.fullIfOlderThan != "always") [
+                  "--full-if-older-than"
+                  cfg.fullIfOlderThan
+                ])
+              )
+            } ${extra}
+          '';
+        serviceConfig = {
+          PrivateTmp = true;
+          ProtectSystem = "strict";
+          ProtectHome = "read-only";
+          StateDirectory = baseNameOf stateDirectory;
+        }
+        // lib.optionalAttrs (localTarget != null) {
+          ReadWritePaths = localTarget;
+        }
+        // lib.optionalAttrs (cfg.secretFile != null) {
+          EnvironmentFile = cfg.secretFile;
+        };
+      }
+      // lib.optionalAttrs (cfg.frequency != null) {
+        startAt = cfg.frequency;
+      };
+
+      tmpfiles.rules = lib.optional (localTarget != null) "d ${localTarget} 0700 root root -";
+    };
+
+    assertions = lib.singleton {
+      # Duplicity will fail if the last file selection option is an include. It
+      # is not always possible to detect but this simple case can be caught.
+      assertion = cfg.include != [ ] -> cfg.exclude != [ ] || cfg.extraFlags != [ ];
+      message = ''
+        Duplicity will fail if you only specify included paths ("Because the
+        default is to include all files, the expression is redundant. Exiting
+        because this probably isn't what you meant.")
+      '';
+    };
+  };
+}

nixos/modules/services/backup/libvirtd-autosnapshot.nix

@@ -0,0 +1,260 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.libvirtd.autoSnapshot;
+
+  # Function to get VM config with defaults
+  getVMConfig =
+    vm:
+    if lib.isString vm then
+      {
+        name = vm;
+        inherit (cfg) snapshotType keep;
+      }
+    else
+      {
+        inherit (vm) name;
+        snapshotType = if vm.snapshotType != null then vm.snapshotType else cfg.snapshotType;
+        keep = if vm.keep != null then vm.keep else cfg.keep;
+      };
+
+  # Main backup script combining all VM scripts
+  backupScript = ''
+    set -eo pipefail
+
+    # Initialize failure tracking
+    failed=""
+
+    # Define the VM snapshot function
+    function snap_vm() {
+      local vmName="$1"
+      local snapshotType="$2"
+      local keep="$3"
+
+      # Add validation for VM name
+      if ! echo "$vmName" | ${pkgs.gnugrep}/bin/grep -qE '^[a-zA-Z0-9_.-]+$'; then
+        echo "Invalid VM name: '$vmName'"
+        failed="$failed $vmName"
+        return
+      fi
+
+      echo "Processing VM: $vmName"
+
+      # Check if VM exists
+      if ! ${pkgs.libvirt}/bin/virsh dominfo "$vmName" >/dev/null 2>&1; then
+        echo "VM '$vmName' does not exist, skipping"
+        return
+      fi
+
+      # Create new snapshot
+      local snapshot_name
+      snapshot_name="${cfg.prefix}_$(date +%Y-%m-%d_%H%M%S)"
+      local snapshot_opts=""
+      [[ "$snapshotType" == "external" ]] && snapshot_opts="--disk-only"
+      if ! ${pkgs.libvirt}/bin/virsh snapshot-create-as \
+        "$vmName" \
+        "$snapshot_name" \
+        "Automatic backup snapshot" \
+        $snapshot_opts \
+        --atomic; then
+        echo "Failed to create snapshot for $vmName"
+        failed="$failed $vmName"
+        return
+      fi
+
+      # List all automatic snapshots for this VM
+      readarray -t SNAPSHOTS < <(${pkgs.libvirt}/bin/virsh snapshot-list "$vmName" --name | ${pkgs.gnugrep}/bin/grep "^${cfg.prefix}_")
+
+      # Count snapshots
+      local snapshot_count=''${#SNAPSHOTS[@]}
+
+      # Delete old snapshots if we have more than the keep limit
+      if [[ $snapshot_count -gt $keep ]]; then
+        # Sort snapshots by date (they're named with date prefix)
+        readarray -t TO_DELETE < <(printf '%s\n' "''${SNAPSHOTS[@]}" | ${pkgs.coreutils}/bin/sort | ${pkgs.coreutils}/bin/head -n -$keep)
+        for snap in "''${TO_DELETE[@]}"; do
+          echo "Removing old snapshot $snap from $vmName"
+
+          # Check if snapshot is internal or external
+          local snapshot_location
+          snapshot_location=$(${pkgs.libvirt}/bin/virsh snapshot-info "$vmName" --snapshotname "$snap" | ${pkgs.gnugrep}/bin/grep "Location:" | ${pkgs.gawk}/bin/awk '{print $2}')
+
+          local delete_opts=""
+          [[ "$snapshot_location" == "internal" ]] && delete_opts="--metadata"
+
+          if ! ${pkgs.libvirt}/bin/virsh snapshot-delete "$vmName" "$snap" $delete_opts; then
+            echo "Failed to remove snapshot $snap from $vmName"
+            failed="$failed $vmName(cleanup)"
+          fi
+        done
+      fi
+    }
+
+    ${
+      if cfg.vms == null then
+        ''
+          # Process all VMs
+          ${pkgs.libvirt}/bin/virsh list --all --name | while read -r vm; do
+            # Skip empty lines
+            [ -z "$vm" ] && continue
+
+            # Call snap_vm function with default settings
+            snap_vm "$vm" ${cfg.snapshotType} ${toString cfg.keep}
+          done
+        ''
+      else
+        ''
+          # Process specific VMs from the list
+          ${lib.concatMapStrings (
+            vm: with getVMConfig vm; "snap_vm '${name}' ${snapshotType} ${toString keep}\n"
+          ) cfg.vms}
+        ''
+    }
+
+    # Report any failures
+    if [ -n "$failed" ]; then
+      echo "Snapshot operation failed for:$failed"
+      exit 1
+    fi
+
+    exit 0
+  '';
+in
+{
+  options = {
+    services.libvirtd.autoSnapshot = {
+      enable = lib.mkEnableOption "LibVirt VM snapshots";
+
+      calendar = lib.mkOption {
+        type = lib.types.str;
+        default = "04:15:00";
+        description = ''
+          When to create snapshots (systemd calendar format).
+          Default is 4:15 AM.
+        '';
+      };
+
+      prefix = lib.mkOption {
+        type = lib.types.str;
+        default = "autosnap";
+        description = ''
+          Prefix for automatic snapshot names.
+          This is used to identify and manage automatic snapshots
+          separately from manual ones.
+        '';
+      };
+
+      keep = lib.mkOption {
+        type = lib.types.int;
+        default = 2;
+        description = "Default number of snapshots to keep for VMs that don't specify a keep value.";
+      };
+
+      snapshotType = lib.mkOption {
+        type = lib.types.enum [
+          "internal"
+          "external"
+        ];
+        default = "internal";
+        description = "Type of snapshot to create (internal or external).";
+      };
+
+      vms = lib.mkOption {
+        type = lib.types.nullOr (
+          lib.types.listOf (
+            lib.types.oneOf [
+              lib.types.str
+              (lib.types.submodule {
+                options = {
+                  name = lib.mkOption {
+                    type = lib.types.str;
+                    description = "Name of the VM";
+                  };
+                  snapshotType = lib.mkOption {
+                    type = lib.types.nullOr (
+                      lib.types.enum [
+                        "internal"
+                        "external"
+                      ]
+                    );
+                    default = null;
+                    description = ''
+                      Type of snapshot to create (internal or external).
+                      If not specified, uses global snapshotType (${toString cfg.snapshotType}).
+                    '';
+                  };
+                  keep = lib.mkOption {
+                    type = lib.types.nullOr lib.types.int;
+                    default = null;
+                    description = ''
+                      Number of snapshots to keep for this VM.
+                      If not specified, uses global keep (${toString cfg.keep}).
+                    '';
+                  };
+                };
+              })
+            ]
+          )
+        );
+        default = null;
+        description = ''
+          If specified only the list of VMs will be snapshotted else all existing one. Each entry can be either:
+          - A string (VM name, uses default settings)
+          - An attribute set with VM configuration
+        '';
+        example = lib.literalExpression ''
+          [
+            "myvm1"              # Uses defaults
+            {
+              name = "myvm2";
+              keep = 30;         # Override retention
+            }
+          ]
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = (cfg.vms == null) || (lib.isList cfg.vms && cfg.vms != [ ]);
+        message = "'services.libvirtd.autoSnapshot.vms' must either be null for all VMs or a non-empty list of VM configurations";
+      }
+      {
+        assertion = config.virtualisation.libvirtd.enable;
+        message = "virtualisation.libvirtd must be enabled to use services.libvirtd.autoSnapshot";
+      }
+    ];
+
+    systemd = {
+      timers.libvirtd-autosnapshot = {
+        description = "LibVirt VM snapshot timer";
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnCalendar = cfg.calendar;
+          AccuracySec = "5m";
+          Unit = "libvirtd-autosnapshot.service";
+        };
+      };
+
+      services.libvirtd-autosnapshot = {
+        description = "LibVirt VM snapshot service";
+        after = [ "libvirtd.service" ];
+        requires = [ "libvirtd.service" ];
+        serviceConfig = {
+          Type = "oneshot";
+          User = "root";
+        };
+        script = backupScript;
+      };
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers._6543 ];
+}

nixos/modules/services/backup/mysql-backup.nix

@@ -0,0 +1,239 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.mysqlBackup;
+  defaultUser = "mysqlbackup";
+
+  # Newer mariadb versions warn of the usage of 'mysqldump' and recommend 'mariadb-dump' (https://mariadb.com/kb/en/mysqldump/)
+  dumpBinary =
+    if
+      (
+        lib.getName config.services.mysql.package == lib.getName pkgs.mariadb
+        && lib.versionAtLeast config.services.mysql.package.version "11.0.0"
+      )
+    then
+      "${config.services.mysql.package}/bin/mariadb-dump"
+    else
+      "${config.services.mysql.package}/bin/mysqldump";
+
+  compressionAlgs = {
+    gzip = rec {
+      pkg = pkgs.gzip;
+      ext = ".gz";
+      minLevel = 1;
+      maxLevel = 9;
+      cmd = compressionLevelFlag: "${pkg}/bin/gzip -c ${cfg.gzipOptions} ${compressionLevelFlag}";
+    };
+    xz = rec {
+      pkg = pkgs.xz;
+      ext = ".xz";
+      minLevel = 0;
+      maxLevel = 9;
+      cmd = compressionLevelFlag: "${pkg}/bin/xz -z -c ${compressionLevelFlag} -";
+    };
+    zstd = rec {
+      pkg = pkgs.zstd;
+      ext = ".zst";
+      minLevel = 1;
+      maxLevel = 19;
+      cmd = compressionLevelFlag: "${pkg}/bin/zstd ${compressionLevelFlag} -";
+    };
+  };
+
+  compressionLevelFlag = lib.optionalString (cfg.compressionLevel != null) (
+    "-" + toString cfg.compressionLevel
+  );
+
+  selectedAlg = compressionAlgs.${cfg.compressionAlg};
+  compressionCmd = selectedAlg.cmd compressionLevelFlag;
+
+  shouldUseSingleTransaction =
+    db:
+    if lib.isBool cfg.singleTransaction then
+      cfg.singleTransaction
+    else
+      lib.elem db cfg.singleTransaction;
+
+  backupScript = ''
+    set -o pipefail
+    failed=""
+    ${lib.concatMapStringsSep "\n" backupDatabaseScript cfg.databases}
+    if [ -n "$failed" ]; then
+      echo "Backup of database(s) failed:$failed"
+      exit 1
+    fi
+  '';
+
+  backupDatabaseScript = db: ''
+    dest="${cfg.location}/${db}${selectedAlg.ext}"
+    if ${dumpBinary} ${lib.optionalString (shouldUseSingleTransaction db) "--single-transaction"} ${db} | ${compressionCmd} > $dest.tmp; then
+      mv $dest.tmp $dest
+      echo "Backed up to $dest"
+    else
+      echo "Failed to back up to $dest"
+      rm -f $dest.tmp
+      failed="$failed ${db}"
+    fi
+  '';
+
+in
+{
+  options = {
+    services.mysqlBackup = {
+      enable = lib.mkEnableOption "MySQL backups";
+
+      calendar = lib.mkOption {
+        type = lib.types.str;
+        default = "01:15:00";
+        description = ''
+          Configured when to run the backup service systemd unit (DayOfWeek Year-Month-Day Hour:Minute:Second).
+        '';
+      };
+
+      compressionAlg = lib.mkOption {
+        type = lib.types.enum (lib.attrNames compressionAlgs);
+        default = "gzip";
+        description = ''
+          Compression algorithm to use for database dumps.
+        '';
+      };
+
+      compressionLevel = lib.mkOption {
+        type = lib.types.nullOr lib.types.int;
+        default = null;
+        description = ''
+          Compression level to use for ${lib.concatStringsSep ", " (lib.init (lib.attrNames compressionAlgs))} or ${lib.last (lib.attrNames compressionAlgs)}.
+          ${lib.concatStringsSep "\n" (
+            lib.mapAttrsToList (
+              name: algo: "- For ${name}: ${toString algo.minLevel}-${toString algo.maxLevel}"
+            ) compressionAlgs
+          )}
+
+          :::{.note}
+          If compression level is also specified in gzipOptions, the gzipOptions value will be overwritten
+          :::
+        '';
+      };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = defaultUser;
+        description = ''
+          User to be used to perform backup.
+        '';
+      };
+
+      databases = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = ''
+          List of database names to dump.
+        '';
+      };
+
+      location = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/backup/mysql";
+        description = ''
+          Location to put the compressed MySQL database dumps.
+        '';
+      };
+
+      singleTransaction = lib.mkOption {
+        default = false;
+        type = lib.types.oneOf [
+          lib.types.bool
+          (lib.types.listOf lib.types.str)
+        ];
+        description = ''
+          Whether to create database dump in a single transaction.
+          Can be either a boolean for all databases or a list of database names.
+        '';
+      };
+
+      gzipOptions = lib.mkOption {
+        default = "--no-name --rsyncable";
+        type = lib.types.str;
+        description = ''
+          Command line options to use when invoking `gzip`.
+          Only used when compression is set to "gzip".
+          If compression level is specified both here and in compressionLevel, the compressionLevel value will take precedence.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # assert config to be correct
+    assertions = [
+      {
+        assertion =
+          cfg.compressionLevel == null
+          || selectedAlg.minLevel <= cfg.compressionLevel && cfg.compressionLevel <= selectedAlg.maxLevel;
+        message = "${cfg.compressionAlg} compression level must be between ${toString selectedAlg.minLevel} and ${toString selectedAlg.maxLevel}";
+      }
+      {
+        assertion =
+          !(lib.isList cfg.singleTransaction)
+          || lib.all (db: lib.elem db cfg.databases) cfg.singleTransaction;
+        message = "All databases in singleTransaction must be included in the databases option";
+      }
+    ];
+
+    # ensure unix user to be used to perform backup exist.
+    users.users = lib.optionalAttrs (cfg.user == defaultUser) {
+      ${defaultUser} = {
+        isSystemUser = true;
+        createHome = false;
+        home = cfg.location;
+        group = "nogroup";
+      };
+    };
+
+    # add the compression tool to the system environment.
+    environment.systemPackages = [ selectedAlg.pkg ];
+
+    # ensure database user to be used to perform backup exist.
+    services.mysql.ensureUsers = [
+      {
+        name = cfg.user;
+        ensurePermissions =
+          let
+            privs = "SELECT, SHOW VIEW, TRIGGER, LOCK TABLES";
+            grant = db: lib.nameValuePair "\\`${db}\\`.*" privs;
+          in
+          lib.listToAttrs (map grant cfg.databases);
+      }
+    ];
+
+    systemd = {
+      timers.mysql-backup = {
+        description = "Mysql backup timer";
+        wantedBy = [ "timers.target" ];
+        timerConfig = {
+          OnCalendar = cfg.calendar;
+          AccuracySec = "5m";
+          Unit = "mysql-backup.service";
+        };
+      };
+      services.mysql-backup = {
+        description = "MySQL backup service";
+        enable = true;
+        serviceConfig = {
+          Type = "oneshot";
+          User = cfg.user;
+        };
+        script = backupScript;
+      };
+      tmpfiles.rules = [
+        "d ${cfg.location} 0700 ${cfg.user} - - -"
+      ];
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers._6543 ];
+}

nixos/modules/services/backup/pgbackrest.nix

@@ -0,0 +1,479 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.pgbackrest;
+
+  settingsFormat = pkgs.formats.ini {
+    listsAsDuplicateKeys = true;
+  };
+
+  # pgBackRest "options"
+  settingsType =
+    with lib.types;
+    attrsOf (oneOf [
+      bool
+      ints.unsigned
+      str
+      (attrsOf str)
+      (listOf str)
+    ]);
+
+  # Applied to both repoNNN-* and pgNNN-* options in global and stanza sections.
+  flattenWithIndex =
+    attrs: prefix:
+    lib.concatMapAttrs (
+      name:
+      let
+        index = lib.lists.findFirstIndex (n: n == name) null (lib.attrNames attrs);
+        index1 = index + 1;
+      in
+      lib.mapAttrs' (option: lib.nameValuePair "${prefix}${toString index1}-${option}")
+    ) attrs;
+
+  # Remove nulls, turn attrsets into lists and bools into y/n
+  normalize =
+    x:
+    lib.pipe x [
+      (lib.filterAttrs (_: v: v != null))
+      (lib.mapAttrs (_: v: if lib.isAttrs v then lib.mapAttrsToList (n': v': "${n'}=${v'}") v else v))
+      (lib.mapAttrs (
+        _: v:
+        if v == true then
+          "y"
+        else if v == false then
+          "n"
+        else
+          v
+      ))
+    ];
+
+  fullConfig = {
+    global = normalize (cfg.settings // flattenWithIndex cfg.repos "repo");
+  }
+  // lib.mapAttrs' (
+    cmd: settings: lib.nameValuePair "global:${cmd}" (normalize settings)
+  ) cfg.commands
+  // lib.mapAttrs (
+    _: cfg': normalize (cfg'.settings // flattenWithIndex cfg'.instances "pg")
+  ) cfg.stanzas;
+
+  namedJobs = lib.listToAttrs (
+    lib.flatten (
+      lib.mapAttrsToList (
+        stanza:
+        { jobs, ... }:
+        lib.mapAttrsToList (
+          job: attrs: lib.nameValuePair "pgbackrest-${stanza}-${job}" (attrs // { inherit stanza job; })
+        ) jobs
+      ) cfg.stanzas
+    )
+  );
+
+  disabledOption = lib.mkOption {
+    default = null;
+    readOnly = true;
+    internal = true;
+  };
+
+  secretPathOption =
+    with lib.types;
+    lib.mkOption {
+      type = nullOr (pathWith {
+        inStore = false;
+        absolute = true;
+      });
+      default = null;
+      internal = true;
+    };
+in
+
+{
+  meta = {
+    maintainers = with lib.maintainers; [ wolfgangwalther ];
+  };
+
+  # TODO: Add enableServer option and corresponding pgBackRest TLS server service.
+  # TODO: Write wrapper around pgbackrest to turn --repo=<name> into --repo=<number>
+  # The following two are dependent on improvements upstream:
+  #   https://github.com/pgbackrest/pgbackrest/issues/2621
+  # TODO: Add support for more repository types
+  # TODO: Support passing encryption key safely
+  options.services.pgbackrest = {
+    enable = lib.mkEnableOption "pgBackRest";
+
+    repos = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (
+          submodule (
+            { config, name, ... }:
+            let
+              setHostForType =
+                type:
+                if name == "localhost" then
+                  null
+                # "posix" is the default repo type, which uses the -host option.
+                # Other types use prefixed options, for example -sftp-host.
+                else if config.type or "posix" != type then
+                  null
+                else
+                  name;
+            in
+            {
+              freeformType = settingsType;
+
+              options.host = lib.mkOption {
+                type = nullOr str;
+                default = setHostForType "posix";
+                defaultText = lib.literalExpression "name";
+                description = "Repository host when operating remotely";
+              };
+
+              options.sftp-host = lib.mkOption {
+                type = nullOr str;
+                default = setHostForType "sftp";
+                defaultText = lib.literalExpression "name";
+                description = "SFTP repository host";
+              };
+
+              options.sftp-private-key-file = lib.mkOption {
+                type = nullOr (pathWith {
+                  inStore = false;
+                  absolute = true;
+                });
+                default = null;
+                description = ''
+                  SFTP private key file.
+
+                  The file must be accessible by both the pgbackrest and the postgres users.
+                '';
+              };
+
+              # The following options should not be used; they would store secrets in the store.
+              options.azure-key = disabledOption;
+              options.cipher-pass = disabledOption;
+              options.s3-key = disabledOption;
+              options.s3-key-secret = disabledOption;
+              options.s3-kms-key-id = disabledOption; # unsure whether that's a secret or not
+              options.s3-sse-customer-key = disabledOption; # unsure whether that's a secret or not
+              options.s3-token = disabledOption;
+              options.sftp-private-key-passphrase = disabledOption;
+
+              # The following options are not fully supported / tested, yet, but point to files with secrets.
+              # Users can already set those options, but we'll force non-store paths.
+              options.gcs-key = secretPathOption;
+              options.host-cert-file = secretPathOption;
+              options.host-key-file = secretPathOption;
+            }
+          )
+        );
+      default = { };
+      description = ''
+        An attribute set of repositories as described in:
+        <https://pgbackrest.org/configuration.html#section-repository>
+
+        Each repository defaults to set `repo-host` to the attribute's name.
+        The special value "localhost" will unset `repo-host`.
+
+        ::: {.note}
+        The prefix `repoNNN-` is added automatically.
+        Example: Use `path` instead of `repo1-path`.
+        :::
+      '';
+      example = lib.literalExpression ''
+        {
+          localhost.path = "/var/lib/backup";
+          "backup.example.com".host-type = "tls";
+        }
+      '';
+    };
+
+    stanzas = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (submodule {
+          options = {
+            jobs = lib.mkOption {
+              type = lib.types.attrsOf (
+                lib.types.submodule {
+                  options.schedule = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      When or how often the backup should run.
+                      Must be in the format described in {manpage}`systemd.time(7)`.
+                    '';
+                  };
+
+                  options.type = lib.mkOption {
+                    type = lib.types.str;
+                    description = ''
+                      Backup type as described in:
+                      <https://pgbackrest.org/command.html#command-backup/category-command/option-type>
+                    '';
+                  };
+                }
+              );
+              default = { };
+              description = ''
+                Backups jobs to schedule for this stanza as described in:
+                <https://pgbackrest.org/user-guide.html#quickstart/schedule-backup>
+              '';
+              example = lib.literalExpression ''
+                {
+                  weekly = { schedule = "Sun, 6:30"; type = "full"; };
+                  daily = { schedule = "Mon..Sat, 6:30"; type = "diff"; };
+                }
+              '';
+            };
+
+            instances = lib.mkOption {
+              type =
+                with lib.types;
+                attrsOf (
+                  submodule (
+                    { name, ... }:
+                    {
+                      freeformType = settingsType;
+                      options.host = lib.mkOption {
+                        type = nullOr str;
+                        default = if name == "localhost" then null else name;
+                        defaultText = lib.literalExpression ''if name == "localhost" then null else name'';
+                        description = "PostgreSQL host for operating remotely.";
+                      };
+
+                      # The following options are not fully supported / tested, yet, but point to files with secrets.
+                      # Users can already set those options, but we'll force non-store paths.
+                      options.host-cert-file = secretPathOption;
+                      options.host-key-file = secretPathOption;
+                    }
+                  )
+                );
+              default = { };
+              description = ''
+                An attribute set of database instances as described in:
+                <https://pgbackrest.org/configuration.html#section-stanza>
+
+                Each instance defaults to set `pg-host` to the attribute's name.
+                The special value "localhost" will unset `pg-host`.
+
+                ::: {.note}
+                The prefix `pgNNN-` is added automatically.
+                Example: Use `user` instead of `pg1-user`.
+                :::
+              '';
+              example = lib.literalExpression ''
+                {
+                  localhost.database = "app";
+                  "postgres.example.com".port = "5433";
+                }
+              '';
+            };
+
+            settings = lib.mkOption {
+              type = lib.types.submodule {
+                freeformType = settingsType;
+
+                # The following options are not fully supported / tested, yet, but point to files with secrets.
+                # Users can already set those options, but we'll force non-store paths.
+                options.tls-server-cert-file = secretPathOption;
+                options.tls-server-key-file = secretPathOption;
+              };
+              default = { };
+              description = ''
+                An attribute set of options as described in:
+                <https://pgbackrest.org/configuration.html>
+
+                All options can be used.
+                Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
+                Stanza options should be set via [`instances`](#opt-services.pgbackrest.stanzas._name_.instances) instead.
+              '';
+              example = lib.literalExpression ''
+                {
+                  process-max = 2;
+                }
+              '';
+            };
+          };
+        });
+      default = { };
+      description = ''
+        An attribute set of stanzas as described in:
+        <https://pgbackrest.org/user-guide.html#quickstart/configure-stanza>
+      '';
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = settingsType;
+
+        # The following options are not fully supported / tested, yet, but point to files with secrets.
+        # Users can already set those options, but we'll force non-store paths.
+        options.tls-server-cert-file = secretPathOption;
+        options.tls-server-key-file = secretPathOption;
+      };
+      default = { };
+      description = ''
+        An attribute set of options as described in:
+        <https://pgbackrest.org/configuration.html>
+
+        All globally available options, i.e. all except stanza options, can be used.
+        Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
+      '';
+      example = lib.literalExpression ''
+        {
+          process-max = 2;
+        }
+      '';
+    };
+
+    commands =
+      lib.genAttrs
+        [
+          # List of commands from https://pgbackrest.org/command.html:
+          "annotate"
+          "archive-get"
+          "archive-push"
+          "backup"
+          "check"
+          "expire"
+          "help"
+          "info"
+          "repo-get"
+          "repo-ls"
+          "restore"
+          "server"
+          "server-ping"
+          "stanza-create"
+          "stanza-delete"
+          "stanza-upgrade"
+          "start"
+          "stop"
+          "verify"
+          "version"
+        ]
+        (
+          command:
+          lib.mkOption {
+            type = lib.types.submodule {
+              freeformType = settingsType;
+
+              # The following options are not fully supported / tested, yet, but point to files with secrets.
+              # Users can already set those options, but we'll force non-store paths.
+              options.tls-server-cert-file = secretPathOption;
+              options.tls-server-key-file = secretPathOption;
+            };
+            default = { };
+            description = ''
+              Options for the '${command}' command.
+
+              An attribute set of options as described in:
+              <https://pgbackrest.org/configuration.html>
+
+              All globally available options, i.e. all except stanza options, can be used.
+              Repository options should be set via [`repos`](#opt-services.pgbackrest.repos) instead.
+            '';
+          }
+        );
+  };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      {
+        services.pgbackrest.settings = {
+          log-level-console = lib.mkDefault "info";
+          log-level-file = lib.mkDefault "off";
+          cmd-ssh = lib.getExe pkgs.openssh;
+        };
+
+        environment.systemPackages = [ pkgs.pgbackrest ];
+        environment.etc."pgbackrest/pgbackrest.conf".source =
+          settingsFormat.generate "pgbackrest.conf" fullConfig;
+
+        users.users.pgbackrest = {
+          name = "pgbackrest";
+          group = "pgbackrest";
+          description = "pgBackRest service user";
+          isSystemUser = true;
+          useDefaultShell = true;
+          createHome = true;
+          home = cfg.repos.localhost.path or "/var/lib/pgbackrest";
+        };
+        users.groups.pgbackrest = { };
+
+        systemd.services = lib.mapAttrs (
+          _:
+          {
+            stanza,
+            job,
+            type,
+            ...
+          }:
+          {
+            description = "pgBackRest job ${job} for stanza ${stanza}";
+
+            serviceConfig = {
+              User = "pgbackrest";
+              Group = "pgbackrest";
+              Type = "oneshot";
+              # stanza-create is idempotent, so safe to always run
+              ExecStartPre = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' stanza-create";
+              ExecStart = "${lib.getExe pkgs.pgbackrest} --stanza='${stanza}' backup --type='${type}'";
+            };
+          }
+        ) namedJobs;
+
+        systemd.timers = lib.mapAttrs (
+          name:
+          {
+            stanza,
+            job,
+            schedule,
+            ...
+          }:
+          {
+            description = "pgBackRest job ${job} for stanza ${stanza}";
+            wantedBy = [ "timers.target" ];
+            after = [ "network-online.target" ];
+            wants = [ "network-online.target" ];
+            timerConfig = {
+              OnCalendar = schedule;
+              Persistent = true;
+              Unit = "${name}.service";
+            };
+          }
+        ) namedJobs;
+      }
+
+      # The default stanza is set up for the local postgresql instance.
+      # It does not backup automatically, the systemd timer still needs to be set.
+      (lib.mkIf config.services.postgresql.enable {
+        services.pgbackrest.stanzas.default = {
+          settings.cmd = lib.getExe pkgs.pgbackrest;
+          instances.localhost = {
+            path = config.services.postgresql.dataDir;
+            user = "postgres";
+          };
+        };
+        # If PostgreSQL runs on the same machine, any restore will have to be done with that user.
+        # Keeping the lock file in a directory writeable by the postgres user prevents errors.
+        services.pgbackrest.commands.restore.lock-path = "/tmp/postgresql";
+        services.postgresql.identMap = ''
+          postgres pgbackrest postgres
+        '';
+        services.postgresql.initdbArgs = [ "--allow-group-access" ];
+        users.users.pgbackrest.extraGroups = [ "postgres" ];
+
+        services.postgresql.settings = {
+          archive_command = ''${lib.getExe pkgs.pgbackrest} --stanza=default archive-push "%p"'';
+          archive_mode = lib.mkDefault "on";
+        };
+        users.groups.pgbackrest.members = [ "postgres" ];
+      })
+    ]
+  );
+}

nixos/modules/services/backup/postgresql-backup.nix

@@ -0,0 +1,211 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.postgresqlBackup;
+
+  postgresqlBackupService =
+    db: dumpCmd:
+    let
+      compressSuffixes = {
+        "none" = "";
+        "gzip" = ".gz";
+        "zstd" = ".zstd";
+      };
+      compressSuffix = lib.getAttr cfg.compression compressSuffixes;
+
+      compressCmd = lib.getAttr cfg.compression {
+        "none" = "cat";
+        "gzip" = "${pkgs.gzip}/bin/gzip -c -${toString cfg.compressionLevel} --rsyncable";
+        "zstd" = "${pkgs.zstd}/bin/zstd -c -${toString cfg.compressionLevel} --rsyncable";
+      };
+
+      mkSqlPath = prefix: suffix: "${cfg.location}/${db}${prefix}.sql${suffix}";
+      curFile = mkSqlPath "" compressSuffix;
+      prevFile = mkSqlPath ".prev" compressSuffix;
+      prevFiles = map (mkSqlPath ".prev") (lib.attrValues compressSuffixes);
+      inProgressFile = mkSqlPath ".in-progress" compressSuffix;
+    in
+    {
+      enable = true;
+
+      description = "Backup of ${db} database(s)";
+
+      requires = [ "postgresql.target" ];
+
+      path = [
+        pkgs.coreutils
+        config.services.postgresql.package
+      ];
+
+      script = ''
+        set -e -o pipefail
+
+        umask 0077 # ensure backup is only readable by postgres user
+
+        if [ -e ${curFile} ]; then
+          rm -f ${toString prevFiles}
+          mv ${curFile} ${prevFile}
+        fi
+
+        ${dumpCmd} \
+          | ${compressCmd} \
+          > ${inProgressFile}
+
+        mv ${inProgressFile} ${curFile}
+      '';
+
+      serviceConfig = {
+        Type = "oneshot";
+        User = "postgres";
+      };
+
+      startAt = cfg.startAt;
+    };
+
+in
+{
+
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "postgresqlBackup" "period" ] ''
+      A systemd timer is now used instead of cron.
+      The starting time can be configured via <literal>services.postgresqlBackup.startAt</literal>.
+    '')
+  ];
+
+  options = {
+    services.postgresqlBackup = {
+      enable = lib.mkEnableOption "PostgreSQL dumps";
+
+      startAt = lib.mkOption {
+        default = "*-*-* 01:15:00";
+        type = with lib.types; either (listOf str) str;
+        description = ''
+          This option defines (see `systemd.time` for format) when the
+          databases should be dumped.
+          The default is to update at 01:15 (at night) every day.
+        '';
+      };
+
+      backupAll = lib.mkOption {
+        default = cfg.databases == [ ];
+        defaultText = lib.literalExpression "services.postgresqlBackup.databases == []";
+        type = lib.types.bool;
+        description = ''
+          Backup all databases using pg_dumpall.
+          This option is mutual exclusive to
+          `services.postgresqlBackup.databases`.
+          The resulting backup dump will have the name all.sql.gz.
+          This option is the default if no databases are specified.
+        '';
+      };
+
+      databases = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = ''
+          List of database names to dump.
+        '';
+      };
+
+      location = lib.mkOption {
+        default = "/var/backup/postgresql";
+        type = lib.types.path;
+        description = ''
+          Path of directory where the PostgreSQL database dumps will be placed.
+        '';
+      };
+
+      pgdumpOptions = lib.mkOption {
+        type = lib.types.separatedString " ";
+        default = "-C";
+        description = ''
+          Command line options for pg_dump. This options is not used if
+          `config.services.postgresqlBackup.backupAll` is enabled. Note that
+          config.services.postgresqlBackup.backupAll is also active, when no
+          databases where specified.
+        '';
+      };
+
+      pgdumpAllOptions = lib.mkOption {
+        type = lib.types.separatedString " ";
+        default = "";
+        description = ''
+          Command line options for pg_dumpall. This options is not used if
+          `config.services.postgresqlBackup.backupAll` is disabled.
+        '';
+      };
+
+      compression = lib.mkOption {
+        type = lib.types.enum [
+          "none"
+          "gzip"
+          "zstd"
+        ];
+        default = "gzip";
+        description = ''
+          The type of compression to use on the generated database dump.
+        '';
+      };
+
+      compressionLevel = lib.mkOption {
+        type = lib.types.ints.between 1 19;
+        default = 6;
+        description = ''
+          The compression level used when compression is enabled.
+          gzip accepts levels 1 to 9. zstd accepts levels 1 to 19.
+        '';
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      {
+        assertions = [
+          {
+            assertion = cfg.backupAll -> cfg.databases == [ ];
+            message = "config.services.postgresqlBackup.backupAll cannot be used together with config.services.postgresqlBackup.databases";
+          }
+          {
+            assertion =
+              cfg.compression == "none"
+              || (cfg.compression == "gzip" && cfg.compressionLevel >= 1 && cfg.compressionLevel <= 9)
+              || (cfg.compression == "zstd" && cfg.compressionLevel >= 1 && cfg.compressionLevel <= 19);
+            message = "config.services.postgresqlBackup.compressionLevel must be set between 1 and 9 for gzip and 1 and 19 for zstd";
+          }
+        ];
+
+        systemd.tmpfiles.rules = [
+          "d '${cfg.location}' 0700 postgres - - -"
+        ];
+      }
+
+      (lib.mkIf cfg.backupAll {
+        systemd.services.postgresqlBackup = postgresqlBackupService "all" "pg_dumpall ${cfg.pgdumpAllOptions}";
+      })
+
+      (lib.mkIf (!cfg.backupAll) {
+        systemd.services = lib.listToAttrs (
+          map (
+            db:
+            let
+              cmd = "pg_dump ${cfg.pgdumpOptions} ${db}";
+            in
+            {
+              name = "postgresqlBackup-${db}";
+              value = postgresqlBackupService db cmd;
+            }
+          ) cfg.databases
+        );
+      })
+    ]
+  );
+
+  meta.maintainers = with lib.maintainers; [ Scrumplex ];
+}

nixos/modules/services/backup/postgresql-wal-receiver.nix

@@ -0,0 +1,214 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  receiverSubmodule = {
+    options = {
+      postgresqlPackage = lib.mkPackageOption pkgs "postgresql" {
+        example = "postgresql_15";
+      };
+
+      directory = lib.mkOption {
+        type = lib.types.path;
+        example = lib.literalExpression "/mnt/pg_wal/main/";
+        description = ''
+          Directory to write the output to.
+        '';
+      };
+
+      statusInterval = lib.mkOption {
+        type = lib.types.int;
+        default = 10;
+        description = ''
+          Specifies the number of seconds between status packets sent back to the server.
+          This allows for easier monitoring of the progress from server.
+          A value of zero disables the periodic status updates completely,
+          although an update will still be sent when requested by the server, to avoid timeout disconnect.
+        '';
+      };
+
+      slot = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "some_slot_name";
+        description = ''
+          Require {command}`pg_receivewal` to use an existing replication slot (see
+          [Section 26.2.6 of the PostgreSQL manual](https://www.postgresql.org/docs/current/warm-standby.html#STREAMING-REPLICATION-SLOTS)).
+          When this option is used, {command}`pg_receivewal` will report a flush position to the server,
+          indicating when each segment has been synchronized to disk so that the server can remove that segment if it is not otherwise needed.
+
+          When the replication client of {command}`pg_receivewal` is configured on the server as a synchronous standby,
+          then using a replication slot will report the flush position to the server, but only when a WAL file is closed.
+          Therefore, that configuration will cause transactions on the primary to wait for a long time and effectively not work satisfactorily.
+          The option {option}`synchronous` must be specified in addition to make this work correctly.
+        '';
+      };
+
+      synchronous = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Flush the WAL data to disk immediately after it has been received.
+          Also send a status packet back to the server immediately after flushing, regardless of {option}`statusInterval`.
+
+          This option should be specified if the replication client of {command}`pg_receivewal` is configured on the server as a synchronous standby,
+          to ensure that timely feedback is sent to the server.
+        '';
+      };
+
+      compress = lib.mkOption {
+        type = lib.types.ints.between 0 9;
+        default = 0;
+        description = ''
+          Enables gzip compression of write-ahead logs, and specifies the compression level
+          (`0` through `9`, `0` being no compression and `9` being best compression).
+          The suffix `.gz` will automatically be added to all filenames.
+
+          This option requires PostgreSQL >= 10.
+        '';
+      };
+
+      connection = lib.mkOption {
+        type = lib.types.str;
+        example = "postgresql://user@somehost";
+        description = ''
+          Specifies parameters used to connect to the server, as a connection string.
+          See [Section 34.1.1 of the PostgreSQL manual](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING) for more information.
+
+          Because {command}`pg_receivewal` doesn't connect to any particular database in the cluster,
+          database name in the connection string will be ignored.
+        '';
+      };
+
+      extraArgs = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        example = lib.literalExpression ''
+          [
+            "--no-sync"
+          ]
+        '';
+        description = ''
+          A list of extra arguments to pass to the {command}`pg_receivewal` command.
+        '';
+      };
+
+      environment = lib.mkOption {
+        type = with lib.types; attrsOf str;
+        default = { };
+        example = lib.literalExpression ''
+          {
+            PGPASSFILE = "/private/passfile";
+            PGSSLMODE = "require";
+          }
+        '';
+        description = ''
+          Environment variables passed to the service.
+          Usable parameters are listed in [Section 34.14 of the PostgreSQL manual](https://www.postgresql.org/docs/current/libpq-envars.html).
+        '';
+      };
+    };
+  };
+
+in
+{
+  options = {
+    services.postgresqlWalReceiver = {
+      receivers = lib.mkOption {
+        type = with lib.types; attrsOf (submodule receiverSubmodule);
+        default = { };
+        example = lib.literalExpression ''
+          {
+            main = {
+              postgresqlPackage = pkgs.postgresql_15;
+              directory = /mnt/pg_wal/main/;
+              slot = "main_wal_receiver";
+              connection = "postgresql://user@somehost";
+            };
+          }
+        '';
+        description = ''
+          PostgreSQL WAL receivers.
+          Stream write-ahead logs from a PostgreSQL server using {command}`pg_receivewal` (formerly {command}`pg_receivexlog`).
+          See [the man page](https://www.postgresql.org/docs/current/app-pgreceivewal.html) for more information.
+        '';
+      };
+    };
+  };
+
+  config =
+    let
+      receivers = config.services.postgresqlWalReceiver.receivers;
+    in
+    lib.mkIf (receivers != { }) {
+      users = {
+        users.postgres = {
+          uid = config.ids.uids.postgres;
+          group = "postgres";
+          description = "PostgreSQL server user";
+        };
+
+        groups.postgres = {
+          gid = config.ids.gids.postgres;
+        };
+      };
+
+      assertions = lib.concatLists (
+        lib.attrsets.mapAttrsToList (name: config: [
+          {
+            assertion = config.compress > 0 -> lib.versionAtLeast config.postgresqlPackage.version "10";
+            message = "Invalid configuration for WAL receiver \"${name}\": compress requires PostgreSQL version >= 10.";
+          }
+        ]) receivers
+      );
+
+      systemd.tmpfiles.rules = lib.mapAttrsToList (name: config: ''
+        d ${lib.escapeShellArg config.directory} 0750 postgres postgres - -
+      '') receivers;
+
+      systemd.services = lib.mapAttrs' (
+        name: config:
+        lib.nameValuePair "postgresql-wal-receiver-${name}" {
+          description = "PostgreSQL WAL receiver (${name})";
+          wantedBy = [ "multi-user.target" ];
+          startLimitIntervalSec = 0; # retry forever, useful in case of network disruption
+
+          serviceConfig = {
+            User = "postgres";
+            Group = "postgres";
+            KillSignal = "SIGINT";
+            Restart = "always";
+            RestartSec = 60;
+          };
+
+          inherit (config) environment;
+
+          script =
+            let
+              receiverCommand =
+                postgresqlPackage:
+                if (lib.versionAtLeast postgresqlPackage.version "10") then
+                  "${postgresqlPackage}/bin/pg_receivewal"
+                else
+                  "${postgresqlPackage}/bin/pg_receivexlog";
+            in
+            ''
+              ${receiverCommand config.postgresqlPackage} \
+                --no-password \
+                --directory=${lib.escapeShellArg config.directory} \
+                --status-interval=${toString config.statusInterval} \
+                --dbname=${lib.escapeShellArg config.connection} \
+                ${lib.optionalString (config.compress > 0) "--compress=${toString config.compress}"} \
+                ${lib.optionalString (config.slot != "") "--slot=${lib.escapeShellArg config.slot}"} \
+                ${lib.optionalString config.synchronous "--synchronous"} \
+                ${lib.concatStringsSep " " config.extraArgs}
+            '';
+        }
+      ) receivers;
+    };
+
+  meta.maintainers = with lib.maintainers; [ euxane ];
+}

nixos/modules/services/backup/restic-rest-server.nix

@@ -0,0 +1,152 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.restic.server;
+in
+{
+  meta.maintainers = [ lib.maintainers.bachp ];
+
+  options.services.restic.server = {
+    enable = lib.mkEnableOption "Restic REST Server";
+
+    listenAddress = lib.mkOption {
+      default = "8000";
+      example = "127.0.0.1:8080";
+      type = lib.types.str;
+      description = "Listen on a specific IP address and port or unix socket.";
+    };
+
+    dataDir = lib.mkOption {
+      default = "/var/lib/restic";
+      type = lib.types.path;
+      description = "The directory for storing the restic repository.";
+    };
+
+    appendOnly = lib.mkOption {
+      default = false;
+      type = lib.types.bool;
+      description = ''
+        Enable append only mode.
+        This mode allows creation of new backups but prevents deletion and modification of existing backups.
+        This can be useful when backing up systems that have a potential of being hacked.
+      '';
+    };
+
+    htpasswd-file = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.path;
+      description = "The path to the servers .htpasswd file. Defaults to `\${dataDir}/.htpasswd`.";
+    };
+
+    privateRepos = lib.mkOption {
+      default = false;
+      type = lib.types.bool;
+      description = ''
+        Enable private repos.
+        Grants access only when a subdirectory with the same name as the user is specified in the repository URL.
+      '';
+    };
+
+    prometheus = lib.mkOption {
+      default = false;
+      type = lib.types.bool;
+      description = "Enable Prometheus metrics at /metrics.";
+    };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Extra commandline options to pass to Restic REST server.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "restic-rest-server" { };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = lib.substring 0 1 cfg.listenAddress != ":";
+        message = "The restic-rest-server now uses systemd socket activation, which expects only the Port number: services.restic.server.listenAddress = \"${
+          lib.substring 1 6 cfg.listenAddress
+        }\";";
+      }
+    ];
+
+    systemd.services.restic-rest-server = {
+      description = "Restic REST Server";
+      after = [
+        "network.target"
+        "restic-rest-server.socket"
+      ];
+      requires = [ "restic-rest-server.socket" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/rest-server \
+          --path ${cfg.dataDir} \
+          ${lib.optionalString (cfg.htpasswd-file != null) "--htpasswd-file ${cfg.htpasswd-file}"} \
+          ${lib.optionalString cfg.appendOnly "--append-only"} \
+          ${lib.optionalString cfg.privateRepos "--private-repos"} \
+          ${lib.optionalString cfg.prometheus "--prometheus"} \
+          ${lib.escapeShellArgs cfg.extraFlags} \
+        '';
+        Type = "simple";
+        User = "restic";
+        Group = "restic";
+
+        # Security hardening
+        CapabilityBoundingSet = "";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateNetwork = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectControlGroups = true;
+        PrivateDevices = true;
+        ReadWritePaths = [ cfg.dataDir ];
+        ReadOnlyPaths = lib.optional (cfg.htpasswd-file != null) cfg.htpasswd-file;
+        RemoveIPC = true;
+        RestrictAddressFamilies = "none";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = "@system-service";
+        UMask = 27;
+      };
+    };
+
+    systemd.sockets.restic-rest-server = {
+      listenStreams = [ cfg.listenAddress ];
+      wantedBy = [ "sockets.target" ];
+    };
+
+    systemd.tmpfiles.rules = lib.mkIf cfg.privateRepos [
+      "f ${cfg.dataDir}/.htpasswd 0700 restic restic -"
+    ];
+
+    users.users.restic = {
+      group = "restic";
+      home = cfg.dataDir;
+      createHome = true;
+      uid = config.ids.uids.restic;
+    };
+
+    users.groups.restic.gid = config.ids.uids.restic;
+  };
+}

nixos/modules/services/backup/restic.nix

@@ -0,0 +1,533 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+let
+  # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers"
+  inherit (utils.systemdUtils.unitOptions) unitOption;
+in
+{
+  options.services.restic.backups = lib.mkOption {
+    description = ''
+      Periodic backups to create with Restic.
+    '';
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, ... }:
+        {
+          options = {
+            passwordFile = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                Read the repository password from a file.
+              '';
+              example = "/etc/nixos/restic-password";
+            };
+
+            environmentFile = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                file containing the credentials to access the repository, in the
+                format of an EnvironmentFile as described by {manpage}`systemd.exec(5)`
+              '';
+            };
+
+            rcloneOptions = lib.mkOption {
+              type =
+                with lib.types;
+                nullOr (
+                  attrsOf (oneOf [
+                    str
+                    bool
+                  ])
+                );
+              default = null;
+              description = ''
+                Options to pass to rclone to control its behavior.
+                See <https://rclone.org/docs/#options> for
+                available options. When specifying option names, strip the
+                leading `--`. To set a flag such as
+                `--drive-use-trash`, which does not take a value,
+                set the value to the Boolean `true`.
+              '';
+              example = {
+                bwlimit = "10M";
+                drive-use-trash = "true";
+              };
+            };
+
+            rcloneConfig = lib.mkOption {
+              type =
+                with lib.types;
+                nullOr (
+                  attrsOf (oneOf [
+                    str
+                    bool
+                  ])
+                );
+              default = null;
+              description = ''
+                Configuration for the rclone remote being used for backup.
+                See the remote's specific options under rclone's docs at
+                <https://rclone.org/docs/>. When specifying
+                option names, use the "config" name specified in the docs.
+                For example, to set `--b2-hard-delete` for a B2
+                remote, use `hard_delete = true` in the
+                attribute set.
+                Warning: Secrets set in here will be world-readable in the Nix
+                store! Consider using the `rcloneConfigFile`
+                option instead to specify secret values separately. Note that
+                options set here will override those set in the config file.
+              '';
+              example = {
+                type = "b2";
+                account = "xxx";
+                key = "xxx";
+                hard_delete = true;
+              };
+            };
+
+            rcloneConfigFile = lib.mkOption {
+              type = with lib.types; nullOr path;
+              default = null;
+              description = ''
+                Path to the file containing rclone configuration. This file
+                must contain configuration for the remote specified in this backup
+                set and also must be readable by root. Options set in
+                `rcloneConfig` will override those set in this
+                file.
+              '';
+            };
+
+            inhibitsSleep = lib.mkOption {
+              default = false;
+              type = lib.types.bool;
+              example = true;
+              description = ''
+                Prevents the system from sleeping while backing up.
+              '';
+            };
+
+            repository = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                repository to backup to.
+              '';
+              example = "sftp:backup@192.168.1.100:/backups/${name}";
+            };
+
+            repositoryFile = lib.mkOption {
+              type = with lib.types; nullOr path;
+              default = null;
+              description = ''
+                Path to the file containing the repository location to backup to.
+              '';
+            };
+
+            paths = lib.mkOption {
+              # This is nullable for legacy reasons only. We should consider making it a pure listOf
+              # after some time has passed since this comment was added.
+              type = lib.types.nullOr (lib.types.listOf lib.types.str);
+              default = [ ];
+              description = ''
+                Which paths to backup, in addition to ones specified via
+                `dynamicFilesFrom`.  If null or an empty array and
+                `dynamicFilesFrom` is also null, no backup command will be run.
+                 This can be used to create a prune-only job.
+              '';
+              example = [
+                "/var/lib/postgresql"
+                "/home/user/backup"
+              ];
+            };
+
+            command = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                Command to pass to --stdin-from-command. If null or an empty array, and `paths`/`dynamicFilesFrom`
+                are also null, no backup command will be run.
+              '';
+              example = [
+                "sudo"
+                "-u"
+                "postgres"
+                "pg_dumpall"
+              ];
+            };
+
+            exclude = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                Patterns to exclude when backing up. See
+                https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for
+                details on syntax.
+              '';
+              example = [
+                "/var/cache"
+                "/home/*/.cache"
+                ".git"
+              ];
+            };
+
+            timerConfig = lib.mkOption {
+              type = lib.types.nullOr (lib.types.attrsOf unitOption);
+              default = {
+                OnCalendar = "daily";
+                Persistent = true;
+              };
+              description = ''
+                When to run the backup. See {manpage}`systemd.timer(5)` for
+                details. If null no timer is created and the backup will only
+                run when explicitly started.
+              '';
+              example = {
+                OnCalendar = "00:05";
+                RandomizedDelaySec = "5h";
+                Persistent = true;
+              };
+            };
+
+            user = lib.mkOption {
+              type = lib.types.str;
+              default = "root";
+              description = ''
+                As which user the backup should run.
+              '';
+              example = "postgresql";
+            };
+
+            extraBackupArgs = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                Extra arguments passed to restic backup.
+              '';
+              example = [
+                "--cleanup-cache"
+                "--exclude-file=/etc/nixos/restic-ignore"
+              ];
+            };
+
+            extraOptions = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                Extra extended options to be passed to the restic --option flag.
+              '';
+              example = [
+                "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'"
+              ];
+            };
+
+            initialize = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Create the repository if it doesn't exist.
+              '';
+            };
+
+            pruneOpts = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                A list of options (--keep-\* et al.) for 'restic forget
+                --prune', to automatically prune old snapshots.  The
+                'forget' command is run *after* the 'backup' command, so
+                keep that in mind when constructing the --keep-\* options.
+              '';
+              example = [
+                "--keep-daily 7"
+                "--keep-weekly 5"
+                "--keep-monthly 12"
+                "--keep-yearly 75"
+              ];
+            };
+
+            runCheck = lib.mkOption {
+              type = lib.types.bool;
+              default = builtins.length config.services.restic.backups.${name}.checkOpts > 0;
+              defaultText = lib.literalExpression ''builtins.length config.services.backups.${name}.checkOpts > 0'';
+              description = "Whether to run the `check` command with the provided `checkOpts` options.";
+              example = true;
+            };
+
+            checkOpts = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ ];
+              description = ''
+                A list of options for 'restic check'.
+              '';
+              example = [
+                "--with-cache"
+              ];
+            };
+
+            dynamicFilesFrom = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                A script that produces a list of files to back up.  The
+                results of this command are given to the '--files-from'
+                option. The result is merged with paths specified via `paths`.
+              '';
+              example = "find /home/matt/git -type d -name .git";
+            };
+
+            backupPrepareCommand = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                A script that must run before starting the backup process.
+              '';
+            };
+
+            backupCleanupCommand = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                A script that must run after finishing the backup process.
+              '';
+            };
+
+            package = lib.mkPackageOption pkgs "restic" { };
+
+            createWrapper = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Whether to generate and add a script to the system path, that has the same environment variables set
+                as the systemd service. This can be used to e.g. mount snapshots or perform other opterations, without
+                having to manually specify most options.
+              '';
+            };
+
+            progressFps = lib.mkOption {
+              type = with lib.types; nullOr numbers.nonnegative;
+              default = null;
+              description = ''
+                Controls the frequency of progress reporting.
+              '';
+              example = 0.1;
+            };
+          };
+        }
+      )
+    );
+    default = { };
+    example = {
+      localbackup = {
+        paths = [ "/home" ];
+        exclude = [ "/home/*/.cache" ];
+        repository = "/mnt/backup-hdd";
+        passwordFile = "/etc/nixos/secrets/restic-password";
+        initialize = true;
+      };
+      remotebackup = {
+        paths = [ "/home" ];
+        repository = "sftp:backup@host:/backups/home";
+        passwordFile = "/etc/nixos/secrets/restic-password";
+        extraOptions = [
+          "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'"
+        ];
+        timerConfig = {
+          OnCalendar = "00:05";
+          RandomizedDelaySec = "5h";
+        };
+      };
+      commandbackup = {
+        command = [
+          "\${lib.getExe pkgs.sudo}"
+          "-u postgres"
+          "\${pkgs.postgresql}/bin/pg_dumpall"
+        ];
+        extraBackupArgs = [ "--tag database" ];
+        repository = "s3:example.com/mybucket";
+        passwordFile = "/etc/nixos/secrets/restic-password";
+        environmentFile = "/etc/nixos/secrets/restic-environment";
+        pruneOpts = [
+          "--keep-daily 14"
+          "--keep-weekly 4"
+          "--keep-monthly 2"
+          "--group-by tags"
+        ];
+      };
+    };
+  };
+
+  config = {
+    assertions = lib.flatten (
+      lib.mapAttrsToList (name: backup: [
+        {
+          assertion =
+            ((backup.repository == null) != (backup.repositoryFile == null))
+            || (backup.environmentFile != null);
+          message = "services.restic.backups.${name}: exactly one of repository, repositoryFile or environmentFile should be set";
+        }
+        {
+          assertion =
+            let
+              fileBackup = (backup.paths != null && backup.paths != [ ]) || backup.dynamicFilesFrom != null;
+              commandBackup = backup.command != [ ];
+            in
+            !(fileBackup && commandBackup);
+          message = "services.restic.backups.${name}: cannot do both a command backup and a file backup at the same time.";
+        }
+        {
+          assertion = (backup.passwordFile != null) || (backup.environmentFile != null);
+          message = "services.restic.backups.${name}: passwordFile or environmentFile must be set";
+        }
+      ]) config.services.restic.backups
+    );
+    systemd.services = lib.mapAttrs' (
+      name: backup:
+      let
+        extraOptions = lib.concatMapStrings (arg: " -o ${arg}") backup.extraOptions;
+        inhibitCmd = lib.concatStringsSep " " [
+          "${pkgs.systemd}/bin/systemd-inhibit"
+          "--mode='block'"
+          "--who='restic'"
+          "--what='sleep'"
+          "--why=${lib.escapeShellArg "Scheduled backup ${name}"} "
+        ];
+        resticCmd = "${lib.optionalString backup.inhibitsSleep inhibitCmd}${lib.getExe backup.package}${extraOptions}";
+        excludeFlags = lib.optional (
+          backup.exclude != [ ]
+        ) "--exclude-file=${pkgs.writeText "exclude-patterns" (lib.concatStringsSep "\n" backup.exclude)}";
+        filesFromTmpFile = "/run/restic-backups-${name}/includes";
+        fileBackup = (backup.dynamicFilesFrom != null) || (backup.paths != null && backup.paths != [ ]);
+        commandBackup = backup.command != [ ];
+        doBackup = fileBackup || commandBackup;
+        pruneCmd = lib.optionals (builtins.length backup.pruneOpts > 0) [
+          (resticCmd + " unlock")
+          (resticCmd + " forget --prune " + (lib.concatStringsSep " " backup.pruneOpts))
+        ];
+        checkCmd = lib.optionals backup.runCheck [
+          (resticCmd + " check " + (lib.concatStringsSep " " backup.checkOpts))
+        ];
+        # Helper functions for rclone remotes
+        rcloneRemoteName = builtins.elemAt (lib.splitString ":" backup.repository) 1;
+        rcloneAttrToOpt = v: "RCLONE_" + lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] v);
+        rcloneAttrToConf = v: "RCLONE_CONFIG_" + lib.toUpper (rcloneRemoteName + "_" + v);
+        toRcloneVal = v: if lib.isBool v then lib.boolToString v else v;
+      in
+      lib.nameValuePair "restic-backups-${name}" (
+        {
+          environment = {
+            # not %C, because that wouldn't work in the wrapper script
+            RESTIC_CACHE_DIR = "/var/cache/restic-backups-${name}";
+            RESTIC_PASSWORD_FILE = backup.passwordFile;
+            RESTIC_REPOSITORY = backup.repository;
+            RESTIC_REPOSITORY_FILE = backup.repositoryFile;
+          }
+          // lib.optionalAttrs (backup.rcloneOptions != null) (
+            lib.mapAttrs' (
+              name: value: lib.nameValuePair (rcloneAttrToOpt name) (toRcloneVal value)
+            ) backup.rcloneOptions
+          )
+          // lib.optionalAttrs (backup.rcloneConfigFile != null) {
+            RCLONE_CONFIG = backup.rcloneConfigFile;
+          }
+          // lib.optionalAttrs (backup.rcloneConfig != null) (
+            lib.mapAttrs' (
+              name: value: lib.nameValuePair (rcloneAttrToConf name) (toRcloneVal value)
+            ) backup.rcloneConfig
+          )
+          // lib.optionalAttrs (backup.progressFps != null) {
+            RESTIC_PROGRESS_FPS = toString backup.progressFps;
+          };
+          path = [ config.programs.ssh.package ];
+          restartIfChanged = false;
+          wants = [ "network-online.target" ];
+          after = [ "network-online.target" ];
+          serviceConfig = {
+            Type = "oneshot";
+            ExecStart =
+              lib.optionals doBackup [
+                "${resticCmd} backup ${
+                  lib.concatStringsSep " " (
+                    backup.extraBackupArgs
+                    ++ lib.optionals fileBackup (excludeFlags ++ [ "--files-from=${filesFromTmpFile}" ])
+                    ++ lib.optionals commandBackup ([ "--stdin-from-command=true --" ] ++ backup.command)
+                  )
+                }"
+              ]
+              ++ pruneCmd
+              ++ checkCmd;
+            User = backup.user;
+            RuntimeDirectory = "restic-backups-${name}";
+            CacheDirectory = "restic-backups-${name}";
+            CacheDirectoryMode = "0700";
+            PrivateTmp = true;
+          }
+          // lib.optionalAttrs (backup.environmentFile != null) {
+            EnvironmentFile = backup.environmentFile;
+          };
+        }
+        // lib.optionalAttrs (backup.initialize || doBackup || backup.backupPrepareCommand != null) {
+          preStart = ''
+            ${lib.optionalString (backup.backupPrepareCommand != null) ''
+              ${pkgs.writeScript "backupPrepareCommand" backup.backupPrepareCommand}
+            ''}
+            ${lib.optionalString backup.initialize ''
+              ${resticCmd} cat config > /dev/null || ${resticCmd} init
+            ''}
+            ${lib.optionalString (backup.paths != null && backup.paths != [ ]) ''
+              cat ${pkgs.writeText "staticPaths" (lib.concatLines backup.paths)} >> ${filesFromTmpFile}
+            ''}
+            ${lib.optionalString (backup.dynamicFilesFrom != null) ''
+              ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} >> ${filesFromTmpFile}
+            ''}
+          '';
+        }
+        // lib.optionalAttrs (doBackup || backup.backupCleanupCommand != null) {
+          postStop = ''
+            ${lib.optionalString (backup.backupCleanupCommand != null) ''
+              ${pkgs.writeScript "backupCleanupCommand" backup.backupCleanupCommand}
+            ''}
+            ${lib.optionalString fileBackup ''
+              rm ${filesFromTmpFile}
+            ''}
+          '';
+        }
+      )
+    ) config.services.restic.backups;
+    systemd.timers = lib.mapAttrs' (
+      name: backup:
+      lib.nameValuePair "restic-backups-${name}" {
+        wantedBy = [ "timers.target" ];
+        inherit (backup) timerConfig;
+      }
+    ) (lib.filterAttrs (_: backup: backup.timerConfig != null) config.services.restic.backups);
+
+    # generate wrapper scripts, as described in the createWrapper option
+    environment.systemPackages = lib.mapAttrsToList (
+      name: backup:
+      let
+        extraOptions = lib.concatMapStrings (arg: " -o ${arg}") backup.extraOptions;
+        resticCmd = "${lib.getExe backup.package}${extraOptions}";
+      in
+      pkgs.writeShellScriptBin "restic-${name}" ''
+        set -a  # automatically export variables
+        ${lib.optionalString (backup.environmentFile != null) "source ${backup.environmentFile}"}
+        # set same environment variables as the systemd service
+        ${lib.pipe config.systemd.services."restic-backups-${name}".environment [
+          (lib.filterAttrs (n: v: v != null && n != "PATH"))
+          (lib.mapAttrs (_: v: "${v}"))
+          lib.toShellVars
+        ]}
+        PATH=${config.systemd.services."restic-backups-${name}".environment.PATH}:$PATH
+
+        exec ${resticCmd} "$@"
+      ''
+    ) (lib.filterAttrs (_: v: v.createWrapper) config.services.restic.backups);
+  };
+}

nixos/modules/services/backup/rsnapshot.nix

@@ -0,0 +1,83 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.rsnapshot;
+  cfgfile = pkgs.writeText "rsnapshot.conf" ''
+    config_version	1.2
+    cmd_cp	${pkgs.coreutils}/bin/cp
+    cmd_rm	${pkgs.coreutils}/bin/rm
+    cmd_rsync	${pkgs.rsync}/bin/rsync
+    cmd_ssh	${pkgs.openssh}/bin/ssh
+    cmd_logger	${pkgs.inetutils}/bin/logger
+    cmd_du	${pkgs.coreutils}/bin/du
+    cmd_rsnapshot_diff	${pkgs.rsnapshot}/bin/rsnapshot-diff
+    lockfile	/run/rsnapshot.pid
+    link_dest	1
+
+    ${cfg.extraConfig}
+  '';
+in
+{
+  options = {
+    services.rsnapshot = {
+      enable = lib.mkEnableOption "rsnapshot backups";
+      enableManualRsnapshot = lib.mkOption {
+        description = "Whether to enable manual usage of the rsnapshot command with this module.";
+        default = true;
+        type = lib.types.bool;
+      };
+
+      extraConfig = lib.mkOption {
+        default = "";
+        example = ''
+          retains	hourly	24
+          retain	daily	365
+          backup	/home/	localhost/
+        '';
+        type = lib.types.lines;
+        description = ''
+          rsnapshot configuration option in addition to the defaults from
+          rsnapshot and this module.
+
+          Note that tabs are required to separate option arguments, and
+          directory names require trailing slashes.
+
+          The "extra" in the option name might be a little misleading right
+          now, as it is required to get a functional configuration.
+        '';
+      };
+
+      cronIntervals = lib.mkOption {
+        default = { };
+        example = {
+          hourly = "0 * * * *";
+          daily = "50 21 * * *";
+        };
+        type = lib.types.attrsOf lib.types.str;
+        description = ''
+          Periodicity at which intervals should be run by cron.
+          Note that the intervals also have to exist in configuration
+          as retain options.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable (
+    lib.mkMerge [
+      {
+        services.cron.systemCronJobs = lib.mapAttrsToList (
+          interval: time: "${time} root ${pkgs.rsnapshot}/bin/rsnapshot -c ${cfgfile} ${interval}"
+        ) cfg.cronIntervals;
+      }
+      (lib.mkIf cfg.enableManualRsnapshot {
+        environment.systemPackages = [ pkgs.rsnapshot ];
+        environment.etc."rsnapshot.conf".source = cfgfile;
+      })
+    ]
+  );
+}

nixos/modules/services/backup/sanoid.nix

@@ -0,0 +1,296 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.sanoid;
+
+  datasetSettingsType =
+    with lib.types;
+    (attrsOf (
+      nullOr (oneOf [
+        str
+        int
+        bool
+        (listOf str)
+      ])
+    ))
+    // {
+      description = "dataset/template options";
+    };
+
+  commonOptions = {
+    hourly = lib.mkOption {
+      description = "Number of hourly snapshots.";
+      type = with lib.types; nullOr ints.unsigned;
+      default = null;
+    };
+
+    daily = lib.mkOption {
+      description = "Number of daily snapshots.";
+      type = with lib.types; nullOr ints.unsigned;
+      default = null;
+    };
+
+    monthly = lib.mkOption {
+      description = "Number of monthly snapshots.";
+      type = with lib.types; nullOr ints.unsigned;
+      default = null;
+    };
+
+    yearly = lib.mkOption {
+      description = "Number of yearly snapshots.";
+      type = with lib.types; nullOr ints.unsigned;
+      default = null;
+    };
+
+    autoprune = lib.mkOption {
+      description = "Whether to automatically prune old snapshots.";
+      type = with lib.types; nullOr bool;
+      default = null;
+    };
+
+    autosnap = lib.mkOption {
+      description = "Whether to automatically take snapshots.";
+      type = with lib.types; nullOr bool;
+      default = null;
+    };
+
+    pre_snapshot_script = lib.mkOption {
+      description = "Script to run before taking snapshot.";
+      type = with lib.types; nullOr str;
+      default = null;
+    };
+
+    post_snapshot_script = lib.mkOption {
+      description = "Script to run after taking snapshot.";
+      type = with lib.types; nullOr str;
+      default = null;
+    };
+
+    pruning_script = lib.mkOption {
+      description = "Script to run after pruning snapshot.";
+      type = with lib.types; nullOr str;
+      default = null;
+    };
+
+    no_inconsistent_snapshot = lib.mkOption {
+      description = "Whether to take a snapshot if the pre script fails";
+      type = with lib.types; nullOr bool;
+      default = null;
+    };
+
+    force_post_snapshot_script = lib.mkOption {
+      description = "Whether to run the post script if the pre script fails";
+      type = with lib.types; nullOr bool;
+      default = null;
+    };
+
+    script_timeout = lib.mkOption {
+      description = "Time limit for pre/post/pruning script execution time (<=0 for infinite).";
+      type = with lib.types; nullOr int;
+      default = null;
+    };
+  };
+
+  datasetOptions = rec {
+    use_template = lib.mkOption {
+      description = "Names of the templates to use for this dataset.";
+      type = lib.types.listOf (
+        lib.types.str
+        // {
+          check = (lib.types.enum (lib.attrNames cfg.templates)).check;
+          description = "configured template name";
+        }
+      );
+      default = [ ];
+    };
+    useTemplate = use_template;
+
+    recursive = lib.mkOption {
+      description = ''
+        Whether to recursively snapshot dataset children.
+        You can also set this to `"zfs"` to handle datasets
+        recursively in an atomic way without the possibility to
+        override settings for child datasets.
+      '';
+      type =
+        with lib.types;
+        oneOf [
+          bool
+          (enum [ "zfs" ])
+        ];
+      default = false;
+    };
+
+    process_children_only = lib.mkOption {
+      description = "Whether to only snapshot child datasets if recursing.";
+      type = lib.types.bool;
+      default = false;
+    };
+    processChildrenOnly = process_children_only;
+  };
+
+  # Extract unique dataset names
+  datasets = lib.unique (lib.attrNames cfg.datasets);
+
+  # Function to build "zfs allow" and "zfs unallow" commands for the
+  # filesystems we've delegated permissions to.
+  buildAllowCommand =
+    zfsAction: permissions: dataset:
+    lib.escapeShellArgs [
+      # Here we explicitly use the booted system to guarantee the stable API needed by ZFS
+      "-+/run/booted-system/sw/bin/zfs"
+      zfsAction
+      "sanoid"
+      (lib.concatStringsSep "," permissions)
+      dataset
+    ];
+
+  configFile =
+    let
+      mkValueString =
+        v: if lib.isList v then lib.concatStringsSep "," v else lib.generators.mkValueStringDefault { } v;
+
+      mkKeyValue =
+        k: v:
+        if v == null then
+          ""
+        else if k == "processChildrenOnly" then
+          ""
+        else if k == "useTemplate" then
+          ""
+        else
+          lib.generators.mkKeyValueDefault { inherit mkValueString; } "=" k v;
+    in
+    lib.generators.toINI { inherit mkKeyValue; } cfg.settings;
+
+in
+{
+
+  # Interface
+
+  options.services.sanoid = {
+    enable = lib.mkEnableOption "Sanoid ZFS snapshotting service";
+
+    package = lib.mkPackageOption pkgs "sanoid" { };
+
+    interval = lib.mkOption {
+      type = lib.types.str;
+      default = "hourly";
+      example = "daily";
+      description = ''
+        Run sanoid at this interval. The default is to run hourly.
+
+        The format is described in
+        {manpage}`systemd.time(7)`.
+      '';
+    };
+
+    datasets = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { config, options, ... }:
+          {
+            freeformType = datasetSettingsType;
+            options = commonOptions // datasetOptions;
+            config.use_template = lib.modules.mkAliasAndWrapDefsWithPriority lib.id (
+              options.useTemplate or { }
+            );
+            config.process_children_only = lib.modules.mkAliasAndWrapDefsWithPriority lib.id (
+              options.processChildrenOnly or { }
+            );
+          }
+        )
+      );
+      default = { };
+      description = "Datasets to snapshot.";
+    };
+
+    templates = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule {
+          freeformType = datasetSettingsType;
+          options = commonOptions;
+        }
+      );
+      default = { };
+      description = "Templates for datasets.";
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.attrsOf datasetSettingsType;
+      description = ''
+        Free-form settings written directly to the config file. See
+        <https://github.com/jimsalterjrs/sanoid/blob/master/sanoid.defaults.conf>
+        for allowed values.
+      '';
+    };
+
+    extraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [
+        "--verbose"
+        "--readonly"
+        "--debug"
+      ];
+      description = ''
+        Extra arguments to pass to sanoid. See
+        <https://github.com/jimsalterjrs/sanoid/#sanoid-command-line-options>
+        for allowed options.
+      '';
+    };
+  };
+
+  # Implementation
+
+  config = lib.mkIf cfg.enable {
+    services.sanoid.settings = lib.mkMerge [
+      (lib.mapAttrs' (d: v: lib.nameValuePair ("template_" + d) v) cfg.templates)
+      (lib.mapAttrs (d: v: v) cfg.datasets)
+    ];
+
+    systemd.services.sanoid = {
+      description = "Sanoid snapshot service";
+      serviceConfig = {
+        ExecStartPre = (
+          map (buildAllowCommand "allow" [
+            "snapshot"
+            "mount"
+            "destroy"
+          ]) datasets
+        );
+        ExecStopPost = (
+          map (buildAllowCommand "unallow" [
+            "snapshot"
+            "mount"
+            "destroy"
+          ]) datasets
+        );
+        ExecStart = lib.escapeShellArgs (
+          [
+            "${cfg.package}/bin/sanoid"
+            "--cron"
+            "--configdir"
+            (pkgs.writeTextDir "sanoid.conf" configFile)
+          ]
+          ++ cfg.extraArgs
+        );
+        User = "sanoid";
+        Group = "sanoid";
+        DynamicUser = true;
+        RuntimeDirectory = "sanoid";
+        CacheDirectory = "sanoid";
+      };
+      # Prevents missing snapshots during DST changes
+      environment.TZ = "UTC";
+      after = [ "zfs.target" ];
+      startAt = cfg.interval;
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ lopsided98 ];
+}

nixos/modules/services/backup/snapraid.nix

@@ -0,0 +1,240 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.snapraid;
+in
+{
+  imports = [
+    # Should have never been on the top-level.
+    (lib.mkRenamedOptionModule [ "snapraid" ] [ "services" "snapraid" ])
+  ];
+
+  options.services.snapraid = with lib.types; {
+    enable = lib.mkEnableOption "SnapRAID";
+    dataDisks = lib.mkOption {
+      default = { };
+      example = {
+        d1 = "/mnt/disk1/";
+        d2 = "/mnt/disk2/";
+        d3 = "/mnt/disk3/";
+      };
+      description = "SnapRAID data disks.";
+      type = attrsOf str;
+    };
+    parityFiles = lib.mkOption {
+      default = [ ];
+      example = [
+        "/mnt/diskp/snapraid.parity"
+        "/mnt/diskq/snapraid.2-parity"
+        "/mnt/diskr/snapraid.3-parity"
+        "/mnt/disks/snapraid.4-parity"
+        "/mnt/diskt/snapraid.5-parity"
+        "/mnt/disku/snapraid.6-parity"
+      ];
+      description = "SnapRAID parity files.";
+      type = listOf str;
+    };
+    contentFiles = lib.mkOption {
+      default = [ ];
+      example = [
+        "/var/snapraid.content"
+        "/mnt/disk1/snapraid.content"
+        "/mnt/disk2/snapraid.content"
+      ];
+      description = "SnapRAID content list files.";
+      type = listOf str;
+    };
+    exclude = lib.mkOption {
+      default = [ ];
+      example = [
+        "*.unrecoverable"
+        "/tmp/"
+        "/lost+found/"
+      ];
+      description = "SnapRAID exclude directives.";
+      type = listOf str;
+    };
+    touchBeforeSync = lib.mkOption {
+      default = true;
+      example = false;
+      description = "Whether {command}`snapraid touch` should be run before {command}`snapraid sync`.";
+      type = bool;
+    };
+    sync.interval = lib.mkOption {
+      default = "01:00";
+      example = "daily";
+      description = "How often to run {command}`snapraid sync`.";
+      type = str;
+    };
+    scrub = {
+      interval = lib.mkOption {
+        default = "Mon *-*-* 02:00:00";
+        example = "weekly";
+        description = "How often to run {command}`snapraid scrub`.";
+        type = str;
+      };
+      plan = lib.mkOption {
+        default = 8;
+        example = 5;
+        description = "Percent of the array that should be checked by {command}`snapraid scrub`.";
+        type = int;
+      };
+      olderThan = lib.mkOption {
+        default = 10;
+        example = 20;
+        description = "Number of days since data was last scrubbed before it can be scrubbed again.";
+        type = int;
+      };
+    };
+    extraConfig = lib.mkOption {
+      default = "";
+      example = ''
+        nohidden
+        blocksize 256
+        hashsize 16
+        autosave 500
+        pool /pool
+      '';
+      description = "Extra config options for SnapRAID.";
+      type = lines;
+    };
+  };
+
+  config =
+    let
+      nParity = builtins.length cfg.parityFiles;
+      mkPrepend = pre: s: pre + s;
+    in
+    lib.mkIf cfg.enable {
+      assertions = [
+        {
+          assertion = nParity <= 6;
+          message = "You can have no more than six SnapRAID parity files.";
+        }
+        {
+          assertion = builtins.length cfg.contentFiles >= nParity + 1;
+          message = "There must be at least one SnapRAID content file for each SnapRAID parity file plus one.";
+        }
+      ];
+
+      environment = {
+        systemPackages = with pkgs; [ snapraid ];
+
+        etc."snapraid.conf" = {
+          text =
+            with cfg;
+            let
+              prependData = mkPrepend "data ";
+              prependContent = mkPrepend "content ";
+              prependExclude = mkPrepend "exclude ";
+            in
+            lib.concatStringsSep "\n" (
+              map prependData ((lib.mapAttrsToList (name: value: name + " " + value)) dataDisks)
+              ++ lib.zipListsWith (a: b: a + b) (
+                [ "parity " ] ++ map (i: toString i + "-parity ") (lib.range 2 6)
+              ) parityFiles
+              ++ map prependContent contentFiles
+              ++ map prependExclude exclude
+            )
+            + "\n"
+            + extraConfig;
+        };
+      };
+
+      systemd.services = with cfg; {
+        snapraid-scrub = {
+          description = "Scrub the SnapRAID array";
+          startAt = scrub.interval;
+          serviceConfig = {
+            Type = "oneshot";
+            ExecStart = "${pkgs.snapraid}/bin/snapraid scrub -p ${toString scrub.plan} -o ${toString scrub.olderThan}";
+            Nice = 19;
+            IOSchedulingPriority = 7;
+            CPUSchedulingPolicy = "batch";
+
+            LockPersonality = true;
+            MemoryDenyWriteExecute = true;
+            NoNewPrivileges = true;
+            PrivateDevices = true;
+            PrivateTmp = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            RestrictAddressFamilies = "none";
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+            SystemCallArchitectures = "native";
+            SystemCallFilter = "@system-service";
+            SystemCallErrorNumber = "EPERM";
+            CapabilityBoundingSet = "CAP_DAC_OVERRIDE";
+
+            ProtectSystem = "strict";
+            ProtectHome = "read-only";
+            ReadWritePaths =
+              # scrub requires access to directories containing content files
+              # to remove them if they are stale
+              let
+                contentDirs = map dirOf contentFiles;
+              in
+              lib.unique (lib.attrValues dataDisks ++ contentDirs);
+          };
+          unitConfig.After = "snapraid-sync.service";
+        };
+        snapraid-sync = {
+          description = "Synchronize the state of the SnapRAID array";
+          startAt = sync.interval;
+          serviceConfig = {
+            Type = "oneshot";
+            ExecStart = "${pkgs.snapraid}/bin/snapraid sync";
+            Nice = 19;
+            IOSchedulingPriority = 7;
+            CPUSchedulingPolicy = "batch";
+
+            LockPersonality = true;
+            MemoryDenyWriteExecute = true;
+            NoNewPrivileges = true;
+            PrivateTmp = true;
+            ProtectClock = true;
+            ProtectControlGroups = true;
+            ProtectHostname = true;
+            ProtectKernelLogs = true;
+            ProtectKernelModules = true;
+            ProtectKernelTunables = true;
+            RestrictAddressFamilies = "none";
+            RestrictNamespaces = true;
+            RestrictRealtime = true;
+            RestrictSUIDSGID = true;
+            SystemCallArchitectures = "native";
+            SystemCallFilter = "@system-service";
+            SystemCallErrorNumber = "EPERM";
+            CapabilityBoundingSet = "CAP_DAC_OVERRIDE" + lib.optionalString cfg.touchBeforeSync " CAP_FOWNER";
+
+            ProtectSystem = "strict";
+            ProtectHome = "read-only";
+            ReadWritePaths =
+              # sync requires access to directories containing content files
+              # to remove them if they are stale
+              let
+                contentDirs = map dirOf contentFiles;
+                # Multiple "split" parity files can be specified in a single
+                # "parityFile", separated by a comma.
+                # https://www.snapraid.it/manual#7.1
+                splitParityFiles = map (s: lib.splitString "," s) parityFiles;
+              in
+              lib.unique (lib.attrValues dataDisks ++ splitParityFiles ++ contentDirs);
+          }
+          // lib.optionalAttrs touchBeforeSync {
+            ExecStartPre = "${pkgs.snapraid}/bin/snapraid touch";
+          };
+        };
+      };
+    };
+}

nixos/modules/services/backup/syncoid.nix

@@ -0,0 +1,481 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.syncoid;
+
+  # Extract local dasaset names (so no datasets containing "@")
+  localDatasetName =
+    d:
+    lib.optionals (d != null) (
+      let
+        m = builtins.match "([^/@]+[^@]*)" d;
+      in
+      lib.optionals (m != null) m
+    );
+
+  # Escape as required by: https://www.freedesktop.org/software/systemd/man/systemd.unit.html
+  escapeUnitName =
+    name:
+    lib.concatMapStrings (s: if lib.isList s then "-" else s) (
+      builtins.split "[^a-zA-Z0-9_.\\-]+" name
+    );
+
+  # Function to build "zfs allow" commands for the filesystems we've delegated
+  # permissions to. It also checks if the target dataset exists before
+  # delegating permissions, if it doesn't exist we delegate it to the parent
+  # dataset (if it exists). This should solve the case of provisoning new
+  # datasets.
+  buildAllowCommand =
+    permissions: dataset:
+
+    "-+${pkgs.writeShellScript "zfs-allow-${dataset}" ''
+      # Here we explicitly use the booted system to guarantee the stable API needed by ZFS
+
+      # Run a ZFS list on the dataset to check if it exists
+      if ${
+        lib.escapeShellArgs [
+          "/run/booted-system/sw/bin/zfs"
+          "list"
+          dataset
+        ]
+      } 2> /dev/null; then
+        ${lib.escapeShellArgs [
+          "/run/booted-system/sw/bin/zfs"
+          "allow"
+          cfg.user
+          (lib.concatStringsSep "," permissions)
+          dataset
+        ]}
+      ${lib.optionalString ((builtins.dirOf dataset) != ".") ''
+        else
+          ${lib.escapeShellArgs [
+            "/run/booted-system/sw/bin/zfs"
+            "allow"
+            cfg.user
+            (lib.concatStringsSep "," permissions)
+            # Remove the last part of the path
+            (builtins.dirOf dataset)
+          ]}
+      ''}
+      fi
+    ''}";
+
+  # Function to build "zfs unallow" commands for the filesystems we've
+  # delegated permissions to. Here we unallow both the target but also
+  # on the parent dataset because at this stage we have no way of
+  # knowing if the allow command did execute on the parent dataset or
+  # not in the pre-hook. We can't run the same if in the post hook
+  # since the dataset should have been created at this point.
+  buildUnallowCommand =
+    permissions: dataset:
+
+    "-+${pkgs.writeShellScript "zfs-unallow-${dataset}" ''
+      # Here we explicitly use the booted system to guarantee the stable API needed by ZFS
+      ${lib.escapeShellArgs [
+        "/run/booted-system/sw/bin/zfs"
+        "unallow"
+        cfg.user
+        (lib.concatStringsSep "," permissions)
+        dataset
+      ]}
+      ${lib.optionalString ((builtins.dirOf dataset) != ".") (
+        lib.escapeShellArgs [
+          "/run/booted-system/sw/bin/zfs"
+          "unallow"
+          cfg.user
+          (lib.concatStringsSep "," permissions)
+          # Remove the last part of the path
+          (builtins.dirOf dataset)
+        ]
+      )}
+    ''}";
+in
+{
+
+  # Interface
+
+  options.services.syncoid = {
+    enable = lib.mkEnableOption "Syncoid ZFS synchronization service";
+
+    package = lib.mkPackageOption pkgs "sanoid" { };
+
+    interval = lib.mkOption {
+      type = with lib.types; either str (listOf str);
+      default = "hourly";
+      example = "*-*-* *:15:00";
+      description = ''
+        Run syncoid at this interval. The default is to run hourly.
+
+        Must be in the format described in {manpage}`systemd.time(7)`.  This is
+        equivalent to adding a corresponding timer unit with
+        {option}`OnCalendar` set to the value given here.
+
+        Set to an empty list to avoid starting syncoid automatically.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "syncoid";
+      example = "backup";
+      description = ''
+        The user for the service. ZFS privilege delegation will be
+        automatically configured for any local pools used by syncoid if this
+        option is set to a user other than root. The user will be given the
+        "hold" and "send" privileges on any pool that has datasets being sent
+        and the "create", "mount", "receive", and "rollback" privileges on
+        any pool that has datasets being received.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "syncoid";
+      example = "backup";
+      description = "The group for the service.";
+    };
+
+    sshKey = lib.mkOption {
+      type = with lib.types; nullOr (coercedTo path toString str);
+      default = null;
+      description = ''
+        SSH private key file to use to login to the remote system. Can be
+        overridden in individual commands.
+      '';
+    };
+
+    localSourceAllow = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      # Permissions snapshot and destroy are in case --no-sync-snap is not used
+      default = [
+        "bookmark"
+        "hold"
+        "send"
+        "snapshot"
+        "destroy"
+        "mount"
+      ];
+      description = ''
+        Permissions granted for the {option}`services.syncoid.user` user
+        for local source datasets. See
+        <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
+        for available permissions.
+      '';
+    };
+
+    localTargetAllow = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [
+        "change-key"
+        "compression"
+        "create"
+        "mount"
+        "mountpoint"
+        "receive"
+        "rollback"
+      ];
+      example = [
+        "create"
+        "mount"
+        "receive"
+        "rollback"
+      ];
+      description = ''
+        Permissions granted for the {option}`services.syncoid.user` user
+        for local target datasets. See
+        <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
+        for available permissions.
+        Make sure to include the `change-key` permission if you send raw encrypted datasets,
+        the `compression` permission if you send raw compressed datasets, and so on.
+        For remote target datasets you'll have to set your remote user permissions by yourself.
+      '';
+    };
+
+    commonArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      example = [ "--no-sync-snap" ];
+      description = ''
+        Arguments to add to every syncoid command, unless disabled for that
+        command. See
+        <https://github.com/jimsalterjrs/sanoid/#syncoid-command-line-options>
+        for available options.
+      '';
+    };
+
+    service = lib.mkOption {
+      type = lib.types.attrs;
+      default = { };
+      description = ''
+        Systemd configuration common to all syncoid services.
+      '';
+    };
+
+    commands = lib.mkOption {
+      type = lib.types.attrsOf (
+        lib.types.submodule (
+          { name, ... }:
+          {
+            options = {
+              source = lib.mkOption {
+                type = lib.types.str;
+                example = "pool/dataset";
+                description = ''
+                  Source ZFS dataset. Can be either local or remote. Defaults to
+                  the attribute name.
+                '';
+              };
+
+              target = lib.mkOption {
+                type = lib.types.str;
+                example = "user@server:pool/dataset";
+                description = ''
+                  Target ZFS dataset. Can be either local
+                  («pool/dataset») or remote
+                  («user@server:pool/dataset»).
+                '';
+              };
+
+              recursive = lib.mkEnableOption ''the transfer of child datasets'';
+
+              sshKey = lib.mkOption {
+                type = with lib.types; nullOr (coercedTo path toString str);
+                description = ''
+                  SSH private key file to use to login to the remote system.
+                  Defaults to {option}`services.syncoid.sshKey` option.
+                '';
+              };
+
+              localSourceAllow = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                description = ''
+                  Permissions granted for the {option}`services.syncoid.user` user
+                  for local source datasets. See
+                  <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
+                  for available permissions.
+                  Defaults to {option}`services.syncoid.localSourceAllow` option.
+                '';
+              };
+
+              localTargetAllow = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                description = ''
+                  Permissions granted for the {option}`services.syncoid.user` user
+                  for local target datasets. See
+                  <https://openzfs.github.io/openzfs-docs/man/8/zfs-allow.8.html>
+                  for available permissions.
+                  Make sure to include the `change-key` permission if you send raw encrypted datasets,
+                  the `compression` permission if you send raw compressed datasets, and so on.
+                  For remote target datasets you'll have to set your remote user permissions by yourself.
+                '';
+              };
+
+              sendOptions = lib.mkOption {
+                type = lib.types.separatedString " ";
+                default = "";
+                example = "Lc e";
+                description = ''
+                  Advanced options to pass to zfs send. Options are specified
+                  without their leading dashes and separated by spaces.
+                '';
+              };
+
+              recvOptions = lib.mkOption {
+                type = lib.types.separatedString " ";
+                default = "";
+                example = "ux recordsize o compression=lz4";
+                description = ''
+                  Advanced options to pass to zfs recv. Options are specified
+                  without their leading dashes and separated by spaces.
+                '';
+              };
+
+              useCommonArgs = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = ''
+                  Whether to add the configured common arguments to this command.
+                '';
+              };
+
+              service = lib.mkOption {
+                type = lib.types.attrs;
+                default = { };
+                description = ''
+                  Systemd configuration specific to this syncoid service.
+                '';
+              };
+
+              extraArgs = lib.mkOption {
+                type = lib.types.listOf lib.types.str;
+                default = [ ];
+                example = [ "--sshport 2222" ];
+                description = "Extra syncoid arguments for this command.";
+              };
+            };
+            config = {
+              source = lib.mkDefault name;
+              sshKey = lib.mkDefault cfg.sshKey;
+              localSourceAllow = lib.mkDefault cfg.localSourceAllow;
+              localTargetAllow = lib.mkDefault cfg.localTargetAllow;
+            };
+          }
+        )
+      );
+      default = { };
+      example = lib.literalExpression ''
+        {
+          "pool/test".target = "root@target:pool/test";
+        }
+      '';
+      description = "Syncoid commands to run.";
+    };
+  };
+
+  # Implementation
+
+  config = lib.mkIf cfg.enable {
+    users = {
+      users = lib.mkIf (cfg.user == "syncoid") {
+        syncoid = {
+          group = cfg.group;
+          isSystemUser = true;
+          # For syncoid to be able to create /var/lib/syncoid/.ssh/
+          # and to use custom ssh_config or known_hosts.
+          home = "/var/lib/syncoid";
+          createHome = false;
+        };
+      };
+      groups = lib.mkIf (cfg.group == "syncoid") {
+        syncoid = { };
+      };
+    };
+
+    systemd.services = lib.mapAttrs' (
+      name: c:
+      lib.nameValuePair "syncoid-${escapeUnitName name}" (
+        lib.mkMerge [
+          {
+            description = "Syncoid ZFS synchronization from ${c.source} to ${c.target}";
+            after = [ "zfs.target" ];
+            startAt = cfg.interval;
+            # syncoid may need zpool to get feature@extensible_dataset
+            path = [ "/run/booted-system/sw/bin/" ];
+            serviceConfig = {
+              ExecStartPre =
+                (map (buildAllowCommand c.localSourceAllow) (localDatasetName c.source))
+                ++ (map (buildAllowCommand c.localTargetAllow) (localDatasetName c.target));
+              ExecStopPost =
+                (map (buildUnallowCommand c.localSourceAllow) (localDatasetName c.source))
+                ++ (map (buildUnallowCommand c.localTargetAllow) (localDatasetName c.target));
+              ExecStart = lib.escapeShellArgs (
+                [ "${cfg.package}/bin/syncoid" ]
+                ++ lib.optionals c.useCommonArgs cfg.commonArgs
+                ++ lib.optional c.recursive "-r"
+                ++ lib.optionals (c.sshKey != null) [
+                  "--sshkey"
+                  c.sshKey
+                ]
+                ++ c.extraArgs
+                ++ [
+                  "--sendoptions"
+                  c.sendOptions
+                  "--recvoptions"
+                  c.recvOptions
+                  "--no-privilege-elevation"
+                  c.source
+                  c.target
+                ]
+              );
+              User = cfg.user;
+              Group = cfg.group;
+              StateDirectory = [ "syncoid" ];
+              StateDirectoryMode = "700";
+              # Prevent SSH control sockets of different syncoid services from interfering
+              PrivateTmp = true;
+              # Permissive access to /proc because syncoid
+              # calls ps(1) to detect ongoing `zfs receive`.
+              ProcSubset = "all";
+              ProtectProc = "default";
+
+              # The following options are only for optimizing:
+              # systemd-analyze security | grep syncoid-'*'
+              AmbientCapabilities = "";
+              CapabilityBoundingSet = "";
+              DeviceAllow = [ "/dev/zfs" ];
+              LockPersonality = true;
+              MemoryDenyWriteExecute = true;
+              NoNewPrivileges = true;
+              PrivateDevices = true;
+              PrivateMounts = true;
+              PrivateNetwork = lib.mkDefault false;
+              PrivateUsers = false; # Enabling this breaks on zfs-2.2.0
+              ProtectClock = true;
+              ProtectControlGroups = true;
+              ProtectHome = true;
+              ProtectHostname = true;
+              ProtectKernelLogs = true;
+              ProtectKernelModules = true;
+              ProtectKernelTunables = true;
+              ProtectSystem = "strict";
+              RemoveIPC = true;
+              RestrictAddressFamilies = [
+                "AF_UNIX"
+                "AF_INET"
+                "AF_INET6"
+              ];
+              RestrictNamespaces = true;
+              RestrictRealtime = true;
+              RestrictSUIDSGID = true;
+              RootDirectory = "/run/syncoid/${escapeUnitName name}";
+              RootDirectoryStartOnly = true;
+              BindPaths = [ "/dev/zfs" ];
+              BindReadOnlyPaths = [
+                builtins.storeDir
+                "/etc"
+                "/run"
+                "/bin/sh"
+              ];
+              # Avoid useless mounting of RootDirectory= in the own RootDirectory= of ExecStart='s mount namespace.
+              InaccessiblePaths = [ "-+/run/syncoid/${escapeUnitName name}" ];
+              MountAPIVFS = true;
+              # Create RootDirectory= in the host's mount namespace.
+              RuntimeDirectory = [ "syncoid/${escapeUnitName name}" ];
+              RuntimeDirectoryMode = "700";
+              SystemCallFilter = [
+                "@system-service"
+                # Groups in @system-service which do not contain a syscall listed by:
+                # perf stat -x, 2>perf.log -e 'syscalls:sys_enter_*' syncoid …
+                # awk >perf.syscalls -F "," '$1 > 0 {sub("syscalls:sys_enter_","",$3); print $3}' perf.log
+                # systemd-analyze syscall-filter | grep -v -e '#' | sed -e ':loop; /^[^ ]/N; s/\n //; t loop' | grep $(printf ' -e \\<%s\\>' $(cat perf.syscalls)) | cut -f 1 -d ' '
+                "~@aio"
+                "~@chown"
+                "~@keyring"
+                "~@memlock"
+                "~@privileged"
+                "~@resources"
+                "~@setuid"
+                "~@timer"
+              ];
+              SystemCallArchitectures = "native";
+              # This is for BindPaths= and BindReadOnlyPaths=
+              # to allow traversal of directories they create in RootDirectory=.
+              UMask = "0066";
+            };
+          }
+          cfg.service
+          c.service
+        ]
+      )
+    ) cfg.commands;
+  };
+
+  meta.maintainers = with lib.maintainers; [
+    julm
+    lopsided98
+  ];
+}

nixos/modules/services/backup/tarsnap.nix

@@ -0,0 +1,451 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  utils,
+  ...
+}:
+let
+  gcfg = config.services.tarsnap;
+  opt = options.services.tarsnap;
+
+  configFile = name: cfg: ''
+    keyfile ${cfg.keyfile}
+    ${lib.optionalString (cfg.cachedir != null) "cachedir ${cfg.cachedir}"}
+    ${lib.optionalString cfg.nodump "nodump"}
+    ${lib.optionalString cfg.printStats "print-stats"}
+    ${lib.optionalString cfg.printStats "humanize-numbers"}
+    ${lib.optionalString (cfg.checkpointBytes != null) ("checkpoint-bytes " + cfg.checkpointBytes)}
+    ${lib.optionalString cfg.aggressiveNetworking "aggressive-networking"}
+    ${lib.concatStringsSep "\n" (map (v: "exclude ${v}") cfg.excludes)}
+    ${lib.concatStringsSep "\n" (map (v: "include ${v}") cfg.includes)}
+    ${lib.optionalString cfg.lowmem "lowmem"}
+    ${lib.optionalString cfg.verylowmem "verylowmem"}
+    ${lib.optionalString (cfg.maxbw != null) "maxbw ${toString cfg.maxbw}"}
+    ${lib.optionalString (cfg.maxbwRateUp != null) "maxbw-rate-up ${toString cfg.maxbwRateUp}"}
+    ${lib.optionalString (cfg.maxbwRateDown != null) "maxbw-rate-down ${toString cfg.maxbwRateDown}"}
+  '';
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "services"
+      "tarsnap"
+      "cachedir"
+    ] "Use services.tarsnap.archives.<name>.cachedir")
+  ];
+
+  options = {
+    services.tarsnap = {
+      enable = lib.mkEnableOption "periodic tarsnap backups";
+
+      package = lib.mkPackageOption pkgs "tarsnap" { };
+
+      keyfile = lib.mkOption {
+        type = lib.types.str;
+        default = "/root/tarsnap.key";
+        description = ''
+          The keyfile which associates this machine with your tarsnap
+          account.
+          Create the keyfile with {command}`tarsnap-keygen`.
+
+          Note that each individual archive (specified below) may also have its
+          own individual keyfile specified. Tarsnap does not allow multiple
+          concurrent backups with the same cache directory and key (starting a
+          new backup will cause another one to fail). If you have multiple
+          archives specified, you should either spread out your backups to be
+          far apart, or specify a separate key for each archive. By default
+          every archive defaults to using
+          `"/root/tarsnap.key"`.
+
+          It's recommended for backups that you generate a key for every archive
+          using {manpage}`tarsnap-keygen(1)`, and then generate a
+          write-only tarsnap key using {manpage}`tarsnap-keymgmt(1)`,
+          and keep your master key(s) for a particular machine off-site.
+
+          The keyfile name should be given as a string and not a path, to
+          avoid the key being copied into the Nix store.
+        '';
+      };
+
+      archives = lib.mkOption {
+        type = lib.types.attrsOf (
+          lib.types.submodule (
+            { config, options, ... }:
+            {
+              options = {
+                keyfile = lib.mkOption {
+                  type = lib.types.str;
+                  default = gcfg.keyfile;
+                  defaultText = lib.literalExpression "config.${opt.keyfile}";
+                  description = ''
+                    Set a specific keyfile for this archive. This defaults to
+                    `"/root/tarsnap.key"` if left unspecified.
+
+                    Use this option if you want to run multiple backups
+                    concurrently - each archive must have a unique key. You can
+                    generate a write-only key derived from your master key (which
+                    is recommended) using {manpage}`tarsnap-keymgmt(1)`.
+
+                    Note: every archive must have an individual master key. You
+                    must generate multiple keys with
+                    {manpage}`tarsnap-keygen(1)`, and then generate write
+                    only keys from those.
+
+                    The keyfile name should be given as a string and not a path, to
+                    avoid the key being copied into the Nix store.
+                  '';
+                };
+
+                cachedir = lib.mkOption {
+                  type = lib.types.nullOr lib.types.path;
+                  default = "/var/cache/tarsnap/${utils.escapeSystemdPath config.keyfile}";
+                  defaultText = lib.literalExpression ''
+                    "/var/cache/tarsnap/''${utils.escapeSystemdPath config.${options.keyfile}}"
+                  '';
+                  description = ''
+                    The cache allows tarsnap to identify previously stored data
+                    blocks, reducing archival time and bandwidth usage.
+
+                    Should the cache become desynchronized or corrupted, tarsnap
+                    will refuse to run until you manually rebuild the cache with
+                    {command}`tarsnap --fsck`.
+
+                    Set to `null` to disable caching.
+                  '';
+                };
+
+                nodump = lib.mkOption {
+                  type = lib.types.bool;
+                  default = true;
+                  description = ''
+                    Exclude files with the `nodump` flag.
+                  '';
+                };
+
+                printStats = lib.mkOption {
+                  type = lib.types.bool;
+                  default = true;
+                  description = ''
+                    Print global archive statistics upon completion.
+                    The output is available via
+                    {command}`systemctl status tarsnap-archive-name`.
+                  '';
+                };
+
+                checkpointBytes = lib.mkOption {
+                  type = lib.types.nullOr lib.types.str;
+                  default = "1GB";
+                  description = ''
+                    Create a checkpoint every `checkpointBytes`
+                    of uploaded data (optionally specified using an SI prefix).
+
+                    1GB is the minimum value. A higher value is recommended,
+                    as checkpointing is expensive.
+
+                    Set to `null` to disable checkpointing.
+                  '';
+                };
+
+                period = lib.mkOption {
+                  type = lib.types.str;
+                  default = "01:15";
+                  example = "hourly";
+                  description = ''
+                    Create archive at this interval.
+
+                    The format is described in
+                    {manpage}`systemd.time(7)`.
+                  '';
+                };
+
+                aggressiveNetworking = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Upload data over multiple TCP connections, potentially
+                    increasing tarsnap's bandwidth utilisation at the cost
+                    of slowing down all other network traffic. Not
+                    recommended unless TCP congestion is the dominant
+                    limiting factor.
+                  '';
+                };
+
+                directories = lib.mkOption {
+                  type = lib.types.listOf lib.types.path;
+                  default = [ ];
+                  description = "List of filesystem paths to archive.";
+                };
+
+                excludes = lib.mkOption {
+                  type = lib.types.listOf lib.types.str;
+                  default = [ ];
+                  description = ''
+                    Exclude files and directories matching these patterns.
+                  '';
+                };
+
+                includes = lib.mkOption {
+                  type = lib.types.listOf lib.types.str;
+                  default = [ ];
+                  description = ''
+                    Include only files and directories matching these
+                    patterns (the empty list includes everything).
+
+                    Exclusions have precedence over inclusions.
+                  '';
+                };
+
+                lowmem = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Reduce memory consumption by not caching small files.
+                    Possibly beneficial if the average file size is smaller
+                    than 1 MB and the number of files is lower than the
+                    total amount of RAM in KB.
+                  '';
+                };
+
+                verylowmem = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Reduce memory consumption by a factor of 2 beyond what
+                    `lowmem` does, at the cost of significantly
+                    slowing down the archiving process.
+                  '';
+                };
+
+                maxbw = lib.mkOption {
+                  type = lib.types.nullOr lib.types.int;
+                  default = null;
+                  description = ''
+                    Abort archival if upstream bandwidth usage in bytes
+                    exceeds this threshold.
+                  '';
+                };
+
+                maxbwRateUp = lib.mkOption {
+                  type = lib.types.nullOr lib.types.int;
+                  default = null;
+                  example = lib.literalExpression "25 * 1000";
+                  description = ''
+                    Upload bandwidth rate limit in bytes.
+                  '';
+                };
+
+                maxbwRateDown = lib.mkOption {
+                  type = lib.types.nullOr lib.types.int;
+                  default = null;
+                  example = lib.literalExpression "50 * 1000";
+                  description = ''
+                    Download bandwidth rate limit in bytes.
+                  '';
+                };
+
+                verbose = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Whether to produce verbose logging output.
+                  '';
+                };
+                explicitSymlinks = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Whether to follow symlinks specified as archives.
+                  '';
+                };
+                followSymlinks = lib.mkOption {
+                  type = lib.types.bool;
+                  default = false;
+                  description = ''
+                    Whether to follow all symlinks in archive trees.
+                  '';
+                };
+              };
+            }
+          )
+        );
+
+        default = { };
+
+        example = lib.literalExpression ''
+          {
+            nixos =
+              { directories = [ "/home" "/root/ssl" ];
+              };
+
+            gamedata =
+              { directories = [ "/var/lib/minecraft" ];
+                period      = "*:30";
+              };
+          }
+        '';
+
+        description = ''
+          Tarsnap archive configurations. Each attribute names an archive
+          to be created at a given time interval, according to the options
+          associated with it. When uploading to the tarsnap server,
+          archive names are suffixed by a 1 second resolution timestamp,
+          with the format `%Y%m%d%H%M%S`.
+
+          For each member of the set is created a timer which triggers the
+          instanced `tarsnap-archive-name` service unit. You may use
+          {command}`systemctl start tarsnap-archive-name` to
+          manually trigger creation of `archive-name` at
+          any time.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf gcfg.enable {
+    assertions =
+      (lib.mapAttrsToList (name: cfg: {
+        assertion = cfg.directories != [ ];
+        message = "Must specify paths for tarsnap to back up";
+      }) gcfg.archives)
+      ++ (lib.mapAttrsToList (name: cfg: {
+        assertion = !(cfg.lowmem && cfg.verylowmem);
+        message = "You cannot set both lowmem and verylowmem";
+      }) gcfg.archives);
+
+    systemd.services =
+      (lib.mapAttrs' (
+        name: cfg:
+        lib.nameValuePair "tarsnap-${name}" {
+          description = "Tarsnap archive '${name}'";
+          requires = [ "network-online.target" ];
+          after = [ "network-online.target" ];
+
+          path = with pkgs; [
+            iputils
+            gcfg.package
+            util-linux
+          ];
+
+          # In order for the persistent tarsnap timer to work reliably, we have to
+          # make sure that the tarsnap server is reachable after systemd starts up
+          # the service - therefore we sleep in a loop until we can ping the
+          # endpoint.
+          preStart = ''
+            while ! ping -4 -q -c 1 v1-0-0-server.tarsnap.com &> /dev/null; do sleep 3; done
+          '';
+
+          script =
+            let
+              tarsnap = ''${lib.getExe gcfg.package} --configfile "/etc/tarsnap/${name}.conf"'';
+              run = ''
+                ${tarsnap} -c -f "${name}-$(date +"%Y%m%d%H%M%S")" \
+                                        ${lib.optionalString cfg.verbose "-v"} \
+                                        ${lib.optionalString cfg.explicitSymlinks "-H"} \
+                                        ${lib.optionalString cfg.followSymlinks "-L"} \
+                                        ${lib.concatStringsSep " " cfg.directories}'';
+              cachedir = lib.escapeShellArg cfg.cachedir;
+            in
+            if (cfg.cachedir != null) then
+              ''
+                mkdir -p ${cachedir}
+                chmod 0700 ${cachedir}
+
+                ( flock 9
+                  if [ ! -e ${cachedir}/firstrun ]; then
+                    ( flock 10
+                      flock -u 9
+                      ${tarsnap} --fsck
+                      flock 9
+                    ) 10>${cachedir}/firstrun
+                  fi
+                ) 9>${cachedir}/lockf
+
+                 exec flock ${cachedir}/firstrun ${run}
+              ''
+            else
+              "exec ${run}";
+
+          serviceConfig = {
+            Type = "oneshot";
+            IOSchedulingClass = "idle";
+            NoNewPrivileges = "true";
+            CapabilityBoundingSet = [ "CAP_DAC_READ_SEARCH" ];
+            PermissionsStartOnly = "true";
+          };
+        }
+      ) gcfg.archives)
+      //
+
+        (lib.mapAttrs' (
+          name: cfg:
+          lib.nameValuePair "tarsnap-restore-${name}" {
+            description = "Tarsnap restore '${name}'";
+            requires = [ "network-online.target" ];
+
+            path = with pkgs; [
+              iputils
+              gcfg.package
+              util-linux
+            ];
+
+            script =
+              let
+                tarsnap = ''${lib.getExe gcfg.package} --configfile "/etc/tarsnap/${name}.conf"'';
+                lastArchive = "$(${tarsnap} --list-archives | sort | tail -1)";
+                run = ''${tarsnap} -x -f "${lastArchive}" ${lib.optionalString cfg.verbose "-v"}'';
+                cachedir = lib.escapeShellArg cfg.cachedir;
+
+              in
+              if (cfg.cachedir != null) then
+                ''
+                  mkdir -p ${cachedir}
+                  chmod 0700 ${cachedir}
+
+                  ( flock 9
+                    if [ ! -e ${cachedir}/firstrun ]; then
+                      ( flock 10
+                        flock -u 9
+                        ${tarsnap} --fsck
+                        flock 9
+                      ) 10>${cachedir}/firstrun
+                    fi
+                  ) 9>${cachedir}/lockf
+
+                   exec flock ${cachedir}/firstrun ${run}
+                ''
+              else
+                "exec ${run}";
+
+            serviceConfig = {
+              Type = "oneshot";
+              IOSchedulingClass = "idle";
+              NoNewPrivileges = "true";
+              CapabilityBoundingSet = [ "CAP_DAC_READ_SEARCH" ];
+              PermissionsStartOnly = "true";
+            };
+          }
+        ) gcfg.archives);
+
+    # Note: the timer must be Persistent=true, so that systemd will start it even
+    # if e.g. your laptop was asleep while the latest interval occurred.
+    systemd.timers = lib.mapAttrs' (
+      name: cfg:
+      lib.nameValuePair "tarsnap-${name}" {
+        timerConfig.OnCalendar = cfg.period;
+        timerConfig.Persistent = "true";
+        wantedBy = [ "timers.target" ];
+      }
+    ) gcfg.archives;
+
+    environment.etc = lib.mapAttrs' (
+      name: cfg:
+      lib.nameValuePair "tarsnap/${name}.conf" {
+        text = configFile name cfg;
+      }
+    ) gcfg.archives;
+
+    environment.systemPackages = [ gcfg.package ];
+  };
+}

nixos/modules/services/backup/tsm.nix

@@ -0,0 +1,124 @@
+{ config, lib, ... }:
+
+let
+
+  inherit (lib.attrsets) hasAttr;
+  inherit (lib.meta) getExe';
+  inherit (lib.modules) mkDefault mkIf;
+  inherit (lib.options) mkEnableOption mkOption;
+  inherit (lib.types) nonEmptyStr nullOr;
+
+  options.services.tsmBackup = {
+    enable = mkEnableOption ''
+      automatic backups with the
+      IBM Storage Protect (Tivoli Storage Manager, TSM) client.
+      This also enables
+      {option}`programs.tsmClient.enable`
+    '';
+    command = mkOption {
+      type = nonEmptyStr;
+      default = "backup";
+      example = "incr";
+      description = ''
+        The actual command passed to the
+        `dsmc` executable to start the backup.
+      '';
+    };
+    servername = mkOption {
+      type = nonEmptyStr;
+      example = "mainTsmServer";
+      description = ''
+        Create a systemd system service
+        `tsm-backup.service` that starts
+        a backup based on the given servername's stanza.
+        Note that this server's
+        {option}`passwdDir` will default to
+        {file}`/var/lib/tsm-backup/password`
+        (but may be overridden);
+        also, the service will use
+        {file}`/var/lib/tsm-backup` as
+        `HOME` when calling
+        `dsmc`.
+      '';
+    };
+    autoTime = mkOption {
+      type = nullOr nonEmptyStr;
+      default = null;
+      example = "12:00";
+      description = ''
+        The backup service will be invoked
+        automatically at the given date/time,
+        which must be in the format described in
+        {manpage}`systemd.time(5)`.
+        The default `null`
+        disables automatic backups.
+      '';
+    };
+  };
+
+  cfg = config.services.tsmBackup;
+  cfgPrg = config.programs.tsmClient;
+
+  assertions = [
+    {
+      assertion = hasAttr cfg.servername cfgPrg.servers;
+      message = "TSM service servername not found in list of servers";
+    }
+    {
+      assertion = cfgPrg.servers.${cfg.servername}.genPasswd;
+      message = "TSM service requires automatic password generation";
+    }
+  ];
+
+in
+
+{
+
+  inherit options;
+
+  config = mkIf cfg.enable {
+    inherit assertions;
+    programs.tsmClient.enable = true;
+    programs.tsmClient.servers.${cfg.servername}.passworddir = mkDefault "/var/lib/tsm-backup/password";
+    systemd.services.tsm-backup = {
+      description = "IBM Storage Protect (Tivoli Storage Manager) Backup";
+      # DSM_LOG needs a trailing slash to have it treated as a directory.
+      # `/var/log` would be littered with TSM log files otherwise.
+      environment.DSM_LOG = "/var/log/tsm-backup/";
+      # TSM needs a HOME dir to store certificates.
+      environment.HOME = "/var/lib/tsm-backup";
+      serviceConfig = {
+        # for exit status description see
+        # https://www.ibm.com/docs/en/storage-protect/8.1.27?topic=clients-client-return-codes
+        SuccessExitStatus = "4 8";
+        # The `-se` option must come after the command.
+        # The `-optfile` option suppresses a `dsm.opt`-not-found warning.
+        ExecStart = "${getExe' cfgPrg.wrappedPackage "dsmc"} ${cfg.command} -se='${cfg.servername}' -optfile=/dev/null";
+        LogsDirectory = "tsm-backup";
+        StateDirectory = "tsm-backup";
+        StateDirectoryMode = "0750";
+        # systemd sandboxing
+        LockPersonality = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        #PrivateTmp = true;  # would break backup of {/var,}/tmp
+        #PrivateUsers = true;  # would block backup of /home/*
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = "read-only";
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "noaccess";
+        ProtectSystem = "strict";
+        RestrictNamespaces = true;
+        RestrictSUIDSGID = true;
+      };
+      startAt = mkIf (cfg.autoTime != null) cfg.autoTime;
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.yarny ];
+
+}

nixos/modules/services/backup/zfs-replication.nix

@@ -0,0 +1,110 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  cfg = config.services.zfs.autoReplication;
+in
+{
+  options = {
+    services.zfs.autoReplication = {
+      enable = lib.mkEnableOption "ZFS snapshot replication";
+
+      package = lib.mkPackageOption pkgs "zfs-replicate" { };
+
+      followDelete = lib.mkOption {
+        description = "Remove remote snapshots that don't have a local correspondent.";
+        default = true;
+        type = lib.types.bool;
+      };
+
+      host = lib.mkOption {
+        description = "Remote host where snapshots should be sent. `lz4` is expected to be installed on this host.";
+        example = "example.com";
+        type = lib.types.str;
+      };
+
+      identityFilePath = lib.mkOption {
+        description = "Path to SSH key used to login to host.";
+        example = "/home/username/.ssh/id_rsa";
+        type = lib.types.path;
+      };
+
+      localFilesystem = lib.mkOption {
+        description = "Local ZFS filesystem from which snapshots should be sent.  Defaults to the attribute name.";
+        example = "pool/file/path";
+        type = lib.types.str;
+      };
+
+      remoteFilesystem = lib.mkOption {
+        description = "Remote ZFS filesystem where snapshots should be sent.";
+        example = "pool/file/path";
+        type = lib.types.str;
+      };
+
+      recursive = lib.mkOption {
+        description = "Recursively discover snapshots to send.";
+        default = true;
+        type = lib.types.bool;
+      };
+
+      username = lib.mkOption {
+        description = "Username used by SSH to login to remote host.";
+        example = "username";
+        type = lib.types.str;
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [
+      pkgs.lz4
+    ];
+
+    systemd.services.zfs-replication = {
+      after = [
+        "zfs-snapshot-daily.service"
+        "zfs-snapshot-frequent.service"
+        "zfs-snapshot-hourly.service"
+        "zfs-snapshot-monthly.service"
+        "zfs-snapshot-weekly.service"
+      ];
+      description = "ZFS Snapshot Replication";
+      documentation = [
+        "https://github.com/alunduil/zfs-replicate"
+      ];
+      restartIfChanged = false;
+      serviceConfig.ExecStart =
+        let
+          args = lib.map lib.escapeShellArg (
+            [
+              "--verbose"
+              "--user"
+              cfg.username
+              "--identity-file"
+              cfg.identityFilePath
+              cfg.host
+              cfg.remoteFilesystem
+              cfg.localFilesystem
+            ]
+            ++ (lib.optional cfg.recursive "--recursive")
+            ++ (lib.optional cfg.followDelete "--follow-delete")
+          );
+        in
+        "${lib.getExe cfg.package} ${lib.concatStringsSep " " args}";
+      wantedBy = [
+        "zfs-snapshot-daily.service"
+        "zfs-snapshot-frequent.service"
+        "zfs-snapshot-hourly.service"
+        "zfs-snapshot-monthly.service"
+        "zfs-snapshot-weekly.service"
+      ];
+    };
+  };
+
+  meta = {
+    maintainers = with lib.maintainers; [ alunduil ];
+  };
+}

nixos/modules/services/backup/znapzend.nix

@@ -0,0 +1,530 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  planDescription = ''
+    The znapzend backup plan to use for the source.
+
+    The plan specifies how often to backup and for how long to keep the
+    backups. It consists of a series of retention periods to interval
+    associations:
+
+    ```
+      retA=>intA,retB=>intB,...
+    ```
+
+    Both intervals and retention periods are expressed in standard units
+    of time or multiples of them. You can use both the full name or a
+    shortcut according to the following listing:
+
+    ```
+      second|sec|s, minute|min, hour|h, day|d, week|w, month|mon|m, year|y
+    ```
+
+    See {manpage}`znapzendzetup(1)` for more info.
+  '';
+  planExample = "1h=>10min,1d=>1h,1w=>1d,1m=>1w,1y=>1m";
+
+  # A type for a string of the form number{b|k|M|G}
+  mbufferSizeType = lib.types.str // {
+    check = x: lib.types.str.check x && builtins.isList (builtins.match "^[0-9]+[bkMG]$" x);
+    description = "string of the form number{b|k|M|G}";
+  };
+
+  enabledFeatures = lib.concatLists (
+    lib.mapAttrsToList (name: enabled: lib.optional enabled name) cfg.features
+  );
+
+  # Type for a string that must contain certain other strings (the list parameter).
+  # Note that these would need regex escaping.
+  stringContainingStrings =
+    list:
+    let
+      matching = s: map (str: builtins.match ".*${str}.*" s) list;
+    in
+    lib.types.str
+    // {
+      check = x: lib.types.str.check x && lib.all lib.isList (matching x);
+      description = "string containing all of the characters ${lib.concatStringsSep ", " list}";
+    };
+
+  timestampType = stringContainingStrings [
+    "%Y"
+    "%m"
+    "%d"
+    "%H"
+    "%M"
+    "%S"
+  ];
+
+  destType =
+    srcConfig:
+    lib.types.submodule (
+      { name, ... }:
+      {
+        options = {
+
+          label = lib.mkOption {
+            type = lib.types.str;
+            description = "Label for this destination. Defaults to the attribute name.";
+          };
+
+          plan = lib.mkOption {
+            type = lib.types.str;
+            description = planDescription;
+            example = planExample;
+          };
+
+          dataset = lib.mkOption {
+            type = lib.types.str;
+            description = "Dataset name to send snapshots to.";
+            example = "tank/main";
+          };
+
+          host = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            description = ''
+              Host to use for the destination dataset. Can be prefixed with
+              `user@` to specify the ssh user.
+            '';
+            default = null;
+            example = "john@example.com";
+          };
+
+          presend = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            description = ''
+              Command to run before sending the snapshot to the destination.
+              Intended to run a remote script via {command}`ssh` on the
+              destination, e.g. to bring up a backup disk or server or to put a
+              zpool online/offline. See also {option}`postsend`.
+            '';
+            default = null;
+            example = "ssh root@bserv zpool import -Nf tank";
+          };
+
+          postsend = lib.mkOption {
+            type = lib.types.nullOr lib.types.str;
+            description = ''
+              Command to run after sending the snapshot to the destination.
+              Intended to run a remote script via {command}`ssh` on the
+              destination, e.g. to bring up a backup disk or server or to put a
+              zpool online/offline. See also {option}`presend`.
+            '';
+            default = null;
+            example = "ssh root@bserv zpool export tank";
+          };
+        };
+
+        config = {
+          label = lib.mkDefault name;
+          plan = lib.mkDefault srcConfig.plan;
+        };
+      }
+    );
+
+  srcType = lib.types.submodule (
+    { name, config, ... }:
+    {
+      options = {
+
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          description = "Whether to enable this source.";
+          default = true;
+        };
+
+        recursive = lib.mkOption {
+          type = lib.types.bool;
+          description = "Whether to do recursive snapshots.";
+          default = false;
+        };
+
+        mbuffer = {
+          enable = lib.mkOption {
+            type = lib.types.bool;
+            description = "Whether to use {command}`mbuffer`.";
+            default = false;
+          };
+
+          port = lib.mkOption {
+            type = lib.types.nullOr lib.types.ints.u16;
+            description = ''
+              Port to use for {command}`mbuffer`.
+
+              If this is null, it will run {command}`mbuffer` through
+              ssh.
+
+              If this is not null, it will run {command}`mbuffer`
+              directly through TCP, which is not encrypted but faster. In that
+              case the given port needs to be open on the destination host.
+            '';
+            default = null;
+          };
+
+          size = lib.mkOption {
+            type = mbufferSizeType;
+            description = ''
+              The size for {command}`mbuffer`.
+              Supports the units b, k, M, G.
+            '';
+            default = "1G";
+            example = "128M";
+          };
+        };
+
+        presnap = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = ''
+            Command to run before snapshots are taken on the source dataset,
+            e.g. for database locking/flushing. See also
+            {option}`postsnap`.
+          '';
+          default = null;
+          example = lib.literalExpression ''
+            '''''${pkgs.mariadb}/bin/mysql -e "set autocommit=0;flush tables with read lock;\\! ''${pkgs.coreutils}/bin/sleep 600" &  ''${pkgs.coreutils}/bin/echo $! > /tmp/mariadblock.pid ; sleep 10'''
+          '';
+        };
+
+        postsnap = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = ''
+            Command to run after snapshots are taken on the source dataset,
+            e.g. for database unlocking. See also {option}`presnap`.
+          '';
+          default = null;
+          example = lib.literalExpression ''
+            "''${pkgs.coreutils}/bin/kill `''${pkgs.coreutils}/bin/cat /tmp/mariadblock.pid`;''${pkgs.coreutils}/bin/rm /tmp/mariadblock.pid"
+          '';
+        };
+
+        timestampFormat = lib.mkOption {
+          type = timestampType;
+          description = ''
+            The timestamp format to use for constructing snapshot names.
+            The syntax is `strftime`-like. The string must
+            consist of the mandatory `%Y %m %d %H %M %S`.
+            Optionally  `- _ . :`  characters as well as any
+            alphanumeric character are allowed. If suffixed by a
+            `Z`, times will be in UTC.
+          '';
+          default = "%Y-%m-%d-%H%M%S";
+          example = "znapzend-%m.%d.%Y-%H%M%SZ";
+        };
+
+        sendDelay = lib.mkOption {
+          type = lib.types.int;
+          description = ''
+            Specify delay (in seconds) before sending snaps to the destination.
+            May be useful if you want to control sending time.
+          '';
+          default = 0;
+          example = 60;
+        };
+
+        plan = lib.mkOption {
+          type = lib.types.str;
+          description = planDescription;
+          example = planExample;
+        };
+
+        dataset = lib.mkOption {
+          type = lib.types.str;
+          description = "The dataset to use for this source.";
+          example = "tank/home";
+        };
+
+        destinations = lib.mkOption {
+          type = lib.types.attrsOf (destType config);
+          description = "Additional destinations.";
+          default = { };
+          example = lib.literalExpression ''
+            {
+              local = {
+                dataset = "btank/backup";
+                presend = "zpool import -N btank";
+                postsend = "zpool export btank";
+              };
+              remote = {
+                host = "john@example.com";
+                dataset = "tank/john";
+              };
+            };
+          '';
+        };
+      };
+
+      config = {
+        dataset = lib.mkDefault name;
+      };
+
+    }
+  );
+
+  ### Generating the configuration from here
+
+  cfg = config.services.znapzend;
+
+  onOff = b: if b then "on" else "off";
+  nullOff = b: if b == null then "off" else toString b;
+  stripSlashes = lib.replaceStrings [ "/" ] [ "." ];
+
+  attrsToFile =
+    config: lib.concatStringsSep "\n" (builtins.attrValues (lib.mapAttrs (n: v: "${n}=${v}") config));
+
+  mkDestAttrs =
+    dst:
+    with dst;
+    lib.mapAttrs' (n: v: lib.nameValuePair "dst_${label}${n}" v) (
+      {
+        "" = lib.optionalString (host != null) "${host}:" + dataset;
+        _plan = plan;
+      }
+      // lib.optionalAttrs (presend != null) {
+        _precmd = presend;
+      }
+      // lib.optionalAttrs (postsend != null) {
+        _pstcmd = postsend;
+      }
+    );
+
+  mkSrcAttrs =
+    srcCfg:
+    with srcCfg;
+    {
+      enabled = onOff enable;
+      # mbuffer is not referenced by its full path to accommodate non-NixOS systems or differing mbuffer versions between source and target
+      mbuffer =
+        with mbuffer;
+        if enable then "mbuffer" + lib.optionalString (port != null) ":${toString port}" else "off";
+      mbuffer_size = mbuffer.size;
+      post_znap_cmd = nullOff postsnap;
+      pre_znap_cmd = nullOff presnap;
+      recursive = onOff recursive;
+      src = dataset;
+      src_plan = plan;
+      tsformat = timestampFormat;
+      zend_delay = toString sendDelay;
+    }
+    // lib.foldr (a: b: a // b) { } (map mkDestAttrs (builtins.attrValues destinations));
+
+  files = lib.mapAttrs' (
+    n: srcCfg:
+    let
+      fileText = attrsToFile (mkSrcAttrs srcCfg);
+    in
+    {
+      name = srcCfg.dataset;
+      value = pkgs.writeText (stripSlashes srcCfg.dataset) fileText;
+    }
+  ) cfg.zetup;
+
+in
+{
+  options = {
+    services.znapzend = {
+      enable = lib.mkEnableOption "ZnapZend ZFS backup daemon";
+
+      logLevel = lib.mkOption {
+        default = "debug";
+        example = "warning";
+        type = lib.types.enum [
+          "debug"
+          "info"
+          "warning"
+          "err"
+          "alert"
+        ];
+        description = ''
+          The log level when logging to file. Any of debug, info, warning, err,
+          alert. Default in daemonized form is debug.
+        '';
+      };
+
+      logTo = lib.mkOption {
+        type = lib.types.str;
+        default = "syslog::daemon";
+        example = "/var/log/znapzend.log";
+        description = ''
+          Where to log to (syslog::\<facility\> or \<filepath\>).
+        '';
+      };
+
+      mailErrorSummaryTo = lib.mkOption {
+        type = lib.types.singleLineStr;
+        default = "";
+        description = ''
+          Email address to send a summary to if "send task(s) failed".
+        '';
+      };
+
+      noDestroy = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Does all changes to the filesystem except destroy.";
+      };
+
+      autoCreation = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Automatically create the destination dataset if it does not exist.";
+      };
+
+      zetup = lib.mkOption {
+        type = lib.types.attrsOf srcType;
+        description = "Znapzend configuration.";
+        default = { };
+        example = lib.literalExpression ''
+          {
+            "tank/home" = {
+              # Make snapshots of tank/home every hour, keep those for 1 day,
+              # keep every days snapshot for 1 month, etc.
+              plan = "1d=>1h,1m=>1d,1y=>1m";
+              recursive = true;
+              # Send all those snapshots to john@example.com:rtank/john as well
+              destinations.remote = {
+                host = "john@example.com";
+                dataset = "rtank/john";
+              };
+            };
+          };
+        '';
+      };
+
+      pure = lib.mkOption {
+        type = lib.types.bool;
+        description = ''
+          Do not persist any stateful znapzend setups. If this option is
+          enabled, your previously set znapzend setups will be cleared and only
+          the ones defined with this module will be applied.
+        '';
+        default = false;
+      };
+
+      features.oracleMode = lib.mkEnableOption ''
+        destroying snapshots one by one instead of using one long argument list.
+        If source and destination are out of sync for a long time, you may have
+        so many snapshots to destroy that the argument gets is too long and the
+        command fails
+      '';
+      features.recvu = lib.mkEnableOption ''
+        recvu feature which uses `-u` on the receiving end to keep the destination
+        filesystem unmounted
+      '';
+      features.compressed = lib.mkEnableOption ''
+        compressed feature which adds the options `-Lce` to
+        the {command}`zfs send` command. When this is enabled, make
+        sure that both the sending and receiving pool have the same relevant
+        features enabled. Using `-c` will skip unnecessary
+        decompress-compress stages, `-L` is for large block
+        support and -e is for embedded data support. see
+        {manpage}`znapzend(1)`
+        and {manpage}`zfs(8)`
+        for more info
+      '';
+      features.sendRaw = lib.mkEnableOption ''
+        sendRaw feature which adds the options `-w` to the
+        {command}`zfs send` command. For encrypted source datasets this
+        instructs zfs not to decrypt before sending which results in a remote
+        backup that can't be read without the encryption key/passphrase, useful
+        when the remote isn't fully trusted or not physically secure. This
+        option must be used consistently, raw incrementals cannot be based on
+        non-raw snapshots and vice versa
+      '';
+      features.skipIntermediates = lib.mkEnableOption ''
+        the skipIntermediates feature to send a single increment
+        between latest common snapshot and the newly made one. It may skip
+        several source snaps if the destination was offline for some time, and
+        it should skip snapshots not managed by znapzend. Normally for online
+        destinations, the new snapshot is sent as soon as it is created on the
+        source, so there are no automatic increments to skip
+      '';
+      features.lowmemRecurse = lib.mkEnableOption ''
+        use lowmemRecurse on systems where you have too many datasets, so a
+        recursive listing of attributes to find backup plans exhausts the
+        memory available to {command}`znapzend`: instead, go the slower
+        way to first list all impacted dataset names, and then query their
+        configs one by one
+      '';
+      features.zfsGetType = lib.mkEnableOption ''
+        using zfsGetType if your {command}`zfs get` supports a
+        `-t` argument for filtering by dataset type at all AND
+        lists properties for snapshots by default when recursing, so that there
+        is too much data to process while searching for backup plans.
+        If these two conditions apply to your system, the time needed for a
+        `--recursive` search for backup plans can literally
+        differ by hundreds of times (depending on the amount of snapshots in
+        that dataset tree... and a decent backup plan will ensure you have a lot
+        of those), so you would benefit from requesting this feature
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.znapzend ];
+
+    systemd.services = {
+      znapzend = {
+        description = "ZnapZend - ZFS Backup System";
+        wantedBy = [ "zfs.target" ];
+        after = [ "zfs.target" ];
+
+        path = with pkgs; [
+          config.boot.zfs.package
+          mbuffer
+          openssh
+        ];
+
+        preStart =
+          lib.optionalString cfg.pure ''
+            echo Resetting znapzend zetups
+            ${pkgs.znapzend}/bin/znapzendzetup list \
+              | grep -oP '(?<=\*\*\* backup plan: ).*(?= \*\*\*)' \
+              | xargs -I{} ${pkgs.znapzend}/bin/znapzendzetup delete "{}"
+          ''
+          + lib.concatStringsSep "\n" (
+            lib.mapAttrsToList (dataset: config: ''
+              echo Importing znapzend zetup ${config} for dataset ${dataset}
+              ${pkgs.znapzend}/bin/znapzendzetup import --write ${dataset} ${config} &
+            '') files
+          )
+          + ''
+            wait
+          '';
+
+        serviceConfig = {
+          # znapzendzetup --import apparently tries to connect to the backup
+          # host 3 times with a timeout of 30 seconds, leading to a startup
+          # delay of >90s when the host is down, which is just above the default
+          # service timeout of 90 seconds. Increase the timeout so it doesn't
+          # make the service fail in that case.
+          TimeoutStartSec = 180;
+          # Needs to have write access to ZFS
+          User = "root";
+          ExecStart =
+            let
+              args = lib.concatStringsSep " " [
+                "--logto=${cfg.logTo}"
+                "--loglevel=${cfg.logLevel}"
+                (lib.optionalString cfg.noDestroy "--nodestroy")
+                (lib.optionalString cfg.autoCreation "--autoCreation")
+                (lib.optionalString (cfg.mailErrorSummaryTo != "") "--mailErrorSummaryTo=${cfg.mailErrorSummaryTo}")
+                (lib.optionalString (
+                  enabledFeatures != [ ]
+                ) "--features=${lib.concatStringsSep "," enabledFeatures}")
+              ];
+            in
+            "${pkgs.znapzend}/bin/znapzend ${args}";
+          ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+          Restart = "on-failure";
+        };
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ SlothOfAnarchy ];
+}

nixos/modules/services/backup/zrepl.nix

@@ -0,0 +1,61 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.zrepl;
+  format = pkgs.formats.yaml { };
+  configFile = format.generate "zrepl.yml" cfg.settings;
+in
+{
+  meta.maintainers = with lib.maintainers; [ cole-h ];
+
+  options = {
+    services.zrepl = {
+      enable = lib.mkEnableOption "zrepl";
+
+      package = lib.mkPackageOption pkgs "zrepl" { };
+
+      settings = lib.mkOption {
+        default = { };
+        description = ''
+          Configuration for zrepl. See <https://zrepl.github.io/configuration.html>
+          for more information.
+        '';
+        type = lib.types.submodule {
+          freeformType = format.type;
+        };
+      };
+    };
+  };
+
+  ### Implementation ###
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    # zrepl looks for its config in this location by default. This
+    # allows the use of e.g. `zrepl signal wakeup <job>` without having
+    # to specify the storepath of the config.
+    environment.etc."zrepl/zrepl.yml".source = configFile;
+
+    systemd.packages = [ cfg.package ];
+
+    # Note that pkgs.zrepl copies and adapts the upstream systemd unit, and
+    # the fields defined here only override certain fields from that unit.
+    systemd.services.zrepl = {
+      requires = [ "local-fs.target" ];
+      wantedBy = [ "zfs.target" ];
+      after = [ "zfs.target" ];
+
+      path = [ config.boot.zfs.package ];
+      restartTriggers = [ configFile ];
+
+      serviceConfig = {
+        Restart = "on-failure";
+      };
+    };
+  };
+}

nixos/modules/services/blockchain/ethereum/erigon.nix

@@ -0,0 +1,143 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.erigon;
+
+  settingsFormat = pkgs.formats.toml { };
+  configFile = settingsFormat.generate "config.toml" cfg.settings;
+in
+{
+
+  options = {
+    services.erigon = {
+      enable = lib.mkEnableOption "Ethereum implementation on the efficiency frontier";
+
+      package = lib.mkPackageOption pkgs "erigon" { };
+
+      extraArgs = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Additional arguments passed to Erigon";
+        default = [ ];
+      };
+
+      secretJwtPath = lib.mkOption {
+        type = lib.types.path;
+        description = ''
+          Path to the secret jwt used for the http api authentication.
+        '';
+        default = "";
+        example = "config.age.secrets.ERIGON_JWT.path";
+      };
+
+      settings = lib.mkOption {
+        description = ''
+          Configuration for Erigon
+          Refer to <https://github.com/ledgerwatch/erigon#usage> for details on supported values.
+        '';
+
+        type = settingsFormat.type;
+
+        example = {
+          datadir = "/var/lib/erigon";
+          chain = "mainnet";
+          http = true;
+          "http.port" = 8545;
+          "http.api" = [
+            "eth"
+            "debug"
+            "net"
+            "trace"
+            "web3"
+            "erigon"
+          ];
+          ws = true;
+          port = 30303;
+          "authrpc.port" = 8551;
+          "torrent.port" = 42069;
+          "private.api.addr" = "localhost:9090";
+          "log.console.verbosity" = 3; # info
+        };
+
+        defaultText = lib.literalExpression ''
+          {
+            datadir = "/var/lib/erigon";
+            chain = "mainnet";
+            http = true;
+            "http.port" = 8545;
+            "http.api" = ["eth" "debug" "net" "trace" "web3" "erigon"];
+            ws = true;
+            port = 30303;
+            "authrpc.port" = 8551;
+            "torrent.port" = 42069;
+            "private.api.addr" = "localhost:9090";
+            "log.console.verbosity" = 3; # info
+          }
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # Default values are the same as in the binary, they are just written here for convenience.
+    services.erigon.settings = {
+      datadir = lib.mkDefault "/var/lib/erigon";
+      chain = lib.mkDefault "mainnet";
+      http = lib.mkDefault true;
+      "http.port" = lib.mkDefault 8545;
+      "http.api" = lib.mkDefault [
+        "eth"
+        "debug"
+        "net"
+        "trace"
+        "web3"
+        "erigon"
+      ];
+      ws = lib.mkDefault true;
+      port = lib.mkDefault 30303;
+      "authrpc.port" = lib.mkDefault 8551;
+      "torrent.port" = lib.mkDefault 42069;
+      "private.api.addr" = lib.mkDefault "localhost:9090";
+      "log.console.verbosity" = lib.mkDefault 3; # info
+    };
+
+    systemd.services.erigon = {
+      description = "Erigon ethereum implemenntation";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      serviceConfig = {
+        LoadCredential = "ERIGON_JWT:${cfg.secretJwtPath}";
+        ExecStart = "${cfg.package}/bin/erigon --config ${configFile} --authrpc.jwtsecret=%d/ERIGON_JWT ${lib.escapeShellArgs cfg.extraArgs}";
+        DynamicUser = true;
+        Restart = "on-failure";
+        StateDirectory = "erigon";
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+      };
+    };
+  };
+}

nixos/modules/services/blockchain/ethereum/geth.nix

@@ -0,0 +1,257 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  eachGeth = config.services.geth;
+
+  gethOpts =
+    {
+      config,
+      lib,
+      name,
+      ...
+    }:
+    {
+
+      options = {
+
+        enable = lib.mkEnableOption "Go Ethereum Node";
+
+        port = lib.mkOption {
+          type = lib.types.port;
+          default = 30303;
+          description = "Port number Go Ethereum will be listening on, both TCP and UDP.";
+        };
+
+        http = {
+          enable = lib.mkEnableOption "Go Ethereum HTTP API";
+          address = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1";
+            description = "Listen address of Go Ethereum HTTP API.";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 8545;
+            description = "Port number of Go Ethereum HTTP API.";
+          };
+
+          apis = lib.mkOption {
+            type = lib.types.nullOr (lib.types.listOf lib.types.str);
+            default = null;
+            description = "APIs to enable over WebSocket";
+            example = [
+              "net"
+              "eth"
+            ];
+          };
+        };
+
+        websocket = {
+          enable = lib.mkEnableOption "Go Ethereum WebSocket API";
+          address = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1";
+            description = "Listen address of Go Ethereum WebSocket API.";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 8546;
+            description = "Port number of Go Ethereum WebSocket API.";
+          };
+
+          apis = lib.mkOption {
+            type = lib.types.nullOr (lib.types.listOf lib.types.str);
+            default = null;
+            description = "APIs to enable over WebSocket";
+            example = [
+              "net"
+              "eth"
+            ];
+          };
+        };
+
+        authrpc = {
+          enable = lib.mkEnableOption "Go Ethereum Auth RPC API";
+          address = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1";
+            description = "Listen address of Go Ethereum Auth RPC API.";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 8551;
+            description = "Port number of Go Ethereum Auth RPC API.";
+          };
+
+          vhosts = lib.mkOption {
+            type = lib.types.nullOr (lib.types.listOf lib.types.str);
+            default = [ "localhost" ];
+            description = "List of virtual hostnames from which to accept requests.";
+            example = [
+              "localhost"
+              "geth.example.org"
+            ];
+          };
+
+          jwtsecret = lib.mkOption {
+            type = lib.types.str;
+            default = "";
+            description = "Path to a JWT secret for authenticated RPC endpoint.";
+            example = "/var/run/geth/jwtsecret";
+          };
+        };
+
+        metrics = {
+          enable = lib.mkEnableOption "Go Ethereum prometheus metrics";
+          address = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1";
+            description = "Listen address of Go Ethereum metrics service.";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 6060;
+            description = "Port number of Go Ethereum metrics service.";
+          };
+        };
+
+        network = lib.mkOption {
+          type = lib.types.nullOr (
+            lib.types.enum [
+              "holesky"
+              "sepolia"
+            ]
+          );
+          default = null;
+          description = "The network to connect to. Mainnet (null) is the default ethereum network.";
+        };
+
+        syncmode = lib.mkOption {
+          type = lib.types.enum [
+            "snap"
+            "fast"
+            "full"
+            "light"
+          ];
+          default = "snap";
+          description = "Blockchain sync mode.";
+        };
+
+        gcmode = lib.mkOption {
+          type = lib.types.enum [
+            "full"
+            "archive"
+          ];
+          default = "full";
+          description = "Blockchain garbage collection mode.";
+        };
+
+        maxpeers = lib.mkOption {
+          type = lib.types.int;
+          default = 50;
+          description = "Maximum peers to connect to.";
+        };
+
+        extraArgs = lib.mkOption {
+          type = lib.types.listOf lib.types.str;
+          description = "Additional arguments passed to Go Ethereum.";
+          default = [ ];
+        };
+
+        package = lib.mkPackageOption pkgs [ "go-ethereum" "geth" ] { };
+      };
+    };
+in
+
+{
+
+  ###### interface
+
+  options = {
+    services.geth = lib.mkOption {
+      type = lib.types.attrsOf (lib.types.submodule gethOpts);
+      default = { };
+      description = "Specification of one or more geth instances.";
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkIf (eachGeth != { }) {
+
+    environment.systemPackages = lib.flatten (
+      lib.mapAttrsToList (gethName: cfg: [
+        cfg.package
+      ]) eachGeth
+    );
+
+    systemd.services = lib.mapAttrs' (
+      gethName: cfg:
+      let
+        stateDir = "goethereum/${gethName}/${if (cfg.network == null) then "mainnet" else cfg.network}";
+        dataDir = "/var/lib/${stateDir}";
+      in
+      (lib.nameValuePair "geth-${gethName}" (
+        lib.mkIf cfg.enable {
+          description = "Go Ethereum node (${gethName})";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+
+          serviceConfig = {
+            DynamicUser = true;
+            Restart = "always";
+            StateDirectory = stateDir;
+
+            # Hardening measures
+            PrivateTmp = "true";
+            ProtectSystem = "full";
+            NoNewPrivileges = "true";
+            PrivateDevices = "true";
+            MemoryDenyWriteExecute = "true";
+          };
+
+          script = ''
+            ${cfg.package}/bin/geth \
+              --nousb \
+              --ipcdisable \
+              ${lib.optionalString (cfg.network != null) ''--${cfg.network}''} \
+              --syncmode ${cfg.syncmode} \
+              --gcmode ${cfg.gcmode} \
+              --port ${toString cfg.port} \
+              --maxpeers ${toString cfg.maxpeers} \
+              ${lib.optionalString cfg.http.enable ''--http --http.addr ${cfg.http.address} --http.port ${toString cfg.http.port}''} \
+              ${
+                lib.optionalString (cfg.http.apis != null) ''--http.api ${lib.concatStringsSep "," cfg.http.apis}''
+              } \
+              ${lib.optionalString cfg.websocket.enable ''--ws --ws.addr ${cfg.websocket.address} --ws.port ${toString cfg.websocket.port}''} \
+              ${
+                lib.optionalString (
+                  cfg.websocket.apis != null
+                ) ''--ws.api ${lib.concatStringsSep "," cfg.websocket.apis}''
+              } \
+              ${lib.optionalString cfg.metrics.enable ''--metrics --metrics.addr ${cfg.metrics.address} --metrics.port ${toString cfg.metrics.port}''} \
+              --authrpc.addr ${cfg.authrpc.address} --authrpc.port ${toString cfg.authrpc.port} --authrpc.vhosts ${lib.concatStringsSep "," cfg.authrpc.vhosts} \
+              ${
+                if (cfg.authrpc.jwtsecret != "") then
+                  ''--authrpc.jwtsecret ${cfg.authrpc.jwtsecret}''
+                else
+                  ''--authrpc.jwtsecret ${dataDir}/geth/jwtsecret''
+              } \
+              ${lib.escapeShellArgs cfg.extraArgs} \
+              --datadir ${dataDir}
+          '';
+        }
+      ))
+    ) eachGeth;
+
+  };
+
+}

nixos/modules/services/blockchain/ethereum/lighthouse.nix

@@ -0,0 +1,332 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.lighthouse;
+in
+{
+  options = {
+    services.lighthouse = {
+      beacon = lib.mkOption {
+        description = "Beacon node";
+        default = { };
+        type = lib.types.submodule {
+          options = {
+            enable = lib.mkEnableOption "Lightouse Beacon node";
+
+            dataDir = lib.mkOption {
+              type = lib.types.str;
+              default = "/var/lib/lighthouse-beacon";
+              description = ''
+                Directory where data will be stored. Each chain will be stored under it's own specific subdirectory.
+              '';
+            };
+
+            address = lib.mkOption {
+              type = lib.types.str;
+              default = "0.0.0.0";
+              description = ''
+                Listen address of Beacon node.
+              '';
+            };
+
+            port = lib.mkOption {
+              type = lib.types.port;
+              default = 9000;
+              description = ''
+                Port number the Beacon node will be listening on.
+              '';
+            };
+
+            openFirewall = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Open the port in the firewall
+              '';
+            };
+
+            disableDepositContractSync = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Explicitly disables syncing of deposit logs from the execution node.
+                This overrides any previous option that depends on it.
+                Useful if you intend to run a non-validating beacon node.
+              '';
+            };
+
+            execution = {
+              address = lib.mkOption {
+                type = lib.types.str;
+                default = "127.0.0.1";
+                description = ''
+                  Listen address for the execution layer.
+                '';
+              };
+
+              port = lib.mkOption {
+                type = lib.types.port;
+                default = 8551;
+                description = ''
+                  Port number the Beacon node will be listening on for the execution layer.
+                '';
+              };
+
+              jwtPath = lib.mkOption {
+                type = lib.types.str;
+                default = "";
+                description = ''
+                  Path for the jwt secret required to connect to the execution layer.
+                '';
+              };
+            };
+
+            http = {
+              enable = lib.mkEnableOption "Beacon node http api";
+              port = lib.mkOption {
+                type = lib.types.port;
+                default = 5052;
+                description = ''
+                  Port number of Beacon node RPC service.
+                '';
+              };
+
+              address = lib.mkOption {
+                type = lib.types.str;
+                default = "127.0.0.1";
+                description = ''
+                  Listen address of Beacon node RPC service.
+                '';
+              };
+            };
+
+            metrics = {
+              enable = lib.mkEnableOption "Beacon node prometheus metrics";
+              address = lib.mkOption {
+                type = lib.types.str;
+                default = "127.0.0.1";
+                description = ''
+                  Listen address of Beacon node metrics service.
+                '';
+              };
+
+              port = lib.mkOption {
+                type = lib.types.port;
+                default = 5054;
+                description = ''
+                  Port number of Beacon node metrics service.
+                '';
+              };
+            };
+
+            extraArgs = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Additional arguments passed to the lighthouse beacon command.
+              '';
+              default = "";
+              example = "";
+            };
+          };
+        };
+      };
+
+      validator = lib.mkOption {
+        description = "Validator node";
+        default = { };
+        type = lib.types.submodule {
+          options = {
+            enable = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = "Enable Lightouse Validator node.";
+            };
+
+            dataDir = lib.mkOption {
+              type = lib.types.str;
+              default = "/var/lib/lighthouse-validator";
+              description = ''
+                Directory where data will be stored. Each chain will be stored under it's own specific subdirectory.
+              '';
+            };
+
+            beaconNodes = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ "http://localhost:5052" ];
+              description = ''
+                Beacon nodes to connect to.
+              '';
+            };
+
+            metrics = {
+              enable = lib.mkEnableOption "Validator node prometheus metrics";
+              address = lib.mkOption {
+                type = lib.types.str;
+                default = "127.0.0.1";
+                description = ''
+                  Listen address of Validator node metrics service.
+                '';
+              };
+
+              port = lib.mkOption {
+                type = lib.types.port;
+                default = 5056;
+                description = ''
+                  Port number of Validator node metrics service.
+                '';
+              };
+            };
+
+            extraArgs = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Additional arguments passed to the lighthouse validator command.
+              '';
+              default = "";
+              example = "";
+            };
+          };
+        };
+      };
+
+      network = lib.mkOption {
+        type = lib.types.enum [
+          "mainnet"
+          "gnosis"
+          "chiado"
+          "sepolia"
+          "holesky"
+        ];
+        default = "mainnet";
+        description = ''
+          The network to connect to. Mainnet is the default ethereum network.
+        '';
+      };
+
+      extraArgs = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          Additional arguments passed to every lighthouse command.
+        '';
+        default = "";
+        example = "";
+      };
+
+      package = lib.mkPackageOption pkgs "lighthouse" { };
+    };
+  };
+
+  config = lib.mkIf (cfg.beacon.enable || cfg.validator.enable) {
+    environment.systemPackages = [ cfg.package ];
+
+    networking.firewall = lib.mkIf cfg.beacon.enable {
+      allowedTCPPorts = lib.mkIf cfg.beacon.openFirewall [ cfg.beacon.port ];
+      allowedUDPPorts = lib.mkIf cfg.beacon.openFirewall [ cfg.beacon.port ];
+    };
+
+    systemd.services.lighthouse-beacon = lib.mkIf cfg.beacon.enable {
+      description = "Lighthouse beacon node (connect to P2P nodes and verify blocks)";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      script = ''
+        # make sure the chain data directory is created on first run
+        mkdir -p ${cfg.beacon.dataDir}/${cfg.network}
+
+        ${lib.getExe cfg.package} beacon_node \
+          --disable-upnp \
+          ${lib.optionalString cfg.beacon.disableDepositContractSync "--disable-deposit-contract-sync"} \
+          --port ${toString cfg.beacon.port} \
+          --listen-address ${cfg.beacon.address} \
+          --network ${cfg.network} \
+          --datadir ${cfg.beacon.dataDir}/${cfg.network} \
+          --execution-endpoint http://${cfg.beacon.execution.address}:${toString cfg.beacon.execution.port} \
+          --execution-jwt ''${CREDENTIALS_DIRECTORY}/LIGHTHOUSE_JWT \
+          ${lib.optionalString cfg.beacon.http.enable ''--http --http-address ${cfg.beacon.http.address} --http-port ${toString cfg.beacon.http.port}''} \
+          ${lib.optionalString cfg.beacon.metrics.enable ''--metrics --metrics-address ${cfg.beacon.metrics.address} --metrics-port ${toString cfg.beacon.metrics.port}''} \
+          ${cfg.extraArgs} ${cfg.beacon.extraArgs}
+      '';
+      serviceConfig = {
+        LoadCredential = "LIGHTHOUSE_JWT:${cfg.beacon.execution.jwtPath}";
+        DynamicUser = true;
+        Restart = "on-failure";
+        StateDirectory = "lighthouse-beacon";
+        ReadWritePaths = [ cfg.beacon.dataDir ];
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+      };
+    };
+
+    systemd.services.lighthouse-validator = lib.mkIf cfg.validator.enable {
+      description = "Lighthouse validtor node (manages validators, using data obtained from the beacon node via a HTTP API)";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      script = ''
+        # make sure the chain data directory is created on first run
+        mkdir -p ${cfg.validator.dataDir}/${cfg.network}
+
+        ${lib.getExe cfg.package} validator_client \
+          --network ${cfg.network} \
+          --beacon-nodes ${lib.concatStringsSep "," cfg.validator.beaconNodes} \
+          --datadir ${cfg.validator.dataDir}/${cfg.network} \
+          ${lib.optionalString cfg.validator.metrics.enable ''--metrics --metrics-address ${cfg.validator.metrics.address} --metrics-port ${toString cfg.validator.metrics.port}''} \
+          ${cfg.extraArgs} ${cfg.validator.extraArgs}
+      '';
+
+      serviceConfig = {
+        Restart = "on-failure";
+        StateDirectory = "lighthouse-validator";
+        ReadWritePaths = [ cfg.validator.dataDir ];
+        CapabilityBoundingSet = "";
+        DynamicUser = true;
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        ProtectHome = true;
+        ProtectClock = true;
+        ProtectProc = "noaccess";
+        ProcSubset = "pid";
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectControlGroups = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        RestrictRealtime = true;
+        RestrictNamespaces = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+      };
+    };
+  };
+}

nixos/modules/services/cluster/corosync/default.nix

@@ -0,0 +1,121 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.corosync;
+in
+{
+  # interface
+  options.services.corosync = {
+    enable = lib.mkEnableOption "corosync";
+
+    package = lib.mkPackageOption pkgs "corosync" { };
+
+    clusterName = lib.mkOption {
+      type = lib.types.str;
+      default = "nixcluster";
+      description = "Name of the corosync cluster.";
+    };
+
+    extraOptions = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [ ];
+      description = "Additional options with which to start corosync.";
+    };
+
+    nodelist = lib.mkOption {
+      description = "Corosync nodelist: all cluster members.";
+      default = [ ];
+      type =
+        with lib.types;
+        listOf (submodule {
+          options = {
+            nodeid = lib.mkOption {
+              type = int;
+              description = "Node ID number";
+            };
+            name = lib.mkOption {
+              type = str;
+              description = "Node name";
+            };
+            ring_addrs = lib.mkOption {
+              type = listOf str;
+              description = "List of addresses, one for each ring.";
+            };
+          };
+        });
+    };
+  };
+
+  # implementation
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    environment.etc."corosync/corosync.conf".text = ''
+      totem {
+        version: 2
+        secauth: on
+        cluster_name: ${cfg.clusterName}
+        transport: knet
+      }
+
+      nodelist {
+        ${lib.concatMapStrings (
+          {
+            nodeid,
+            name,
+            ring_addrs,
+          }:
+          ''
+            node {
+              nodeid: ${toString nodeid}
+              name: ${name}
+              ${lib.concatStrings (
+                lib.imap0 (i: addr: ''
+                  ring${toString i}_addr: ${addr}
+                '') ring_addrs
+              )}
+            }
+          ''
+        ) cfg.nodelist}
+      }
+
+      quorum {
+        # only corosync_votequorum is supported
+        provider: corosync_votequorum
+        wait_for_all: 0
+        ${lib.optionalString (builtins.length cfg.nodelist < 3) ''
+          two_node: 1
+        ''}
+      }
+
+      logging {
+        to_syslog: yes
+      }
+    '';
+
+    environment.etc."corosync/uidgid.d/root".text = ''
+      # allow pacemaker connection by root
+      uidgid {
+        uid: 0
+        gid: 0
+      }
+    '';
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.corosync = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "corosync";
+        StateDirectoryMode = "0700";
+      };
+    };
+
+    environment.etc."sysconfig/corosync".text = lib.optionalString (cfg.extraOptions != [ ]) ''
+      COROSYNC_OPTIONS="${lib.escapeShellArgs cfg.extraOptions}"
+    '';
+  };
+}

nixos/modules/services/cluster/druid/default.nix

@@ -0,0 +1,299 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.druid;
+  inherit (lib)
+    concatStrings
+    concatStringsSep
+    mapAttrsToList
+    concatMap
+    attrByPath
+    mkIf
+    mkMerge
+    mkEnableOption
+    mkOption
+    types
+    mkPackageOption
+    ;
+
+  druidServiceOption = serviceName: {
+    enable = mkEnableOption serviceName;
+
+    restartIfChanged = mkOption {
+      type = types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on clusters running critical applications.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = false;
+    };
+
+    config = mkOption {
+      default = { };
+      type = types.attrsOf types.anything;
+      description = ''
+        (key=value) Configuration to be written to runtime.properties of the druid ${serviceName}
+        <https://druid.apache.org/docs/latest/configuration/index.html>
+      '';
+      example = {
+        "druid.plainTextPort" = "8082";
+        "druid.service" = "servicename";
+      };
+    };
+
+    jdk = mkPackageOption pkgs "JDK" { default = [ "jdk17_headless" ]; };
+
+    jvmArgs = mkOption {
+      type = types.str;
+      default = "";
+      description = "Arguments to pass to the JVM";
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Open firewall ports for ${serviceName}.";
+    };
+
+    internalConfig = mkOption {
+      default = { };
+      type = types.attrsOf types.anything;
+      internal = true;
+      description = "Internal Option to add to runtime.properties for ${serviceName}.";
+    };
+  };
+
+  druidServiceConfig =
+    {
+      name,
+      serviceOptions ? cfg."${name}",
+      allowedTCPPorts ? [ ],
+      tmpDirs ? [ ],
+      extraConfig ? { },
+    }:
+    (mkIf serviceOptions.enable (mkMerge [
+      {
+        systemd = {
+          services."druid-${name}" = {
+            after = [ "network.target" ];
+
+            description = "Druid ${name}";
+
+            wantedBy = [ "multi-user.target" ];
+
+            inherit (serviceOptions) restartIfChanged;
+
+            path = [
+              cfg.package
+              serviceOptions.jdk
+            ];
+
+            script =
+              let
+                cfgFile =
+                  fileName: properties:
+                  pkgs.writeTextDir fileName (
+                    concatStringsSep "\n" (mapAttrsToList (n: v: "${n}=${toString v}") properties)
+                  );
+
+                commonConfigFile = cfgFile "common.runtime.properties" cfg.commonConfig;
+
+                configFile = cfgFile "runtime.properties" (serviceOptions.config // serviceOptions.internalConfig);
+
+                extraClassPath = concatStrings (map (path: ":" + path) cfg.extraClassPaths);
+
+                extraConfDir = concatStrings (map (dir: ":" + dir + "/*") cfg.extraConfDirs);
+              in
+              ''
+                run-java -Dlog4j.configurationFile=file:${cfg.log4j} \
+                  -Ddruid.extensions.directory=${cfg.package}/extensions \
+                  -Ddruid.extensions.hadoopDependenciesDir=${cfg.package}/hadoop-dependencies \
+                  -classpath  ${commonConfigFile}:${configFile}:${cfg.package}/lib/\*${extraClassPath}${extraConfDir} \
+                  ${serviceOptions.jvmArgs} \
+                  org.apache.druid.cli.Main server ${name}
+              '';
+
+            serviceConfig = {
+              User = "druid";
+              SyslogIdentifier = "druid-${name}";
+              Restart = "always";
+            };
+          };
+
+          tmpfiles.rules = concatMap (x: [ "d ${x} 0755 druid druid" ]) (cfg.commonTmpDirs ++ tmpDirs);
+        };
+        networking.firewall.allowedTCPPorts = mkIf (attrByPath [
+          "openFirewall"
+        ] false serviceOptions) allowedTCPPorts;
+
+        users = {
+          users.druid = {
+            description = "Druid user";
+            group = "druid";
+            isNormalUser = true;
+          };
+          groups.druid = { };
+        };
+      }
+      extraConfig
+    ]));
+in
+{
+  options.services.druid = {
+    package = mkPackageOption pkgs "apache-druid" { default = [ "druid" ]; };
+
+    commonConfig = mkOption {
+      default = { };
+
+      type = types.attrsOf types.anything;
+
+      description = "(key=value) Configuration to be written to common.runtime.properties";
+
+      example = {
+        "druid.zk.service.host" = "localhost:2181";
+        "druid.metadata.storage.type" = "mysql";
+        "druid.metadata.storage.connector.connectURI" = "jdbc:mysql://localhost:3306/druid";
+        "druid.extensions.loadList" = ''[ "mysql-metadata-storage" ]'';
+      };
+    };
+
+    commonTmpDirs = mkOption {
+      default = [ "/var/log/druid/requests" ];
+      type = types.listOf types.str;
+      description = "Common List of directories used by druid processes";
+    };
+
+    log4j = mkOption {
+      type = types.path;
+      description = "Log4j Configuration for the druid process";
+    };
+
+    extraClassPaths = mkOption {
+      default = [ ];
+      type = types.listOf types.str;
+      description = "Extra classpath to include in the jvm";
+    };
+
+    extraConfDirs = mkOption {
+      default = [ ];
+      type = types.listOf types.path;
+      description = "Extra Conf Dirs to include in the jvm";
+    };
+
+    overlord = druidServiceOption "Druid Overlord";
+
+    coordinator = druidServiceOption "Druid Coordinator";
+
+    broker = druidServiceOption "Druid Broker";
+
+    historical = (druidServiceOption "Druid Historical") // {
+      segmentLocations = mkOption {
+
+        default = null;
+
+        description = "Locations where the historical will store its data.";
+
+        type =
+          with types;
+          nullOr (
+            listOf (submodule {
+              options = {
+                path = mkOption {
+                  type = path;
+                  description = "the path to store the segments";
+                };
+
+                maxSize = mkOption {
+                  type = str;
+                  description = "Max size the druid historical can occupy";
+                };
+
+                freeSpacePercent = mkOption {
+                  type = float;
+                  default = 1.0;
+                  description = "Druid Historical will fail to write if it exceeds this value";
+                };
+              };
+            })
+          );
+
+      };
+    };
+
+    middleManager = druidServiceOption "Druid middleManager";
+    router = druidServiceOption "Druid Router";
+  };
+  config = mkMerge [
+    (druidServiceConfig rec {
+      name = "overlord";
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8090 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "coordinator";
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8081 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "broker";
+
+      tmpDirs = [ (attrByPath [ "druid.lookup.snapshotWorkingDir" ] "" cfg."${name}".config) ];
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8082 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "historical";
+
+      tmpDirs = [
+        (attrByPath [ "druid.lookup.snapshotWorkingDir" ] "" cfg."${name}".config)
+      ]
+      ++ (map (x: x.path) cfg."${name}".segmentLocations);
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8083 cfg."${name}".config) ];
+
+      extraConfig.services.druid.historical.internalConfig."druid.segmentCache.locations" =
+        builtins.toJSON cfg.historical.segmentLocations;
+    })
+
+    (druidServiceConfig rec {
+      name = "middleManager";
+
+      tmpDirs = [
+        "/var/log/druid/indexer"
+      ]
+      ++ [ (attrByPath [ "druid.indexer.task.baseTaskDir" ] "" cfg."${name}".config) ];
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8091 cfg."${name}".config) ];
+
+      extraConfig = {
+        services.druid.middleManager.internalConfig = {
+          "druid.indexer.runner.javaCommand" = "${cfg.middleManager.jdk}/bin/java";
+          "druid.indexer.runner.javaOpts" =
+            (attrByPath [ "druid.indexer.runner.javaOpts" ] "" cfg.middleManager.config)
+            + " -Dlog4j.configurationFile=file:${cfg.log4j}";
+        };
+
+        networking.firewall.allowedTCPPortRanges = mkIf cfg.middleManager.openFirewall [
+          {
+            from = attrByPath [ "druid.indexer.runner.startPort" ] 8100 cfg.middleManager.config;
+            to = attrByPath [ "druid.indexer.runner.endPort" ] 65535 cfg.middleManager.config;
+          }
+        ];
+      };
+    })
+
+    (druidServiceConfig rec {
+      name = "router";
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8888 cfg."${name}".config) ];
+    })
+  ];
+
+}

nixos/modules/services/cluster/hadoop/conf.nix

@@ -0,0 +1,58 @@
+{
+  cfg,
+  pkgs,
+  lib,
+}:
+let
+  propertyXml =
+    name: value:
+    lib.optionalString (value != null) ''
+      <property>
+        <name>${name}</name>
+        <value>${builtins.toString value}</value>
+      </property>
+    '';
+  siteXml =
+    fileName: properties:
+    pkgs.writeTextDir fileName ''
+      <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+      <!-- generated by NixOS -->
+      <configuration>
+        ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}
+      </configuration>
+    '';
+  cfgLine = name: value: ''
+    ${name}=${builtins.toString value}
+  '';
+  cfgFile =
+    fileName: properties:
+    pkgs.writeTextDir fileName ''
+      # generated by NixOS
+      ${builtins.concatStringsSep "" (pkgs.lib.mapAttrsToList cfgLine properties)}
+    '';
+  userFunctions = ''
+    hadoop_verify_logdir() {
+      echo Skipping verification of log directory
+    }
+  '';
+  hadoopEnv = ''
+    export HADOOP_LOG_DIR=/tmp/hadoop/$USER
+  '';
+in
+pkgs.runCommand "hadoop-conf" { } (
+  with cfg;
+  ''
+    mkdir -p $out/
+    cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/
+    cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/
+    cp ${siteXml "hbase-site.xml" (hbaseSiteDefault // hbaseSite // hbaseSiteInternal)}/* $out/
+    cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/
+    cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/
+    cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/
+    cp ${cfgFile "container-executor.cfg" containerExecutorCfg}/* $out/
+    cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
+    cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/
+    cp ${log4jProperties} $out/log4j.properties
+    ${lib.concatMapStringsSep "\n" (dir: "cp -f -r ${dir}/* $out/") extraConfDirs}
+  ''
+)

nixos/modules/services/cluster/hadoop/default.nix

@@ -0,0 +1,232 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  opt = options.services.hadoop;
+in
+{
+  imports = [
+    ./yarn.nix
+    ./hdfs.nix
+    ./hbase.nix
+  ];
+
+  options.services.hadoop = {
+    coreSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "fs.defaultFS" = "hdfs://localhost";
+        }
+      '';
+      description = ''
+        Hadoop core-site.xml definition
+        <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml>
+      '';
+    };
+    coreSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to core-site.xml based on module options
+      '';
+    };
+
+    hdfsSiteDefault = lib.mkOption {
+      default = {
+        "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-address" = "0.0.0.0:9870";
+        "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-bind-host" = "0.0.0.0";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for hdfs-site.xml
+      '';
+    };
+    hdfsSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "dfs.nameservices" = "namenode1";
+        }
+      '';
+      description = ''
+        Additional options and overrides for hdfs-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml>
+      '';
+    };
+    hdfsSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hdfs-site.xml based on module options
+      '';
+    };
+
+    mapredSiteDefault = lib.mkOption {
+      default = {
+        "mapreduce.framework.name" = "yarn";
+        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+      };
+      defaultText = lib.literalExpression ''
+        {
+          "mapreduce.framework.name" = "yarn";
+          "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+        }
+      '';
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for mapred-site.xml
+      '';
+    };
+    mapredSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
+        }
+      '';
+      description = ''
+        Additional options and overrides for mapred-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml>
+      '';
+    };
+
+    yarnSiteDefault = lib.mkOption {
+      default = {
+        "yarn.nodemanager.admin-env" = "PATH=$PATH";
+        "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
+        "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
+        "yarn.nodemanager.bind-host" = "0.0.0.0";
+        "yarn.nodemanager.container-executor.class" =
+          "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
+        "yarn.nodemanager.env-whitelist" =
+          "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
+        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
+        "yarn.nodemanager.linux-container-executor.path" =
+          "/run/wrappers/yarn-nodemanager/bin/container-executor";
+        "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
+        "yarn.resourcemanager.bind-host" = "0.0.0.0";
+        "yarn.resourcemanager.scheduler.class" =
+          "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for yarn-site.xml
+      '';
+    };
+    yarnSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
+        }
+      '';
+      description = ''
+        Additional options and overrides for yarn-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml>
+      '';
+    };
+    yarnSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to yarn-site.xml based on module options
+      '';
+    };
+
+    httpfsSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "hadoop.http.max.threads" = 500;
+        }
+      '';
+      description = ''
+        Hadoop httpfs-site.xml definition
+        <https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html>
+      '';
+    };
+
+    log4jProperties = lib.mkOption {
+      default = "${cfg.package}/etc/hadoop/log4j.properties";
+      defaultText = lib.literalExpression ''
+        "''${config.${opt.package}}/etc/hadoop/log4j.properties"
+      '';
+      type = lib.types.path;
+      example = lib.literalExpression ''
+        "''${pkgs.hadoop}/etc/hadoop/log4j.properties";
+      '';
+      description = "log4j.properties file added to HADOOP_CONF_DIR";
+    };
+
+    containerExecutorCfg = lib.mkOption {
+      default = {
+        # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
+        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
+        "min.user.id" = 1000;
+        "feature.terminal.enabled" = 1;
+        "feature.mount-cgroup.enabled" = 1;
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        options.services.hadoop.containerExecutorCfg.default // {
+          "feature.terminal.enabled" = 0;
+        }
+      '';
+      description = ''
+        Yarn container-executor.cfg definition
+        <https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html>
+      '';
+    };
+
+    extraConfDirs = lib.mkOption {
+      default = [ ];
+      type = lib.types.listOf lib.types.path;
+      example = lib.literalExpression ''
+        [
+          ./extraHDFSConfs
+          ./extraYARNConfs
+        ]
+      '';
+      description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
+    };
+
+    gatewayRole.enable = lib.mkEnableOption "gateway role for deploying hadoop configs";
+
+    package = lib.mkPackageOption pkgs "hadoop" { };
+  };
+
+  config = lib.mkIf cfg.gatewayRole.enable {
+    users.groups.hadoop = {
+      gid = config.ids.gids.hadoop;
+    };
+    environment = {
+      systemPackages = [ cfg.package ];
+      etc."hadoop-conf".source =
+        let
+          hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+        in
+        "${hadoopConf}";
+      variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/";
+    };
+  };
+}

nixos/modules/services/cluster/hadoop/hbase.nix

@@ -0,0 +1,246 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  mkIfNotNull = x: lib.mkIf (x != null) x;
+  # generic hbase role options
+  hbaseRoleOption =
+    name: extraOpts:
+    {
+      enable = lib.mkEnableOption "HBase ${name}";
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open firewall ports for HBase ${name}.";
+      };
+
+      restartIfChanged = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Restart ${name} con config change.";
+      };
+
+      extraFlags = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        example = lib.literalExpression ''[ "--backup" ]'';
+        description = "Extra flags for the ${name} service.";
+      };
+
+      environment = lib.mkOption {
+        type = with lib.types; attrsOf str;
+        default = { };
+        example = lib.literalExpression ''
+          {
+            HBASE_MASTER_OPTS = "-Dcom.sun.management.jmxremote.ssl=true";
+          }
+        '';
+        description = "Environment variables passed to ${name}.";
+      };
+    }
+    // extraOpts;
+  # generic hbase role configs
+  hbaseRoleConfig =
+    name: ports:
+    (lib.mkIf cfg.hbase."${name}".enable {
+      services.hadoop.gatewayRole = {
+        enable = true;
+        enableHbaseCli = lib.mkDefault true;
+      };
+
+      systemd.services."hbase-${lib.toLower name}" = {
+        description = "HBase ${name}";
+        wantedBy = [ "multi-user.target" ];
+        path =
+          with cfg;
+          [ hbase.package ] ++ lib.optional (with cfg.hbase.master; enable && initHDFS) package;
+        preStart = lib.mkIf (with cfg.hbase.master; enable && initHDFS) (
+          lib.concatStringsSep "\n" (
+            map (x: "HADOOP_USER_NAME=hdfs hdfs --config /etc/hadoop-conf ${x}") [
+              "dfsadmin -safemode wait"
+              "dfs -mkdir -p ${cfg.hbase.rootdir}"
+              "dfs -chown hbase ${cfg.hbase.rootdir}"
+            ]
+          )
+        );
+
+        inherit (cfg.hbase."${name}") environment;
+        script = lib.concatStringsSep " " (
+          [
+            "hbase --config /etc/hadoop-conf/"
+            "${lib.toLower name} start"
+          ]
+          ++ cfg.hbase."${name}".extraFlags
+          ++ map (x: "--${lib.toLower x} ${toString cfg.hbase.${name}.${x}}") (
+            lib.filter (x: lib.hasAttr x cfg.hbase.${name}) [
+              "port"
+              "infoPort"
+            ]
+          )
+        );
+
+        serviceConfig = {
+          User = "hbase";
+          SyslogIdentifier = "hbase-${lib.toLower name}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
+
+      networking = {
+        firewall.allowedTCPPorts = lib.mkIf cfg.hbase."${name}".openFirewall ports;
+        hosts = lib.mkIf (with cfg.hbase.regionServer; enable && overrideHosts) {
+          "127.0.0.2" = lib.mkForce [ ];
+          "::1" = lib.mkForce [ ];
+        };
+      };
+
+    });
+in
+{
+  options.services.hadoop = {
+
+    gatewayRole.enableHbaseCli = lib.mkEnableOption "HBase CLI tools";
+
+    hbaseSiteDefault = lib.mkOption {
+      default = {
+        "hbase.regionserver.ipc.address" = "0.0.0.0";
+        "hbase.master.ipc.address" = "0.0.0.0";
+        "hbase.master.info.bindAddress" = "0.0.0.0";
+        "hbase.regionserver.info.bindAddress" = "0.0.0.0";
+
+        "hbase.cluster.distributed" = "true";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for hbase-site.xml
+      '';
+    };
+    hbaseSite = lib.mkOption {
+      default = { };
+      type = with lib.types; attrsOf anything;
+      example = lib.literalExpression ''
+        {
+          "hbase.hregion.max.filesize" = 20*1024*1024*1024;
+          "hbase.table.normalization.enabled" = "true";
+        }
+      '';
+      description = ''
+        Additional options and overrides for hbase-site.xml
+        <https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml>
+      '';
+    };
+    hbaseSiteInternal = lib.mkOption {
+      default = { };
+      type = with lib.types; attrsOf anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hbase-site.xml based on module options
+      '';
+    };
+
+    hbase = {
+
+      package = lib.mkPackageOption pkgs "hbase" { };
+
+      rootdir = lib.mkOption {
+        description = ''
+          This option will set "hbase.rootdir" in hbase-site.xml and determine
+          the directory shared by region servers and into which HBase persists.
+          The URL should be 'fully-qualified' to include the filesystem scheme.
+          If a core-site.xml is provided, the FS scheme defaults to the value
+          of "fs.defaultFS".
+
+          Filesystems other than HDFS (like S3, QFS, Swift) are also supported.
+        '';
+        type = lib.types.str;
+        example = "hdfs://nameservice1/hbase";
+        default = "/hbase";
+      };
+      zookeeperQuorum = lib.mkOption {
+        description = ''
+          This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
+          Comma separated list of servers in the ZooKeeper ensemble.
+        '';
+        type = with lib.types; nullOr commas;
+        example = "zk1.internal,zk2.internal,zk3.internal";
+        default = null;
+      };
+    }
+    // (
+      let
+        ports = port: infoPort: {
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = port;
+            description = "RPC port";
+          };
+          infoPort = lib.mkOption {
+            type = lib.types.port;
+            default = infoPort;
+            description = "web UI port";
+          };
+        };
+      in
+      lib.mapAttrs hbaseRoleOption {
+        master.initHDFS = lib.mkEnableOption "initialization of the hbase directory on HDFS";
+        regionServer.overrideHosts = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
+            Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
+            or /etc/hosts entries.
+          '';
+        };
+        thrift = ports 9090 9095;
+        rest = ports 8080 8085;
+      }
+    );
+  };
+
+  config = lib.mkMerge (
+    [
+
+      (lib.mkIf cfg.gatewayRole.enable {
+
+        environment.systemPackages = lib.mkIf cfg.gatewayRole.enableHbaseCli [ cfg.hbase.package ];
+
+        services.hadoop.hbaseSiteInternal = with cfg.hbase; {
+          "hbase.zookeeper.quorum" = mkIfNotNull zookeeperQuorum;
+        };
+
+        users.users.hbase = {
+          description = "Hadoop HBase user";
+          group = "hadoop";
+          isSystemUser = true;
+        };
+      })
+    ]
+    ++ (lib.mapAttrsToList hbaseRoleConfig {
+      master = [
+        16000
+        16010
+      ];
+      regionServer = [
+        16020
+        16030
+      ];
+      thrift = with cfg.hbase.thrift; [
+        port
+        infoPort
+      ];
+      rest = with cfg.hbase.rest; [
+        port
+        infoPort
+      ];
+    })
+  );
+}

nixos/modules/services/cluster/hadoop/hdfs.nix

@@ -0,0 +1,237 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+
+  # Config files for hadoop services
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+
+  # Generator for HDFS service options
+  hadoopServiceOption =
+    {
+      serviceName,
+      firewallOption ? true,
+      extraOpts ? null,
+    }:
+    {
+      enable = lib.mkEnableOption serviceName;
+      restartIfChanged = lib.mkOption {
+        type = lib.types.bool;
+        description = ''
+          Automatically restart the service on config change.
+          This can be set to false to defer restarts on clusters running critical applications.
+          Please consider the security implications of inadvertently running an older version,
+          and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+        '';
+        default = false;
+      };
+      extraFlags = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        description = "Extra command line flags to pass to ${serviceName}";
+        example = [
+          "-Dcom.sun.management.jmxremote"
+          "-Dcom.sun.management.jmxremote.port=8010"
+        ];
+      };
+      extraEnv = lib.mkOption {
+        type = with lib.types; attrsOf str;
+        default = { };
+        description = "Extra environment variables for ${serviceName}";
+      };
+    }
+    // (lib.optionalAttrs firewallOption {
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open firewall ports for ${serviceName}.";
+      };
+    })
+    // (lib.optionalAttrs (extraOpts != null) extraOpts);
+
+  # Generator for HDFS service configs
+  hadoopServiceConfig =
+    {
+      name,
+      serviceOptions ? cfg.hdfs."${lib.toLower name}",
+      description ? "Hadoop HDFS ${name}",
+      User ? "hdfs",
+      allowedTCPPorts ? [ ],
+      preStart ? "",
+      environment ? { },
+      extraConfig ? { },
+    }:
+    (
+
+      lib.mkIf serviceOptions.enable (
+        lib.mkMerge [
+          {
+            systemd.services."hdfs-${lib.toLower name}" = {
+              inherit description preStart;
+              environment = environment // serviceOptions.extraEnv;
+              wantedBy = [ "multi-user.target" ];
+              inherit (serviceOptions) restartIfChanged;
+              serviceConfig = {
+                inherit User;
+                SyslogIdentifier = "hdfs-${lib.toLower name}";
+                ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} ${lib.toLower name} ${lib.escapeShellArgs serviceOptions.extraFlags}";
+                Restart = "always";
+              };
+            };
+
+            services.hadoop.gatewayRole.enable = true;
+
+            networking.firewall.allowedTCPPorts = lib.mkIf (
+              (builtins.hasAttr "openFirewall" serviceOptions) && serviceOptions.openFirewall
+            ) allowedTCPPorts;
+          }
+          extraConfig
+        ]
+      )
+    );
+
+in
+{
+  options.services.hadoop.hdfs = {
+
+    namenode = hadoopServiceOption { serviceName = "HDFS NameNode"; } // {
+      formatOnInit = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Format HDFS namenode on first start. This is useful for quickly spinning up
+          ephemeral HDFS clusters with a single namenode.
+          For HA clusters, initialization involves multiple steps across multiple nodes.
+          Follow this guide to initialize an HA cluster manually:
+          <https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html>
+        '';
+      };
+    };
+
+    datanode = hadoopServiceOption { serviceName = "HDFS DataNode"; } // {
+      dataDirs = lib.mkOption {
+        default = null;
+        description = "Tier and path definitions for datanode storage.";
+        type =
+          with lib.types;
+          nullOr (
+            listOf (submodule {
+              options = {
+                type = lib.mkOption {
+                  type = enum [
+                    "SSD"
+                    "DISK"
+                    "ARCHIVE"
+                    "RAM_DISK"
+                  ];
+                  description = ''
+                    Storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS storage policies.
+                  '';
+                };
+                path = lib.mkOption {
+                  type = path;
+                  example = [ "/var/lib/hadoop/hdfs/dn" ];
+                  description = "Determines where on the local filesystem a data node should store its blocks.";
+                };
+              };
+            })
+          );
+      };
+    };
+
+    journalnode = hadoopServiceOption { serviceName = "HDFS JournalNode"; };
+
+    zkfc = hadoopServiceOption {
+      serviceName = "HDFS ZooKeeper failover controller";
+      firewallOption = false;
+    };
+
+    httpfs = hadoopServiceOption { serviceName = "HDFS JournalNode"; } // {
+      tempPath = lib.mkOption {
+        type = lib.types.path;
+        default = "/tmp/hadoop/httpfs";
+        description = "HTTPFS_TEMP path used by HTTPFS";
+      };
+    };
+
+  };
+
+  config = lib.mkMerge [
+    (hadoopServiceConfig {
+      name = "NameNode";
+      allowedTCPPorts = [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+        8022 # namenode.servicerpc-address
+        8019 # dfs.ha.zkfc.port
+      ];
+      preStart = (
+        lib.mkIf cfg.hdfs.namenode.formatOnInit "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true"
+      );
+    })
+
+    (hadoopServiceConfig {
+      name = "DataNode";
+      # port numbers for datanode changed between hadoop 2 and 3
+      allowedTCPPorts =
+        if lib.versionAtLeast cfg.package.version "3" then
+          [
+            9864 # datanode.http.address
+            9866 # datanode.address
+            9867 # datanode.ipc.address
+          ]
+        else
+          [
+            50075 # datanode.http.address
+            50010 # datanode.address
+            50020 # datanode.ipc.address
+          ];
+      extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = lib.mkIf (
+        cfg.hdfs.datanode.dataDirs != null
+      ) (lib.concatMapStringsSep "," (x: "[" + x.type + "]file://" + x.path) cfg.hdfs.datanode.dataDirs);
+    })
+
+    (hadoopServiceConfig {
+      name = "JournalNode";
+      allowedTCPPorts = [
+        8480 # dfs.journalnode.http-address
+        8485 # dfs.journalnode.rpc-address
+      ];
+    })
+
+    (hadoopServiceConfig {
+      name = "zkfc";
+      description = "Hadoop HDFS ZooKeeper failover controller";
+    })
+
+    (hadoopServiceConfig {
+      name = "HTTPFS";
+      environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
+      preStart = "mkdir -p $HTTPFS_TEMP";
+      User = "httpfs";
+      allowedTCPPorts = [
+        14000 # httpfs.http.port
+      ];
+    })
+
+    (lib.mkIf cfg.gatewayRole.enable {
+      users.users.hdfs = {
+        description = "Hadoop HDFS user";
+        group = "hadoop";
+        uid = config.ids.uids.hdfs;
+      };
+    })
+    (lib.mkIf cfg.hdfs.httpfs.enable {
+      users.users.httpfs = {
+        description = "Hadoop HTTPFS user";
+        group = "hadoop";
+        isSystemUser = true;
+      };
+    })
+
+  ];
+}

nixos/modules/services/cluster/hadoop/yarn.nix

@@ -0,0 +1,226 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  restartIfChanged = lib.mkOption {
+    type = lib.types.bool;
+    description = ''
+      Automatically restart the service on config change.
+      This can be set to false to defer restarts on clusters running critical applications.
+      Please consider the security implications of inadvertently running an older version,
+      and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+    '';
+    default = false;
+  };
+  extraFlags = lib.mkOption {
+    type = with lib.types; listOf str;
+    default = [ ];
+    description = "Extra command line flags to pass to the service";
+    example = [
+      "-Dcom.sun.management.jmxremote"
+      "-Dcom.sun.management.jmxremote.port=8010"
+    ];
+  };
+  extraEnv = lib.mkOption {
+    type = with lib.types; attrsOf str;
+    default = { };
+    description = "Extra environment variables";
+  };
+in
+{
+  options.services.hadoop.yarn = {
+    resourcemanager = {
+      enable = lib.mkEnableOption "Hadoop YARN ResourceManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open firewall ports for resourcemanager
+        '';
+      };
+    };
+    nodemanager = {
+      enable = lib.mkEnableOption "Hadoop YARN NodeManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
+      resource = {
+        cpuVCores = lib.mkOption {
+          description = "Number of vcores that can be allocated for containers.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationVCores = lib.mkOption {
+          description = "The maximum virtual CPU cores any container can be allocated.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        memoryMB = lib.mkOption {
+          description = "Amount of physical memory, in MB, that can be allocated for containers.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationMB = lib.mkOption {
+          description = "The maximum physical memory any container can be allocated.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+      };
+
+      useCGroups = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Use cgroups to enforce resource limits on containers
+        '';
+      };
+
+      localDir = lib.mkOption {
+        description = "List of directories to store localized files in.";
+        type = with lib.types; nullOr (listOf path);
+        example = [ "/var/lib/hadoop/yarn/nm" ];
+        default = null;
+      };
+
+      addBinBash = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Add /bin/bash. This is needed by the linux container executor's launch script.
+        '';
+      };
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open firewall ports for nodemanager.
+          Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.gatewayRole.enable {
+      users.users.yarn = {
+        description = "Hadoop YARN user";
+        group = "hadoop";
+        uid = config.ids.uids.yarn;
+      };
+    })
+
+    (lib.mkIf cfg.yarn.resourcemanager.enable {
+      systemd.services.yarn-resourcemanager = {
+        description = "Hadoop YARN ResourceManager";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.yarn.resourcemanager) restartIfChanged;
+        environment = cfg.yarn.resourcemanager.extraEnv;
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-resourcemanager";
+          ExecStart =
+            "${cfg.package}/bin/yarn --config ${hadoopConf} "
+            + " resourcemanager ${lib.escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.gatewayRole.enable = true;
+
+      networking.firewall.allowedTCPPorts = (
+        lib.mkIf cfg.yarn.resourcemanager.openFirewall [
+          8088 # resourcemanager.webapp.address
+          8030 # resourcemanager.scheduler.address
+          8031 # resourcemanager.resource-tracker.address
+          8032 # resourcemanager.address
+          8033 # resourcemanager.admin.address
+        ]
+      );
+    })
+
+    (lib.mkIf cfg.yarn.nodemanager.enable {
+      # Needed because yarn hardcodes /bin/bash in container start scripts
+      # These scripts can't be patched, they are generated at runtime
+      systemd.tmpfiles.rules = [
+        (lib.mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
+      ];
+
+      systemd.services.yarn-nodemanager = {
+        description = "Hadoop YARN NodeManager";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.yarn.nodemanager) restartIfChanged;
+        environment = cfg.yarn.nodemanager.extraEnv;
+
+        preStart = ''
+          # create log dir
+          mkdir -p /var/log/hadoop/yarn/nodemanager
+          chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
+
+          # set up setuid container executor binary
+          umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
+          rm -rf /run/wrappers/yarn-nodemanager/ || true
+          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
+          cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
+          chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
+          chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
+          cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
+        '';
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-nodemanager";
+          PermissionsStartOnly = true;
+          ExecStart =
+            "${cfg.package}/bin/yarn --config ${hadoopConf} "
+            + " nodemanager ${lib.escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.gatewayRole.enable = true;
+
+      services.hadoop.yarnSiteInternal =
+        with cfg.yarn.nodemanager;
+        lib.mkMerge [
+          {
+            "yarn.nodemanager.local-dirs" = lib.mkIf (localDir != null) (concatStringsSep "," localDir);
+            "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
+            "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
+            "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
+            "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
+          }
+          (lib.mkIf useCGroups (
+            lib.warnIf (lib.versionOlder cfg.package.version "3.5.0")
+              ''
+                hadoop < 3.5.0 does not support cgroup v2
+                setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended
+                see: https://issues.apache.org/jira/browse/YARN-11669
+              ''
+              {
+                "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
+                "yarn.nodemanager.linux-container-executor.resources-handler.class" =
+                  "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
+                "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
+                "yarn.nodemanager.linux-container-executor.cgroups.mount-path" =
+                  "/run/wrappers/yarn-nodemanager/cgroup";
+              }
+          ))
+        ];
+
+      networking.firewall.allowedTCPPortRanges = [
+        (lib.mkIf (cfg.yarn.nodemanager.openFirewall) {
+          from = 1024;
+          to = 65535;
+        })
+      ];
+    })
+
+  ];
+}

nixos/modules/services/cluster/k3s/default.nix

@@ -0,0 +1,913 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.k3s;
+  removeOption =
+    config: instruction:
+    lib.mkRemovedOptionModule (
+      [
+        "services"
+        "k3s"
+      ]
+      ++ config
+    ) instruction;
+
+  manifestDir = "/var/lib/rancher/k3s/server/manifests";
+  chartDir = "/var/lib/rancher/k3s/server/static/charts";
+  imageDir = "/var/lib/rancher/k3s/agent/images";
+  containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
+  yamlFormat = pkgs.formats.yaml { };
+  yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
+  # Manifests need a valid YAML suffix to be respected by k3s
+  mkManifestTarget =
+    name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
+  # Produces a list containing all duplicate manifest names
+  duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
+    builtins.attrNames cfg.manifests
+  );
+  # Produces a list containing all duplicate chart names
+  duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
+    builtins.attrNames cfg.charts
+  );
+
+  # Converts YAML -> JSON -> Nix
+  fromYaml =
+    path:
+    builtins.fromJSON (
+      builtins.readFile (
+        pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ pkgs.yq-go ]; } ''
+          yq --no-colors --output-format json ${path} > $out
+        ''
+      )
+    );
+
+  # Replace prefixes and characters that are problematic in file names
+  cleanHelmChartName =
+    name:
+    let
+      woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
+    in
+    lib.replaceStrings
+      [
+        "/"
+        ":"
+      ]
+      [
+        "-"
+        "-"
+      ]
+      woPrefix;
+
+  # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
+  fetchHelm =
+    {
+      name,
+      repo,
+      version,
+      hash ? lib.fakeHash,
+    }:
+    let
+      isOci = lib.hasPrefix "oci://" repo;
+      pullCmd = if isOci then repo else "--repo ${repo} ${name}";
+      name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
+    in
+    pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
+      {
+        inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+        nativeBuildInputs = with pkgs; [
+          kubernetes-helm
+          cacert
+          # Helm requires HOME to refer to a writable dir
+          writableTmpDirAsHomeHook
+        ];
+      }
+      ''
+        helm pull ${pullCmd} --version ${version}
+        mv ./*.tgz $out
+      '';
+
+  # Returns the path to a YAML manifest file
+  mkExtraDeployManifest =
+    x:
+    # x is a derivation that provides a YAML file
+    if lib.isDerivation x then
+      x.outPath
+    # x is an attribute set that needs to be converted to a YAML file
+    else if builtins.isAttrs x then
+      (yamlFormat.generate "extra-deploy-chart-manifest" x)
+    # assume x is a path to a YAML file
+    else
+      x;
+
+  # Generate a HelmChart custom resource.
+  mkHelmChartCR =
+    name: value:
+    let
+      chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
+      # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
+      valuesContent = builtins.toJSON chartValues;
+    in
+    # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
+    # values
+    lib.recursiveUpdate {
+      apiVersion = "helm.cattle.io/v1";
+      kind = "HelmChart";
+      metadata = {
+        inherit name;
+        namespace = "kube-system";
+      };
+      spec = {
+        inherit valuesContent;
+        inherit (value) targetNamespace createNamespace;
+        chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
+      };
+    } value.extraFieldDefinitions;
+
+  # Generate a HelmChart custom resource together with extraDeploy manifests. This
+  # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
+  # deploys.
+  mkAutoDeployChartManifest = name: value: {
+    # target is the final name of the link created for the manifest file
+    target = mkManifestTarget name;
+    inherit (value) enable package;
+    # source is a store path containing the complete manifest file
+    source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
+      [
+        (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
+      ]
+      # alternate the YAML doc separator (---) and extraDeploy manifests to create
+      # multi document YAMLs
+      ++ (lib.concatMap (x: [
+        yamlDocSeparator
+        (mkExtraDeployManifest x)
+      ]) value.extraDeploy)
+    );
+  };
+
+  autoDeployChartsModule = lib.types.submodule (
+    { config, ... }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          example = false;
+          description = ''
+            Whether to enable the installation of this Helm chart. Note that setting
+            this option to `false` will not uninstall the chart from the cluster, if
+            it was previously installed. Please use the the `--disable` flag or `.skip`
+            files to delete/disable Helm charts, as mentioned in the
+            [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
+          '';
+        };
+
+        repo = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "https://kubernetes.github.io/ingress-nginx";
+          description = ''
+            The repo of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        name = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "ingress-nginx";
+          description = ''
+            The name of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        version = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "4.7.0";
+          description = ''
+            The version of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        hash = lib.mkOption {
+          type = lib.types.str;
+          example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
+          default = "";
+          description = ''
+            The hash of the packaged Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        package = lib.mkOption {
+          type = with lib.types; either path package;
+          example = lib.literalExpression "../my-helm-chart.tgz";
+          description = ''
+            The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
+            and `hash` in case of conflicts.
+          '';
+        };
+
+        targetNamespace = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          default = "default";
+          example = "kube-system";
+          description = "The namespace in which the Helm chart gets installed.";
+        };
+
+        createNamespace = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          example = true;
+          description = "Whether to create the target namespace if not present.";
+        };
+
+        values = lib.mkOption {
+          type = with lib.types; either path attrs;
+          default = { };
+          example = {
+            replicaCount = 3;
+            hostName = "my-host";
+            server = {
+              name = "nginx";
+              port = 80;
+            };
+          };
+          description = ''
+            Override default chart values via Nix expressions. This is equivalent to setting
+            values in a `values.yaml` file.
+
+            WARNING: The values (including secrets!) specified here are exposed unencrypted
+            in the world-readable nix store.
+          '';
+        };
+
+        extraDeploy = lib.mkOption {
+          type = with lib.types; listOf (either path attrs);
+          default = [ ];
+          example = lib.literalExpression ''
+            [
+              ../manifests/my-extra-deployment.yaml
+              {
+                apiVersion = "v1";
+                kind = "Service";
+                metadata = {
+                  name = "app-service";
+                };
+                spec = {
+                  selector = {
+                    "app.kubernetes.io/name" = "MyApp";
+                  };
+                  ports = [
+                    {
+                      name = "name-of-service-port";
+                      protocol = "TCP";
+                      port = 80;
+                      targetPort = "http-web-svc";
+                    }
+                  ];
+                };
+              }
+            ];
+          '';
+          description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
+        };
+
+        extraFieldDefinitions = lib.mkOption {
+          inherit (yamlFormat) type;
+          default = { };
+          example = {
+            spec = {
+              bootstrap = true;
+              helmVersion = "v2";
+              backOffLimit = 3;
+              jobImage = "custom-helm-controller:v0.0.1";
+            };
+          };
+          description = ''
+            Extra HelmChart field definitions that are merged with the rest of the HelmChart
+            custom resource. This can be used to set advanced fields or to overwrite
+            generated fields. See <https://docs.k3s.io/helm#helmchart-field-definitions>
+            for possible fields.
+          '';
+        };
+      };
+
+      config.package = lib.mkDefault (fetchHelm {
+        inherit (config)
+          repo
+          name
+          version
+          hash
+          ;
+      });
+    }
+  );
+
+  manifestModule = lib.types.submodule (
+    {
+      name,
+      config,
+      options,
+      ...
+    }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether this manifest file should be generated.";
+        };
+
+        target = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "manifest.yaml";
+          description = ''
+            Name of the symlink (relative to {file}`${manifestDir}`).
+            Defaults to the attribute name.
+          '';
+        };
+
+        content = lib.mkOption {
+          type = with lib.types; nullOr (either attrs (listOf attrs));
+          default = null;
+          description = ''
+            Content of the manifest file. A single attribute set will
+            generate a single document YAML file. A list of attribute sets
+            will generate multiple documents separated by `---` in a single
+            YAML file.
+          '';
+        };
+
+        source = lib.mkOption {
+          type = lib.types.path;
+          example = lib.literalExpression "./manifests/app.yaml";
+          description = ''
+            Path of the source `.yaml` file.
+          '';
+        };
+      };
+
+      config = {
+        target = lib.mkDefault (mkManifestTarget name);
+        source = lib.mkIf (config.content != null) (
+          let
+            name' = "k3s-manifest-" + builtins.baseNameOf name;
+            docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
+            mkSource =
+              value:
+              if builtins.isList value then
+                pkgs.concatText name' (
+                  lib.concatMap (x: [
+                    yamlDocSeparator
+                    (yamlFormat.generate docName x)
+                  ]) value
+                )
+              else
+                yamlFormat.generate name' value;
+          in
+          lib.mkDerivedConfig options.content mkSource
+        );
+      };
+    }
+  );
+in
+{
+  imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
+
+  # interface
+  options.services.k3s = {
+    enable = lib.mkEnableOption "k3s";
+
+    package = lib.mkPackageOption pkgs "k3s" { };
+
+    role = lib.mkOption {
+      description = ''
+        Whether k3s should run as a server or agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - Starts by default as a standalone server using an embedded sqlite datastore.
+        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
+        - Configure `serverAddr` to join an already-initialized HA cluster.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
+      '';
+      default = "server";
+      type = lib.types.enum [
+        "server"
+        "agent"
+      ];
+    };
+
+    serverAddr = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The k3s server to connect to.
+
+        Servers and agents need to communicate each other. Read
+        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
+        to know how to configure the firewall.
+      '';
+      example = "https://10.0.0.10:6443";
+      default = "";
+    };
+
+    clusterInit = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Initialize HA cluster using an embedded etcd datastore.
+
+        If this option is `false` and `role` is `server`
+
+        On a server that was using the default embedded sqlite backend,
+        enabling this option will migrate to an embedded etcd DB.
+
+        If an HA cluster using the embedded etcd datastore was already initialized,
+        this option has no effect.
+
+        This option only makes sense in a server that is not connecting to another server.
+
+        If you are configuring an HA cluster with an embedded etcd,
+        the 1st server must have `clusterInit = true`
+        and other servers must connect to it using `serverAddr`.
+      '';
+    };
+
+    token = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The k3s token to use when connecting to a server.
+
+        WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the tokenFile option instead.
+      '';
+      default = "";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing k3s token to use when connecting to the server.";
+      default = null;
+    };
+
+    extraFlags = lib.mkOption {
+      description = "Extra flags to pass to the k3s command.";
+      type = with lib.types; either str (listOf str);
+      default = [ ];
+      example = [
+        "--disable traefik"
+        "--cluster-cidr 10.24.0.0/16"
+      ];
+    };
+
+    disableAgent = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Only run the server. This option only makes sense for a server.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = ''
+        File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
+      '';
+      default = null;
+    };
+
+    configPath = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
+    };
+
+    manifests = lib.mkOption {
+      type = lib.types.attrsOf manifestModule;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          deployment.source = ../manifests/deployment.yaml;
+          my-service = {
+            enable = false;
+            target = "app-service.yaml";
+            content = {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "app-service";
+              };
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
+              };
+            };
+          };
+
+          nginx.content = [
+            {
+              apiVersion = "v1";
+              kind = "Pod";
+              metadata = {
+                name = "nginx";
+                labels = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+              };
+              spec = {
+                containers = [
+                  {
+                    name = "nginx";
+                    image = "nginx:1.14.2";
+                    ports = [
+                      {
+                        containerPort = 80;
+                        name = "http-web-svc";
+                      }
+                    ];
+                  }
+                ];
+              };
+            }
+            {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "nginx-service";
+              };
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
+              };
+            }
+          ];
+        };
+      '';
+      description = ''
+        Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
+        Note that deleting manifest files will not remove or otherwise modify the resources
+        it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
+        as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
+        This option only makes sense on server nodes (`role = server`).
+        Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
+        for further information.
+      '';
+    };
+
+    charts = lib.mkOption {
+      type = with lib.types; attrsOf (either path package);
+      default = { };
+      example = lib.literalExpression ''
+        nginx = ../charts/my-nginx-chart.tgz;
+        redis = ../charts/my-redis-chart.tgz;
+      '';
+      description = ''
+        Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
+        The attribute name will be used as the link target (relative to {file}`${chartDir}`).
+        The specified charts will only be placed on the file system and made available to the
+        Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
+        option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
+        to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
+      '';
+    };
+
+    containerdConfigTemplate = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = lib.literalExpression ''
+        # Base K3s config
+        {{ template "base" . }}
+
+        # Add a custom runtime
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
+          BinaryName = "/path/to/custom-container-runtime"
+      '';
+      description = ''
+        Config template for containerd, to be placed at
+        `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
+        See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
+      '';
+    };
+
+    images = lib.mkOption {
+      type = with lib.types; listOf package;
+      default = [ ];
+      example = lib.literalExpression ''
+        [
+          (pkgs.dockerTools.pullImage {
+            imageName = "docker.io/bitnami/keycloak";
+            imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
+            hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
+            finalImageTag = "21.1.2-debian-11-r0";
+          })
+
+          config.services.k3s.package.airgap-images
+        ]
+      '';
+      description = ''
+        List of derivations that provide container images.
+        All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
+        by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
+        use, if you want to pre-provision this node with all k3s container images. This option
+        only makes sense on nodes with an enabled agent.
+      '';
+    };
+
+    gracefulNodeShutdown = {
+      enable = lib.mkEnableOption ''
+        graceful node shutdowns where the kubelet attempts to detect
+        node system shutdown and terminates pods running on the node. See the
+        [documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
+        for further information.
+      '';
+
+      shutdownGracePeriod = lib.mkOption {
+        type = lib.types.nonEmptyStr;
+        default = "30s";
+        example = "1m30s";
+        description = ''
+          Specifies the total duration that the node should delay the shutdown by. This is the total
+          grace period for pod termination for both regular and critical pods.
+        '';
+      };
+
+      shutdownGracePeriodCriticalPods = lib.mkOption {
+        type = lib.types.nonEmptyStr;
+        default = "10s";
+        example = "15s";
+        description = ''
+          Specifies the duration used to terminate critical pods during a node shutdown. This should be
+          less than `shutdownGracePeriod`.
+        '';
+      };
+    };
+
+    extraKubeletConfig = lib.mkOption {
+      type = with lib.types; attrsOf anything;
+      default = { };
+      example = {
+        podsPerCore = 3;
+        memoryThrottlingFactor = 0.69;
+        containerLogMaxSize = "5Mi";
+      };
+      description = ''
+        Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
+        configuration that can be configured via a file is defined by the
+        [KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
+        struct. See the
+        [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
+        for further information.
+      '';
+    };
+
+    extraKubeProxyConfig = lib.mkOption {
+      type = with lib.types; attrsOf anything;
+      default = { };
+      example = {
+        mode = "nftables";
+        clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
+      };
+      description = ''
+        Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
+        configuration that can be configured via a file is defined by the
+        [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
+        struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
+        set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
+      '';
+    };
+
+    autoDeployCharts = lib.mkOption {
+      type = lib.types.attrsOf autoDeployChartsModule;
+      apply = lib.mapAttrs mkAutoDeployChartManifest;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          harbor = {
+            name = "harbor";
+            repo = "https://helm.goharbor.io";
+            version = "1.14.0";
+            hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
+            values = {
+              existingSecretAdminPassword = "harbor-admin";
+              expose = {
+                tls = {
+                  enabled = true;
+                  certSource = "secret";
+                  secret.secretName = "my-tls-secret";
+                };
+                ingress = {
+                  hosts.core = "example.com";
+                  className = "nginx";
+                };
+              };
+            };
+          };
+          nginx = {
+            repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
+            version = "20.0.0";
+            hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
+          };
+          custom-chart = {
+            package = ../charts/my-chart.tgz;
+            values = ../values/my-values.yaml;
+            extraFieldDefinitions = {
+              spec.timeout = "60s";
+            };
+          };
+        }
+      '';
+      description = ''
+        Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
+        attribute names that are also used in the [](#opt-services.k3s.manifests) and
+        [](#opt-services.k3s.charts) options. Manifests with the same name will override
+        auto deploying charts with the same name. Similiarly, charts with the same name will
+        overwrite the Helm chart contained in auto deploying charts. This option only makes
+        sense on server nodes (`role = server`). See the
+        [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
+      '';
+    };
+  };
+
+  # implementation
+
+  config = lib.mkIf cfg.enable {
+    warnings =
+      (lib.optional (cfg.role != "server" && cfg.manifests != { })
+        "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
+        "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
+        "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (duplicateManifests != [ ])
+        "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
+      )
+      ++ (lib.optional (duplicateCharts != [ ])
+        "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
+      )
+      ++ (lib.optional (
+        cfg.disableAgent && cfg.images != [ ]
+      ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
+      ++ (lib.optional (
+        cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
+      ) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
+      ++ (lib.optional
+        (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
+        "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
+      );
+
+    assertions = [
+      {
+        assertion = cfg.role == "agent" -> !cfg.disableAgent;
+        message = "k3s: disableAgent must be false if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !cfg.clusterInit;
+        message = "k3s: clusterInit must be false if role is 'agent'";
+      }
+    ];
+
+    environment.systemPackages = [ config.services.k3s.package ];
+
+    # Use systemd-tmpfiles to activate k3s content
+    systemd.tmpfiles.settings."10-k3s" =
+      let
+        # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
+        enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
+        # Merge charts with charts contained in enabled auto deploying charts
+        helmCharts =
+          (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
+            lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
+          ))
+          // cfg.charts;
+        # Make a systemd-tmpfiles rule for a manifest
+        mkManifestRule = manifest: {
+          name = "${manifestDir}/${manifest.target}";
+          value = {
+            "L+".argument = "${manifest.source}";
+          };
+        };
+        # Ensure that all chart targets have a .tgz suffix
+        mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
+        # Make a systemd-tmpfiles rule for a chart
+        mkChartRule = target: source: {
+          name = "${chartDir}/${mkChartTarget target}";
+          value = {
+            "L+".argument = "${source}";
+          };
+        };
+        # Make a systemd-tmpfiles rule for a container image
+        mkImageRule = image: {
+          name = "${imageDir}/${image.name}";
+          value = {
+            "L+".argument = "${image}";
+          };
+        };
+      in
+      (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
+      // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
+      // (builtins.listToAttrs (map mkImageRule cfg.images))
+      // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
+        ${containerdConfigTemplateFile} = {
+          "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
+        };
+      });
+
+    systemd.services.k3s =
+      let
+        kubeletParams =
+          (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
+            inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
+          })
+          // cfg.extraKubeletConfig;
+        kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
+          {
+            apiVersion = "kubelet.config.k8s.io/v1beta1";
+            kind = "KubeletConfiguration";
+          }
+          // kubeletParams
+        );
+
+        kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
+          {
+            apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
+            kind = "KubeProxyConfiguration";
+          }
+          // cfg.extraKubeProxyConfig
+        );
+      in
+      {
+        description = "k3s service";
+        after = [
+          "firewall.service"
+          "network-online.target"
+        ];
+        wants = [
+          "firewall.service"
+          "network-online.target"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
+        serviceConfig = {
+          # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
+          Type = if cfg.role == "agent" then "exec" else "notify";
+          KillMode = "process";
+          Delegate = "yes";
+          Restart = "always";
+          RestartSec = "5s";
+          LimitNOFILE = 1048576;
+          LimitNPROC = "infinity";
+          LimitCORE = "infinity";
+          TasksMax = "infinity";
+          EnvironmentFile = cfg.environmentFile;
+          ExecStart = lib.concatStringsSep " \\\n " (
+            [ "${cfg.package}/bin/k3s ${cfg.role}" ]
+            ++ (lib.optional cfg.clusterInit "--cluster-init")
+            ++ (lib.optional cfg.disableAgent "--disable-agent")
+            ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
+            ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
+            ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
+            ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
+            ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
+            ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
+            ++ (lib.flatten cfg.extraFlags)
+          );
+        };
+      };
+  };
+
+  meta.maintainers = lib.teams.k3s.members;
+}

nixos/modules/services/cluster/kubernetes/addon-manager.nix

@@ -0,0 +1,184 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  cfg = top.addonManager;
+
+  isRBACEnabled = lib.elem "RBAC" top.apiserver.authorizationMode;
+
+  addons = pkgs.runCommand "kubernetes-addons" { } ''
+    mkdir -p $out
+    # since we are mounting the addons to the addon manager, they need to be copied
+    ${lib.concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (
+      lib.mapAttrsToList (name: addon: pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)) (
+        cfg.addons
+      )
+    )}
+  '';
+in
+{
+  ###### interface
+  options.services.kubernetes.addonManager = with lib.types; {
+
+    bootstrapAddons = lib.mkOption {
+      description = ''
+        Bootstrap addons are like regular addons, but they are applied with cluster-admin rights.
+        They are applied at addon-manager startup only.
+      '';
+      default = { };
+      type = attrsOf attrs;
+      example = lib.literalExpression ''
+        {
+          "my-service" = {
+            "apiVersion" = "v1";
+            "kind" = "Service";
+            "metadata" = {
+              "name" = "my-service";
+              "namespace" = "default";
+            };
+            "spec" = { ... };
+          };
+        }
+      '';
+    };
+
+    addons = lib.mkOption {
+      description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
+      default = { };
+      type = attrsOf (either attrs (listOf attrs));
+      example = lib.literalExpression ''
+        {
+          "my-service" = {
+            "apiVersion" = "v1";
+            "kind" = "Service";
+            "metadata" = {
+              "name" = "my-service";
+              "namespace" = "default";
+            };
+            "spec" = { ... };
+          };
+        }
+        // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dns.nix> { cfg = config.services.kubernetes; };
+      '';
+    };
+
+    enable = lib.mkEnableOption "Kubernetes addon manager";
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    environment.etc."kubernetes/addons".source = "${addons}/";
+
+    systemd.services.kube-addon-manager = {
+      description = "Kubernetes addon manager";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      environment.ADDON_PATH = "/etc/kubernetes/addons/";
+      path = [ pkgs.gawk ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = "${top.package}/bin/kube-addons";
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+        Restart = "on-failure";
+        RestartSec = 10;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddons = lib.mkIf isRBACEnabled (
+      let
+        name = "system:kube-addon-manager";
+        namespace = "kube-system";
+      in
+      {
+
+        kube-addon-manager-r = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "Role";
+          metadata = {
+            inherit name namespace;
+          };
+          rules = [
+            {
+              apiGroups = [ "*" ];
+              resources = [ "*" ];
+              verbs = [ "*" ];
+            }
+          ];
+        };
+
+        kube-addon-manager-rb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "RoleBinding";
+          metadata = {
+            inherit name namespace;
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "Role";
+            inherit name;
+          };
+          subjects = [
+            {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "User";
+              inherit name;
+            }
+          ];
+        };
+
+        kube-addon-manager-cluster-lister-cr = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRole";
+          metadata = {
+            name = "${name}:cluster-lister";
+          };
+          rules = [
+            {
+              apiGroups = [ "*" ];
+              resources = [ "*" ];
+              verbs = [ "list" ];
+            }
+          ];
+        };
+
+        kube-addon-manager-cluster-lister-crb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRoleBinding";
+          metadata = {
+            name = "${name}:cluster-lister";
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "ClusterRole";
+            name = "${name}:cluster-lister";
+          };
+          subjects = [
+            {
+              kind = "User";
+              inherit name;
+            }
+          ];
+        };
+      }
+    );
+
+    services.kubernetes.pki.certs = {
+      addonManager = top.lib.mkCert {
+        name = "kube-addon-manager";
+        CN = "system:kube-addon-manager";
+        action = "systemctl restart kube-addon-manager.service";
+      };
+    };
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/addons/dns.nix

@@ -0,0 +1,401 @@
+{
+  config,
+  options,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  version = "1.10.1";
+  cfg = config.services.kubernetes.addons.dns;
+  ports = {
+    dns = 10053;
+    health = 10054;
+    metrics = 10055;
+  };
+in
+{
+  options.services.kubernetes.addons.dns = {
+    enable = lib.mkEnableOption "kubernetes dns addon";
+
+    clusterIp = lib.mkOption {
+      description = "Dns addon clusterIP";
+
+      # this default is also what kubernetes users
+      default =
+        (lib.concatStringsSep "." (
+          lib.take 3 (lib.splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange)
+        ))
+        + ".254";
+      defaultText = lib.literalMD ''
+        The `x.y.z.254` IP of
+        `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
+      '';
+      type = lib.types.str;
+    };
+
+    clusterDomain = lib.mkOption {
+      description = "Dns cluster domain";
+      default = "cluster.local";
+      type = lib.types.str;
+    };
+
+    replicas = lib.mkOption {
+      description = "Number of DNS pod replicas to deploy in the cluster.";
+      default = 2;
+      type = lib.types.int;
+    };
+
+    reconcileMode = lib.mkOption {
+      description = ''
+        Controls the addon manager reconciliation mode for the DNS addon.
+
+        Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
+
+        See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
+      '';
+      default = "Reconcile";
+      type = lib.types.enum [
+        "Reconcile"
+        "EnsureExists"
+      ];
+    };
+
+    coredns = lib.mkOption {
+      description = "Docker image to seed for the CoreDNS container.";
+      type = lib.types.attrs;
+      default = {
+        imageName = "coredns/coredns";
+        imageDigest = "sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e";
+        finalImageTag = version;
+        sha256 = "0wg696920smmal7552a2zdhfncndn5kfammfa8bk8l7dz9bhk0y1";
+      };
+    };
+
+    corefile = lib.mkOption {
+      description = ''
+        Custom coredns corefile configuration.
+
+        See: <https://coredns.io/manual/toc/#configuration>.
+      '';
+      type = lib.types.str;
+      default = ''
+        .:${toString ports.dns} {
+          errors
+          health :${toString ports.health}
+          kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+          }
+          prometheus :${toString ports.metrics}
+          forward . /etc/resolv.conf
+          cache 30
+          loop
+          reload
+          loadbalance
+        }'';
+      defaultText = lib.literalExpression ''
+        '''
+          .:${toString ports.dns} {
+            errors
+            health :${toString ports.health}
+            kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
+              pods insecure
+              fallthrough in-addr.arpa ip6.arpa
+            }
+            prometheus :${toString ports.metrics}
+            forward . /etc/resolv.conf
+            cache 30
+            loop
+            reload
+            loadbalance
+          }
+        '''
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.kubernetes.kubelet.seedDockerImages = lib.singleton (
+      pkgs.dockerTools.pullImage cfg.coredns
+    );
+
+    services.kubernetes.addonManager.bootstrapAddons = {
+      coredns-cr = {
+        apiVersion = "rbac.authorization.k8s.io/v1";
+        kind = "ClusterRole";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/bootstrapping" = "rbac-defaults";
+          };
+          name = "system:coredns";
+        };
+        rules = [
+          {
+            apiGroups = [ "" ];
+            resources = [
+              "endpoints"
+              "services"
+              "pods"
+              "namespaces"
+            ];
+            verbs = [
+              "list"
+              "watch"
+            ];
+          }
+          {
+            apiGroups = [ "" ];
+            resources = [ "nodes" ];
+            verbs = [ "get" ];
+          }
+          {
+            apiGroups = [ "discovery.k8s.io" ];
+            resources = [ "endpointslices" ];
+            verbs = [
+              "list"
+              "watch"
+            ];
+          }
+        ];
+      };
+
+      coredns-crb = {
+        apiVersion = "rbac.authorization.k8s.io/v1";
+        kind = "ClusterRoleBinding";
+        metadata = {
+          annotations = {
+            "rbac.authorization.kubernetes.io/autoupdate" = "true";
+          };
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/bootstrapping" = "rbac-defaults";
+          };
+          name = "system:coredns";
+        };
+        roleRef = {
+          apiGroup = "rbac.authorization.k8s.io";
+          kind = "ClusterRole";
+          name = "system:coredns";
+        };
+        subjects = [
+          {
+            kind = "ServiceAccount";
+            name = "coredns";
+            namespace = "kube-system";
+          }
+        ];
+      };
+    };
+
+    services.kubernetes.addonManager.addons = {
+      coredns-sa = {
+        apiVersion = "v1";
+        kind = "ServiceAccount";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+      };
+
+      coredns-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+        data = {
+          Corefile = cfg.corefile;
+        };
+      };
+
+      coredns-deploy = {
+        apiVersion = "apps/v1";
+        kind = "Deployment";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "CoreDNS";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+        spec = {
+          replicas = cfg.replicas;
+          selector = {
+            matchLabels = {
+              k8s-app = "kube-dns";
+            };
+          };
+          strategy = {
+            rollingUpdate = {
+              maxUnavailable = 1;
+            };
+            type = "RollingUpdate";
+          };
+          template = {
+            metadata = {
+              labels = {
+                k8s-app = "kube-dns";
+              };
+            };
+            spec = {
+              containers = [
+                {
+                  args = [
+                    "-conf"
+                    "/etc/coredns/Corefile"
+                  ];
+                  image = with cfg.coredns; "${imageName}:${finalImageTag}";
+                  imagePullPolicy = "Never";
+                  livenessProbe = {
+                    failureThreshold = 5;
+                    httpGet = {
+                      path = "/health";
+                      port = ports.health;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 60;
+                    successThreshold = 1;
+                    timeoutSeconds = 5;
+                  };
+                  name = "coredns";
+                  ports = [
+                    {
+                      containerPort = ports.dns;
+                      name = "dns";
+                      protocol = "UDP";
+                    }
+                    {
+                      containerPort = ports.dns;
+                      name = "dns-tcp";
+                      protocol = "TCP";
+                    }
+                    {
+                      containerPort = ports.metrics;
+                      name = "metrics";
+                      protocol = "TCP";
+                    }
+                  ];
+                  resources = {
+                    limits = {
+                      memory = "170Mi";
+                    };
+                    requests = {
+                      cpu = "100m";
+                      memory = "70Mi";
+                    };
+                  };
+                  securityContext = {
+                    allowPrivilegeEscalation = false;
+                    capabilities = {
+                      drop = [ "all" ];
+                    };
+                    readOnlyRootFilesystem = true;
+                  };
+                  volumeMounts = [
+                    {
+                      mountPath = "/etc/coredns";
+                      name = "config-volume";
+                      readOnly = true;
+                    }
+                  ];
+                }
+              ];
+              dnsPolicy = "Default";
+              nodeSelector = {
+                "beta.kubernetes.io/os" = "linux";
+              };
+              serviceAccountName = "coredns";
+              tolerations = [
+                {
+                  effect = "NoSchedule";
+                  key = "node-role.kubernetes.io/master";
+                }
+                {
+                  key = "CriticalAddonsOnly";
+                  operator = "Exists";
+                }
+              ];
+              volumes = [
+                {
+                  configMap = {
+                    items = [
+                      {
+                        key = "Corefile";
+                        path = "Corefile";
+                      }
+                    ];
+                    name = "coredns";
+                  };
+                  name = "config-volume";
+                }
+              ];
+            };
+          };
+        };
+      };
+
+      coredns-svc = {
+        apiVersion = "v1";
+        kind = "Service";
+        metadata = {
+          annotations = {
+            "prometheus.io/port" = toString ports.metrics;
+            "prometheus.io/scrape" = "true";
+          };
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "CoreDNS";
+          };
+          name = "kube-dns";
+          namespace = "kube-system";
+        };
+        spec = {
+          clusterIP = cfg.clusterIp;
+          ports = [
+            {
+              name = "dns";
+              port = 53;
+              targetPort = ports.dns;
+              protocol = "UDP";
+            }
+            {
+              name = "dns-tcp";
+              port = 53;
+              targetPort = ports.dns;
+              protocol = "TCP";
+            }
+          ];
+          selector = {
+            k8s-app = "kube-dns";
+          };
+        };
+      };
+    };
+
+    services.kubernetes.kubelet.clusterDns = lib.mkDefault [ cfg.clusterIp ];
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/apiserver.nix

@@ -0,0 +1,541 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.apiserver;
+
+  isRBACEnabled = lib.elem "RBAC" cfg.authorizationMode;
+
+  apiserverServiceIP = (
+    lib.concatStringsSep "." (lib.take 3 (lib.splitString "." cfg.serviceClusterIpRange)) + ".1"
+  );
+in
+{
+
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "apiserver" "admissionControl" ]
+      [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "apiserver" "address" ]
+      [ "services" "kubernetes" "apiserver" "bindAddress" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecureBindAddress" ] "")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecurePort" ] "")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "servers" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "servers" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "keyFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "certFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "certFile" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "caFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "caFile" ]
+    )
+  ];
+
+  ###### interface
+  options.services.kubernetes.apiserver = with lib.types; {
+
+    advertiseAddress = lib.mkOption {
+      description = ''
+        Kubernetes apiserver IP address on which to advertise the apiserver
+        to members of the cluster. This address must be reachable by the rest
+        of the cluster.
+      '';
+      default = null;
+      type = nullOr str;
+    };
+
+    allowPrivileged = lib.mkOption {
+      description = "Whether to allow privileged containers on Kubernetes.";
+      default = false;
+      type = bool;
+    };
+
+    authorizationMode = lib.mkOption {
+      description = ''
+        Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
+      '';
+      default = [
+        "RBAC"
+        "Node"
+      ]; # Enabling RBAC by default, although kubernetes default is AllowAllow
+      type = listOf (enum [
+        "AlwaysAllow"
+        "AlwaysDeny"
+        "ABAC"
+        "Webhook"
+        "RBAC"
+        "Node"
+      ]);
+    };
+
+    authorizationPolicy = lib.mkOption {
+      description = ''
+        Kubernetes apiserver authorization policy file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
+      '';
+      default = [ ];
+      type = listOf attrs;
+    };
+
+    basicAuthFile = lib.mkOption {
+      description = ''
+        Kubernetes apiserver basic authentication file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    bindAddress = lib.mkOption {
+      description = ''
+        The IP address on which to listen for the --secure-port port.
+        The associated interface(s) must be reachable by the rest
+        of the cluster, and by CLI/web clients.
+      '';
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    clientCaFile = lib.mkOption {
+      description = "Kubernetes apiserver CA file for client auth.";
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    disableAdmissionPlugins = lib.mkOption {
+      description = ''
+        Kubernetes admission control plugins to disable. See
+        <https://kubernetes.io/docs/admin/admission-controllers/>
+      '';
+      default = [ ];
+      type = listOf str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes apiserver";
+
+    enableAdmissionPlugins = lib.mkOption {
+      description = ''
+        Kubernetes admission control plugins to enable. See
+        <https://kubernetes.io/docs/admin/admission-controllers/>
+      '';
+      default = [
+        "NamespaceLifecycle"
+        "LimitRanger"
+        "ServiceAccount"
+        "ResourceQuota"
+        "DefaultStorageClass"
+        "DefaultTolerationSeconds"
+        "NodeRestriction"
+      ];
+      example = [
+        "NamespaceLifecycle"
+        "NamespaceExists"
+        "LimitRanger"
+        "SecurityContextDeny"
+        "ServiceAccount"
+        "ResourceQuota"
+        "PodSecurityPolicy"
+        "NodeRestriction"
+        "DefaultStorageClass"
+      ];
+      type = listOf str;
+    };
+
+    etcd = {
+      servers = lib.mkOption {
+        description = "List of etcd servers.";
+        default = [ "http://127.0.0.1:2379" ];
+        type = types.listOf types.str;
+      };
+
+      keyFile = lib.mkOption {
+        description = "Etcd key file.";
+        default = null;
+        type = types.nullOr types.path;
+      };
+
+      certFile = lib.mkOption {
+        description = "Etcd cert file.";
+        default = null;
+        type = types.nullOr types.path;
+      };
+
+      caFile = lib.mkOption {
+        description = "Etcd ca file.";
+        default = top.caFile;
+        defaultText = lib.literalExpression "config.${otop.caFile}";
+        type = types.nullOr types.path;
+      };
+    };
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes apiserver extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    extraSANs = lib.mkOption {
+      description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert.";
+      default = [ ];
+      type = listOf str;
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeletClientCaFile = lib.mkOption {
+      description = "Path to a cert file for connecting to kubelet.";
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    kubeletClientCertFile = lib.mkOption {
+      description = "Client certificate to use for connections to kubelet.";
+      default = null;
+      type = nullOr path;
+    };
+
+    kubeletClientKeyFile = lib.mkOption {
+      description = "Key to use for connections to kubelet.";
+      default = null;
+      type = nullOr path;
+    };
+
+    preferredAddressTypes = lib.mkOption {
+      description = "List of the preferred NodeAddressTypes to use for kubelet connections.";
+      type = nullOr str;
+      default = null;
+    };
+
+    proxyClientCertFile = lib.mkOption {
+      description = "Client certificate to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
+    proxyClientKeyFile = lib.mkOption {
+      description = "Key to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
+    runtimeConfig = lib.mkOption {
+      description = ''
+        Api runtime configuration. See
+        <https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/>
+      '';
+      default = "authentication.k8s.io/v1beta1=true";
+      example = "api/all=false,api/v1=true";
+      type = str;
+    };
+
+    storageBackend = lib.mkOption {
+      description = ''
+        Kubernetes apiserver storage backend.
+      '';
+      default = "etcd3";
+      type = enum [
+        "etcd2"
+        "etcd3"
+      ];
+    };
+
+    securePort = lib.mkOption {
+      description = "Kubernetes apiserver secure port.";
+      default = 6443;
+      type = int;
+    };
+
+    apiAudiences = lib.mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "api,https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountIssuer = lib.mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountSigningKeyFile = lib.mkOption {
+      description = ''
+        Path to the file that contains the current private key of the service
+        account token issuer. The issuer will sign issued ID tokens with this
+        private key.
+      '';
+      type = path;
+    };
+
+    serviceAccountKeyFile = lib.mkOption {
+      description = ''
+        File containing PEM-encoded x509 RSA or ECDSA private or public keys,
+        used to verify ServiceAccount tokens. The specified file can contain
+        multiple keys, and the flag can be specified multiple times with
+        different files. If unspecified, --tls-private-key-file is used.
+        Must be specified when --service-account-signing-key is provided
+      '';
+      type = path;
+    };
+
+    serviceClusterIpRange = lib.mkOption {
+      description = ''
+        A CIDR notation IP range from which to assign service cluster IPs.
+        This must not overlap with any IP ranges assigned to nodes for pods.
+      '';
+      default = "10.0.0.0/24";
+      type = str;
+    };
+
+    tlsCertFile = lib.mkOption {
+      description = "Kubernetes apiserver certificate file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = lib.mkOption {
+      description = "Kubernetes apiserver private key file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tokenAuthFile = lib.mkOption {
+      description = ''
+        Kubernetes apiserver token authentication file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+    webhookConfig = lib.mkOption {
+      description = ''
+        Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
+        See <https://kubernetes.io/docs/reference/access-authn-authz/webhook/>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkMerge [
+
+    (lib.mkIf cfg.enable {
+      systemd.services.kube-apiserver = {
+        description = "Kubernetes APIServer Service";
+        wantedBy = [ "kubernetes.target" ];
+        after = [ "network.target" ];
+        serviceConfig = {
+          Slice = "kubernetes.slice";
+          ExecStart = ''
+            ${top.package}/bin/kube-apiserver \
+            --allow-privileged=${lib.boolToString cfg.allowPrivileged} \
+            --authorization-mode=${lib.concatStringsSep "," cfg.authorizationMode} \
+              ${lib.optionalString (lib.elem "ABAC" cfg.authorizationMode) "--authorization-policy-file=${pkgs.writeText "kube-auth-policy.jsonl" (lib.concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy)}"} \
+              ${lib.optionalString (lib.elem "Webhook" cfg.authorizationMode) "--authorization-webhook-config-file=${cfg.webhookConfig}"} \
+            --bind-address=${cfg.bindAddress} \
+            ${lib.optionalString (cfg.advertiseAddress != null) "--advertise-address=${cfg.advertiseAddress}"} \
+            ${lib.optionalString (cfg.clientCaFile != null) "--client-ca-file=${cfg.clientCaFile}"} \
+            --disable-admission-plugins=${lib.concatStringsSep "," cfg.disableAdmissionPlugins} \
+            --enable-admission-plugins=${lib.concatStringsSep "," cfg.enableAdmissionPlugins} \
+            --etcd-servers=${lib.concatStringsSep "," cfg.etcd.servers} \
+            ${lib.optionalString (cfg.etcd.caFile != null) "--etcd-cafile=${cfg.etcd.caFile}"} \
+            ${lib.optionalString (cfg.etcd.certFile != null) "--etcd-certfile=${cfg.etcd.certFile}"} \
+            ${lib.optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \
+            ${
+              lib.optionalString (cfg.featureGates != { })
+                "--feature-gates=${
+                  (lib.concatStringsSep "," (
+                    builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                  ))
+                }"
+            } \
+            ${lib.optionalString (cfg.basicAuthFile != null) "--basic-auth-file=${cfg.basicAuthFile}"} \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientCaFile != null
+              ) "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientCertFile != null
+              ) "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientKeyFile != null
+              ) "--kubelet-client-key=${cfg.kubeletClientKeyFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.preferredAddressTypes != null
+              ) "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.proxyClientCertFile != null
+              ) "--proxy-client-cert-file=${cfg.proxyClientCertFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.proxyClientKeyFile != null
+              ) "--proxy-client-key-file=${cfg.proxyClientKeyFile}"
+            } \
+            ${lib.optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \
+            --secure-port=${toString cfg.securePort} \
+            --api-audiences=${toString cfg.apiAudiences} \
+            --service-account-issuer=${toString cfg.serviceAccountIssuer} \
+            --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
+            --service-account-key-file=${cfg.serviceAccountKeyFile} \
+            --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
+            --storage-backend=${cfg.storageBackend} \
+            ${lib.optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \
+            ${lib.optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \
+            ${lib.optionalString (cfg.tokenAuthFile != null) "--token-auth-file=${cfg.tokenAuthFile}"} \
+            ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+            ${cfg.extraOpts}
+          '';
+          WorkingDirectory = top.dataDir;
+          User = "kubernetes";
+          Group = "kubernetes";
+          AmbientCapabilities = "cap_net_bind_service";
+          Restart = "on-failure";
+          RestartSec = 5;
+        };
+
+        unitConfig = {
+          StartLimitIntervalSec = 0;
+        };
+      };
+
+      services.etcd = {
+        clientCertAuth = lib.mkDefault true;
+        peerClientCertAuth = lib.mkDefault true;
+        listenClientUrls = lib.mkDefault [ "https://0.0.0.0:2379" ];
+        listenPeerUrls = lib.mkDefault [ "https://0.0.0.0:2380" ];
+        advertiseClientUrls = lib.mkDefault [ "https://${top.masterAddress}:2379" ];
+        initialCluster = lib.mkDefault [ "${top.masterAddress}=https://${top.masterAddress}:2380" ];
+        name = lib.mkDefault top.masterAddress;
+        initialAdvertisePeerUrls = lib.mkDefault [ "https://${top.masterAddress}:2380" ];
+      };
+
+      services.kubernetes.addonManager.bootstrapAddons = lib.mkIf isRBACEnabled {
+
+        apiserver-kubelet-api-admin-crb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRoleBinding";
+          metadata = {
+            name = "system:kube-apiserver:kubelet-api-admin";
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "ClusterRole";
+            name = "system:kubelet-api-admin";
+          };
+          subjects = [
+            {
+              kind = "User";
+              name = "system:kube-apiserver";
+            }
+          ];
+        };
+
+      };
+
+      services.kubernetes.pki.certs = with top.lib; {
+        apiServer = mkCert {
+          name = "kube-apiserver";
+          CN = "kubernetes";
+          hosts = [
+            "kubernetes.default.svc"
+            "kubernetes.default.svc.${top.addons.dns.clusterDomain}"
+            cfg.advertiseAddress
+            top.masterAddress
+            apiserverServiceIP
+            "127.0.0.1"
+          ]
+          ++ cfg.extraSANs;
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverProxyClient = mkCert {
+          name = "kube-apiserver-proxy-client";
+          CN = "front-proxy-client";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverKubeletClient = mkCert {
+          name = "kube-apiserver-kubelet-client";
+          CN = "system:kube-apiserver";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverEtcdClient = mkCert {
+          name = "kube-apiserver-etcd-client";
+          CN = "etcd-client";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        clusterAdmin = mkCert {
+          name = "cluster-admin";
+          CN = "cluster-admin";
+          fields = {
+            O = "system:masters";
+          };
+          privateKeyOwner = "root";
+        };
+        etcd = mkCert {
+          name = "etcd";
+          CN = top.masterAddress;
+          hosts = [
+            "etcd.local"
+            "etcd.${top.addons.dns.clusterDomain}"
+            top.masterAddress
+            cfg.advertiseAddress
+          ];
+          privateKeyOwner = "etcd";
+          action = "systemctl restart etcd.service";
+        };
+      };
+
+    })
+
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/controller-manager.nix

@@ -0,0 +1,182 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.controllerManager;
+in
+{
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "controllerManager" "address" ]
+      [ "services" "kubernetes" "controllerManager" "bindAddress" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "controllerManager" "insecurePort" ] "")
+  ];
+
+  ###### interface
+  options.services.kubernetes.controllerManager = with lib.types; {
+
+    allocateNodeCIDRs = lib.mkOption {
+      description = "Whether to automatically allocate CIDR ranges for cluster nodes.";
+      default = true;
+      type = bool;
+    };
+
+    bindAddress = lib.mkOption {
+      description = "Kubernetes controller manager listening address.";
+      default = "127.0.0.1";
+      type = str;
+    };
+
+    clusterCidr = lib.mkOption {
+      description = "Kubernetes CIDR Range for Pods in cluster.";
+      default = top.clusterCidr;
+      defaultText = lib.literalExpression "config.${otop.clusterCidr}";
+      type = str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes controller manager";
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes controller manager extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager";
+
+    leaderElect = lib.mkOption {
+      description = "Whether to start leader election before executing main loop.";
+      type = bool;
+      default = true;
+    };
+
+    rootCaFile = lib.mkOption {
+      description = ''
+        Kubernetes controller manager certificate authority file included in
+        service account's token secret.
+      '';
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    securePort = lib.mkOption {
+      description = "Kubernetes controller manager secure listening port.";
+      default = 10252;
+      type = int;
+    };
+
+    serviceAccountKeyFile = lib.mkOption {
+      description = ''
+        Kubernetes controller manager PEM-encoded private RSA key file used to
+        sign service account tokens
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsCertFile = lib.mkOption {
+      description = "Kubernetes controller-manager certificate file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = lib.mkOption {
+      description = "Kubernetes controller-manager private key file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    systemd.services.kube-controller-manager = {
+      description = "Kubernetes Controller Manager Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      serviceConfig = {
+        RestartSec = "30s";
+        Restart = "on-failure";
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-controller-manager \
+                    --allocate-node-cidrs=${lib.boolToString cfg.allocateNodeCIDRs} \
+                    --bind-address=${cfg.bindAddress} \
+                    ${lib.optionalString (cfg.clusterCidr != null) "--cluster-cidr=${cfg.clusterCidr}"} \
+                    ${
+                      lib.optionalString (cfg.featureGates != { })
+                        "--feature-gates=${
+                          lib.concatStringsSep "," (
+                            builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                          )
+                        }"
+                    } \
+                    --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
+                    --leader-elect=${lib.boolToString cfg.leaderElect} \
+                    ${lib.optionalString (cfg.rootCaFile != null) "--root-ca-file=${cfg.rootCaFile}"} \
+                    --secure-port=${toString cfg.securePort} \
+                    ${
+                      lib.optionalString (
+                        cfg.serviceAccountKeyFile != null
+                      ) "--service-account-private-key-file=${cfg.serviceAccountKeyFile}"
+                    } \
+                    ${lib.optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \
+                    ${
+                      lib.optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"
+                    } \
+                    ${lib.optionalString (lib.elem "RBAC" top.apiserver.authorizationMode) "--use-service-account-credentials"} \
+                    ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                    ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+      path = top.path;
+    };
+
+    services.kubernetes.pki.certs = with top.lib; {
+      controllerManager = mkCert {
+        name = "kube-controller-manager";
+        CN = "kube-controller-manager";
+        action = "systemctl restart kube-controller-manager.service";
+      };
+      controllerManagerClient = mkCert {
+        name = "kube-controller-manager-client";
+        CN = "system:kube-controller-manager";
+        action = "systemctl restart kube-controller-manager.service";
+      };
+    };
+
+    services.kubernetes.controllerManager.kubeconfig.server = lib.mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/default.nix

@@ -0,0 +1,356 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.kubernetes;
+  opt = options.services.kubernetes;
+
+  defaultContainerdSettings = {
+    version = 2;
+    root = "/var/lib/containerd";
+    state = "/run/containerd";
+    oom_score = 0;
+
+    grpc = {
+      address = "/run/containerd/containerd.sock";
+    };
+
+    plugins."io.containerd.grpc.v1.cri" = {
+      sandbox_image = "pause:latest";
+
+      cni = {
+        bin_dir = "/opt/cni/bin";
+        max_conf_num = 0;
+      };
+
+      containerd.runtimes.runc = {
+        runtime_type = "io.containerd.runc.v2";
+        options.SystemdCgroup = true;
+      };
+    };
+  };
+
+  mkKubeConfig =
+    name: conf:
+    pkgs.writeText "${name}-kubeconfig" (
+      builtins.toJSON {
+        apiVersion = "v1";
+        kind = "Config";
+        clusters = [
+          {
+            name = "local";
+            cluster.certificate-authority = conf.caFile or cfg.caFile;
+            cluster.server = conf.server;
+          }
+        ];
+        users = [
+          {
+            inherit name;
+            user = {
+              client-certificate = conf.certFile;
+              client-key = conf.keyFile;
+            };
+          }
+        ];
+        contexts = [
+          {
+            context = {
+              cluster = "local";
+              user = name;
+            };
+            name = "local";
+          }
+        ];
+        current-context = "local";
+      }
+    );
+
+  caCert = secret "ca";
+
+  etcdEndpoints = [ "https://${cfg.masterAddress}:2379" ];
+
+  mkCert =
+    {
+      name,
+      CN,
+      hosts ? [ ],
+      fields ? { },
+      action ? "",
+      privateKeyOwner ? "kubernetes",
+      privateKeyGroup ? "kubernetes",
+    }:
+    rec {
+      inherit
+        name
+        caCert
+        CN
+        hosts
+        fields
+        action
+        ;
+      cert = secret name;
+      key = secret "${name}-key";
+      privateKeyOptions = {
+        owner = privateKeyOwner;
+        group = privateKeyGroup;
+        mode = "0600";
+        path = key;
+      };
+    };
+
+  secret = name: "${cfg.secretsPath}/${name}.pem";
+
+  mkKubeConfigOptions = prefix: {
+    server = lib.mkOption {
+      description = "${prefix} kube-apiserver server address.";
+      type = lib.types.str;
+    };
+
+    caFile = lib.mkOption {
+      description = "${prefix} certificate authority file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = cfg.caFile;
+      defaultText = lib.literalExpression "config.${opt.caFile}";
+    };
+
+    certFile = lib.mkOption {
+      description = "${prefix} client certificate file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    keyFile = lib.mkOption {
+      description = "${prefix} client key file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+  };
+in
+{
+
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "services"
+      "kubernetes"
+      "addons"
+      "dashboard"
+    ] "Removed due to it being an outdated version")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
+  ];
+
+  ###### interface
+
+  options.services.kubernetes = {
+    roles = lib.mkOption {
+      description = ''
+        Kubernetes role that this machine should take.
+
+        Master role will enable etcd, apiserver, scheduler, controller manager
+        addon manager, flannel and proxy services.
+        Node role will enable flannel, docker, kubelet and proxy services.
+      '';
+      default = [ ];
+      type = lib.types.listOf (
+        lib.types.enum [
+          "master"
+          "node"
+        ]
+      );
+    };
+
+    package = lib.mkPackageOption pkgs "kubernetes" { };
+
+    kubeconfig = mkKubeConfigOptions "Default kubeconfig";
+
+    apiserverAddress = lib.mkOption {
+      description = ''
+        Clusterwide accessible address for the kubernetes apiserver,
+        including protocol and optional port.
+      '';
+      example = "https://kubernetes-apiserver.example.com:6443";
+      type = lib.types.str;
+    };
+
+    caFile = lib.mkOption {
+      description = "Default kubernetes certificate authority";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    dataDir = lib.mkOption {
+      description = "Kubernetes root directory for managing kubelet files.";
+      default = "/var/lib/kubernetes";
+      type = lib.types.path;
+    };
+
+    easyCerts = lib.mkOption {
+      description = "Automatically setup x509 certificates and keys for the entire cluster.";
+      default = false;
+      type = lib.types.bool;
+    };
+
+    featureGates = lib.mkOption {
+      description = "List set of feature gates.";
+      default = { };
+      type = lib.types.attrsOf lib.types.bool;
+    };
+
+    masterAddress = lib.mkOption {
+      description = "Clusterwide available network address or hostname for the kubernetes master server.";
+      example = "master.example.com";
+      type = lib.types.str;
+    };
+
+    path = lib.mkOption {
+      description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+    };
+
+    clusterCidr = lib.mkOption {
+      description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
+      default = "10.1.0.0/16";
+      type = lib.types.nullOr lib.types.str;
+    };
+
+    lib = lib.mkOption {
+      description = "Common functions for the kubernetes modules.";
+      default = {
+        inherit mkCert;
+        inherit mkKubeConfig;
+        inherit mkKubeConfigOptions;
+      };
+      type = lib.types.attrs;
+    };
+
+    secretsPath = lib.mkOption {
+      description = "Default location for kubernetes secrets. Not a store location.";
+      type = lib.types.path;
+      default = cfg.dataDir + "/secrets";
+      defaultText = lib.literalExpression ''
+        config.${opt.dataDir} + "/secrets"
+      '';
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkMerge [
+
+    (lib.mkIf cfg.easyCerts {
+      services.kubernetes.pki.enable = lib.mkDefault true;
+      services.kubernetes.caFile = caCert;
+    })
+
+    (lib.mkIf (lib.elem "master" cfg.roles) {
+      services.kubernetes.apiserver.enable = lib.mkDefault true;
+      services.kubernetes.scheduler.enable = lib.mkDefault true;
+      services.kubernetes.controllerManager.enable = lib.mkDefault true;
+      services.kubernetes.addonManager.enable = lib.mkDefault true;
+      services.kubernetes.proxy.enable = lib.mkDefault true;
+      services.etcd.enable = true; # Cannot mkDefault because of flannel default options
+      services.kubernetes.kubelet = {
+        enable = lib.mkDefault true;
+        taints = lib.mkIf (!(lib.elem "node" cfg.roles)) {
+          master = {
+            key = "node-role.kubernetes.io/master";
+            value = "true";
+            effect = "NoSchedule";
+          };
+        };
+      };
+    })
+
+    (lib.mkIf (lib.all (el: el == "master") cfg.roles) {
+      # if this node is only a master make it unschedulable by default
+      services.kubernetes.kubelet.unschedulable = lib.mkDefault true;
+    })
+
+    (lib.mkIf (lib.elem "node" cfg.roles) {
+      services.kubernetes.kubelet.enable = lib.mkDefault true;
+      services.kubernetes.proxy.enable = lib.mkDefault true;
+    })
+
+    # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
+    (lib.mkIf (cfg.roles != [ ]) {
+      services.kubernetes.flannel.enable = lib.mkDefault true;
+      services.flannel.etcd.endpoints = lib.mkDefault etcdEndpoints;
+      services.kubernetes.easyCerts = lib.mkDefault true;
+    })
+
+    (lib.mkIf cfg.apiserver.enable {
+      services.kubernetes.pki.etcClusterAdminKubeconfig = lib.mkDefault "kubernetes/cluster-admin.kubeconfig";
+      services.kubernetes.apiserver.etcd.servers = lib.mkDefault etcdEndpoints;
+    })
+
+    (lib.mkIf cfg.kubelet.enable {
+      virtualisation.containerd = {
+        enable = lib.mkDefault true;
+        settings = lib.mapAttrsRecursive (name: lib.mkDefault) defaultContainerdSettings;
+      };
+    })
+
+    (lib.mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
+      services.kubernetes.pki.certs = {
+        serviceAccount = mkCert {
+          name = "service-account";
+          CN = "system:service-account-signer";
+          action = ''
+            systemctl restart \
+              kube-apiserver.service \
+              kube-controller-manager.service
+          '';
+        };
+      };
+    })
+
+    (lib.mkIf
+      (
+        cfg.apiserver.enable
+        || cfg.scheduler.enable
+        || cfg.controllerManager.enable
+        || cfg.kubelet.enable
+        || cfg.proxy.enable
+        || cfg.addonManager.enable
+      )
+      {
+        systemd.targets.kubernetes = {
+          description = "Kubernetes";
+          wantedBy = [ "multi-user.target" ];
+        };
+
+        systemd.tmpfiles.rules = [
+          "d /opt/cni/bin 0755 root root -"
+          "d /run/kubernetes 0755 kubernetes kubernetes -"
+          "d ${cfg.dataDir} 0755 kubernetes kubernetes -"
+        ];
+
+        users.users.kubernetes = {
+          uid = config.ids.uids.kubernetes;
+          description = "Kubernetes user";
+          group = "kubernetes";
+          home = cfg.dataDir;
+          createHome = true;
+          homeMode = "755";
+        };
+        users.groups.kubernetes.gid = config.ids.gids.kubernetes;
+
+        # dns addon is enabled by default
+        services.kubernetes.addons.dns.enable = lib.mkDefault true;
+
+        services.kubernetes.apiserverAddress = lib.mkDefault "https://${
+          if cfg.apiserver.advertiseAddress != null then
+            cfg.apiserver.advertiseAddress
+          else
+            "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"
+        }";
+      }
+    )
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  cfg = top.flannel;
+
+  # we want flannel to use kubernetes itself as configuration backend, not direct etcd
+  storageBackend = "kubernetes";
+in
+{
+  ###### interface
+  options.services.kubernetes.flannel = {
+    enable = lib.mkEnableOption "flannel networking";
+
+    openFirewallPorts = lib.mkOption {
+      description = ''Whether to open the Flannel UDP ports in the firewall on all interfaces.'';
+      type = lib.types.bool;
+      default = true;
+    };
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    services.flannel = {
+
+      enable = lib.mkDefault true;
+      network = lib.mkDefault top.clusterCidr;
+      inherit storageBackend;
+      nodeName = config.services.kubernetes.kubelet.hostname;
+    };
+
+    services.kubernetes.kubelet = {
+      cni.config = lib.mkDefault [
+        {
+          name = "mynet";
+          type = "flannel";
+          cniVersion = "0.3.1";
+          delegate = {
+            isDefaultGateway = true;
+            hairpinMode = true;
+            bridge = "mynet";
+          };
+        }
+      ];
+    };
+
+    networking = {
+      firewall.allowedUDPPorts = lib.mkIf cfg.openFirewallPorts [
+        8285 # flannel udp
+        8472 # flannel vxlan
+      ];
+      dhcpcd.denyInterfaces = [
+        "mynet*"
+        "flannel*"
+      ];
+    };
+
+    services.kubernetes.pki.certs = {
+      flannelClient = top.lib.mkCert {
+        name = "flannel-client";
+        CN = "flannel-client";
+        action = "systemctl restart flannel.service";
+      };
+    };
+
+    # give flannel some kubernetes rbac permissions if applicable
+    services.kubernetes.addonManager.bootstrapAddons =
+      lib.mkIf ((storageBackend == "kubernetes") && (lib.elem "RBAC" top.apiserver.authorizationMode))
+        {
+
+          flannel-cr = {
+            apiVersion = "rbac.authorization.k8s.io/v1";
+            kind = "ClusterRole";
+            metadata = {
+              name = "flannel";
+            };
+            rules = [
+              {
+                apiGroups = [ "" ];
+                resources = [ "pods" ];
+                verbs = [ "get" ];
+              }
+              {
+                apiGroups = [ "" ];
+                resources = [ "nodes" ];
+                verbs = [
+                  "list"
+                  "watch"
+                ];
+              }
+              {
+                apiGroups = [ "" ];
+                resources = [ "nodes/status" ];
+                verbs = [ "patch" ];
+              }
+            ];
+          };
+
+          flannel-crb = {
+            apiVersion = "rbac.authorization.k8s.io/v1";
+            kind = "ClusterRoleBinding";
+            metadata = {
+              name = "flannel";
+            };
+            roleRef = {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "ClusterRole";
+              name = "flannel";
+            };
+            subjects = [
+              {
+                kind = "User";
+                name = "flannel-client";
+              }
+            ];
+          };
+
+        };
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -0,0 +1,444 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.kubelet;
+
+  cniConfig =
+    if cfg.cni.config != [ ] && cfg.cni.configDir != null then
+      throw "Verbatim CNI-config and CNI configDir cannot both be set."
+    else if cfg.cni.configDir != null then
+      cfg.cni.configDir
+    else
+      (pkgs.buildEnv {
+        name = "kubernetes-cni-config";
+        paths = imap (
+          i: entry: pkgs.writeTextDir "${toString (10 + i)}-${entry.type}.conf" (builtins.toJSON entry)
+        ) cfg.cni.config;
+      });
+
+  infraContainer = pkgs.dockerTools.buildImage {
+    name = "pause";
+    tag = "latest";
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ top.package.pause ];
+    };
+    config.Cmd = [ "/bin/pause" ];
+  };
+
+  kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
+
+  # Flag based settings are deprecated, use the `--config` flag with a
+  # `KubeletConfiguration` struct.
+  # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
+  #
+  # NOTE: registerWithTaints requires a []core/v1.Taint, therefore requires
+  # additional work to be put in config format.
+  #
+  kubeletConfig = pkgs.writeText "kubelet-config" (
+    builtins.toJSON (
+      {
+        apiVersion = "kubelet.config.k8s.io/v1beta1";
+        kind = "KubeletConfiguration";
+        address = cfg.address;
+        port = cfg.port;
+        authentication = {
+          x509 = lib.optionalAttrs (cfg.clientCaFile != null) { clientCAFile = cfg.clientCaFile; };
+          webhook = {
+            enabled = true;
+            cacheTTL = "10s";
+          };
+        };
+        authorization = {
+          mode = "Webhook";
+        };
+        cgroupDriver = "systemd";
+        hairpinMode = "hairpin-veth";
+        registerNode = cfg.registerNode;
+        containerRuntimeEndpoint = cfg.containerRuntimeEndpoint;
+        healthzPort = cfg.healthz.port;
+        healthzBindAddress = cfg.healthz.bind;
+      }
+      // lib.optionalAttrs (cfg.tlsCertFile != null) { tlsCertFile = cfg.tlsCertFile; }
+      // lib.optionalAttrs (cfg.tlsKeyFile != null) { tlsPrivateKeyFile = cfg.tlsKeyFile; }
+      // lib.optionalAttrs (cfg.clusterDomain != "") { clusterDomain = cfg.clusterDomain; }
+      // lib.optionalAttrs (cfg.clusterDns != [ ]) { clusterDNS = cfg.clusterDns; }
+      // lib.optionalAttrs (cfg.featureGates != { }) { featureGates = cfg.featureGates; }
+      // lib.optionalAttrs (cfg.extraConfig != { }) cfg.extraConfig
+    )
+  );
+
+  manifestPath = "kubernetes/manifests";
+
+  taintOptions =
+    with lib.types;
+    { name, ... }:
+    {
+      options = {
+        key = mkOption {
+          description = "Key of taint.";
+          default = name;
+          defaultText = literalMD "Name of this submodule.";
+          type = str;
+        };
+        value = mkOption {
+          description = "Value of taint.";
+          type = str;
+        };
+        effect = mkOption {
+          description = "Effect of taint.";
+          example = "NoSchedule";
+          type = enum [
+            "NoSchedule"
+            "PreferNoSchedule"
+            "NoExecute"
+          ];
+        };
+      };
+    };
+
+  taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
+    mapAttrsToList (n: v: v) cfg.taints
+  );
+in
+{
+  imports = [
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "containerRuntime" ] "")
+  ];
+
+  ###### interface
+  options.services.kubernetes.kubelet = with lib.types; {
+
+    address = mkOption {
+      description = "Kubernetes kubelet info server listening address.";
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    clusterDns = mkOption {
+      description = "Use alternative DNS.";
+      default = [ "10.1.0.1" ];
+      type = listOf str;
+    };
+
+    clusterDomain = mkOption {
+      description = "Use alternative domain.";
+      default = config.services.kubernetes.addons.dns.clusterDomain;
+      defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
+      type = str;
+    };
+
+    clientCaFile = mkOption {
+      description = "Kubernetes apiserver CA file for client authentication.";
+      default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    cni = {
+      packages = mkOption {
+        description = "List of network plugin packages to install.";
+        type = listOf package;
+        default = [ ];
+      };
+
+      config = mkOption {
+        description = "Kubernetes CNI configuration.";
+        type = listOf attrs;
+        default = [ ];
+        example = literalExpression ''
+          [{
+            "cniVersion": "0.3.1",
+            "name": "mynet",
+            "type": "bridge",
+            "bridge": "cni0",
+            "isGateway": true,
+            "ipMasq": true,
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.22.0.0/16",
+                "routes": [
+                    { "dst": "0.0.0.0/0" }
+                ]
+            }
+          } {
+            "cniVersion": "0.3.1",
+            "type": "loopback"
+          }]
+        '';
+      };
+
+      configDir = mkOption {
+        description = "Path to Kubernetes CNI configuration directory.";
+        type = nullOr path;
+        default = null;
+      };
+    };
+
+    containerRuntimeEndpoint = mkOption {
+      description = "Endpoint at which to find the container runtime api interface/socket";
+      type = str;
+      default = "unix:///run/containerd/containerd.sock";
+    };
+
+    enable = mkEnableOption "Kubernetes kubelet";
+
+    extraOpts = mkOption {
+      description = "Kubernetes kubelet extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    extraConfig = mkOption {
+      description = ''
+        Kubernetes kubelet extra configuration file entries.
+
+        See also [Set Kubelet Parameters Via A Configuration File](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
+        and [Kubelet Configuration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
+      '';
+      default = { };
+      type = attrsOf ((pkgs.formats.json { }).type);
+    };
+
+    featureGates = mkOption {
+      description = "Attribute set of feature gate";
+      default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    healthz = {
+      bind = mkOption {
+        description = "Kubernetes kubelet healthz listening address.";
+        default = "127.0.0.1";
+        type = str;
+      };
+
+      port = mkOption {
+        description = "Kubernetes kubelet healthz port.";
+        default = 10248;
+        type = port;
+      };
+    };
+
+    hostname = mkOption {
+      description = "Kubernetes kubelet hostname override.";
+      defaultText = literalExpression "config.networking.fqdnOrHostName";
+      type = str;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
+
+    manifests = mkOption {
+      description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
+      type = attrsOf attrs;
+      default = { };
+    };
+
+    nodeIp = mkOption {
+      description = "IP address of the node. If set, kubelet will use this IP address for the node.";
+      default = null;
+      type = nullOr str;
+    };
+
+    registerNode = mkOption {
+      description = "Whether to auto register kubelet with API server.";
+      default = true;
+      type = bool;
+    };
+
+    port = mkOption {
+      description = "Kubernetes kubelet info server listening port.";
+      default = 10250;
+      type = port;
+    };
+
+    seedDockerImages = mkOption {
+      description = "List of docker images to preload on system";
+      default = [ ];
+      type = listOf package;
+    };
+
+    taints = mkOption {
+      description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
+      default = { };
+      type = attrsOf (submodule [ taintOptions ]);
+    };
+
+    tlsCertFile = mkOption {
+      description = "File containing x509 Certificate for HTTPS.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = mkOption {
+      description = "File containing x509 private key matching tlsCertFile.";
+      default = null;
+      type = nullOr path;
+    };
+
+    unschedulable = mkOption {
+      description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
+      default = false;
+      type = bool;
+    };
+
+    verbosity = mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = mkMerge [
+    (mkIf cfg.enable {
+
+      environment.etc."cni/net.d".source = cniConfig;
+
+      services.kubernetes.kubelet.seedDockerImages = [ infraContainer ];
+
+      boot.kernel.sysctl = {
+        "net.bridge.bridge-nf-call-iptables" = 1;
+        "net.ipv4.ip_forward" = 1;
+        "net.bridge.bridge-nf-call-ip6tables" = 1;
+      };
+
+      systemd.services.kubelet = {
+        description = "Kubernetes Kubelet Service";
+        wantedBy = [ "kubernetes.target" ];
+        after = [
+          "containerd.service"
+          "network.target"
+          "kube-apiserver.service"
+        ];
+        path =
+          with pkgs;
+          [
+            gitMinimal
+            openssh
+            util-linuxMinimal
+            iproute2
+            ethtool
+            thin-provisioning-tools
+            iptables
+            socat
+          ]
+          ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package
+          ++ top.path;
+        preStart = ''
+          ${concatMapStrings (img: ''
+            echo "Seeding container image: ${img}"
+            ${
+              if (lib.hasSuffix "gz" img) then
+                ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+              else
+                ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+            }
+          '') cfg.seedDockerImages}
+
+          rm /opt/cni/bin/* || true
+          ${concatMapStrings (package: ''
+            echo "Linking cni package: ${package}"
+            ln -fs ${package}/bin/* /opt/cni/bin
+          '') cfg.cni.packages}
+        '';
+        serviceConfig = {
+          Slice = "kubernetes.slice";
+          CPUAccounting = true;
+          MemoryAccounting = true;
+          Restart = "on-failure";
+          RestartSec = "1000ms";
+          ExecStart = ''
+            ${top.package}/bin/kubelet \
+                        --config=${kubeletConfig} \
+                        --hostname-override=${cfg.hostname} \
+                        --kubeconfig=${kubeconfig} \
+                        ${optionalString (cfg.nodeIp != null) "--node-ip=${cfg.nodeIp}"} \
+                        --pod-infra-container-image=pause \
+                        ${optionalString (cfg.manifests != { }) "--pod-manifest-path=/etc/${manifestPath}"} \
+                        ${optionalString (taints != "") "--register-with-taints=${taints}"} \
+                        --root-dir=${top.dataDir} \
+                        ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                        ${cfg.extraOpts}
+          '';
+          WorkingDirectory = top.dataDir;
+        };
+        unitConfig = {
+          StartLimitIntervalSec = 0;
+        };
+      };
+
+      # Always include cni plugins
+      services.kubernetes.kubelet.cni.packages = [
+        pkgs.cni-plugins
+        pkgs.cni-plugin-flannel
+      ];
+
+      boot.kernelModules = [
+        "br_netfilter"
+        "overlay"
+      ];
+
+      services.kubernetes.kubelet.hostname = mkDefault (lib.toLower config.networking.fqdnOrHostName);
+
+      services.kubernetes.pki.certs = with top.lib; {
+        kubelet = mkCert {
+          name = "kubelet";
+          CN = top.kubelet.hostname;
+          action = "systemctl restart kubelet.service";
+
+        };
+        kubeletClient = mkCert {
+          name = "kubelet-client";
+          CN = "system:node:${top.kubelet.hostname}";
+          fields = {
+            O = "system:nodes";
+          };
+          action = "systemctl restart kubelet.service";
+        };
+      };
+
+      services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
+    })
+
+    (mkIf (cfg.enable && cfg.manifests != { }) {
+      environment.etc = mapAttrs' (
+        name: manifest:
+        nameValuePair "${manifestPath}/${name}.json" {
+          text = builtins.toJSON manifest;
+          mode = "0755";
+        }
+      ) cfg.manifests;
+    })
+
+    (mkIf (cfg.unschedulable && cfg.enable) {
+      services.kubernetes.kubelet.taints.unschedulable = {
+        value = "true";
+        effect = "NoSchedule";
+      };
+    })
+
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -0,0 +1,437 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  cfg = top.pki;
+
+  csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (
+    builtins.toJSON {
+      key = {
+        algo = "rsa";
+        size = 2048;
+      };
+      names = singleton cfg.caSpec;
+    }
+  );
+
+  csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (
+    builtins.toJSON {
+      key = {
+        algo = "rsa";
+        size = 2048;
+      };
+      CN = top.masterAddress;
+      hosts = [ top.masterAddress ] ++ cfg.cfsslAPIExtraSANs;
+    }
+  );
+
+  cfsslAPITokenBaseName = "apitoken.secret";
+  cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
+  certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
+  cfsslAPITokenLength = 32;
+
+  clusterAdminKubeconfig =
+    with cfg.certs.clusterAdmin;
+    top.lib.mkKubeConfig "cluster-admin" {
+      server = top.apiserverAddress;
+      certFile = cert;
+      keyFile = key;
+    };
+
+  remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
+in
+{
+  ###### interface
+  options.services.kubernetes.pki = with lib.types; {
+
+    enable = mkEnableOption "easyCert issuer service";
+
+    certs = mkOption {
+      description = "List of certificate specs to feed to cert generator.";
+      default = { };
+      type = attrs;
+    };
+
+    genCfsslCACert = mkOption {
+      description = ''
+        Whether to automatically generate cfssl CA certificate and key,
+        if they don't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    genCfsslAPICerts = mkOption {
+      description = ''
+        Whether to automatically generate cfssl API webserver TLS cert and key,
+        if they don't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    cfsslAPIExtraSANs = mkOption {
+      description = ''
+        Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
+      '';
+      default = [ ];
+      example = [ "subdomain.example.com" ];
+      type = listOf str;
+    };
+
+    genCfsslAPIToken = mkOption {
+      description = ''
+        Whether to automatically generate cfssl API-token secret,
+        if they doesn't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    pkiTrustOnBootstrap = mkOption {
+      description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
+      default = true;
+      type = bool;
+    };
+
+    caCertPathPrefix = mkOption {
+      description = ''
+        Path-prefrix for the CA-certificate to be used for cfssl signing.
+        Suffixes ".pem" and "-key.pem" will be automatically appended for
+        the public and private keys respectively.
+      '';
+      default = "${config.services.cfssl.dataDir}/ca";
+      defaultText = literalExpression ''"''${config.services.cfssl.dataDir}/ca"'';
+      type = str;
+    };
+
+    caSpec = mkOption {
+      description = "Certificate specification for the auto-generated CAcert.";
+      default = {
+        CN = "kubernetes-cluster-ca";
+        O = "NixOS";
+        OU = "services.kubernetes.pki.caSpec";
+        L = "auto-generated";
+      };
+      type = attrs;
+    };
+
+    etcClusterAdminKubeconfig = mkOption {
+      description = ''
+        Symlink a kubeconfig with cluster-admin privileges to environment path
+        (/etc/\<path\>).
+      '';
+      default = null;
+      type = nullOr str;
+    };
+
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable (
+    let
+      cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
+      cfsslCert = "${cfsslCertPathPrefix}.pem";
+      cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+    in
+    {
+
+      services.cfssl = mkIf (top.apiserver.enable) {
+        enable = true;
+        address = "0.0.0.0";
+        tlsCert = cfsslCert;
+        tlsKey = cfsslKey;
+        configFile = toString (
+          pkgs.writeText "cfssl-config.json" (
+            builtins.toJSON {
+              signing = {
+                profiles = {
+                  default = {
+                    usages = [ "digital signature" ];
+                    auth_key = "default";
+                    expiry = "720h";
+                  };
+                };
+              };
+              auth_keys = {
+                default = {
+                  type = "standard";
+                  key = "file:${cfsslAPITokenPath}";
+                };
+              };
+            }
+          )
+        );
+      };
+
+      systemd.services.cfssl.preStart =
+        with pkgs;
+        with config.services.cfssl;
+        mkIf (top.apiserver.enable) (
+          concatStringsSep "\n" [
+            "set -e"
+            (optionalString cfg.genCfsslCACert ''
+              if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
+                ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
+                  ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
+              fi
+            '')
+            (optionalString cfg.genCfsslAPICerts ''
+              if [ ! -f "${dataDir}/cfssl.pem" ]; then
+                ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
+                  ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
+              fi
+            '')
+            (optionalString cfg.genCfsslAPIToken ''
+              if [ ! -f "${cfsslAPITokenPath}" ]; then
+                install -o cfssl -m 400 <(head -c ${
+                  toString (cfsslAPITokenLength / 2)
+                } /dev/urandom | od -An -t x | tr -d ' ') "${cfsslAPITokenPath}"
+              fi
+            '')
+          ]
+        );
+
+      systemd.services.kube-certmgr-bootstrap = {
+        description = "Kubernetes certmgr bootstrapper";
+        wantedBy = [ "certmgr.service" ];
+        after = [ "cfssl.target" ];
+        script = concatStringsSep "\n" [
+          ''
+            set -e
+
+            # If there's a cfssl (cert issuer) running locally, then don't rely on user to
+            # manually paste it in place. Just symlink.
+            # otherwise, create the target file, ready for users to insert the token
+
+            mkdir -p "$(dirname "${certmgrAPITokenPath}")"
+            if [ -f "${cfsslAPITokenPath}" ]; then
+              ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
+            elif [ ! -f "${certmgrAPITokenPath}" ]; then
+              # Don't remove the token if it already exists
+              install -m 600 /dev/null "${certmgrAPITokenPath}"
+            fi
+          ''
+          (optionalString (cfg.pkiTrustOnBootstrap) ''
+            if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
+              ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
+                ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
+            fi
+          '')
+        ];
+        serviceConfig = {
+          RestartSec = "10s";
+          Restart = "on-failure";
+        };
+      };
+
+      services.certmgr = {
+        enable = true;
+        package = pkgs.certmgr;
+        svcManager = "command";
+        specs =
+          let
+            mkSpec = _: cert: {
+              inherit (cert) action;
+              authority = {
+                inherit remote;
+                root_ca = cert.caCert;
+                profile = "default";
+                auth_key_file = certmgrAPITokenPath;
+              };
+              certificate = {
+                path = cert.cert;
+              };
+              private_key = cert.privateKeyOptions;
+              request = {
+                hosts = [ cert.CN ] ++ cert.hosts;
+                inherit (cert) CN;
+                key = {
+                  algo = "rsa";
+                  size = 2048;
+                };
+                names = [ cert.fields ];
+              };
+            };
+          in
+          mapAttrs mkSpec cfg.certs;
+      };
+
+      #TODO: Get rid of kube-addon-manager in the future for the following reasons
+      # - it is basically just a shell script wrapped around kubectl
+      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+      # - it is designed to be used with k8s system components only
+      # - it would be better with a more Nix-oriented way of managing addons
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [
+        {
+          environment.KUBECONFIG =
+            with cfg.certs.addonManager;
+            top.lib.mkKubeConfig "addon-manager" {
+              server = top.apiserverAddress;
+              certFile = cert;
+              keyFile = key;
+            };
+        }
+
+        (optionalAttrs (top.addonManager.bootstrapAddons != { }) {
+          serviceConfig.PermissionsStartOnly = true;
+          preStart =
+            with pkgs;
+            let
+              files = mapAttrsToList (
+                n: v: writeText "${n}.json" (builtins.toJSON v)
+              ) top.addonManager.bootstrapAddons;
+            in
+            ''
+              export KUBECONFIG=${clusterAdminKubeconfig}
+              ${top.package}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+            '';
+        })
+      ]);
+
+      environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (
+        cfg.etcClusterAdminKubeconfig != null
+      ) clusterAdminKubeconfig;
+
+      environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
+        (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
+          set -e
+          exec 1>&2
+
+          if [ $# -gt 0 ]; then
+            echo "Usage: $(basename $0)"
+            echo ""
+            echo "No args. Apitoken must be provided on stdin."
+            echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
+            exit 1
+          fi
+
+          if [ $(id -u) != 0 ]; then
+            echo "Run as root please."
+            exit 1
+          fi
+
+          read -r token
+          if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
+            echo "Token must be of length ${toString cfsslAPITokenLength}."
+            exit 1
+          fi
+
+          install -m 0600 <(echo $token) ${certmgrAPITokenPath}
+
+          echo "Restarting certmgr..." >&1
+          systemctl restart certmgr
+
+          echo "Waiting for certs to appear..." >&1
+
+          ${optionalString top.kubelet.enable ''
+            while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
+            echo "Restarting kubelet..." >&1
+            systemctl restart kubelet
+          ''}
+
+          ${optionalString top.proxy.enable ''
+            while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
+            echo "Restarting kube-proxy..." >&1
+            systemctl restart kube-proxy
+          ''}
+
+          ${optionalString top.flannel.enable ''
+            while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
+            echo "Restarting flannel..." >&1
+            systemctl restart flannel
+          ''}
+
+          echo "Node joined successfully"
+        '')
+      ];
+
+      # isolate etcd on loopback at the master node
+      # easyCerts doesn't support multimaster clusters anyway atm.
+      services.etcd = with cfg.certs.etcd; {
+        listenClientUrls = [ "https://127.0.0.1:2379" ];
+        listenPeerUrls = [ "https://127.0.0.1:2380" ];
+        advertiseClientUrls = [ "https://etcd.local:2379" ];
+        initialCluster = [ "${top.masterAddress}=https://etcd.local:2380" ];
+        initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
+        certFile = mkDefault cert;
+        keyFile = mkDefault key;
+        trustedCaFile = mkDefault caCert;
+      };
+      networking.extraHosts = mkIf (config.services.etcd.enable) ''
+        127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
+      '';
+
+      services.flannel = with cfg.certs.flannelClient; {
+        kubeconfig = top.lib.mkKubeConfig "flannel" {
+          server = top.apiserverAddress;
+          certFile = cert;
+          keyFile = key;
+        };
+      };
+
+      services.kubernetes = {
+
+        apiserver = mkIf top.apiserver.enable (
+          with cfg.certs.apiServer;
+          {
+            etcd = with cfg.certs.apiserverEtcdClient; {
+              servers = [ "https://etcd.local:2379" ];
+              certFile = mkDefault cert;
+              keyFile = mkDefault key;
+              caFile = mkDefault caCert;
+            };
+            clientCaFile = mkDefault caCert;
+            tlsCertFile = mkDefault cert;
+            tlsKeyFile = mkDefault key;
+            serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
+            serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
+            kubeletClientCaFile = mkDefault caCert;
+            kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
+            kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
+            proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
+            proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
+          }
+        );
+        controllerManager = mkIf top.controllerManager.enable {
+          serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
+          rootCaFile = cfg.certs.controllerManagerClient.caCert;
+          kubeconfig = with cfg.certs.controllerManagerClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        scheduler = mkIf top.scheduler.enable {
+          kubeconfig = with cfg.certs.schedulerClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        kubelet = mkIf top.kubelet.enable {
+          clientCaFile = mkDefault cfg.certs.kubelet.caCert;
+          tlsCertFile = mkDefault cfg.certs.kubelet.cert;
+          tlsKeyFile = mkDefault cfg.certs.kubelet.key;
+          kubeconfig = with cfg.certs.kubeletClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        proxy = mkIf top.proxy.enable {
+          kubeconfig = with cfg.certs.kubeProxyClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+      };
+    }
+  );
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/proxy.nix

@@ -0,0 +1,120 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.proxy;
+in
+{
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "kubernetes" "proxy" "address" ]
+      [ "services" "kubernetes" "proxy" "bindAddress" ]
+    )
+  ];
+
+  ###### interface
+  options.services.kubernetes.proxy = with lib.types; {
+
+    bindAddress = mkOption {
+      description = "Kubernetes proxy listening address.";
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    enable = mkEnableOption "Kubernetes proxy";
+
+    extraOpts = mkOption {
+      description = "Kubernetes proxy extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    hostname = mkOption {
+      description = "Kubernetes proxy hostname override.";
+      default = config.networking.hostName;
+      defaultText = literalExpression "config.networking.hostName";
+      type = str;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes proxy";
+
+    verbosity = mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable {
+    systemd.services.kube-proxy = {
+      description = "Kubernetes Proxy Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      path = with pkgs; [
+        iptables
+        conntrack-tools
+      ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-proxy \
+          --bind-address=${cfg.bindAddress} \
+          ${optionalString (top.clusterCidr != null) "--cluster-cidr=${top.clusterCidr}"} \
+          ${
+            optionalString (cfg.featureGates != { })
+              "--feature-gates=${
+                concatStringsSep "," (
+                  builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates)
+                )
+              }"
+          } \
+          --hostname-override=${cfg.hostname} \
+          --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \
+          ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+          ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.proxy.hostname = with config.networking; mkDefault hostName;
+
+    services.kubernetes.pki.certs = {
+      kubeProxyClient = top.lib.mkCert {
+        name = "kube-proxy-client";
+        CN = "system:kube-proxy";
+        action = "systemctl restart kube-proxy.service";
+      };
+    };
+
+    services.kubernetes.proxy.kubeconfig.server = mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/scheduler.nix

@@ -0,0 +1,111 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.scheduler;
+in
+{
+  ###### interface
+  options.services.kubernetes.scheduler = with lib.types; {
+
+    address = lib.mkOption {
+      description = "Kubernetes scheduler listening address.";
+      default = "127.0.0.1";
+      type = str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes scheduler";
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes scheduler extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler";
+
+    leaderElect = lib.mkOption {
+      description = "Whether to start leader election before executing main loop.";
+      type = bool;
+      default = true;
+    };
+
+    port = lib.mkOption {
+      description = "Kubernetes scheduler listening port.";
+      default = 10251;
+      type = port;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    systemd.services.kube-scheduler = {
+      description = "Kubernetes Scheduler Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-scheduler \
+                    --bind-address=${cfg.address} \
+                    ${
+                      lib.optionalString (cfg.featureGates != { })
+                        "--feature-gates=${
+                          lib.concatStringsSep "," (
+                            builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                          )
+                        }"
+                    } \
+                    --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
+                    --leader-elect=${lib.boolToString cfg.leaderElect} \
+                    --secure-port=${toString cfg.port} \
+                    ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                    ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.pki.certs = {
+      schedulerClient = top.lib.mkCert {
+        name = "kube-scheduler-client";
+        CN = "system:kube-scheduler";
+        action = "systemctl restart kube-scheduler.service";
+      };
+    };
+
+    services.kubernetes.scheduler.kubeconfig.server = lib.mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/pacemaker/default.nix

@@ -0,0 +1,52 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.pacemaker;
+in
+{
+  # interface
+  options.services.pacemaker = {
+    enable = lib.mkEnableOption "pacemaker";
+
+    package = lib.mkPackageOption pkgs "pacemaker" { };
+  };
+
+  # implementation
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.services.corosync.enable;
+        message = ''
+          Enabling services.pacemaker requires a services.corosync configuration.
+        '';
+      }
+    ];
+
+    environment.systemPackages = [ cfg.package ];
+
+    # required by pacemaker
+    users.users.hacluster = {
+      isSystemUser = true;
+      group = "pacemaker";
+      home = "/var/lib/pacemaker";
+    };
+    users.groups.pacemaker = { };
+
+    systemd.tmpfiles.rules = [
+      "d /var/log/pacemaker 0700 hacluster pacemaker -"
+    ];
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.pacemaker = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "pacemaker";
+        StateDirectoryMode = "0700";
+      };
+    };
+  };
+}

nixos/modules/services/cluster/patroni/default.nix

@@ -0,0 +1,303 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.patroni;
+  defaultUser = "patroni";
+  defaultGroup = "patroni";
+  format = pkgs.formats.yaml { };
+
+  configFileName = "patroni-${cfg.scope}-${cfg.name}.yaml";
+  configFile = format.generate configFileName cfg.settings;
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "patroni" "raft" ] ''
+      Raft has been deprecated by upstream.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "patroni" "raftPort" ] ''
+      Raft has been deprecated by upstream.
+    '')
+  ];
+
+  options.services.patroni = {
+
+    enable = lib.mkEnableOption "Patroni";
+
+    postgresqlPackage = lib.mkOption {
+      type = lib.types.package;
+      example = lib.literalExpression "pkgs.postgresql_14";
+      description = ''
+        PostgreSQL package to use.
+        Plugins can be enabled like this `pkgs.postgresql_14.withPackages (p: [ p.pg_safeupdate p.postgis ])`.
+      '';
+    };
+
+    postgresqlDataDir = lib.mkOption {
+      type = lib.types.path;
+      defaultText = lib.literalExpression ''"/var/lib/postgresql/''${config.services.patroni.postgresqlPackage.psqlSchema}"'';
+      example = "/var/lib/postgresql/14";
+      default = "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}";
+      description = ''
+        The data directory for PostgreSQL. If left as the default value
+        this directory will automatically be created before the PostgreSQL server starts, otherwise
+        the sysadmin is responsible for ensuring the directory exists with appropriate ownership
+        and permissions.
+      '';
+    };
+
+    postgresqlPort = lib.mkOption {
+      type = lib.types.port;
+      default = 5432;
+      description = ''
+        The port on which PostgreSQL listens.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = defaultUser;
+      example = "postgres";
+      description = ''
+        The user for the service. If left as the default value this user will automatically be created,
+        otherwise the sysadmin is responsible for ensuring the user exists.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = defaultGroup;
+      example = "postgres";
+      description = ''
+        The group for the service. If left as the default value this group will automatically be created,
+        otherwise the sysadmin is responsible for ensuring the group exists.
+      '';
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/patroni";
+      description = ''
+        Folder where Patroni data will be written, this is where the pgpass password file will be written.
+      '';
+    };
+
+    scope = lib.mkOption {
+      type = lib.types.str;
+      example = "cluster1";
+      description = ''
+        Cluster name.
+      '';
+    };
+
+    name = lib.mkOption {
+      type = lib.types.str;
+      example = "node1";
+      description = ''
+        The name of the host. Must be unique for the cluster.
+      '';
+    };
+
+    namespace = lib.mkOption {
+      type = lib.types.str;
+      default = "/service";
+      description = ''
+        Path within the configuration store where Patroni will keep information about the cluster.
+      '';
+    };
+
+    nodeIp = lib.mkOption {
+      type = lib.types.str;
+      example = "192.168.1.1";
+      description = ''
+        IP address of this node.
+      '';
+    };
+
+    otherNodesIps = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [
+        "192.168.1.2"
+        "192.168.1.3"
+      ];
+      description = ''
+        IP addresses of the other nodes.
+      '';
+    };
+
+    restApiPort = lib.mkOption {
+      type = lib.types.port;
+      default = 8008;
+      description = ''
+        The port on Patroni's REST api listens.
+      '';
+    };
+
+    softwareWatchdog = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        This will configure Patroni to use the software watchdog built into the Linux kernel
+        as described in the [documentation](https://patroni.readthedocs.io/en/latest/watchdog.html#setting-up-software-watchdog-on-linux).
+      '';
+    };
+
+    settings = lib.mkOption {
+      type = format.type;
+      default = { };
+      example = {
+        bootstrap = {
+          initdb = [
+            "encoding=UTF-8"
+            "data-checksums"
+          ];
+        };
+        postgresql = {
+          parameters = {
+            unix_socket_directories = "/tmp";
+          };
+        };
+      };
+      description = ''
+        The primary patroni configuration. See the [documentation](https://patroni.readthedocs.io/en/latest/yaml_configuration.html)
+        for possible values.
+        Secrets should be passed in by using the `environmentFiles` option.
+      '';
+    };
+
+    environmentFiles = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (
+          nullOr (oneOf [
+            str
+            path
+            package
+          ])
+        );
+      default = { };
+      example = {
+        PATRONI_REPLICATION_PASSWORD = "/secret/file";
+        PATRONI_SUPERUSER_PASSWORD = "/secret/file";
+      };
+      description = "Environment variables made available to Patroni as files content, useful for providing secrets from files.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion =
+          !(
+            cfg.enable
+            && config.services.postgresql.enable
+            && cfg.postgresqlDataDir == config.services.postgresql.dataDir
+          );
+        message = ''
+          Both services.patroni and services.postgresql are enabled and
+          services.patroni.postgresqlDataDir == services.postgresql.dataDir
+          Disable one or the other, or configure them to use different directories.
+        '';
+      }
+    ];
+
+    services.patroni.settings = {
+      scope = cfg.scope;
+      name = cfg.name;
+      namespace = cfg.namespace;
+
+      restapi = {
+        listen = "${cfg.nodeIp}:${toString cfg.restApiPort}";
+        connect_address = "${cfg.nodeIp}:${toString cfg.restApiPort}";
+      };
+
+      postgresql = {
+        listen = "${cfg.nodeIp}:${toString cfg.postgresqlPort}";
+        connect_address = "${cfg.nodeIp}:${toString cfg.postgresqlPort}";
+        data_dir = cfg.postgresqlDataDir;
+        bin_dir = "${cfg.postgresqlPackage}/bin";
+        pgpass = "${cfg.dataDir}/pgpass";
+      };
+
+      watchdog = lib.mkIf cfg.softwareWatchdog {
+        mode = "required";
+        device = "/dev/watchdog";
+        safety_margin = 5;
+      };
+    };
+
+    users = {
+      users = lib.mkIf (cfg.user == defaultUser) {
+        patroni = {
+          group = cfg.group;
+          isSystemUser = true;
+        };
+      };
+      groups = lib.mkIf (cfg.group == defaultGroup) {
+        patroni = { };
+      };
+    };
+
+    systemd.services = {
+      patroni = {
+        description = "Runners to orchestrate a high-availability PostgreSQL";
+
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+
+        script = ''
+          ${lib.concatStringsSep "\n" (
+            lib.attrValues (
+              lib.mapAttrs (name: path: ''export ${name}="$(< ${lib.escapeShellArg path})"'') cfg.environmentFiles
+            )
+          )}
+          exec ${pkgs.patroni}/bin/patroni ${configFile}
+        '';
+
+        serviceConfig = lib.mkMerge [
+          {
+            User = cfg.user;
+            Group = cfg.group;
+            Type = "simple";
+            Restart = "on-failure";
+            TimeoutSec = 30;
+            ExecReload = "${pkgs.coreutils}/bin/kill -s HUP $MAINPID";
+            KillMode = "process";
+          }
+          (lib.mkIf
+            (
+              cfg.postgresqlDataDir == "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}"
+              && cfg.dataDir == "/var/lib/patroni"
+            )
+            {
+              StateDirectory = "patroni postgresql postgresql/${cfg.postgresqlPackage.psqlSchema}";
+              StateDirectoryMode = "0750";
+            }
+          )
+        ];
+      };
+    };
+
+    boot.kernelModules = lib.mkIf cfg.softwareWatchdog [ "softdog" ];
+
+    services.udev.extraRules = lib.mkIf cfg.softwareWatchdog ''
+      KERNEL=="watchdog", OWNER="${cfg.user}", GROUP="${cfg.group}", MODE="0600"
+    '';
+
+    environment.systemPackages = [
+      pkgs.patroni
+      cfg.postgresqlPackage
+    ];
+
+    environment.etc."${configFileName}".source = configFile;
+
+    environment.sessionVariables = {
+      PATRONICTL_CONFIG_FILE = "/etc/${configFileName}";
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.phfroidmont ];
+}

nixos/modules/services/cluster/rke2/default.nix

@@ -0,0 +1,338 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.rke2;
+in
+{
+  imports = [ ];
+
+  options.services.rke2 = {
+    enable = lib.mkEnableOption "rke2";
+
+    package = lib.mkPackageOption pkgs "rke2" { };
+
+    role = lib.mkOption {
+      type = lib.types.enum [
+        "server"
+        "agent"
+      ];
+      description = ''
+        Whether rke2 should run as a server or agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - any optionals is allowed.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
+        - `token` or `tokenFile` is required.
+        - `agentToken` or `agentTokenFile` or `disable` or `cni` are not allowed.
+      '';
+      default = "server";
+    };
+
+    configPath = lib.mkOption {
+      type = lib.types.path;
+      description = "Load configuration from FILE.";
+      default = "/etc/rancher/rke2/config.yaml";
+    };
+
+    debug = lib.mkOption {
+      type = lib.types.bool;
+      description = "Turn on debug logs.";
+      default = false;
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      description = "The folder to hold state in.";
+      default = "/var/lib/rancher/rke2";
+    };
+
+    token = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        Shared secret used to join a server or agent to a cluster.
+
+        > WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `tokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing rke2 token to use when connecting to the server.";
+      default = null;
+    };
+
+    disable = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Do not deploy packaged components and delete any deployed components.";
+      default = [ ];
+    };
+
+    nodeName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = "Node name.";
+      default = null;
+    };
+
+    nodeLabel = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Registering and starting kubelet with set of labels.";
+      default = [ ];
+    };
+
+    nodeTaint = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Registering kubelet with set of taints.";
+      default = [ ];
+    };
+
+    nodeIP = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = "IPv4/IPv6 addresses to advertise for node.";
+      default = null;
+    };
+
+    agentToken = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        Shared secret used to join agents to the cluster, but not servers.
+
+        > **WARNING**: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `agentTokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    agentTokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing rke2 agent token to use when connecting to the server.";
+      default = null;
+    };
+
+    serverAddr = lib.mkOption {
+      type = lib.types.str;
+      description = "The rke2 server to connect to, used to join a cluster.";
+      example = "https://10.0.0.10:6443";
+      default = "";
+    };
+
+    selinux = lib.mkOption {
+      type = lib.types.bool;
+      description = "Enable SELinux in containerd.";
+      default = false;
+    };
+
+    cni = lib.mkOption {
+      type = lib.types.enum [
+        "none"
+        "canal"
+        "cilium"
+        "calico"
+        "flannel"
+      ];
+      description = ''
+        CNI Plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`.
+
+        All CNI plugins get installed via a helm chart after the main components are up and running
+        and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm).
+
+        [Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options)
+
+        > **WARNING**: Flannel support in RKE2 is currently experimental.
+      '';
+      default = "canal";
+    };
+
+    cisHardening = lib.mkOption {
+      type = lib.types.bool;
+      description = ''
+        Enable CIS Hardening for RKE2.
+
+        It will set the configurations and controls required to address Kubernetes benchmark controls
+        from the Center for Internet Security (CIS).
+
+        Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide).
+
+        > **NOTICE**:
+        >
+        > You may need restart the `systemd-sysctl` muaually by:
+        >
+        > ```shell
+        > sudo systemctl restart systemd-sysctl
+        > ```
+      '';
+      default = false;
+    };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        Extra flags to pass to the rke2 service/agent.
+
+        Here you can find all the available flags:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+      '';
+      example = [
+        "--disable-kube-proxy"
+        "--cluster-cidr=10.24.0.0/16"
+      ];
+      default = [ ];
+    };
+
+    environmentVars = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      description = ''
+        Environment variables for configuring the rke2 service/agent.
+
+        Here you can find all the available environment variables:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+
+        Besides the options above, you can also active environment variables by edit/create those files:
+
+        - `/etc/default/rke2`
+        - `/etc/sysconfig/rke2`
+        - `/usr/local/lib/systemd/system/rke2.env`
+      '';
+      # See: https://github.com/rancher/rke2/blob/master/bundle/lib/systemd/system/rke2-server.env#L1
+      default = {
+        HOME = "/root";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.serverAddr != "");
+        message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
+      }
+      {
+        assertion =
+          cfg.role == "agent"
+          -> (builtins.pathExists cfg.configPath || cfg.tokenFile != null || cfg.token != "");
+        message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.agentTokenFile != null || cfg.agentToken != "");
+        message = "agentToken or agentTokenFile should NOT be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.disable != [ ]);
+        message = "disable should not be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.cni != "canal");
+        message = "cni should not be set if role is 'agent'";
+      }
+    ];
+
+    environment.systemPackages = [ config.services.rke2.package ];
+    # To configure NetworkManager to ignore calico/flannel related network interfaces.
+    # See: https://docs.rke2.io/known_issues#networkmanager
+    environment.etc."NetworkManager/conf.d/rke2-canal.conf" = {
+      enable = config.networking.networkmanager.enable;
+      text = ''
+        [keyfile]
+        unmanaged-devices=interface-name:cali*;interface-name:flannel*
+      '';
+    };
+    # See: https://docs.rke2.io/security/hardening_guide#set-kernel-parameters
+    boot.kernel.sysctl = lib.mkIf cfg.cisHardening {
+      "vm.panic_on_oom" = 0;
+      "vm.overcommit_memory" = 1;
+      "kernel.panic" = 10;
+      "kernel.panic_on_oops" = 1;
+    };
+
+    systemd.services."rke2-${cfg.role}" = {
+      description = "Rancher Kubernetes Engine v2";
+      documentation = [ "https://github.com/rancher/rke2#readme" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = if cfg.role == "agent" then "exec" else "notify";
+        EnvironmentFile = [
+          "-/etc/default/%N"
+          "-/etc/sysconfig/%N"
+          "-/usr/local/lib/systemd/system/%N.env"
+        ];
+        Environment = lib.mapAttrsToList (k: v: "${k}=${v}") cfg.environmentVars;
+        KillMode = "process";
+        Delegate = "yes";
+        LimitNOFILE = 1048576;
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        TasksMax = "infinity";
+        TimeoutStartSec = 0;
+        Restart = "always";
+        RestartSec = "5s";
+        ExecStartPre = [
+          # There is a conflict between RKE2 and `nm-cloud-setup.service`. This service add a routing table that
+          # interfere with the CNI plugin's configuration. This script checks if the service is enabled and if so,
+          # failed the RKE2 start.
+          # See: https://github.com/rancher/rke2/issues/1053
+          (pkgs.writeScript "check-nm-cloud-setup.sh" ''
+            #! ${pkgs.runtimeShell}
+            set -x
+            ! /run/current-system/systemd/bin/systemctl is-enabled --quiet nm-cloud-setup.service
+          '')
+          "-${pkgs.kmod}/bin/modprobe br_netfilter"
+          "-${pkgs.kmod}/bin/modprobe overlay"
+        ];
+        ExecStart = "${cfg.package}/bin/rke2 '${cfg.role}' ${
+          lib.escapeShellArgs (
+            (lib.optional (cfg.configPath != "/etc/rancher/rke2/config.yaml") "--config=${cfg.configPath}")
+            ++ (lib.optional cfg.debug "--debug")
+            ++ (lib.optional (cfg.dataDir != "/var/lib/rancher/rke2") "--data-dir=${cfg.dataDir}")
+            ++ (lib.optional (cfg.token != "") "--token=${cfg.token}")
+            ++ (lib.optional (cfg.tokenFile != null) "--token-file=${cfg.tokenFile}")
+            ++ (lib.optionals (cfg.role == "server" && cfg.disable != [ ]) (
+              map (d: "--disable=${d}") cfg.disable
+            ))
+            ++ (lib.optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}")
+            ++ (lib.optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel))
+            ++ (lib.optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint))
+            ++ (lib.optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}")
+            ++ (lib.optional (cfg.role == "server" && cfg.agentToken != "") "--agent-token=${cfg.agentToken}")
+            ++ (lib.optional (
+              cfg.role == "server" && cfg.agentTokenFile != null
+            ) "--agent-token-file=${cfg.agentTokenFile}")
+            ++ (lib.optional (cfg.serverAddr != "") "--server=${cfg.serverAddr}")
+            ++ (lib.optional cfg.selinux "--selinux")
+            ++ (lib.optional (cfg.role == "server" && cfg.cni != "canal") "--cni=${cfg.cni}")
+            ++ (lib.optional cfg.cisHardening "--profile=${
+              if cfg.package.version >= "1.25" then "cis-1.23" else "cis-1.6"
+            }")
+            ++ cfg.extraFlags
+          )
+        }";
+        ExecStopPost =
+          let
+            killProcess = pkgs.writeScript "kill-process.sh" ''
+              #! ${pkgs.runtimeShell}
+              /run/current-system/systemd/bin/systemd-cgls /system.slice/$1 | \
+              ${pkgs.gnugrep}/bin/grep -Eo '[0-9]+ (containerd|kubelet)' | \
+              ${pkgs.gawk}/bin/awk '{print $1}' | \
+              ${pkgs.findutils}/bin/xargs -r ${pkgs.util-linux}/bin/kill
+            '';
+          in
+          "-${killProcess} %n";
+      };
+    };
+  };
+}

nixos/modules/services/cluster/spark/default.nix

@@ -0,0 +1,177 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.spark;
+in
+{
+  options = {
+    services.spark = {
+      master = {
+        enable = lib.mkEnableOption "Spark master service";
+        bind = lib.mkOption {
+          type = lib.types.str;
+          description = "Address the spark master binds to.";
+          default = "127.0.0.1";
+          example = "0.0.0.0";
+        };
+        restartIfChanged = lib.mkOption {
+          type = lib.types.bool;
+          description = ''
+            Automatically restart master service on config change.
+            This can be set to false to defer restarts on clusters running critical applications.
+            Please consider the security implications of inadvertently running an older version,
+            and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+          '';
+          default = true;
+        };
+        extraEnvironment = lib.mkOption {
+          type = lib.types.attrsOf lib.types.str;
+          description = "Extra environment variables to pass to spark master. See spark-standalone documentation.";
+          default = { };
+          example = {
+            SPARK_MASTER_WEBUI_PORT = 8181;
+            SPARK_MASTER_OPTS = "-Dspark.deploy.defaultCores=5";
+          };
+        };
+      };
+      worker = {
+        enable = lib.mkEnableOption "Spark worker service";
+        workDir = lib.mkOption {
+          type = lib.types.path;
+          description = "Spark worker work dir.";
+          default = "/var/lib/spark";
+        };
+        master = lib.mkOption {
+          type = lib.types.str;
+          description = "Address of the spark master.";
+          default = "127.0.0.1:7077";
+        };
+        restartIfChanged = lib.mkOption {
+          type = lib.types.bool;
+          description = ''
+            Automatically restart worker service on config change.
+            This can be set to false to defer restarts on clusters running critical applications.
+            Please consider the security implications of inadvertently running an older version,
+            and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+          '';
+          default = true;
+        };
+        extraEnvironment = lib.mkOption {
+          type = lib.types.attrsOf lib.types.str;
+          description = "Extra environment variables to pass to spark worker.";
+          default = { };
+          example = {
+            SPARK_WORKER_CORES = 5;
+            SPARK_WORKER_MEMORY = "2g";
+          };
+        };
+      };
+      confDir = lib.mkOption {
+        type = lib.types.path;
+        description = "Spark configuration directory. Spark will use the configuration files (spark-defaults.conf, spark-env.sh, log4j.properties, etc) from this directory.";
+        default = "${cfg.package}/conf";
+        defaultText = lib.literalExpression ''"''${package}/conf"'';
+      };
+      logDir = lib.mkOption {
+        type = lib.types.path;
+        description = "Spark log directory.";
+        default = "/var/log/spark";
+      };
+      package = lib.mkPackageOption pkgs "spark" {
+        example = ''
+          spark.overrideAttrs (super: rec {
+            pname = "spark";
+            version = "2.4.4";
+
+            src = pkgs.fetchzip {
+              url    = "mirror://apache/spark/"''${pname}-''${version}/''${pname}-''${version}-bin-without-hadoop.tgz";
+              sha256 = "1a9w5k0207fysgpxx6db3a00fs5hdc2ncx99x4ccy2s0v5ndc66g";
+            };
+          })
+        '';
+      };
+    };
+  };
+  config = lib.mkIf (cfg.worker.enable || cfg.master.enable) {
+    environment.systemPackages = [ cfg.package ];
+    systemd = {
+      services = {
+        spark-master = lib.mkIf cfg.master.enable {
+          path = with pkgs; [
+            procps
+            openssh
+            net-tools
+          ];
+          description = "spark master service.";
+          after = [ "network.target" ];
+          wantedBy = [ "multi-user.target" ];
+          restartIfChanged = cfg.master.restartIfChanged;
+          environment = cfg.master.extraEnvironment // {
+            SPARK_MASTER_HOST = cfg.master.bind;
+            SPARK_CONF_DIR = cfg.confDir;
+            SPARK_LOG_DIR = cfg.logDir;
+          };
+          serviceConfig = {
+            Type = "forking";
+            User = "spark";
+            Group = "spark";
+            WorkingDirectory = "${cfg.package}/";
+            ExecStart = "${cfg.package}/sbin/start-master.sh";
+            ExecStop = "${cfg.package}/sbin/stop-master.sh";
+            TimeoutSec = 300;
+            Restart = "always";
+          };
+          unitConfig = {
+            StartLimitBurst = 10;
+          };
+        };
+        spark-worker = lib.mkIf cfg.worker.enable {
+          path = with pkgs; [
+            procps
+            openssh
+            net-tools
+            rsync
+          ];
+          description = "spark master service.";
+          after = [ "network.target" ];
+          wantedBy = [ "multi-user.target" ];
+          restartIfChanged = cfg.worker.restartIfChanged;
+          environment = cfg.worker.extraEnvironment // {
+            SPARK_MASTER = cfg.worker.master;
+            SPARK_CONF_DIR = cfg.confDir;
+            SPARK_LOG_DIR = cfg.logDir;
+            SPARK_WORKER_DIR = cfg.worker.workDir;
+          };
+          serviceConfig = {
+            Type = "forking";
+            User = "spark";
+            WorkingDirectory = "${cfg.package}/";
+            ExecStart = "${cfg.package}/sbin/start-worker.sh spark://${cfg.worker.master}";
+            ExecStop = "${cfg.package}/sbin/stop-worker.sh";
+            TimeoutSec = 300;
+            Restart = "always";
+          };
+          unitConfig = {
+            StartLimitBurst = 10;
+          };
+        };
+      };
+      tmpfiles.rules = [
+        "d '${cfg.worker.workDir}' - spark spark - -"
+        "d '${cfg.logDir}' - spark spark - -"
+      ];
+    };
+    users = {
+      users.spark = {
+        description = "spark user.";
+        group = "spark";
+        isSystemUser = true;
+      };
+      groups.spark = { };
+    };
+  };
+}

nixos/modules/services/cluster/temporal/default.nix

@@ -0,0 +1,146 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.temporal;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+  usingDefaultDataDir = cfg.dataDir == "/var/lib/temporal";
+  usingDefaultUserAndGroup = cfg.user == "temporal" && cfg.group == "temporal";
+in
+{
+  meta.maintainers = [ lib.maintainers.jpds ];
+
+  options.services.temporal = {
+    enable = lib.mkEnableOption "Temporal";
+
+    package = lib.mkPackageOption pkgs "Temporal" {
+      default = [ "temporal" ];
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = settingsFormat.type;
+      };
+
+      description = ''
+        Temporal configuration.
+
+        See <https://docs.temporal.io/references/configuration> for more
+        information about Temporal configuration options
+      '';
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/temporal";
+      apply = lib.converge (lib.removeSuffix "/");
+      description = ''
+        Data directory for Temporal. If you change this, you need to
+        manually create the directory. You also need to create the
+        `temporal` user and group, or change
+        [](#opt-services.temporal.user) and
+        [](#opt-services.temporal.group) to existing ones with
+        access to the directory.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "temporal";
+      description = ''
+        The user Temporal runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "temporal";
+      description = ''
+        The group temporal runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    restartIfChanged = lib.mkOption {
+      type = lib.types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on a server or cluster.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = true;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."temporal/temporal-server.yaml".source =
+      settingsFormat.generate "temporal-server.yaml" cfg.settings;
+
+    systemd.services.temporal = {
+      description = "Temporal server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      inherit (cfg) restartIfChanged;
+      restartTriggers = [ config.environment.etc."temporal/temporal-server.yaml".source ];
+      environment = {
+        HOME = cfg.dataDir;
+      };
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/temporal-server --root / --config /etc/temporal/ -e temporal-server start
+        '';
+        User = cfg.user;
+        Group = cfg.group;
+        Restart = "on-failure";
+        DynamicUser = usingDefaultUserAndGroup && usingDefaultDataDir;
+        CapabilityBoundingSet = [ "" ];
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectControlGroups = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ReadWritePaths = [
+          cfg.dataDir
+        ];
+        RestrictAddressFamilies = [
+          "AF_NETLINK"
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          # 1. allow a reasonable set of syscalls
+          "@system-service @resources"
+          # 2. and deny unreasonable ones
+          "~@privileged"
+          # 3. then allow the required subset within denied groups
+          "@chown"
+        ];
+      }
+      // (lib.optionalAttrs usingDefaultDataDir {
+        StateDirectory = "temporal";
+        StateDirectoryMode = "0700";
+      });
+    };
+  };
+}

nixos/modules/services/computing/boinc/client.nix

@@ -0,0 +1,115 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.boinc;
+  allowRemoteGuiRpcFlag = lib.optionalString cfg.allowRemoteGuiRpc "--allow_remote_gui_rpc";
+
+  fhsEnv = pkgs.buildFHSEnv {
+    name = "boinc-fhs-env";
+    targetPkgs = pkgs': [ cfg.package ] ++ cfg.extraEnvPackages;
+    runScript = "/bin/boinc_client";
+  };
+  fhsEnvExecutable = "${fhsEnv}/bin/${fhsEnv.name}";
+
+in
+{
+  options.services.boinc = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable the BOINC distributed computing client. If this
+        option is set to true, the boinc_client daemon will be run as a
+        background service. The boinccmd command can be used to control the
+        daemon.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "boinc" {
+      example = "boinc-headless";
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/boinc";
+      description = ''
+        The directory in which to store BOINC's configuration and data files.
+      '';
+    };
+
+    allowRemoteGuiRpc = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        If set to true, any remote host can connect to and control this BOINC
+        client (subject to password authentication). If instead set to false,
+        only the hosts listed in {var}`dataDir`/remote_hosts.cfg will be allowed to
+        connect.
+
+        See also: <https://boinc.berkeley.edu/wiki/Controlling_BOINC_remotely#Remote_access>
+      '';
+    };
+
+    extraEnvPackages = lib.mkOption {
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+      example = lib.literalExpression "[ pkgs.virtualbox ]";
+      description = ''
+        Additional packages to make available in the environment in which
+        BOINC will run. Common choices are:
+
+        - {var}`pkgs.virtualbox`:
+          The VirtualBox virtual machine framework. Required by some BOINC
+          projects, such as ATLAS@home.
+        - {var}`pkgs.ocl-icd`:
+          OpenCL infrastructure library. Required by BOINC projects that
+          use OpenCL, in addition to a device-specific OpenCL driver.
+        - {var}`pkgs.linuxPackages.nvidia_x11`:
+          Provides CUDA libraries. Required by BOINC projects that use
+          CUDA. Note that this requires an NVIDIA graphics device to be
+          present on the system.
+
+          Also provides OpenCL drivers for NVIDIA GPUs;
+          {var}`pkgs.ocl-icd` is also needed in this case.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    users.users.boinc = {
+      group = "boinc";
+      createHome = false;
+      description = "BOINC Client";
+      home = cfg.dataDir;
+      isSystemUser = true;
+    };
+    users.groups.boinc = { };
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - boinc boinc - -"
+    ];
+
+    systemd.services.boinc = {
+      description = "BOINC Client";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        exec ${fhsEnvExecutable} --dir ${cfg.dataDir} ${allowRemoteGuiRpcFlag}
+      '';
+      serviceConfig = {
+        User = "boinc";
+        Nice = 10;
+      };
+    };
+  };
+
+  meta = {
+    maintainers = [ ];
+  };
+}

nixos/modules/services/computing/foldingathome/client.nix

@@ -0,0 +1,95 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.foldingathome;
+
+  args = [
+    "--team"
+    "${toString cfg.team}"
+  ]
+  ++ lib.optionals (cfg.user != null) [
+    "--user"
+    cfg.user
+  ]
+  ++ cfg.extraArgs;
+in
+{
+  imports = [
+    (lib.mkRenamedOptionModule [ "services" "foldingAtHome" ] [ "services" "foldingathome" ])
+    (lib.mkRenamedOptionModule
+      [ "services" "foldingathome" "nickname" ]
+      [ "services" "foldingathome" "user" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "foldingathome" "config" ] ''
+      Use <literal>services.foldingathome.extraArgs instead<literal>
+    '')
+  ];
+  options.services.foldingathome = {
+    enable = lib.mkEnableOption "Folding@home client";
+
+    package = lib.mkPackageOption pkgs "fahclient" { };
+
+    user = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        The user associated with the reported computation results. This will
+        be used in the ranking statistics.
+      '';
+    };
+
+    team = lib.mkOption {
+      type = lib.types.int;
+      default = 236565;
+      description = ''
+        The team ID associated with the reported computation results. This
+        will be used in the ranking statistics.
+
+        By default, use the NixOS folding@home team ID is being used.
+      '';
+    };
+
+    daemonNiceLevel = lib.mkOption {
+      type = lib.types.ints.between (-20) 19;
+      default = 0;
+      description = ''
+        Daemon process priority for FAHClient.
+        0 is the default Unix process priority, 19 is the lowest.
+      '';
+    };
+
+    extraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Extra startup options for the FAHClient. Run
+        `fah-client --help` to find all the available options.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.foldingathome = {
+      description = "Folding@home client";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        exec ${lib.getExe cfg.package} ${lib.escapeShellArgs args}
+      '';
+      serviceConfig = {
+        DynamicUser = true;
+        StateDirectory = "foldingathome";
+        Nice = cfg.daemonNiceLevel;
+        WorkingDirectory = "%S/foldingathome";
+      };
+    };
+  };
+
+  meta = {
+    maintainers = with lib.maintainers; [ zimbatm ];
+  };
+}

nixos/modules/services/computing/slurm/slurm.nix

@@ -0,0 +1,493 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.slurm;
+  opt = options.services.slurm;
+  # configuration file can be generated by https://slurm.schedmd.com/configurator.html
+
+  defaultUser = "slurm";
+
+  configFile = pkgs.writeTextDir "slurm.conf" ''
+    ClusterName=${cfg.clusterName}
+    StateSaveLocation=${cfg.stateSaveLocation}
+    SlurmUser=${cfg.user}
+    ${lib.optionalString (cfg.controlMachine != null) "controlMachine=${cfg.controlMachine}"}
+    ${lib.optionalString (cfg.controlAddr != null) "controlAddr=${cfg.controlAddr}"}
+    ${toString (map (x: "NodeName=${x}\n") cfg.nodeName)}
+    ${toString (map (x: "PartitionName=${x}\n") cfg.partitionName)}
+    PlugStackConfig=${plugStackConfig}/plugstack.conf
+    ProctrackType=${cfg.procTrackType}
+    ${cfg.extraConfig}
+  '';
+
+  plugStackConfig = pkgs.writeTextDir "plugstack.conf" ''
+    ${lib.optionalString cfg.enableSrunX11 "optional ${pkgs.slurm-spank-x11}/lib/x11.so"}
+    ${cfg.extraPlugstackConfig}
+  '';
+
+  cgroupConfig = pkgs.writeTextDir "cgroup.conf" ''
+    ${cfg.extraCgroupConfig}
+  '';
+
+  mpiConf = pkgs.writeTextDir "mpi.conf" ''
+    PMIxCliTmpDirBase=${cfg.mpi.PmixCliTmpDirBase}
+    ${cfg.mpi.extraMpiConfig}
+  '';
+
+  slurmdbdConf = pkgs.writeText "slurmdbd.conf" ''
+    DbdHost=${cfg.dbdserver.dbdHost}
+    SlurmUser=${cfg.user}
+    StorageType=accounting_storage/mysql
+    StorageUser=${cfg.dbdserver.storageUser}
+    ${cfg.dbdserver.extraConfig}
+  '';
+
+  # slurm expects some additional config files to be
+  # in the same directory as slurm.conf
+  etcSlurm = pkgs.symlinkJoin {
+    name = "etc-slurm";
+    paths = [
+      configFile
+      cgroupConfig
+      plugStackConfig
+      mpiConf
+    ]
+    ++ cfg.extraConfigPaths;
+  };
+in
+
+{
+
+  ###### interface
+
+  meta.maintainers = [ lib.maintainers.markuskowa ];
+
+  options = {
+
+    services.slurm = {
+
+      server = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = ''
+            Whether to enable the slurm control daemon.
+            Note that the standard authentication method is "munge".
+            The "munge" service needs to be provided with a password file in order for
+            slurm to work properly (see `services.munge.password`).
+          '';
+        };
+      };
+
+      dbdserver = {
+        enable = lib.mkEnableOption "SlurmDBD service";
+
+        dbdHost = lib.mkOption {
+          type = lib.types.str;
+          default = config.networking.hostName;
+          defaultText = lib.literalExpression "config.networking.hostName";
+          description = ''
+            Hostname of the machine where `slurmdbd`
+            is running (i.e. name returned by `hostname -s`).
+          '';
+        };
+
+        storageUser = lib.mkOption {
+          type = lib.types.str;
+          default = cfg.user;
+          defaultText = lib.literalExpression "config.${opt.user}";
+          description = ''
+            Database user name.
+          '';
+        };
+
+        storagePassFile = lib.mkOption {
+          type = with lib.types; nullOr str;
+          default = null;
+          description = ''
+            Path to file with database password. The content of this will be used to
+            create the password for the `StoragePass` option.
+          '';
+        };
+
+        extraConfig = lib.mkOption {
+          type = lib.types.lines;
+          default = "";
+          description = ''
+            Extra configuration for `slurmdbd.conf` See also:
+            {manpage}`slurmdbd.conf(8)`.
+          '';
+        };
+      };
+
+      client = {
+        enable = lib.mkEnableOption "slurm client daemon";
+      };
+
+      enableStools = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to provide a slurm.conf file.
+          Enable this option if you do not run a slurm daemon on this host
+          (i.e. `server.enable` and `client.enable` are `false`)
+          but you still want to run slurm commands from this host.
+        '';
+      };
+
+      package =
+        lib.mkPackageOption pkgs "slurm" {
+          example = "slurm-full";
+        }
+        // {
+          default = pkgs.slurm.override { enableX11 = !cfg.enableSrunX11; };
+        };
+
+      controlMachine = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = null;
+        description = ''
+          The short hostname of the machine where SLURM control functions are
+          executed (i.e. the name returned by the command "hostname -s", use "tux001"
+          rather than "tux001.my.com").
+        '';
+      };
+
+      controlAddr = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = cfg.controlMachine;
+        defaultText = lib.literalExpression "config.${opt.controlMachine}";
+        example = null;
+        description = ''
+          Name that ControlMachine should be referred to in establishing a
+          communications path.
+        '';
+      };
+
+      clusterName = lib.mkOption {
+        type = lib.types.str;
+        default = "default";
+        example = "myCluster";
+        description = ''
+          Necessary to distinguish accounting records in a multi-cluster environment.
+        '';
+      };
+
+      nodeName = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = lib.literalExpression ''[ "linux[1-32] CPUs=1 State=UNKNOWN" ];'';
+        description = ''
+          Name that SLURM uses to refer to a node (or base partition for BlueGene
+          systems). Typically this would be the string that "/bin/hostname -s"
+          returns. Note that now you have to write node's parameters after the name.
+        '';
+      };
+
+      partitionName = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        example = lib.literalExpression ''[ "debug Nodes=linux[1-32] Default=YES MaxTime=INFINITE State=UP" ];'';
+        description = ''
+          Name by which the partition may be referenced. Note that now you have
+          to write the partition's parameters after the name.
+        '';
+      };
+
+      enableSrunX11 = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          If enabled srun will accept the option "--x11" to allow for X11 forwarding
+          from within an interactive session or a batch job. This activates the
+          slurm-spank-x11 module. Note that this option also enables
+          {option}`services.openssh.forwardX11` on the client.
+
+          This option requires slurm to be compiled without native X11 support.
+          The default behavior is to re-compile the slurm package with native X11
+          support disabled if this option is set to true.
+
+          To use the native X11 support add `PrologFlags=X11` in {option}`extraConfig`.
+          Note that this method will only work RSA SSH host keys.
+        '';
+      };
+
+      procTrackType = lib.mkOption {
+        type = lib.types.str;
+        default = "proctrack/linuxproc";
+        description = ''
+          Plugin to be used for process tracking on a job step basis.
+          The slurmd daemon uses this mechanism to identify all processes
+          which are children of processes it spawns for a user job step.
+        '';
+      };
+
+      stateSaveLocation = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/spool/slurmctld";
+        description = ''
+          Directory into which the Slurm controller, slurmctld, saves its state.
+        '';
+      };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = defaultUser;
+        description = ''
+          Set this option when you want to run the slurmctld daemon
+          as something else than the default slurm user "slurm".
+          Note that the UID of this user needs to be the same
+          on all nodes.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Extra configuration options that will be added verbatim at
+          the end of the slurm configuration file.
+        '';
+      };
+
+      mpi = {
+        PmixCliTmpDirBase = lib.mkOption {
+          default = "/tmp/pmix";
+          type = lib.types.str;
+          description = ''
+            Base path for PMIx temporary files.
+          '';
+        };
+
+        extraMpiConfig = lib.mkOption {
+          default = "";
+          type = lib.types.lines;
+          description = ''
+            Extra configuration for that will be added to `mpi.conf`.
+          '';
+        };
+      };
+
+      extraPlugstackConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Extra configuration that will be added to the end of `plugstack.conf`.
+        '';
+      };
+
+      extraCgroupConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Extra configuration for `cgroup.conf`. This file is
+          used when `procTrackType=proctrack/cgroup`.
+        '';
+      };
+
+      extraConfigPaths = lib.mkOption {
+        type = with lib.types; listOf path;
+        default = [ ];
+        description = ''
+          Slurm expects config files for plugins in the same path
+          as `slurm.conf`. Add extra nix store
+          paths that should be merged into same directory as
+          `slurm.conf`.
+        '';
+      };
+
+      etcSlurm = lib.mkOption {
+        type = lib.types.path;
+        internal = true;
+        default = etcSlurm;
+        defaultText = lib.literalMD ''
+          Directory created from generated config files and
+          `config.${opt.extraConfigPaths}`.
+        '';
+        description = ''
+          Path to directory with slurm config files. This option is set by default from the
+          Slurm module and is meant to make the Slurm config file available to other modules.
+        '';
+      };
+
+    };
+
+  };
+
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "slurm" "dbdserver" "storagePass" ] ''
+      This option has been removed so that the database password is not exposed via the nix store.
+      Use services.slurm.dbdserver.storagePassFile to provide the database password.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "slurm" "dbdserver" "configFile" ] ''
+      This option has been removed. Use services.slurm.dbdserver.storagePassFile
+      and services.slurm.dbdserver.extraConfig instead.
+    '')
+  ];
+
+  ###### implementation
+
+  config =
+    let
+      wrappedSlurm = pkgs.stdenv.mkDerivation {
+        name = "wrappedSlurm";
+
+        builder = pkgs.writeText "builder.sh" ''
+          mkdir -p $out/bin
+          find  ${lib.getBin cfg.package}/bin -type f -executable | while read EXE
+          do
+            exename="$(basename $EXE)"
+            wrappername="$out/bin/$exename"
+            cat > "$wrappername" <<EOT
+          #!/bin/sh
+          if [ -z "$SLURM_CONF" ]
+          then
+            SLURM_CONF="${cfg.etcSlurm}/slurm.conf" "$EXE" "\$@"
+          else
+            "$EXE" "\$0"
+          fi
+          EOT
+            chmod +x "$wrappername"
+          done
+
+          mkdir -p $out/share
+          ln -s ${lib.getBin cfg.package}/share/man $out/share/man
+        '';
+      };
+
+    in
+    lib.mkIf (cfg.enableStools || cfg.client.enable || cfg.server.enable || cfg.dbdserver.enable) {
+
+      environment.systemPackages = [ wrappedSlurm ];
+
+      services.munge.enable = lib.mkDefault true;
+
+      # use a static uid as default to ensure it is the same on all nodes
+      users.users.slurm = lib.mkIf (cfg.user == defaultUser) {
+        name = defaultUser;
+        group = "slurm";
+        uid = config.ids.uids.slurm;
+      };
+
+      users.groups.slurm.gid = config.ids.uids.slurm;
+
+      systemd.services.slurmd = lib.mkIf (cfg.client.enable) {
+        path =
+          with pkgs;
+          [
+            wrappedSlurm
+            coreutils
+          ]
+          ++ lib.optional cfg.enableSrunX11 slurm-spank-x11;
+
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "systemd-tmpfiles-clean.service"
+          "munge.service"
+          "network-online.target"
+          "remote-fs.target"
+        ];
+        wants = [ "network-online.target" ];
+
+        serviceConfig = {
+          Type = "forking";
+          KillMode = "process";
+          ExecStart = "${wrappedSlurm}/bin/slurmd";
+          PIDFile = "/run/slurmd.pid";
+          ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+          LimitMEMLOCK = "infinity";
+          Delegate = "Yes";
+        };
+      };
+
+      systemd.tmpfiles.rules = lib.optionals cfg.client.enable [
+        "d /var/spool/slurmd 755 root root -"
+        "d ${cfg.mpi.PmixCliTmpDirBase} 755 root root -"
+      ];
+
+      services.openssh.settings.X11Forwarding = lib.mkIf cfg.client.enable (lib.mkDefault true);
+
+      systemd.services.slurmctld = lib.mkIf (cfg.server.enable) {
+        path =
+          with pkgs;
+          [
+            wrappedSlurm
+            munge
+            coreutils
+          ]
+          ++ lib.optional cfg.enableSrunX11 slurm-spank-x11;
+
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "network.target"
+          "munged.service"
+        ];
+        requires = [ "munged.service" ];
+
+        serviceConfig = {
+          Type = "forking";
+          ExecStart = "${wrappedSlurm}/bin/slurmctld";
+          PIDFile = "/run/slurmctld.pid";
+          ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        };
+
+        preStart = ''
+          mkdir -p ${cfg.stateSaveLocation}
+          chown -R ${cfg.user}:slurm ${cfg.stateSaveLocation}
+        '';
+      };
+
+      systemd.services.slurmdbd =
+        let
+          # slurm strips the last component off the path
+          configPath = "$RUNTIME_DIRECTORY/slurmdbd.conf";
+        in
+        lib.mkIf (cfg.dbdserver.enable) {
+          path = with pkgs; [
+            wrappedSlurm
+            munge
+            coreutils
+          ];
+
+          wantedBy = [ "multi-user.target" ];
+          after = [
+            "network.target"
+            "munged.service"
+            "mysql.service"
+          ];
+          requires = [
+            "munged.service"
+            "mysql.service"
+          ];
+
+          preStart = ''
+            install -m 600 -o ${cfg.user} -T ${slurmdbdConf} ${configPath}
+            ${lib.optionalString (cfg.dbdserver.storagePassFile != null) ''
+              echo "StoragePass=$(cat ${cfg.dbdserver.storagePassFile})" \
+                >> ${configPath}
+            ''}
+          '';
+
+          script = ''
+            export SLURM_CONF=${configPath}
+            exec ${cfg.package}/bin/slurmdbd -D
+          '';
+
+          serviceConfig = {
+            RuntimeDirectory = "slurmdbd";
+            Type = "simple";
+            PIDFile = "/run/slurmdbd.pid";
+            ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+          };
+        };
+
+    };
+
+}

nixos/modules/services/computing/torque/mom.nix

@@ -0,0 +1,73 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+
+  cfg = config.services.torque.mom;
+  torque = pkgs.torque;
+
+  momConfig = pkgs.writeText "torque-mom-config" ''
+    $pbsserver ${cfg.serverNode}
+    $logevent 225
+  '';
+
+in
+{
+  options = {
+
+    services.torque.mom = {
+      enable = lib.mkEnableOption "torque computing node";
+
+      serverNode = lib.mkOption {
+        type = lib.types.str;
+        description = "Hostname running pbs server.";
+      };
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.torque ];
+
+    systemd.services.torque-mom-init = {
+      path = with pkgs; [
+        torque
+        util-linux
+        procps
+        inetutils
+      ];
+
+      script = ''
+        pbs_mkdirs -v aux
+        pbs_mkdirs -v mom
+        hostname > /var/spool/torque/server_name
+        cp -v ${momConfig} /var/spool/torque/mom_priv/config
+      '';
+
+      serviceConfig.Type = "oneshot";
+      unitConfig.ConditionPathExists = "!/var/spool/torque";
+    };
+
+    systemd.services.torque-mom = {
+      path = [ torque ];
+
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "torque-mom-init.service" ];
+      after = [
+        "torque-mom-init.service"
+        "network.target"
+      ];
+
+      serviceConfig = {
+        Type = "forking";
+        ExecStart = "${torque}/bin/pbs_mom";
+        PIDFile = "/var/spool/torque/mom_priv/mom.lock";
+      };
+    };
+
+  };
+}

nixos/modules/services/computing/torque/server.nix

@@ -0,0 +1,111 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.torque.server;
+  torque = pkgs.torque;
+in
+{
+  options = {
+
+    services.torque.server = {
+
+      enable = lib.mkEnableOption "torque server";
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.torque ];
+
+    systemd.services.torque-server-init = {
+      path = with pkgs; [
+        torque
+        util-linux
+        procps
+        inetutils
+      ];
+
+      script = ''
+        tmpsetup=$(mktemp -t torque-XXXX)
+        cp -p ${torque}/bin/torque.setup $tmpsetup
+        sed -i $tmpsetup -e 's/pbs_server -t create/pbs_server -f -t create/'
+
+        pbs_mkdirs -v aux
+        pbs_mkdirs -v server
+        hostname > /var/spool/torque/server_name
+        cp -prv ${torque}/var/spool/torque/* /var/spool/torque/
+        $tmpsetup root
+
+        sleep 1
+        rm -f $tmpsetup
+        kill $(pgrep pbs_server) 2>/dev/null
+        kill $(pgrep trqauthd) 2>/dev/null
+      '';
+
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+
+      unitConfig = {
+        ConditionPathExists = "!/var/spool/torque";
+      };
+    };
+
+    systemd.services.trqauthd = {
+      path = [ torque ];
+
+      requires = [ "torque-server-init.service" ];
+      after = [ "torque-server-init.service" ];
+
+      serviceConfig = {
+        Type = "forking";
+        ExecStart = "${torque}/bin/trqauthd";
+      };
+    };
+
+    systemd.services.torque-server = {
+      documentation = [ "man:pbs_server(8)" ];
+      path = [ torque ];
+
+      wantedBy = [ "multi-user.target" ];
+      wants = [
+        "torque-scheduler.service"
+        "trqauthd.service"
+      ];
+      before = [ "trqauthd.service" ];
+      requires = [ "torque-server-init.service" ];
+      after = [
+        "torque-server-init.service"
+        "network.target"
+      ];
+
+      serviceConfig = {
+        Type = "forking";
+        ExecStart = "${torque}/bin/pbs_server";
+        ExecStop = "${torque}/bin/qterm";
+        PIDFile = "/var/spool/torque/server_priv/server.lock";
+      };
+    };
+
+    systemd.services.torque-scheduler = {
+      documentation = [ "man:pbs_sched(8)" ];
+      path = [ torque ];
+
+      requires = [ "torque-server-init.service" ];
+      after = [ "torque-server-init.service" ];
+
+      serviceConfig = {
+        Type = "forking";
+        ExecStart = "${torque}/bin/pbs_sched";
+        PIDFile = "/var/spool/torque/sched_priv/sched.lock";
+      };
+    };
+
+  };
+}

nixos/modules/services/continuous-integration/buildbot/master.nix

@@ -0,0 +1,317 @@
+# NixOS module for Buildbot continuous integration server.
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.buildbot-master;
+  opt = options.services.buildbot-master;
+
+  package = cfg.package.python.pkgs.toPythonModule cfg.package;
+  python = cfg.package.python;
+
+  escapeStr = lib.escape [ "'" ];
+
+  defaultMasterCfg = pkgs.writeText "master.cfg" ''
+    from buildbot.plugins import *
+    ${cfg.extraImports}
+    factory = util.BuildFactory()
+    c = BuildmasterConfig = dict(
+     workers       = [${lib.concatStringsSep "," cfg.workers}],
+     protocols     = { 'pb': {'port': ${toString cfg.pbPort} } },
+     title         = '${escapeStr cfg.title}',
+     titleURL      = '${escapeStr cfg.titleUrl}',
+     buildbotURL   = '${escapeStr cfg.buildbotUrl}',
+     db            = dict(db_url='${escapeStr cfg.dbUrl}'),
+     www           = dict(port=${toString cfg.port}),
+     change_source = [ ${lib.concatStringsSep "," cfg.changeSource} ],
+     schedulers    = [ ${lib.concatStringsSep "," cfg.schedulers} ],
+     builders      = [ ${lib.concatStringsSep "," cfg.builders} ],
+     services      = [ ${lib.concatStringsSep "," cfg.reporters} ],
+     configurators = [ ${lib.concatStringsSep "," cfg.configurators} ],
+    )
+    for step in [ ${lib.concatStringsSep "," cfg.factorySteps} ]:
+      factory.addStep(step)
+
+    ${cfg.extraConfig}
+  '';
+
+  tacFile = pkgs.writeText "buildbot-master.tac" ''
+    import os
+
+    from twisted.application import service
+    from buildbot.master import BuildMaster
+
+    basedir = '${cfg.buildbotDir}'
+
+    configfile = '${cfg.masterCfg}'
+
+    # Default umask for server
+    umask = None
+
+    # note: this line is matched against to check that this is a buildmaster
+    # directory; do not edit it.
+    application = service.Application('buildmaster')
+
+    m = BuildMaster(basedir, configfile, umask)
+    m.setServiceParent(application)
+  '';
+
+in
+{
+  options = {
+    services.buildbot-master = {
+
+      factorySteps = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Factory Steps";
+        default = [ ];
+        example = [
+          "steps.Git(repourl='https://github.com/buildbot/pyflakes.git', mode='incremental')"
+          "steps.ShellCommand(command=['trial', 'pyflakes'])"
+        ];
+      };
+
+      changeSource = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Change Sources.";
+        default = [ ];
+        example = [
+          "changes.GitPoller('https://github.com/buildbot/pyflakes.git', workdir='gitpoller-workdir', branch='master', pollinterval=300)"
+        ];
+      };
+
+      configurators = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Configurator Steps, see <https://docs.buildbot.net/latest/manual/configuration/configurators.html>";
+        default = [ ];
+        example = [
+          "util.JanitorConfigurator(logHorizon=timedelta(weeks=4), hour=12, dayOfWeek=6)"
+        ];
+      };
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot continuous integration server.";
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra configuration to append to master.cfg";
+        default = "c['buildbotNetUsageData'] = None";
+      };
+
+      extraImports = lib.mkOption {
+        type = lib.types.lines;
+        description = "Extra python imports to prepend to master.cfg";
+        default = "";
+        example = "from buildbot.process.project import Project";
+      };
+
+      masterCfg = lib.mkOption {
+        type = lib.types.path;
+        description = "Optionally pass master.cfg path. Other options in this configuration will be ignored.";
+        default = defaultMasterCfg;
+        defaultText = lib.literalMD ''generated configuration file'';
+        example = "/etc/nixos/buildbot/master.cfg";
+      };
+
+      schedulers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Schedulers.";
+        default = [
+          "schedulers.SingleBranchScheduler(name='all', change_filter=util.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=['runtests'])"
+          "schedulers.ForceScheduler(name='force',builderNames=['runtests'])"
+        ];
+      };
+
+      builders = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Builders.";
+        default = [
+          "util.BuilderConfig(name='runtests',workernames=['example-worker'],factory=factory)"
+        ];
+      };
+
+      workers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "List of Workers.";
+        default = [ "worker.Worker('example-worker', 'pass')" ];
+      };
+
+      reporters = lib.mkOption {
+        default = [ ];
+        type = lib.types.listOf lib.types.str;
+        description = "List of reporter objects used to present build status to various users.";
+      };
+
+      user = lib.mkOption {
+        default = "buildbot";
+        type = lib.types.str;
+        description = "User the buildbot server should execute under.";
+      };
+
+      group = lib.mkOption {
+        default = "buildbot";
+        type = lib.types.str;
+        description = "Primary group of buildbot user.";
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "List of extra groups that the buildbot user should be a part of.";
+      };
+
+      home = lib.mkOption {
+        default = "/home/buildbot";
+        type = lib.types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = lib.mkOption {
+        default = "${cfg.home}/master";
+        defaultText = lib.literalExpression ''"''${config.${opt.home}}/master"'';
+        type = lib.types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      pbPort = lib.mkOption {
+        default = 9989;
+        type = lib.types.either lib.types.str lib.types.port;
+        example = "'tcp:9990:interface=127.0.0.1'";
+        description = ''
+          The buildmaster will listen on a TCP port of your choosing
+          for connections from workers.
+          It can also use this port for connections from remote Change Sources,
+          status clients, and debug tools.
+          This port should be visible to the outside world, and you’ll need to tell
+          your worker admins about your choice.
+          If put in (single) quotes, this can also be used as a connection string,
+          as defined in the [ConnectionStrings guide](https://twistedmatrix.com/documents/current/core/howto/endpoints.html).
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        default = "0.0.0.0";
+        type = lib.types.str;
+        description = "Specifies the bind address on which the buildbot HTTP interface listens.";
+      };
+
+      buildbotUrl = lib.mkOption {
+        default = "http://localhost:8010/";
+        type = lib.types.str;
+        description = "Specifies the Buildbot URL.";
+      };
+
+      title = lib.mkOption {
+        default = "Buildbot";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Title.";
+      };
+
+      titleUrl = lib.mkOption {
+        default = "Buildbot";
+        type = lib.types.str;
+        description = "Specifies the Buildbot TitleURL.";
+      };
+
+      dbUrl = lib.mkOption {
+        default = "sqlite:///state.sqlite";
+        type = lib.types.str;
+        description = "Specifies the database connection string.";
+      };
+
+      port = lib.mkOption {
+        default = 8010;
+        type = lib.types.port;
+        description = "Specifies port number on which the buildbot HTTP interface listens.";
+      };
+
+      package = lib.mkPackageOption pkgs "buildbot-full" {
+        example = "buildbot";
+      };
+
+      packages = lib.mkOption {
+        default = [ pkgs.git ];
+        defaultText = lib.literalExpression "[ pkgs.git ]";
+        type = lib.types.listOf lib.types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+
+      pythonPackages = lib.mkOption {
+        type = lib.types.functionTo (lib.types.listOf lib.types.package);
+        default = pythonPackages: [ ];
+        defaultText = lib.literalExpression "pythonPackages: with pythonPackages; [ ]";
+        description = "Packages to add the to the PYTHONPATH of the buildbot process.";
+        example = lib.literalExpression "pythonPackages: with pythonPackages; [ requests ]";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.groups = lib.optionalAttrs (cfg.group == "buildbot") {
+      buildbot = { };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "buildbot") {
+      buildbot = {
+        description = "Buildbot User.";
+        isNormalUser = true;
+        createHome = true;
+        inherit (cfg) home group extraGroups;
+        useDefaultShell = true;
+      };
+    };
+
+    systemd.services.buildbot-master = {
+      description = "Buildbot Continuous Integration Server.";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.packages ++ cfg.pythonPackages python.pkgs;
+      environment.PYTHONPATH = "${
+        python.withPackages (self: cfg.pythonPackages self ++ [ package ])
+      }/${python.sitePackages}";
+
+      preStart = ''
+        mkdir -vp "${cfg.buildbotDir}"
+        # Link the tac file so buildbot command line tools recognize the directory
+        ln -sf "${tacFile}" "${cfg.buildbotDir}/buildbot.tac"
+        ${cfg.package}/bin/buildbot create-master --db "${cfg.dbUrl}" "${cfg.buildbotDir}"
+        rm -f buildbot.tac.new master.cfg.sample
+      '';
+
+      serviceConfig = {
+        Type = "simple";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd -o --nodaemon --pidfile= --logfile - --python ${cfg.buildbotDir}/buildbot.tac";
+        # To reload on upgrade, set the following in your configuration:
+        # systemd.services.buildbot-master.reloadIfChanged = true;
+        ExecReload = [
+          "${pkgs.coreutils}/bin/ln -sf ${tacFile} ${cfg.buildbotDir}/buildbot.tac"
+          "${pkgs.coreutils}/bin/kill -HUP $MAINPID"
+        ];
+      };
+    };
+  };
+
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "buildbot-master" "bpPort" ]
+      [ "services" "buildbot-master" "pbPort" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "buildbot-master" "status" ] ''
+      Since Buildbot 0.9.0, status targets are deprecated and ignored.
+      Review your configuration and migrate to reporters (available at services.buildbot-master.reporters).
+    '')
+  ];
+
+  meta.maintainers = lib.teams.buildbot.members;
+}

nixos/modules/services/continuous-integration/buildbot/worker.nix

@@ -0,0 +1,201 @@
+# NixOS module for Buildbot Worker.
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.buildbot-worker;
+  opt = options.services.buildbot-worker;
+
+  package = pkgs.python3.pkgs.toPythonModule cfg.package;
+  python = package.pythonModule;
+
+  tacFile = pkgs.writeText "aur-buildbot-worker.tac" ''
+    import os
+    from io import open
+
+    from buildbot_worker.bot import Worker
+    from twisted.application import service
+
+    basedir = '${cfg.buildbotDir}'
+
+    # note: this line is matched against to check that this is a worker
+    # directory; do not edit it.
+    application = service.Application('buildbot-worker')
+
+    master_url_split = '${cfg.masterUrl}'.split(':')
+    buildmaster_host = master_url_split[0]
+    port = int(master_url_split[1])
+    workername = '${cfg.workerUser}'
+
+    with open('${cfg.workerPassFile}', 'r', encoding='utf-8') as passwd_file:
+        passwd = passwd_file.read().strip('\r\n')
+    keepalive = ${toString cfg.keepalive}
+    umask = None
+    maxdelay = 300
+    numcpus = None
+    allow_shutdown = None
+
+    s = Worker(buildmaster_host, port, workername, passwd, basedir,
+               keepalive, umask=umask, maxdelay=maxdelay,
+               numcpus=numcpus, allow_shutdown=allow_shutdown)
+    s.setServiceParent(application)
+  '';
+
+in
+{
+  options = {
+    services.buildbot-worker = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Whether to enable the Buildbot Worker.";
+      };
+
+      user = lib.mkOption {
+        default = "bbworker";
+        type = lib.types.str;
+        description = "User the buildbot Worker should execute under.";
+      };
+
+      group = lib.mkOption {
+        default = "bbworker";
+        type = lib.types.str;
+        description = "Primary group of buildbot Worker user.";
+      };
+
+      extraGroups = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = "List of extra groups that the Buildbot Worker user should be a part of.";
+      };
+
+      home = lib.mkOption {
+        default = "/home/bbworker";
+        type = lib.types.path;
+        description = "Buildbot home directory.";
+      };
+
+      buildbotDir = lib.mkOption {
+        default = "${cfg.home}/worker";
+        defaultText = lib.literalExpression ''"''${config.${opt.home}}/worker"'';
+        type = lib.types.path;
+        description = "Specifies the Buildbot directory.";
+      };
+
+      workerUser = lib.mkOption {
+        default = "example-worker";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker user.";
+      };
+
+      workerPass = lib.mkOption {
+        default = "pass";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker password.";
+      };
+
+      workerPassFile = lib.mkOption {
+        type = lib.types.path;
+        description = "File used to store the Buildbot Worker password";
+      };
+
+      hostMessage = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = "Description of this worker";
+      };
+
+      adminMessage = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = "Name of the administrator of this worker";
+      };
+
+      masterUrl = lib.mkOption {
+        default = "localhost:9989";
+        type = lib.types.str;
+        description = "Specifies the Buildbot Worker connection string.";
+      };
+
+      keepalive = lib.mkOption {
+        default = 600;
+        type = lib.types.int;
+        description = ''
+          This is a number that indicates how frequently keepalive messages should be sent
+          from the worker to the buildmaster, expressed in seconds.
+        '';
+      };
+
+      package = lib.mkPackageOption pkgs "buildbot-worker" { };
+
+      packages = lib.mkOption {
+        default = with pkgs; [ git ];
+        defaultText = lib.literalExpression "[ pkgs.git ]";
+        type = lib.types.listOf lib.types.package;
+        description = "Packages to add to PATH for the buildbot process.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.buildbot-worker.workerPassFile = lib.mkDefault (
+      pkgs.writeText "buildbot-worker-password" cfg.workerPass
+    );
+
+    users.groups = lib.optionalAttrs (cfg.group == "bbworker") {
+      bbworker = { };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "bbworker") {
+      bbworker = {
+        description = "Buildbot Worker User.";
+        isNormalUser = true;
+        createHome = true;
+        home = cfg.home;
+        group = cfg.group;
+        extraGroups = cfg.extraGroups;
+        useDefaultShell = true;
+      };
+    };
+
+    systemd.services.buildbot-worker = {
+      description = "Buildbot Worker.";
+      after = [
+        "network.target"
+        "buildbot-master.service"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.packages;
+      environment.PYTHONPATH = "${python.withPackages (p: [ package ])}/${python.sitePackages}";
+
+      preStart = ''
+        mkdir -vp "${cfg.buildbotDir}/info"
+        ${lib.optionalString (cfg.hostMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-host" cfg.hostMessage}" "${cfg.buildbotDir}/info/host"
+        ''}
+        ${lib.optionalString (cfg.adminMessage != null) ''
+          ln -sf "${pkgs.writeText "buildbot-worker-admin" cfg.adminMessage}" "${cfg.buildbotDir}/info/admin"
+        ''}
+      '';
+
+      serviceConfig = {
+        Type = "simple";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = cfg.home;
+
+        # NOTE: call twistd directly with stdout logging for systemd
+        ExecStart = "${python.pkgs.twisted}/bin/twistd --nodaemon --pidfile= --logfile - --python ${tacFile}";
+      };
+
+    };
+  };
+
+  meta.maintainers = lib.teams.buildbot.members;
+
+}

nixos/modules/services/continuous-integration/buildkite-agents.nix

@@ -0,0 +1,262 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.buildkite-agents;
+
+  hooksDir =
+    hooks:
+    let
+      mkHookEntry = name: text: ''
+        ln --symbolic ${pkgs.writeShellApplication { inherit name text; }}/bin/${name} $out/${name}
+      '';
+    in
+    pkgs.runCommand "buildkite-agent-hooks"
+      {
+        preferLocalBuild = true;
+      }
+      ''
+        mkdir $out
+        ${lib.concatStringsSep "\n" (lib.mapAttrsToList mkHookEntry hooks)}
+      '';
+
+  buildkiteOptions =
+    {
+      name ? "",
+      config,
+      ...
+    }:
+    {
+      options = {
+        enable = lib.mkOption {
+          default = true;
+          type = lib.types.bool;
+          description = "Whether to enable this buildkite agent";
+        };
+
+        package = lib.mkPackageOption pkgs "buildkite-agent" { };
+
+        dataDir = lib.mkOption {
+          default = "/var/lib/buildkite-agent-${name}";
+          description = "The workdir for the agent";
+          type = lib.types.str;
+        };
+
+        extraGroups = lib.mkOption {
+          default = [ "keys" ];
+          description = "Groups the user for this buildkite agent should belong to";
+          type = lib.types.listOf lib.types.str;
+        };
+
+        runtimePackages = lib.mkOption {
+          default = [
+            pkgs.bash
+            pkgs.gnutar
+            pkgs.gzip
+            pkgs.git
+            pkgs.nix
+          ];
+          defaultText = lib.literalExpression "[ pkgs.bash pkgs.gnutar pkgs.gzip pkgs.git pkgs.nix ]";
+          description = "Add programs to the buildkite-agent environment";
+          type = lib.types.listOf lib.types.package;
+        };
+
+        tokenPath = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            The token from your Buildkite "Agents" page.
+
+            A run-time path to the token file, which is supposed to be provisioned
+            outside of Nix store.
+          '';
+        };
+
+        name = lib.mkOption {
+          type = lib.types.str;
+          default = "%hostname-${name}-%n";
+          description = ''
+            The name of the agent as seen in the buildkite dashboard.
+          '';
+        };
+
+        tags = lib.mkOption {
+          type = lib.types.attrsOf (lib.types.either lib.types.str (lib.types.listOf lib.types.str));
+          default = { };
+          example = {
+            queue = "default";
+            docker = "true";
+            ruby2 = "true";
+          };
+          description = ''
+            Tags for the agent.
+          '';
+        };
+
+        extraConfig = lib.mkOption {
+          type = lib.types.lines;
+          default = "";
+          example = "debug=true";
+          description = ''
+            Extra lines to be added verbatim to the configuration file.
+          '';
+        };
+
+        privateSshKeyPath = lib.mkOption {
+          type = lib.types.nullOr lib.types.path;
+          default = null;
+          ## maximum care is taken so that secrets (ssh keys and the CI token)
+          ## don't end up in the Nix store.
+          apply = final: if final == null then null else toString final;
+
+          description = ''
+            OpenSSH private key
+
+            A run-time path to the key file, which is supposed to be provisioned
+            outside of Nix store.
+          '';
+        };
+
+        hooks = lib.mkOption {
+          type = lib.types.attrsOf lib.types.lines;
+          default = { };
+          example = lib.literalExpression ''
+            {
+              environment = '''
+                export SECRET_VAR=`head -1 /run/keys/secret`
+              ''';
+            }'';
+          description = ''
+            "Agent" hooks to install.
+            See <https://buildkite.com/docs/agent/v3/hooks> for possible options.
+          '';
+        };
+
+        hooksPath = lib.mkOption {
+          type = lib.types.path;
+          default = hooksDir config.hooks;
+          defaultText = lib.literalMD "generated from {option}`services.buildkite-agents.<name>.hooks`";
+          description = ''
+            Path to the directory storing the hooks.
+            Consider using {option}`services.buildkite-agents.<name>.hooks.<name>`
+            instead.
+          '';
+        };
+
+        shell = lib.mkOption {
+          type = lib.types.str;
+          default = "${pkgs.bash}/bin/bash -e -c";
+          defaultText = lib.literalExpression ''"''${pkgs.bash}/bin/bash -e -c"'';
+          description = ''
+            Command that buildkite-agent 3 will execute when it spawns a shell.
+          '';
+        };
+      };
+    };
+  enabledAgents = lib.filterAttrs (n: v: v.enable) cfg;
+  mapAgents = function: lib.mkMerge (lib.mapAttrsToList function enabledAgents);
+in
+{
+  options.services.buildkite-agents = lib.mkOption {
+    type = lib.types.attrsOf (lib.types.submodule buildkiteOptions);
+    default = { };
+    description = ''
+      Attribute set of buildkite agents.
+      The attribute key is combined with the hostname and a unique integer to
+      create the final agent name. This can be overridden by setting the `name`
+      attribute.
+    '';
+  };
+
+  config.users.users = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = {
+        name = "buildkite-agent-${name}";
+        home = cfg.dataDir;
+        createHome = true;
+        description = "Buildkite agent user";
+        extraGroups = cfg.extraGroups;
+        isSystemUser = true;
+        group = "buildkite-agent-${name}";
+      };
+    }
+  );
+  config.users.groups = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = { };
+    }
+  );
+
+  config.systemd.services = mapAgents (
+    name: cfg: {
+      "buildkite-agent-${name}" = {
+        description = "Buildkite Agent";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = cfg.runtimePackages ++ [
+          cfg.package
+          pkgs.coreutils
+        ];
+        environment = config.networking.proxy.envVars // {
+          HOME = cfg.dataDir;
+          NIX_REMOTE = "daemon";
+        };
+
+        ## NB: maximum care is taken so that secrets (ssh keys and the CI token)
+        ##     don't end up in the Nix store.
+        preStart =
+          let
+            sshDir = "${cfg.dataDir}/.ssh";
+            tagStr =
+              name: value:
+              if lib.isList value then
+                lib.concatStringsSep "," (builtins.map (v: "${name}=${v}") value)
+              else
+                "${name}=${value}";
+            tagsStr = lib.concatStringsSep "," (lib.mapAttrsToList tagStr cfg.tags);
+          in
+          lib.optionalString (cfg.privateSshKeyPath != null) ''
+            mkdir -m 0700 -p "${sshDir}"
+            install -m600 "${toString cfg.privateSshKeyPath}" "${sshDir}/id_rsa"
+          ''
+          + ''
+            cat > "${cfg.dataDir}/buildkite-agent.cfg" <<EOF
+            token="$(cat ${toString cfg.tokenPath})"
+            name="${cfg.name}"
+            shell="${cfg.shell}"
+            tags="${tagsStr}"
+            build-path="${cfg.dataDir}/builds"
+            hooks-path="${cfg.hooksPath}"
+            ${cfg.extraConfig}
+            EOF
+          '';
+
+        serviceConfig = {
+          ExecStart = "${cfg.package}/bin/buildkite-agent start --config ${cfg.dataDir}/buildkite-agent.cfg";
+          User = "buildkite-agent-${name}";
+          RestartSec = 5;
+          Restart = "on-failure";
+          TimeoutSec = 10;
+          # set a long timeout to give buildkite-agent a chance to finish current builds
+          TimeoutStopSec = "2 min";
+          KillMode = "mixed";
+        };
+      };
+    }
+  );
+
+  config.assertions = mapAgents (
+    name: cfg: [
+      {
+        assertion = cfg.hooksPath != hooksDir cfg.hooks -> cfg.hooks == { };
+        message = ''
+          Options `services.buildkite-agents.${name}.hooksPath' and
+          `services.buildkite-agents.${name}.hooks.<name>' are mutually exclusive.
+        '';
+      }
+    ]
+  );
+}

nixos/modules/services/continuous-integration/gitea-actions-runner.nix

@@ -0,0 +1,283 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    any
+    attrValues
+    concatStringsSep
+    escapeShellArg
+    hasInfix
+    hasSuffix
+    optionalAttrs
+    optionals
+    literalExpression
+    mapAttrs'
+    mkEnableOption
+    mkOption
+    mkPackageOption
+    mkIf
+    nameValuePair
+    types
+    ;
+
+  inherit (utils)
+    escapeSystemdPath
+    ;
+
+  cfg = config.services.gitea-actions-runner;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+  # Check whether any runner instance label requires a container runtime
+  # Empty label strings result in the upstream defined defaultLabels, which require docker
+  # https://gitea.com/gitea/act_runner/src/tag/v0.1.5/internal/app/cmd/register.go#L93-L98
+  hasDockerScheme =
+    instance: instance.labels == [ ] || any (label: hasInfix ":docker:" label) instance.labels;
+  wantsContainerRuntime = any hasDockerScheme (attrValues cfg.instances);
+
+  hasHostScheme = instance: any (label: hasSuffix ":host" label) instance.labels;
+
+  # provide shorthands for whether container runtimes are enabled
+  hasDocker = config.virtualisation.docker.enable;
+  hasPodman = config.virtualisation.podman.enable;
+
+  tokenXorTokenFile =
+    instance:
+    (instance.token == null && instance.tokenFile != null)
+    || (instance.token != null && instance.tokenFile == null);
+in
+{
+  meta.maintainers = with lib.maintainers; [
+    hexa
+  ];
+
+  options.services.gitea-actions-runner = with types; {
+    package = mkPackageOption pkgs "gitea-actions-runner" { };
+
+    instances = mkOption {
+      default = { };
+      description = ''
+        Gitea Actions Runner instances.
+      '';
+      type = attrsOf (submodule {
+        options = {
+          enable = mkEnableOption "Gitea Actions Runner instance";
+
+          name = mkOption {
+            type = str;
+            example = literalExpression "config.networking.hostName";
+            description = ''
+              The name identifying the runner instance towards the Gitea/Forgejo instance.
+            '';
+          };
+
+          url = mkOption {
+            type = str;
+            example = "https://forge.example.com";
+            description = ''
+              Base URL of your Gitea/Forgejo instance.
+            '';
+          };
+
+          token = mkOption {
+            type = nullOr str;
+            default = null;
+            description = ''
+              Plain token to register at the configured Gitea/Forgejo instance.
+            '';
+          };
+
+          tokenFile = mkOption {
+            type = nullOr (either str path);
+            default = null;
+            description = ''
+              Path to an environment file, containing the `TOKEN` environment
+              variable, that holds a token to register at the configured
+              Gitea/Forgejo instance.
+            '';
+          };
+
+          labels = mkOption {
+            type = listOf str;
+            example = literalExpression ''
+              [
+                # provide a debian base with nodejs for actions
+                "debian-latest:docker://node:18-bullseye"
+                # fake the ubuntu name, because node provides no ubuntu builds
+                "ubuntu-latest:docker://node:18-bullseye"
+                # provide native execution on the host
+                #"native:host"
+              ]
+            '';
+            description = ''
+              Labels used to map jobs to their runtime environment. Changing these
+              labels currently requires a new registration token.
+
+              Many common actions require bash, git and nodejs, as well as a filesystem
+              that follows the filesystem hierarchy standard.
+            '';
+          };
+          settings = mkOption {
+            description = ''
+              Configuration for `act_runner daemon`.
+              See <https://gitea.com/gitea/act_runner/src/branch/main/internal/pkg/config/config.example.yaml> for an example configuration
+            '';
+
+            type = types.submodule {
+              freeformType = settingsFormat.type;
+            };
+
+            default = { };
+          };
+
+          hostPackages = mkOption {
+            type = listOf package;
+            default = with pkgs; [
+              bash
+              coreutils
+              curl
+              gawk
+              gitMinimal
+              gnused
+              nodejs
+              wget
+            ];
+            defaultText = literalExpression ''
+              with pkgs; [
+                bash
+                coreutils
+                curl
+                gawk
+                gitMinimal
+                gnused
+                nodejs
+                wget
+              ]
+            '';
+            description = ''
+              List of packages, that are available to actions, when the runner is configured
+              with a host execution label.
+            '';
+          };
+        };
+      });
+    };
+  };
+
+  config = mkIf (cfg.instances != { }) {
+    assertions = [
+      {
+        assertion = any tokenXorTokenFile (attrValues cfg.instances);
+        message = "Instances of gitea-actions-runner can have `token` or `tokenFile`, not both.";
+      }
+      {
+        assertion = wantsContainerRuntime -> hasDocker || hasPodman;
+        message = "Label configuration on gitea-actions-runner instance requires either docker or podman.";
+      }
+    ];
+
+    systemd.services =
+      let
+        mkRunnerService =
+          name: instance:
+          let
+            wantsContainerRuntime = hasDockerScheme instance;
+            wantsHost = hasHostScheme instance;
+            wantsDocker = wantsContainerRuntime && config.virtualisation.docker.enable;
+            wantsPodman = wantsContainerRuntime && config.virtualisation.podman.enable;
+            configFile = settingsFormat.generate "config.yaml" instance.settings;
+          in
+          nameValuePair "gitea-runner-${escapeSystemdPath name}" {
+            inherit (instance) enable;
+            description = "Gitea Actions Runner";
+            wants = [ "network-online.target" ];
+            after = [
+              "network-online.target"
+            ]
+            ++ optionals wantsDocker [
+              "docker.service"
+            ]
+            ++ optionals wantsPodman [
+              "podman.service"
+            ];
+            wantedBy = [
+              "multi-user.target"
+            ];
+            environment =
+              optionalAttrs (instance.token != null) {
+                TOKEN = "${instance.token}";
+              }
+              // optionalAttrs wantsPodman {
+                DOCKER_HOST = "unix:///run/podman/podman.sock";
+              }
+              // {
+                HOME = "/var/lib/gitea-runner/${name}";
+              };
+            path =
+              with pkgs;
+              [
+                coreutils
+              ]
+              ++ lib.optionals wantsHost instance.hostPackages;
+            serviceConfig = {
+              DynamicUser = true;
+              User = "gitea-runner";
+              StateDirectory = "gitea-runner";
+              WorkingDirectory = "-/var/lib/gitea-runner/${name}";
+
+              # gitea-runner might fail when gitea is restarted during upgrade.
+              Restart = "on-failure";
+              RestartSec = 2;
+
+              ExecStartPre = [
+                (pkgs.writeShellScript "gitea-register-runner-${name}" ''
+                  export INSTANCE_DIR="$STATE_DIRECTORY/${name}"
+                  mkdir -vp "$INSTANCE_DIR"
+                  cd "$INSTANCE_DIR"
+
+                  # force reregistration on changed labels
+                  export LABELS_FILE="$INSTANCE_DIR/.labels"
+                  export LABELS_WANTED="$(echo ${escapeShellArg (concatStringsSep "\n" instance.labels)} | sort)"
+                  export LABELS_CURRENT="$(cat $LABELS_FILE 2>/dev/null || echo 0)"
+
+                  if [ ! -e "$INSTANCE_DIR/.runner" ] || [ "$LABELS_WANTED" != "$LABELS_CURRENT" ]; then
+                    # remove existing registration file, so that changing the labels forces a re-registration
+                    rm -v "$INSTANCE_DIR/.runner" || true
+
+                    # perform the registration
+                    ${cfg.package}/bin/act_runner register --no-interactive \
+                      --instance ${escapeShellArg instance.url} \
+                      --token "$TOKEN" \
+                      --name ${escapeShellArg instance.name} \
+                      --labels ${escapeShellArg (concatStringsSep "," instance.labels)} \
+                      --config ${configFile}
+
+                    # and write back the configured labels
+                    echo "$LABELS_WANTED" > "$LABELS_FILE"
+                  fi
+
+                '')
+              ];
+              ExecStart = "${cfg.package}/bin/act_runner daemon --config ${configFile}";
+              SupplementaryGroups =
+                optionals wantsDocker [
+                  "docker"
+                ]
+                ++ optionals wantsPodman [
+                  "podman"
+                ];
+            }
+            // optionalAttrs (instance.tokenFile != null) {
+              EnvironmentFile = instance.tokenFile;
+            };
+          };
+      in
+      mapAttrs' mkRunnerService cfg.instances;
+  };
+}

nixos/modules/services/continuous-integration/github-runner/options.nix

@@ -0,0 +1,282 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  options.services.github-runners = lib.mkOption {
+    description = ''
+      Multiple GitHub Runners.
+    '';
+    example = {
+      runner1 = {
+        enable = true;
+        url = "https://github.com/owner/repo";
+        name = "runner1";
+        tokenFile = "/secrets/token1";
+      };
+
+      runner2 = {
+        enable = true;
+        url = "https://github.com/owner/repo";
+        name = "runner2";
+        tokenFile = "/secrets/token2";
+      };
+    };
+    default = { };
+    type = lib.types.attrsOf (
+      lib.types.submodule (
+        { name, config, ... }:
+        {
+          options = {
+            enable = lib.mkOption {
+              default = false;
+              example = true;
+              description = ''
+                Whether to enable GitHub Actions runner.
+
+                Note: GitHub recommends using self-hosted runners with private repositories only. Learn more here:
+                [About self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners).
+              '';
+              type = lib.types.bool;
+            };
+
+            url = lib.mkOption {
+              type = lib.types.str;
+              description = ''
+                Repository to add the runner to.
+
+                Changing this option triggers a new runner registration.
+
+                IMPORTANT: If your token is org-wide (not per repository), you need to
+                provide a github org link, not a single repository, so do it like this
+                `https://github.com/nixos`, not like this
+                `https://github.com/nixos/nixpkgs`.
+                Otherwise, you are going to get a `404 NotFound`
+                from `POST https://api.github.com/actions/runner-registration`
+                in the configure script.
+              '';
+              example = "https://github.com/nixos/nixpkgs";
+            };
+
+            tokenFile = lib.mkOption {
+              type = lib.types.path;
+              description = ''
+                The full path to a file which contains either
+
+                * a fine-grained personal access token (PAT),
+                * a classic PAT
+                * or a runner registration token
+
+                Changing this option or the `tokenFile`’s content triggers a new runner registration.
+
+                We suggest using the fine-grained PATs. A runner registration token is valid
+                only for 1 hour after creation, so the next time the runner configuration changes
+                this will give you hard-to-debug HTTP 404 errors in the configure step.
+
+                The file should contain exactly one line with the token without any newline.
+                (Use `echo -n '…token…' > …token file…` to make sure no newlines sneak in.)
+
+                If the file contains a PAT, the service creates a new registration token
+                on startup as needed.
+                If a registration token is given, it can be used to re-register a runner of the same
+                name but is time-limited as noted above.
+
+                For fine-grained PATs:
+
+                Give it "Read and Write access to organization/repository self hosted runners",
+                depending on whether it is organization wide or per-repository. You might have to
+                experiment a little, fine-grained PATs are a `beta` Github feature and still subject
+                to change; nonetheless they are the best option at the moment.
+
+                For classic PATs:
+
+                Make sure the PAT has a scope of `admin:org` for organization-wide registrations
+                or a scope of `repo` for a single repository.
+
+                For runner registration tokens:
+
+                Nothing special needs to be done, but updating will break after one hour,
+                so these are not recommended.
+              '';
+              example = "/run/secrets/github-runner/nixos.token";
+            };
+
+            name = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Name of the runner to configure. If null, defaults to the hostname.
+
+                Changing this option triggers a new runner registration.
+              '';
+              example = "nixos";
+              default = name;
+            };
+
+            runnerGroup = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Name of the runner group to add this runner to (defaults to the default runner group).
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = null;
+            };
+
+            extraLabels = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              description = ''
+                Extra labels in addition to the default (unless disabled through the `noDefaultLabels` option).
+
+                Changing this option triggers a new runner registration.
+              '';
+              example = lib.literalExpression ''[ "nixos" ]'';
+              default = [ ];
+            };
+
+            noDefaultLabels = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Disables adding the default labels. Also see the `extraLabels` option.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = false;
+            };
+
+            replace = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                Replace any existing runner with the same name.
+
+                Without this flag, registering a new runner with the same name fails.
+              '';
+              default = false;
+            };
+
+            extraPackages = lib.mkOption {
+              type = lib.types.listOf lib.types.package;
+              description = ''
+                Extra packages to add to `PATH` of the service to make them available to workflows.
+              '';
+              default = [ ];
+            };
+
+            extraEnvironment = lib.mkOption {
+              type = lib.types.attrs;
+              description = ''
+                Extra environment variables to set for the runner, as an attrset.
+              '';
+              example = {
+                GIT_CONFIG = "/path/to/git/config";
+              };
+              default = { };
+            };
+
+            serviceOverrides = lib.mkOption {
+              type = lib.types.attrs;
+              description = ''
+                Modify the systemd service. Can be used to, e.g., adjust the sandboxing options.
+                See {manpage}`systemd.exec(5)` for more options.
+              '';
+              example = {
+                ProtectHome = false;
+                RestrictAddressFamilies = [ "AF_PACKET" ];
+              };
+              default = { };
+            };
+
+            package = lib.mkPackageOption pkgs "github-runner" { } // {
+              apply =
+                # Support old github-runner versions which don't have the `nodeRuntimes` arg yet.
+                pkg: pkg.override (old: lib.optionalAttrs (old ? nodeRuntimes) { inherit (config) nodeRuntimes; });
+            };
+
+            ephemeral = lib.mkOption {
+              type = lib.types.bool;
+              description = ''
+                If enabled, causes the following behavior:
+
+                - Passes the `--ephemeral` flag to the runner configuration script
+                - De-registers and stops the runner with GitHub after it has processed one job
+                - On stop, systemd wipes the runtime directory (this always happens, even without using the ephemeral option)
+                - Restarts the service after its successful exit
+                - On start, wipes the state directory and configures a new runner
+
+                You should only enable this option if `tokenFile` points to a file which contains a
+                personal access token (PAT). If you're using the option with a registration token, restarting the
+                service will fail as soon as the registration token expired.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = false;
+            };
+
+            user = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                User under which to run the service.
+
+                If this option and the `group` option is set to `null`,
+                the service runs as a dynamically allocated user.
+
+                Also see the `group` option for an overview on the effects of the `user` and `group` settings.
+              '';
+              default = null;
+              defaultText = lib.literalExpression "username";
+            };
+
+            group = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              description = ''
+                Group under which to run the service.
+
+                The effect of this option depends on the value of the `user` option:
+
+                - `group == null` and `user == null`:
+                  The service runs with a dynamically allocated user and group.
+                - `group == null` and `user != null`:
+                  The service runs as the given user and its default group.
+                - `group != null` and `user == null`:
+                  This configuration is invalid. In this case, the service would use the given group
+                  but run as root implicitly. If this is really what you want, set `user = "root"` explicitly.
+              '';
+              default = null;
+              defaultText = lib.literalExpression "groupname";
+            };
+
+            workDir = lib.mkOption {
+              type = with lib.types; nullOr str;
+              description = ''
+                Working directory, available as `$GITHUB_WORKSPACE` during workflow runs
+                and used as a default for [repository checkouts](https://github.com/actions/checkout).
+                The service cleans this directory on every service start.
+
+                A value of `null` will default to the systemd `RuntimeDirectory`.
+
+                Changing this option triggers a new runner registration.
+              '';
+              default = null;
+            };
+
+            nodeRuntimes = lib.mkOption {
+              type =
+                with lib.types;
+                nonEmptyListOf (enum [
+                  "node20"
+                  "node24"
+                ]);
+              default = [
+                "node20"
+                "node24"
+              ];
+              description = ''
+                List of Node.js runtimes the runner should support.
+              '';
+            };
+          };
+        }
+      )
+    );
+  };
+}