push sheeet

nixos/modules/services/monitoring/alerta.nix

@@ -0,0 +1,117 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.alerta;
+
+  alertaConf = pkgs.writeTextFile {
+    name = "alertad.conf";
+    text = ''
+      DATABASE_URL = '${cfg.databaseUrl}'
+      DATABASE_NAME = '${cfg.databaseName}'
+      LOG_FILE = '${cfg.logDir}/alertad.log'
+      LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+      CORS_ORIGINS = [ ${lib.concatMapStringsSep ", " (s: "\"" + s + "\"") cfg.corsOrigins} ];
+      AUTH_REQUIRED = ${if cfg.authenticationRequired then "True" else "False"}
+      SIGNUP_ENABLED = ${if cfg.signupEnabled then "True" else "False"}
+      ${cfg.extraConfig}
+    '';
+  };
+in
+{
+  options.services.alerta = {
+    enable = lib.mkEnableOption "alerta";
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 5000;
+      description = "Port of Alerta";
+    };
+
+    bind = lib.mkOption {
+      type = lib.types.str;
+      default = "0.0.0.0";
+      description = "Address to bind to. The default is to bind to all addresses";
+    };
+
+    logDir = lib.mkOption {
+      type = lib.types.path;
+      description = "Location where the logfiles are stored";
+      default = "/var/log/alerta";
+    };
+
+    databaseUrl = lib.mkOption {
+      type = lib.types.str;
+      description = "URL of the MongoDB or PostgreSQL database to connect to";
+      default = "mongodb://localhost";
+    };
+
+    databaseName = lib.mkOption {
+      type = lib.types.str;
+      description = "Name of the database instance to connect to";
+      default = "monitoring";
+    };
+
+    corsOrigins = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "List of URLs that can access the API for Cross-Origin Resource Sharing (CORS)";
+      default = [
+        "http://localhost"
+        "http://localhost:5000"
+      ];
+    };
+
+    authenticationRequired = lib.mkOption {
+      type = lib.types.bool;
+      description = "Whether users must authenticate when using the web UI or command-line tool";
+      default = false;
+    };
+
+    signupEnabled = lib.mkOption {
+      type = lib.types.bool;
+      description = "Whether to prevent sign-up of new users via the web UI";
+      default = true;
+    };
+
+    extraConfig = lib.mkOption {
+      description = "These lines go into alertad.conf verbatim.";
+      default = "";
+      type = lib.types.lines;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.tmpfiles.settings."10-alerta".${cfg.logDir}.d = {
+      user = "alerta";
+      group = "alerta";
+    };
+
+    systemd.services.alerta = {
+      description = "Alerta Monitoring System";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "networking.target" ];
+      environment = {
+        ALERTA_SVR_CONF_FILE = alertaConf;
+      };
+      serviceConfig = {
+        ExecStart = "${pkgs.alerta-server}/bin/alertad run --port ${toString cfg.port} --host ${cfg.bind}";
+        User = "alerta";
+        Group = "alerta";
+      };
+    };
+
+    environment.systemPackages = [ pkgs.alerta ];
+
+    users.users.alerta = {
+      uid = config.ids.uids.alerta;
+      description = "Alerta user";
+    };
+
+    users.groups.alerta = {
+      gid = config.ids.gids.alerta;
+    };
+  };
+}

nixos/modules/services/monitoring/alloy.nix

@@ -0,0 +1,100 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  cfg = config.services.alloy;
+in
+{
+  meta = {
+    maintainers = with lib.maintainers; [
+      flokli
+      hbjydev
+    ];
+  };
+
+  options.services.alloy = {
+    enable = lib.mkEnableOption "Grafana Alloy";
+
+    package = lib.mkPackageOption pkgs "grafana-alloy" { };
+
+    configPath = lib.mkOption {
+      type = lib.types.path;
+      default = "/etc/alloy";
+      description = ''
+        Alloy configuration file/directory path.
+
+        We default to `/etc/alloy` here, and expect the user to configure a
+        configuration file via `environment.etc."alloy/config.alloy"`.
+
+        This allows config reload, contrary to specifying a store path.
+
+        All `.alloy` files in the same directory (ignoring subdirs) are also
+        honored and are added to `systemd.services.alloy.reloadTriggers` to
+        enable config reload during nixos-rebuild switch.
+
+        This can also point to another directory containing `*.alloy` files, or
+        a single Alloy file in the Nix store (at the cost of reload).
+
+        Component names must be unique across all Alloy configuration files, and
+        configuration blocks must not be repeated.
+
+        Alloy will continue to run if subsequent reloads of the configuration
+        file fail, potentially marking components as unhealthy depending on
+        the nature of the failure. When this happens, Alloy will continue
+        functioning in the last valid state.
+      '';
+    };
+
+    environmentFile = lib.mkOption {
+      type = with lib.types; nullOr path;
+      default = null;
+      example = "/run/secrets/alloy.env";
+      description = ''
+        EnvironmentFile as defined in {manpage}`systemd.exec(5)`.
+      '';
+    };
+
+    extraFlags = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [ ];
+      example = [
+        "--server.http.listen-addr=127.0.0.1:12346"
+        "--disable-reporting"
+      ];
+      description = ''
+        Extra command-line flags passed to {command}`alloy run`.
+
+        See <https://grafana.com/docs/alloy/latest/reference/cli/run/>
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.alloy = {
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      reloadTriggers = lib.mapAttrsToList (_: v: v.source or null) (
+        lib.filterAttrs (n: _: lib.hasPrefix "alloy/" n && lib.hasSuffix ".alloy" n) config.environment.etc
+      );
+      serviceConfig = {
+        Restart = "always";
+        DynamicUser = true;
+        RestartSec = 2;
+        SupplementaryGroups = [
+          # allow to read the systemd journal for loki log forwarding
+          "systemd-journal"
+        ];
+        ExecStart = "${lib.getExe cfg.package} run ${cfg.configPath} ${lib.escapeShellArgs cfg.extraFlags}";
+        ExecReload = "${pkgs.coreutils}/bin/kill -SIGHUP $MAINPID";
+        ConfigurationDirectory = "alloy";
+        StateDirectory = "alloy";
+        WorkingDirectory = "%S/alloy";
+        Type = "simple";
+        EnvironmentFile = lib.mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/amazon-cloudwatch-agent.nix

@@ -0,0 +1,214 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  cfg = config.services.amazon-cloudwatch-agent;
+
+  tomlFormat = pkgs.formats.toml { };
+  jsonFormat = pkgs.formats.json { };
+
+  # See https://docs.aws.amazon.com/prescriptive-guidance/latest/implementing-logging-monitoring-cloudwatch/create-store-cloudwatch-configurations.html#store-cloudwatch-configuration-s3.
+  #
+  # We don't use the multiple JSON configuration files feature,
+  # but "config-translator" will log a benign error if the "-input-dir" option is omitted or is a non-existent directory.
+  #
+  # Create an empty directory to hide this benign error log. This prevents false-positives if users filter for "error" in the agent logs.
+  configurationDirectory = pkgs.runCommand "amazon-cloudwatch-agent.d" { } "mkdir $out";
+in
+{
+  options.services.amazon-cloudwatch-agent = {
+    enable = lib.mkEnableOption "Amazon CloudWatch Agent";
+    package = lib.mkPackageOption pkgs "amazon-cloudwatch-agent" { };
+    commonConfigurationFile = lib.mkOption {
+      type = lib.types.path;
+      default = tomlFormat.generate "common-config.toml" cfg.commonConfiguration;
+      defaultText = lib.literalExpression ''tomlFormat.generate "common-config.toml" cfg.commonConfiguration'';
+      description = ''
+        Amazon CloudWatch Agent common configuration. See
+        <https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/install-CloudWatch-Agent-commandline-fleet.html#CloudWatch-Agent-profile-instance-first>
+        for supported values.
+
+        {option}`commonConfigurationFile` takes precedence over {option}`commonConfiguration`.
+
+        Note: Restricted evaluation blocks access to paths outside the Nix store.
+        This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
+        As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
+        `systemctl restart amazon-cloudwatch-agent.service` must be used instead.
+      '';
+      example = "/etc/amazon-cloudwatch-agent/amazon-cloudwatch-agent.json";
+    };
+    commonConfiguration = lib.mkOption {
+      type = tomlFormat.type;
+      default = { };
+      description = ''
+        See {option}`commonConfigurationFile`.
+
+        {option}`commonConfigurationFile` takes precedence over {option}`commonConfiguration`.
+      '';
+      example = {
+        credentials = {
+          shared_credential_profile = "profile_name";
+          shared_credential_file = "/path/to/credentials";
+        };
+        proxy = {
+          http_proxy = "http_url";
+          https_proxy = "https_url";
+          no_proxy = "domain";
+        };
+      };
+    };
+    configurationFile = lib.mkOption {
+      type = lib.types.path;
+      default = jsonFormat.generate "amazon-cloudwatch-agent.json" cfg.configuration;
+      defaultText = lib.literalExpression ''jsonFormat.generate "amazon-cloudwatch-agent.json" cfg.configuration'';
+      description = ''
+        Amazon CloudWatch Agent configuration file. See
+        <https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html>
+        for supported values.
+
+        The following options aren't supported:
+        * `agent.run_as_user`
+          * Use {option}`user` instead.
+
+        {option}`configurationFile` takes precedence over {option}`configuration`.
+
+        Note: Restricted evaluation blocks access to paths outside the Nix store.
+        This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
+        As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
+        `systemctl restart amazon-cloudwatch-agent.service` must be used instead.
+      '';
+      example = "/etc/amazon-cloudwatch-agent/amazon-cloudwatch-agent.json";
+    };
+    configuration = lib.mkOption {
+      type = jsonFormat.type;
+      default = { };
+      description = ''
+        See {option}`configurationFile`.
+
+        {option}`configurationFile` takes precedence over {option}`configuration`.
+      '';
+      # Subset of "CloudWatch agent configuration file: Complete examples" and "CloudWatch agent configuration file: Traces section" in the description link.
+      #
+      # Log file path changed from "/opt/aws/amazon-cloudwatch-agent/logs" to "/var/log/amazon-cloudwatch-agent" to follow the FHS.
+      example = {
+        agent = {
+          metrics_collection_interval = 10;
+          logfile = "/var/log/amazon-cloudwatch-agent/amazon-cloudwatch-agent.log";
+        };
+        metrics = {
+          namespace = "MyCustomNamespace";
+          metrics_collected = {
+            cpu = {
+              resource = [ "*" ];
+              measurement = [
+                {
+                  name = "cpu_usage_idle";
+                  rename = "CPU_USAGE_IDLE";
+                  unit = "Percent";
+                }
+                {
+                  name = "cpu_usage_nice";
+                  unit = "Percent";
+                }
+                "cpu_usage_guest"
+              ];
+              totalcpu = false;
+              metrics_collection_interval = 10;
+              append_dimensions = {
+                customized_dimension_key_1 = "customized_dimension_value_1";
+                customized_dimension_key_2 = "customized_dimension_value_2";
+              };
+            };
+          };
+        };
+        logs = {
+          logs_collected = {
+            files = {
+              collect_list = [
+                {
+                  file_path = "/var/log/amazon-cloudwatch-agent/amazon-cloudwatch-agent.log";
+                  log_group_name = "amazon-cloudwatch-agent.log";
+                  log_stream_name = "{instance_id}";
+                  timezone = "UTC";
+                }
+              ];
+            };
+          };
+          log_stream_name = "log_stream_name";
+          force_flush_interval = 15;
+        };
+        traces = {
+          traces_collected = {
+            xray = { };
+            oltp = { };
+          };
+        };
+      };
+    };
+    # Replaces "agent.run_as_user" from the configuration file.
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "root";
+      description = ''
+        The user that runs the Amazon CloudWatch Agent.
+      '';
+      example = "amazon-cloudwatch-agent";
+    };
+    mode = lib.mkOption {
+      type = lib.types.str;
+      default = "auto";
+      description = ''
+        Amazon CloudWatch Agent mode. Indicates whether the agent is running in EC2 ("ec2"), on-premises ("onPremise"),
+        or if it should guess based on metadata endpoints like IMDS or the ECS task metadata endpoint ("auto").
+      '';
+      example = "onPremise";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # See https://github.com/aws/amazon-cloudwatch-agent/blob/v1.300049.1/packaging/dependencies/amazon-cloudwatch-agent.service.
+    systemd.services.amazon-cloudwatch-agent = {
+      description = "Amazon CloudWatch Agent";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "simple";
+        # "start-amazon-cloudwatch-agent" assumes the package is installed at "/opt/aws/amazon-cloudwatch-agent" so we can't use it.
+        #
+        # See https://github.com/aws/amazon-cloudwatch-agent/issues/1319.
+        #
+        # This program:
+        # 1. Switches to a non-root user if configured.
+        # 2. Runs "config-translator" to translate the input JSON configuration files into separate TOML (for CloudWatch Logs + Metrics),
+        #    YAML (for X-Ray + OpenTelemetry), and JSON (for environment variables) configuration files.
+        # 3. Runs "amazon-cloudwatch-agent" with the paths to these generated files.
+        #
+        # Re-implementing with systemd options.
+        User = cfg.user;
+        RuntimeDirectory = "amazon-cloudwatch-agent";
+        LogsDirectory = "amazon-cloudwatch-agent";
+        ExecStartPre = builtins.concatStringsSep " " [
+          "${cfg.package}/bin/config-translator"
+          "-config ${cfg.commonConfigurationFile}"
+          "-input ${cfg.configurationFile}"
+          "-input-dir ${configurationDirectory}"
+          "-mode ${cfg.mode}"
+          "-output \${RUNTIME_DIRECTORY}/amazon-cloudwatch-agent.toml"
+        ];
+        ExecStart = builtins.concatStringsSep " " [
+          "${cfg.package}/bin/amazon-cloudwatch-agent"
+          "-config \${RUNTIME_DIRECTORY}/amazon-cloudwatch-agent.toml"
+          "-envconfig \${RUNTIME_DIRECTORY}/env-config.json"
+          "-otelconfig \${RUNTIME_DIRECTORY}/amazon-cloudwatch-agent.yaml"
+          "-pidfile \${RUNTIME_DIRECTORY}/amazon-cloudwatch-agent.pid"
+        ];
+        KillMode = "process";
+        Restart = "on-failure";
+        RestartSec = 60;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/apcupsd.nix

@@ -0,0 +1,227 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.apcupsd;
+
+  configFile = pkgs.writeText "apcupsd.conf" ''
+    ## apcupsd.conf v1.1 ##
+    # apcupsd complains if the first line is not like above.
+    ${cfg.configText}
+    SCRIPTDIR ${toString scriptDir}
+  '';
+
+  # List of events from "man apccontrol"
+  eventList = [
+    "annoyme"
+    "battattach"
+    "battdetach"
+    "changeme"
+    "commfailure"
+    "commok"
+    "doreboot"
+    "doshutdown"
+    "emergency"
+    "failing"
+    "killpower"
+    "loadlimit"
+    "mainsback"
+    "onbattery"
+    "offbattery"
+    "powerout"
+    "remotedown"
+    "runlimit"
+    "timeout"
+    "startselftest"
+    "endselftest"
+  ];
+
+  shellCmdsForEventScript = eventname: commands: ''
+    echo "#!${pkgs.runtimeShell}" > "$out/${eventname}"
+    echo '${commands}' >> "$out/${eventname}"
+    chmod a+x "$out/${eventname}"
+  '';
+
+  eventToShellCmds =
+    event:
+    if builtins.hasAttr event cfg.hooks then
+      (shellCmdsForEventScript event (builtins.getAttr event cfg.hooks))
+    else
+      "";
+
+  scriptDir = pkgs.runCommand "apcupsd-scriptdir" { preferLocalBuild = true; } (
+    ''
+      mkdir "$out"
+      # Copy SCRIPTDIR from apcupsd package
+      cp -r ${pkgs.apcupsd}/etc/apcupsd/* "$out"/
+      # Make the files writeable (nix will unset the write bits afterwards)
+      chmod u+w "$out"/*
+      # Remove the sample event notification scripts, because they don't work
+      # anyways (they try to send mail to "root" with the "mail" command)
+      (cd "$out" && rm changeme commok commfailure onbattery offbattery)
+      # Remove the sample apcupsd.conf file (we're generating our own)
+      rm "$out/apcupsd.conf"
+      # Set the SCRIPTDIR= line in apccontrol to the dir we're creating now
+      sed -i -e "s|^SCRIPTDIR=.*|SCRIPTDIR=$out|" "$out/apccontrol"
+    ''
+    + lib.concatStringsSep "\n" (map eventToShellCmds eventList)
+
+  );
+
+  # Ensure the CLI uses our generated configFile
+  wrappedBinaries =
+    pkgs.runCommand "apcupsd-wrapped-binaries"
+      {
+        preferLocalBuild = true;
+        nativeBuildInputs = [ pkgs.makeWrapper ];
+      }
+      ''
+        for p in "${lib.getBin pkgs.apcupsd}/bin/"*; do
+            bname=$(basename "$p")
+            makeWrapper "$p" "$out/bin/$bname" --add-flags "-f ${configFile}"
+        done
+      '';
+
+  apcupsdWrapped = pkgs.symlinkJoin {
+    name = "apcupsd-wrapped";
+    # Put wrappers first so they "win"
+    paths = [
+      wrappedBinaries
+      pkgs.apcupsd
+    ];
+  };
+in
+
+{
+
+  ###### interface
+
+  options = {
+
+    services.apcupsd = {
+
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Whether to enable the APC UPS daemon. apcupsd monitors your UPS and
+          permits orderly shutdown of your computer in the event of a power
+          failure. User manual: http://www.apcupsd.com/manual/manual.html.
+          Note that apcupsd runs as root (to allow shutdown of computer).
+          You can check the status of your UPS with the "apcaccess" command.
+        '';
+      };
+
+      configText = lib.mkOption {
+        default = ''
+          UPSTYPE usb
+          NISIP 127.0.0.1
+          BATTERYLEVEL 50
+          MINUTES 5
+        '';
+        type = lib.types.lines;
+        description = ''
+          Contents of the runtime configuration file, apcupsd.conf. The default
+          settings makes apcupsd autodetect USB UPSes, limit network access to
+          localhost and shutdown the system when the battery level is below 50
+          percent, or when the UPS has calculated that it has 5 minutes or less
+          of remaining power-on time. See man apcupsd.conf for details.
+        '';
+      };
+
+      hooks = lib.mkOption {
+        default = { };
+        example = {
+          doshutdown = "# shell commands to notify that the computer is shutting down";
+        };
+        type = lib.types.attrsOf lib.types.lines;
+        description = ''
+          Each attribute in this option names an apcupsd event and the string
+          value it contains will be executed in a shell, in response to that
+          event (prior to the default action). See "man apccontrol" for the
+          list of events and what they represent.
+
+          A hook script can stop apccontrol from doing its default action by
+          exiting with value 99. Do not do this unless you know what you're
+          doing.
+        '';
+      };
+
+    };
+
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion =
+          let
+            hooknames = builtins.attrNames cfg.hooks;
+          in
+          lib.all (x: lib.elem x eventList) hooknames;
+        message = ''
+          One (or more) attribute names in services.apcupsd.hooks are invalid.
+          Current attribute names: ${toString (builtins.attrNames cfg.hooks)}
+          Valid attribute names  : ${toString eventList}
+        '';
+      }
+    ];
+
+    # Give users access to the "apcaccess" tool
+    environment.systemPackages = [ apcupsdWrapped ];
+
+    # NOTE 1: apcupsd runs as root because it needs permission to run
+    # "shutdown"
+    #
+    # NOTE 2: When apcupsd calls "wall", it prints an error because stdout is
+    # not connected to a tty (it is connected to the journal):
+    #   wall: cannot get tty name: Inappropriate ioctl for device
+    # The message still gets through.
+    systemd.services.apcupsd = {
+      description = "APC UPS Daemon";
+      wantedBy = [ "multi-user.target" ];
+      preStart = "mkdir -p /run/apcupsd/";
+      serviceConfig = {
+        ExecStart = "${pkgs.apcupsd}/bin/apcupsd -b -f ${configFile} -d1";
+        # TODO: When apcupsd has initiated a shutdown, systemd always ends up
+        # waiting for it to stop ("A stop job is running for UPS daemon"). This
+        # is weird, because in the journal one can clearly see that apcupsd has
+        # received the SIGTERM signal and has already quit (or so it seems).
+        # This reduces the wait time from 90 seconds (default) to just 5. Then
+        # systemd kills it with SIGKILL.
+        TimeoutStopSec = 5;
+      };
+      unitConfig.Documentation = "man:apcupsd(8)";
+    };
+
+    # A special service to tell the UPS to power down/hibernate just before the
+    # computer shuts down. (The UPS has a built in delay before it actually
+    # shuts off power.) Copied from here:
+    # http://forums.opensuse.org/english/get-technical-help-here/applications/479499-apcupsd-systemd-killpower-issues.html
+    systemd.services.apcupsd-killpower = {
+      description = "APC UPS Kill Power";
+      after = [ "shutdown.target" ]; # append umount.target?
+      before = [ "final.target" ];
+      wantedBy = [ "shutdown.target" ];
+      unitConfig = {
+        ConditionPathExists = "/run/apcupsd/powerfail";
+        DefaultDependencies = "no";
+      };
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.apcupsd}/bin/apcupsd --killpower -f ${configFile}";
+        TimeoutSec = "infinity";
+        StandardOutput = "tty";
+        RemainAfterExit = "yes";
+      };
+    };
+
+  };
+
+}

nixos/modules/services/monitoring/arbtt.nix

@@ -0,0 +1,52 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.arbtt;
+in
+{
+  options = {
+    services.arbtt = {
+      enable = lib.mkEnableOption "Arbtt statistics capture service";
+
+      package = lib.mkPackageOption pkgs [ "haskellPackages" "arbtt" ] { };
+
+      logFile = lib.mkOption {
+        type = lib.types.str;
+        default = "%h/.arbtt/capture.log";
+        example = "/home/username/.arbtt-capture.log";
+        description = ''
+          The log file for captured samples.
+        '';
+      };
+
+      sampleRate = lib.mkOption {
+        type = lib.types.int;
+        default = 60;
+        example = 120;
+        description = ''
+          The sampling interval in seconds.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.user.services.arbtt = {
+      description = "arbtt statistics capture service";
+      wantedBy = [ "graphical-session.target" ];
+      partOf = [ "graphical-session.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        ExecStart = "${cfg.package}/bin/arbtt-capture --logfile=${cfg.logFile} --sample-rate=${toString cfg.sampleRate}";
+        Restart = "always";
+      };
+    };
+  };
+
+  meta.maintainers = [ ];
+}

nixos/modules/services/monitoring/below.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.below;
+  cfgContents = lib.concatStringsSep "\n" (
+    lib.mapAttrsToList (n: v: ''${n} = "${v}"'') (
+      lib.filterAttrs (_k: v: v != null) {
+        log_dir = cfg.dirs.log;
+        store_dir = cfg.dirs.store;
+        cgroup_filter_out = cfg.cgroupFilterOut;
+      }
+    )
+  );
+
+  mkDisableOption =
+    n:
+    lib.mkOption {
+      type = lib.types.bool;
+      default = true;
+      description = "Whether to enable ${n}.";
+    };
+  optionalType =
+    ty: x:
+    lib.mkOption (
+      x
+      // {
+        description = x.description;
+        type = (lib.types.nullOr ty);
+        default = null;
+      }
+    );
+  optionalPath = optionalType lib.types.path;
+  optionalStr = optionalType lib.types.str;
+  optionalInt = optionalType lib.types.int;
+in
+{
+  options = {
+    services.below = {
+      enable = lib.mkEnableOption "'below' resource monitor";
+
+      cgroupFilterOut = optionalStr {
+        description = "A regexp matching the full paths of cgroups whose data shouldn't be collected";
+        example = "user.slice.*";
+      };
+      collect = {
+        diskStats = mkDisableOption "dist_stat collection";
+        ioStats = lib.mkEnableOption "io.stat collection for cgroups";
+        exitStats = mkDisableOption "eBPF-based exitstats";
+      };
+      compression.enable = lib.mkEnableOption "data compression";
+      retention = {
+        size = optionalInt {
+          description = ''
+            Size limit for below's data, in bytes. Data is deleted oldest-first, in 24h 'shards'.
+
+            ::: {.note}
+            The size limit may be exceeded by at most the size of the active shard, as:
+            - the active shard cannot be deleted;
+            - the size limit is only enforced when a new shard is created.
+            :::
+          '';
+        };
+        time = optionalInt {
+          description = ''
+            Retention time, in seconds.
+
+            ::: {.note}
+            As data is stored in 24 hour shards which are discarded as a whole,
+            only data expired by 24h (or more) is guaranteed to be discarded.
+            :::
+
+            ::: {.note}
+            If `retention.size` is set, data may be discarded earlier than the specified time.
+            :::
+          '';
+        };
+      };
+      dirs = {
+        log = optionalPath { description = "Where to store below's logs"; };
+        store = optionalPath {
+          description = "Where to store below's data";
+          example = "/var/lib/below";
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.below ];
+    # /etc/below.conf is also referred to by the `below` CLI tool,
+    #  so this can't be a store-only file whose path is passed to the service
+    environment.etc."below/below.conf".text = cfgContents;
+
+    systemd = {
+      packages = [ pkgs.below ];
+      services.below = {
+        # Workaround for https://github.com/NixOS/nixpkgs/issues/81138
+        wantedBy = [ "multi-user.target" ];
+        restartTriggers = [ cfgContents ];
+
+        serviceConfig.ExecStart = [
+          ""
+          (
+            "${lib.getExe pkgs.below} record "
+            + (lib.concatStringsSep " " (
+              lib.optional (!cfg.collect.diskStats) "--disable-disk-stat"
+              ++ lib.optional cfg.collect.ioStats "--collect-io-stat"
+              ++ lib.optional (!cfg.collect.exitStats) "--disable-exitstats"
+              ++ lib.optional cfg.compression.enable "--compress"
+              ++
+
+                lib.optional (cfg.retention.size != null) "--store-size-limit ${toString cfg.retention.size}"
+              ++ lib.optional (cfg.retention.time != null) "--retain-for-s ${toString cfg.retention.time}"
+            ))
+          )
+        ];
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ nicoo ];
+}

nixos/modules/services/monitoring/bosun.nix

@@ -0,0 +1,155 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.bosun;
+
+  configFile = pkgs.writeText "bosun.conf" ''
+    ${lib.optionalString (cfg.opentsdbHost != null) "tsdbHost = ${cfg.opentsdbHost}"}
+    ${lib.optionalString (cfg.influxHost != null) "influxHost = ${cfg.influxHost}"}
+    httpListen = ${cfg.listenAddress}
+    stateFile = ${cfg.stateFile}
+    ledisDir = ${cfg.ledisDir}
+    checkFrequency = ${cfg.checkFrequency}
+
+    ${cfg.extraConfig}
+  '';
+
+in
+{
+
+  options = {
+
+    services.bosun = {
+
+      enable = lib.mkEnableOption "bosun";
+
+      package = lib.mkPackageOption pkgs "bosun" { };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = "bosun";
+        description = ''
+          User account under which bosun runs.
+        '';
+      };
+
+      group = lib.mkOption {
+        type = lib.types.str;
+        default = "bosun";
+        description = ''
+          Group account under which bosun runs.
+        '';
+      };
+
+      opentsdbHost = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = "localhost:4242";
+        description = ''
+          Host and port of the OpenTSDB database that stores bosun data.
+          To disable opentsdb you can pass null as parameter.
+        '';
+      };
+
+      influxHost = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        example = "localhost:8086";
+        description = ''
+          Host and port of the influxdb database.
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        type = lib.types.str;
+        default = ":8070";
+        description = ''
+          The host address and port that bosun's web interface will listen on.
+        '';
+      };
+
+      stateFile = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/bosun/bosun.state";
+        description = ''
+          Path to bosun's state file.
+        '';
+      };
+
+      ledisDir = lib.mkOption {
+        type = lib.types.path;
+        default = "/var/lib/bosun/ledis_data";
+        description = ''
+          Path to bosun's ledis data dir
+        '';
+      };
+
+      checkFrequency = lib.mkOption {
+        type = lib.types.str;
+        default = "5m";
+        description = ''
+          Bosun's check frequency
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = ''
+          Extra configuration options for Bosun. You should describe your
+          desired templates, alerts, macros, etc through this configuration
+          option.
+
+          A detailed description of the supported syntax can be found at-spi2-atk
+          <https://bosun.org/configuration.html>
+        '';
+      };
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.services.bosun = {
+      description = "bosun metrics collector (part of Bosun)";
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = ''
+        mkdir -p "$(dirname "${cfg.stateFile}")";
+        touch "${cfg.stateFile}"
+        touch "${cfg.stateFile}.tmp"
+
+        mkdir -p "${cfg.ledisDir}";
+
+        if [ "$(id -u)" = 0 ]; then
+          chown ${cfg.user}:${cfg.group} "${cfg.stateFile}"
+          chown ${cfg.user}:${cfg.group} "${cfg.stateFile}.tmp"
+          chown ${cfg.user}:${cfg.group} "${cfg.ledisDir}"
+        fi
+      '';
+
+      serviceConfig = {
+        PermissionsStartOnly = true;
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart = ''
+          ${cfg.package}/bin/bosun -c ${configFile}
+        '';
+      };
+    };
+
+    users.users.bosun = {
+      description = "bosun user";
+      group = "bosun";
+      uid = config.ids.uids.bosun;
+    };
+
+    users.groups.bosun.gid = config.ids.gids.bosun;
+
+  };
+
+}

nixos/modules/services/monitoring/cadvisor.nix

@@ -0,0 +1,150 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.cadvisor;
+
+in
+{
+  options = {
+    services.cadvisor = {
+      enable = lib.mkEnableOption "Cadvisor service";
+
+      listenAddress = lib.mkOption {
+        default = "127.0.0.1";
+        type = lib.types.str;
+        description = "Cadvisor listening host";
+      };
+
+      port = lib.mkOption {
+        default = 8080;
+        type = lib.types.port;
+        description = "Cadvisor listening port";
+      };
+
+      storageDriver = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = "influxdb";
+        description = "Cadvisor storage driver.";
+      };
+
+      storageDriverHost = lib.mkOption {
+        default = "localhost:8086";
+        type = lib.types.str;
+        description = "Cadvisor storage driver host.";
+      };
+
+      storageDriverDb = lib.mkOption {
+        default = "root";
+        type = lib.types.str;
+        description = "Cadvisord storage driver database name.";
+      };
+
+      storageDriverUser = lib.mkOption {
+        default = "root";
+        type = lib.types.str;
+        description = "Cadvisor storage driver username.";
+      };
+
+      storageDriverPassword = lib.mkOption {
+        default = "root";
+        type = lib.types.str;
+        description = ''
+          Cadvisor storage driver password.
+
+          Warning: this password is stored in the world-readable Nix store. It's
+          recommended to use the {option}`storageDriverPasswordFile` option
+          since that gives you control over the security of the password.
+          {option}`storageDriverPasswordFile` also takes precedence over {option}`storageDriverPassword`.
+        '';
+      };
+
+      storageDriverPasswordFile = lib.mkOption {
+        type = lib.types.str;
+        description = ''
+          File that contains the cadvisor storage driver password.
+
+          {option}`storageDriverPasswordFile` takes precedence over {option}`storageDriverPassword`
+
+          Warning: when {option}`storageDriverPassword` is non-empty this defaults to a file in the
+          world-readable Nix store that contains the value of {option}`storageDriverPassword`.
+
+          It's recommended to override this with a path not in the Nix store.
+          Tip: use [nixops key management](https://nixos.org/nixops/manual/#idm140737318306400)
+        '';
+      };
+
+      storageDriverSecure = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = "Cadvisor storage driver, enable secure communication.";
+      };
+
+      extraOptions = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = ''
+          Additional cadvisor options.
+
+          See <https://github.com/google/cadvisor/blob/master/docs/runtime_options.md> for available options.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    {
+      services.cadvisor.storageDriverPasswordFile = lib.mkIf (cfg.storageDriverPassword != "") (
+        lib.mkDefault (
+          toString (
+            pkgs.writeTextFile {
+              name = "cadvisor-storage-driver-password";
+              text = cfg.storageDriverPassword;
+            }
+          )
+        )
+      );
+    }
+
+    (lib.mkIf cfg.enable {
+      systemd.services.cadvisor = {
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "network.target"
+          "docker.service"
+          "influxdb.service"
+        ];
+
+        path = lib.optionals config.boot.zfs.enabled [ pkgs.zfs ];
+
+        postStart = lib.mkBefore ''
+          until ${pkgs.curl.bin}/bin/curl -s -o /dev/null 'http://${cfg.listenAddress}:${toString cfg.port}/containers/'; do
+            sleep 1;
+          done
+        '';
+
+        script = ''
+          exec ${pkgs.cadvisor}/bin/cadvisor \
+            -logtostderr=true \
+            -listen_ip="${cfg.listenAddress}" \
+            -port="${toString cfg.port}" \
+            ${lib.escapeShellArgs cfg.extraOptions} \
+            ${lib.optionalString (cfg.storageDriver != null) ''
+              -storage_driver "${cfg.storageDriver}" \
+              -storage_driver_host "${cfg.storageDriverHost}" \
+              -storage_driver_db "${cfg.storageDriverDb}" \
+              -storage_driver_user "${cfg.storageDriverUser}" \
+              -storage_driver_password "$(cat "${cfg.storageDriverPasswordFile}")" \
+              ${lib.optionalString cfg.storageDriverSecure "-storage_driver_secure"}
+            ''}
+        '';
+
+        serviceConfig.TimeoutStartSec = 300;
+      };
+    })
+  ];
+}

nixos/modules/services/monitoring/certspotter.md

@@ -0,0 +1,78 @@
+# Cert Spotter {#module-services-certspotter}
+
+Cert Spotter is a tool for monitoring [Certificate Transparency](https://en.wikipedia.org/wiki/Certificate_Transparency)
+logs.
+
+## Service Configuration {#modules-services-certspotter-service-configuration}
+
+A basic config that notifies you of all certificate changes for your
+domain would look as follows:
+
+```nix
+{
+  services.certspotter = {
+    enable = true;
+    # replace example.org with your domain name
+    watchlist = [ ".example.org" ];
+    emailRecipients = [ "webmaster@example.org" ];
+  };
+
+  # Configure an SMTP client
+  programs.msmtp.enable = true;
+  # Or you can use any other module that provides sendmail, like
+  # services.nullmailer, services.opensmtpd, services.postfix
+}
+```
+
+In this case, the leading dot in `".example.org"` means that Cert
+Spotter should monitor not only `example.org`, but also all of its
+subdomains.
+
+## Operation {#modules-services-certspotter-operation}
+
+**By default, NixOS configures Cert Spotter to skip all certificates
+issued before its first launch**, because checking the entire
+Certificate Transparency logs requires downloading tens of terabytes of
+data. If you want to check the *entire* logs for previously issued
+certificates, you have to set `services.certspotter.startAtEnd` to
+`false` and remove all previously saved log state in
+`/var/lib/certspotter/logs`. The downloaded logs aren't saved, so if you
+add a new domain to the watchlist and want Cert Spotter to go through
+the logs again, you will have to remove `/var/lib/certspotter/logs`
+again.
+
+After catching up with the logs, Cert Spotter will start monitoring live
+logs. As of October 2023, it uses around **20 Mbps** of traffic on
+average.
+
+## Hooks {#modules-services-certspotter-hooks}
+
+Cert Spotter supports running custom hooks instead of (or in addition
+to) sending emails. Hooks are shell scripts that will be passed certain
+environment variables.
+
+To see hook documentation, see Cert Spotter's man pages:
+
+```ShellSession
+nix-shell -p certspotter --run 'man 8 certspotter-script'
+```
+
+For example, you can remove `emailRecipients` and send email
+notifications manually using the following hook:
+
+```nix
+{
+  services.certspotter.hooks = [
+    (pkgs.writeShellScript "certspotter-hook" ''
+      function print_email() {
+        echo "Subject: [certspotter] $SUMMARY"
+        echo "Mime-Version: 1.0"
+        echo "Content-Type: text/plain; charset=US-ASCII"
+        echo
+        cat "$TEXT_FILENAME"
+      }
+      print_email | ${config.services.certspotter.sendmailPath} -i webmaster@example.org
+    '')
+  ];
+}
+```

nixos/modules/services/monitoring/certspotter.nix

@@ -0,0 +1,160 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.certspotter;
+
+  configDir = pkgs.linkFarm "certspotter-config" (
+    lib.toList {
+      name = "watchlist";
+      path = pkgs.writeText "certspotter-watchlist" (builtins.concatStringsSep "\n" cfg.watchlist);
+    }
+    ++ lib.optional (cfg.emailRecipients != [ ]) {
+      name = "email_recipients";
+      path = pkgs.writeText "certspotter-email_recipients" (
+        builtins.concatStringsSep "\n" cfg.emailRecipients
+      );
+    }
+    # always generate hooks dir when no emails are provided to allow running cert spotter with no hooks/emails
+    ++ lib.optional (cfg.emailRecipients == [ ] || cfg.hooks != [ ]) {
+      name = "hooks.d";
+      path = pkgs.linkFarm "certspotter-hooks" (
+        lib.imap1 (i: path: {
+          inherit path;
+          name = "hook${toString i}";
+        }) cfg.hooks
+      );
+    }
+  );
+in
+{
+  options.services.certspotter = {
+    enable = lib.mkEnableOption "Cert Spotter, a Certificate Transparency log monitor";
+
+    package = lib.mkPackageOption pkgs "certspotter" { };
+
+    startAtEnd = lib.mkOption {
+      type = lib.types.bool;
+      description = ''
+        Whether to skip certificates issued before the first launch of Cert Spotter.
+        Setting this to `false` will cause Cert Spotter to download tens of terabytes of data.
+      '';
+      default = true;
+    };
+
+    sendmailPath = lib.mkOption {
+      type = with lib.types; nullOr path;
+      description = ''
+        Path to the `sendmail` binary. By default, the local sendmail wrapper is used
+        (see {option}`services.mail.sendmailSetuidWrapper`}).
+      '';
+      example = lib.literalExpression ''"''${pkgs.system-sendmail}/bin/sendmail"'';
+    };
+
+    watchlist = lib.mkOption {
+      type = with lib.types; listOf str;
+      description = "Domain names to watch. To monitor a domain with all subdomains, prefix its name with `.` (e.g. `.example.org`).";
+      default = [ ];
+      example = [
+        ".example.org"
+        "another.example.com"
+      ];
+    };
+
+    emailRecipients = lib.mkOption {
+      type = with lib.types; listOf str;
+      description = "A list of email addresses to send certificate updates to.";
+      default = [ ];
+    };
+
+    hooks = lib.mkOption {
+      type = with lib.types; listOf path;
+      description = ''
+        Scripts to run upon the detection of a new certificate. See `man 8 certspotter-script` or
+        [the GitHub page](https://github.com/SSLMate/certspotter/blob/${
+          pkgs.certspotter.src.rev or "master"
+        }/man/certspotter-script.md)
+        for more info.
+      '';
+      default = [ ];
+      example = lib.literalExpression ''
+        [
+          (pkgs.writeShellScript "certspotter-hook" '''
+            echo "Event summary: $SUMMARY."
+          ''')
+        ]
+      '';
+    };
+
+    extraFlags = lib.mkOption {
+      type = with lib.types; listOf str;
+      description = "Extra command-line arguments to pass to Cert Spotter";
+      example = [ "-no_save" ];
+      default = [ ];
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = (cfg.emailRecipients != [ ]) -> (cfg.sendmailPath != null);
+        message = ''
+          You must configure the sendmail setuid wrapper (services.mail.sendmailSetuidWrapper)
+          or services.certspotter.sendmailPath
+        '';
+      }
+    ];
+
+    services.certspotter.sendmailPath =
+      let
+        inherit (config.security) wrapperDir;
+        inherit (config.services.mail) sendmailSetuidWrapper;
+      in
+      lib.mkMerge [
+        (lib.mkIf (sendmailSetuidWrapper != null) (
+          lib.mkOptionDefault "${wrapperDir}/${sendmailSetuidWrapper.program}"
+        ))
+        (lib.mkIf (sendmailSetuidWrapper == null) (lib.mkOptionDefault null))
+      ];
+
+    users.users.certspotter = {
+      description = "Cert Spotter user";
+      group = "certspotter";
+      home = "/var/lib/certspotter";
+      isSystemUser = true;
+    };
+    users.groups.certspotter = { };
+
+    systemd.services.certspotter = {
+      description = "Cert Spotter - Certificate Transparency Monitor";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      environment.CERTSPOTTER_CONFIG_DIR = configDir;
+      environment.SENDMAIL_PATH =
+        if cfg.sendmailPath != null then cfg.sendmailPath else "/run/current-system/sw/bin/false";
+      script = ''
+        export CERTSPOTTER_STATE_DIR="$STATE_DIRECTORY"
+        cd "$CERTSPOTTER_STATE_DIR"
+        ${lib.optionalString cfg.startAtEnd ''
+          if [[ ! -d logs ]]; then
+            # Don't download certificates issued before the first launch
+            exec ${cfg.package}/bin/certspotter -start_at_end ${lib.escapeShellArgs cfg.extraFlags}
+          fi
+        ''}
+        exec ${cfg.package}/bin/certspotter ${lib.escapeShellArgs cfg.extraFlags}
+      '';
+      serviceConfig = {
+        User = "certspotter";
+        Group = "certspotter";
+        StateDirectory = "certspotter";
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ chayleaf ];
+  meta.doc = ./certspotter.md;
+}

nixos/modules/services/monitoring/cockpit.nix

@@ -0,0 +1,140 @@
+{
+  pkgs,
+  config,
+  lib,
+  ...
+}:
+
+let
+  cfg = config.services.cockpit;
+  inherit (lib)
+    types
+    mkEnableOption
+    mkOption
+    mkIf
+    mkPackageOption
+    ;
+  settingsFormat = pkgs.formats.ini { };
+in
+{
+  options = {
+    services.cockpit = {
+      enable = mkEnableOption "Cockpit";
+
+      package = mkPackageOption pkgs "Cockpit" {
+        default = [ "cockpit" ];
+      };
+
+      allowed-origins = lib.mkOption {
+        type = types.listOf types.str;
+
+        default = [ ];
+
+        description = ''
+          List of allowed origins.
+
+          Maps to the WebService.Origins setting and allows merging from multiple modules.
+        '';
+      };
+
+      settings = lib.mkOption {
+        type = settingsFormat.type;
+
+        default = { };
+
+        description = ''
+          Settings for cockpit that will be saved in /etc/cockpit/cockpit.conf.
+
+          See the [documentation](https://cockpit-project.org/guide/latest/cockpit.conf.5.html), that is also available with `man cockpit.conf.5` for details.
+        '';
+      };
+
+      showBanner = mkOption {
+        description = "Whether to add the Cockpit banner to the issue and motd files.";
+        type = types.bool;
+        default = true;
+        example = false;
+      };
+
+      port = mkOption {
+        description = "Port where cockpit will listen.";
+        type = types.port;
+        default = 9090;
+      };
+
+      openFirewall = mkOption {
+        description = "Open port for cockpit.";
+        type = types.bool;
+        default = false;
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # expose cockpit-bridge system-wide
+    environment.systemPackages = [ cfg.package ];
+
+    # allow cockpit to find its plugins
+    environment.pathsToLink = [ "/share/cockpit" ];
+
+    environment.etc = {
+      # generate cockpit settings
+      "cockpit/cockpit.conf".source = settingsFormat.generate "cockpit.conf" cfg.settings;
+
+      # Add "Web console: ..." line to issue and MOTD
+      "issue.d/cockpit.issue" = {
+        enable = cfg.showBanner;
+        source = "/run/cockpit/issue";
+      };
+      "motd.d/cockpit" = {
+        enable = cfg.showBanner;
+        source = "/run/cockpit/issue";
+      };
+    };
+
+    security.pam.services.cockpit = {
+      startSession = true;
+    };
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
+
+    systemd.packages = [ cfg.package ];
+
+    systemd.sockets.cockpit = {
+      wantedBy = [ "multi-user.target" ];
+      listenStreams = [
+        "" # workaround so it doesn't listen on both ports caused by the runtime merging
+        (toString cfg.port)
+      ];
+    };
+
+    # Enable connecting to remote hosts from the login page
+    systemd.services = mkIf (cfg.settings ? LoginTo -> cfg.settings.LoginTo) {
+      "cockpit-wsinstance-http".path = [
+        config.programs.ssh.package
+        cfg.package
+      ];
+      "cockpit-wsinstance-https@".path = [
+        config.programs.ssh.package
+        cfg.package
+      ];
+    };
+
+    systemd.tmpfiles.rules = [
+      # From $out/lib/tmpfiles.d/cockpit-tmpfiles.conf
+      "C /run/cockpit/inactive.motd 0640 root root - ${cfg.package}/share/cockpit/motd/inactive.motd"
+      "f /run/cockpit/active.motd   0640 root root -"
+      "L+ /run/cockpit/motd - - - - inactive.motd"
+      "d /etc/cockpit/ws-certs.d 0600 root root 0"
+    ];
+
+    services.cockpit.allowed-origins = [
+      "https://localhost:${toString config.services.cockpit.port}"
+    ];
+
+    services.cockpit.settings.WebService.Origins =
+      builtins.concatStringsSep " " config.services.cockpit.allowed-origins;
+  };
+
+  meta.maintainers = pkgs.cockpit.meta.maintainers;
+}

nixos/modules/services/monitoring/collectd.nix

@@ -0,0 +1,168 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.collectd;
+
+  baseDirLine = ''BaseDir "${cfg.dataDir}"'';
+  unvalidated_conf = pkgs.writeText "collectd-unvalidated.conf" cfg.extraConfig;
+
+  conf =
+    if cfg.validateConfig then
+      pkgs.runCommand "collectd.conf" { } ''
+        echo testing ${unvalidated_conf}
+        cp ${unvalidated_conf} collectd.conf
+        # collectd -t fails if BaseDir does not exist.
+        substituteInPlace collectd.conf --replace ${lib.escapeShellArgs [ baseDirLine ]} 'BaseDir "."'
+        ${package}/bin/collectd -t -C collectd.conf
+        cp ${unvalidated_conf} $out
+      ''
+    else
+      unvalidated_conf;
+
+  package = if cfg.buildMinimalPackage then minimalPackage else cfg.package;
+
+  minimalPackage = cfg.package.override {
+    enabledPlugins = [ "syslog" ] ++ builtins.attrNames cfg.plugins;
+  };
+
+in
+{
+  options.services.collectd = with lib.types; {
+    enable = lib.mkEnableOption "collectd agent";
+
+    validateConfig = lib.mkOption {
+      default = true;
+      description = ''
+        Validate the syntax of collectd configuration file at build time.
+        Disable this if you use the Include directive on files unavailable in
+        the build sandbox, or when cross-compiling.
+      '';
+      type = types.bool;
+    };
+
+    package = lib.mkPackageOption pkgs "collectd" { };
+
+    buildMinimalPackage = lib.mkOption {
+      default = false;
+      description = ''
+        Build a minimal collectd package with only the configured `services.collectd.plugins`
+      '';
+      type = bool;
+    };
+
+    user = lib.mkOption {
+      default = "collectd";
+      description = ''
+        User under which to run collectd.
+      '';
+      type = nullOr str;
+    };
+
+    dataDir = lib.mkOption {
+      default = "/var/lib/collectd";
+      description = ''
+        Data directory for collectd agent.
+      '';
+      type = path;
+    };
+
+    autoLoadPlugin = lib.mkOption {
+      default = false;
+      description = ''
+        Enable plugin autoloading.
+      '';
+      type = bool;
+    };
+
+    include = lib.mkOption {
+      default = [ ];
+      description = ''
+        Additional paths to load config from.
+      '';
+      type = listOf str;
+    };
+
+    plugins = lib.mkOption {
+      default = { };
+      example = {
+        cpu = "";
+        memory = "";
+        network = "Server 192.168.1.1 25826";
+      };
+      description = ''
+        Attribute set of plugin names to plugin config segments
+      '';
+      type = attrsOf lines;
+    };
+
+    extraConfig = lib.mkOption {
+      default = "";
+      description = ''
+        Extra configuration for collectd. Use mkBefore to add lines before the
+        default config, and mkAfter to add them below.
+      '';
+      type = lines;
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    # 1200 is after the default (1000) but before mkAfter (1500).
+    services.collectd.extraConfig = lib.mkOrder 1200 ''
+      ${baseDirLine}
+      AutoLoadPlugin ${lib.boolToString cfg.autoLoadPlugin}
+      Hostname "${config.networking.hostName}"
+
+      LoadPlugin syslog
+      <Plugin "syslog">
+        LogLevel "info"
+        NotifyLevel "OKAY"
+      </Plugin>
+
+      ${lib.concatStrings (
+        lib.mapAttrsToList (plugin: pluginConfig: ''
+          LoadPlugin ${plugin}
+          <Plugin "${plugin}">
+          ${pluginConfig}
+          </Plugin>
+        '') cfg.plugins
+      )}
+
+      ${lib.concatMapStrings (f: ''
+        Include "${f}"
+      '') cfg.include}
+    '';
+
+    systemd.tmpfiles.rules = [
+      "d '${cfg.dataDir}' - ${cfg.user} - - -"
+    ];
+
+    systemd.services.collectd = {
+      description = "Collectd Monitoring Agent";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        ExecStart = "${package}/sbin/collectd -C ${conf} -f";
+        User = cfg.user;
+        Restart = "on-failure";
+        RestartSec = 3;
+      };
+    };
+
+    users.users = lib.optionalAttrs (cfg.user == "collectd") {
+      collectd = {
+        isSystemUser = true;
+        group = "collectd";
+      };
+    };
+
+    users.groups = lib.optionalAttrs (cfg.user == "collectd") {
+      collectd = { };
+    };
+  };
+}

nixos/modules/services/monitoring/das_watchdog.nix

@@ -0,0 +1,40 @@
+# A general watchdog for the linux operating system that should run in the
+# background at all times to ensure a realtime process won't hang the machine
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  inherit (pkgs) das_watchdog;
+
+in
+{
+  ###### interface
+
+  options = {
+    services.das_watchdog.enable = lib.mkEnableOption "realtime watchdog";
+  };
+
+  ###### implementation
+
+  config = lib.mkIf config.services.das_watchdog.enable {
+    environment.systemPackages = [ das_watchdog ];
+    systemd.services.das_watchdog = {
+      description = "Watchdog to ensure a realtime process won't hang the machine";
+      after = [
+        "multi-user.target"
+        "sound.target"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        User = "root";
+        Type = "simple";
+        ExecStart = "${das_watchdog}/bin/das_watchdog";
+        RemainAfterExit = true;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/datadog-agent.nix

@@ -0,0 +1,355 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.datadog-agent;
+
+  ddConf = {
+    skip_ssl_validation = false;
+    confd_path = "/etc/datadog-agent/conf.d";
+    additional_checksd = "/etc/datadog-agent/checks.d";
+    use_dogstatsd = true;
+  }
+  // lib.optionalAttrs (cfg.logLevel != null) { log_level = cfg.logLevel; }
+  // lib.optionalAttrs (cfg.hostname != null) { inherit (cfg) hostname; }
+  // lib.optionalAttrs (cfg.ddUrl != null) { dd_url = cfg.ddUrl; }
+  // lib.optionalAttrs (cfg.site != null) { site = cfg.site; }
+  // lib.optionalAttrs (cfg.tags != null) { tags = lib.concatStringsSep ", " cfg.tags; }
+  // lib.optionalAttrs (cfg.enableLiveProcessCollection) {
+    process_config = {
+      dd_agent_bin = "${datadogPkg}/bin/agent";
+      process_collection.enabled = "true";
+      container_collection.enabled = "true";
+    };
+  }
+  // lib.optionalAttrs (cfg.enableTraceAgent) {
+    apm_config = {
+      enabled = true;
+    };
+  }
+  // cfg.extraConfig;
+
+  # Generate Datadog configuration files for each configured checks.
+  # This works because check configurations have predictable paths,
+  # and because JSON is a valid subset of YAML.
+  makeCheckConfigs =
+    entries:
+    lib.mapAttrs' (name: conf: {
+      name = "datadog-agent/conf.d/${name}.d/conf.yaml";
+      value.source = pkgs.writeText "${name}-check-conf.yaml" (builtins.toJSON conf);
+    }) entries;
+
+  defaultChecks = {
+    disk = cfg.diskCheck;
+    network = cfg.networkCheck;
+  };
+
+  # Assemble all check configurations and the top-level agent
+  # configuration.
+  etcfiles =
+    with pkgs;
+    with builtins;
+    {
+      "datadog-agent/datadog.yaml" = {
+        source = writeText "datadog.yaml" (toJSON ddConf);
+      };
+    }
+    // makeCheckConfigs (cfg.checks // defaultChecks);
+
+  # Apply the configured extraIntegrations to the provided agent
+  # package. See the documentation of `dd-agent/integrations-core.nix`
+  # for detailed information on this.
+  datadogPkg = cfg.package.override {
+    pythonPackages = pkgs.datadog-integrations-core cfg.extraIntegrations;
+  };
+in
+{
+  options.services.datadog-agent = {
+    enable = lib.mkEnableOption "Datadog-agent v7 monitoring service";
+
+    package = lib.mkPackageOption pkgs "datadog-agent" {
+      extraDescription = ''
+        ::: {.note}
+        The provided package is expected to have an overridable `pythonPackages`-attribute
+        which configures the Python environment with the Datadog checks.
+        :::
+      '';
+    };
+
+    apiKeyFile = lib.mkOption {
+      description = ''
+        Path to a file containing the Datadog API key to associate the
+        agent with your account.
+      '';
+      example = "/run/keys/datadog_api_key";
+      type = lib.types.path;
+    };
+
+    ddUrl = lib.mkOption {
+      description = ''
+        Custom dd_url to configure the agent with. Useful if traffic to datadog
+        needs to go through a proxy.
+        Don't use this to point to another datadog site (EU) - use site instead.
+      '';
+      default = null;
+      example = "http://haproxy.example.com:3834";
+      type = lib.types.nullOr lib.types.str;
+    };
+
+    site = lib.mkOption {
+      description = ''
+        The datadog site to point the agent towards.
+        Set to datadoghq.eu to point it to their EU site.
+      '';
+      default = null;
+      example = "datadoghq.eu";
+      type = lib.types.nullOr lib.types.str;
+    };
+
+    tags = lib.mkOption {
+      description = "The tags to mark this Datadog agent";
+      example = [
+        "test"
+        "service"
+      ];
+      default = null;
+      type = lib.types.nullOr (lib.types.listOf lib.types.str);
+    };
+
+    hostname = lib.mkOption {
+      description = "The hostname to show in the Datadog dashboard (optional)";
+      default = null;
+      example = "mymachine.mydomain";
+      type = lib.types.nullOr lib.types.str;
+    };
+
+    logLevel = lib.mkOption {
+      description = "Logging verbosity.";
+      default = null;
+      type = lib.types.nullOr (
+        lib.types.enum [
+          "DEBUG"
+          "INFO"
+          "WARN"
+          "ERROR"
+        ]
+      );
+    };
+
+    extraIntegrations = lib.mkOption {
+      default = { };
+      type = lib.types.attrs;
+
+      description = ''
+        Extra integrations from the Datadog core-integrations
+        repository that should be built and included.
+
+        By default the included integrations are disk, mongo, network,
+        nginx and postgres.
+
+        To include additional integrations the name of the derivation
+        and a function to filter its dependencies from the Python
+        package set must be provided.
+      '';
+
+      example = lib.literalExpression ''
+        {
+          ntp = pythonPackages: [ pythonPackages.ntplib ];
+        }
+      '';
+    };
+
+    extraConfig = lib.mkOption {
+      default = { };
+      type = lib.types.attrs;
+      description = ''
+        Extra configuration options that will be merged into the
+        main config file {file}`datadog.yaml`.
+      '';
+    };
+
+    enableLiveProcessCollection = lib.mkOption {
+      description = ''
+        Whether to enable the live process collection agent.
+      '';
+      default = false;
+      type = lib.types.bool;
+    };
+
+    processAgentPackage = lib.mkOption {
+      default = pkgs.datadog-process-agent;
+      defaultText = lib.literalExpression "pkgs.datadog-process-agent";
+      description = ''
+        Which DataDog v7 agent package to use. Note that the provided
+        package is expected to have an overridable `pythonPackages`-attribute
+        which configures the Python environment with the Datadog
+        checks.
+      '';
+      type = lib.types.package;
+    };
+
+    enableTraceAgent = lib.mkOption {
+      description = ''
+        Whether to enable the trace agent.
+      '';
+      default = false;
+      type = lib.types.bool;
+    };
+
+    checks = lib.mkOption {
+      description = ''
+        Configuration for all Datadog checks. Keys of this attribute
+        set will be used as the name of the check to create the
+        appropriate configuration in `conf.d/$check.d/conf.yaml`.
+
+        The configuration is converted into JSON from the plain Nix
+        language configuration, meaning that you should write
+        configuration adhering to Datadog's documentation - but in Nix
+        language.
+
+        Refer to the implementation of this module (specifically the
+        definition of `defaultChecks`) for an example.
+
+        Note: The 'disk' and 'network' check are configured in
+        separate options because they exist by default. Attempting to
+        override their configuration here will have no effect.
+      '';
+
+      example = {
+        http_check = {
+          init_config = null; # sic!
+          instances = [
+            {
+              name = "some-service";
+              url = "http://localhost:1337/healthz";
+              tags = [ "some-service" ];
+            }
+          ];
+        };
+      };
+
+      default = { };
+
+      # sic! The structure of the values is up to the check, so we can
+      # not usefully constrain the type further.
+      type = with lib.types; attrsOf attrs;
+    };
+
+    diskCheck = lib.mkOption {
+      description = "Disk check config";
+      type = lib.types.attrs;
+      default = {
+        init_config = { };
+        instances = [ { use_mount = "false"; } ];
+      };
+    };
+
+    networkCheck = lib.mkOption {
+      description = "Network check config";
+      type = lib.types.attrs;
+      default = {
+        init_config = { };
+        # Network check only supports one configured instance
+        instances = [
+          {
+            collect_connection_state = false;
+            excluded_interfaces = [
+              "lo"
+              "lo0"
+            ];
+          }
+        ];
+      };
+    };
+  };
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [
+      datadogPkg
+      pkgs.sysstat
+      pkgs.procps
+      pkgs.iproute2
+    ];
+
+    users.users.datadog = {
+      description = "Datadog Agent User";
+      uid = config.ids.uids.datadog;
+      group = "datadog";
+      home = "/var/log/datadog/";
+      createHome = true;
+    };
+
+    users.groups.datadog.gid = config.ids.gids.datadog;
+
+    systemd.services =
+      let
+        makeService =
+          attrs:
+          lib.recursiveUpdate {
+            path = [
+              datadogPkg
+              pkgs.sysstat
+              pkgs.procps
+              pkgs.iproute2
+            ];
+            wantedBy = [ "multi-user.target" ];
+            serviceConfig = {
+              User = "datadog";
+              Group = "datadog";
+              Restart = "always";
+              RestartSec = 2;
+            };
+            restartTriggers = [ datadogPkg ] ++ map (x: x.source) (lib.attrValues etcfiles);
+          } attrs;
+      in
+      {
+        datadog-agent = makeService {
+          description = "Datadog agent monitor";
+          preStart = ''
+            chown -R datadog: /etc/datadog-agent
+            rm -f /etc/datadog-agent/auth_token
+          '';
+          script = ''
+            export DD_API_KEY=$(head -n 1 ${cfg.apiKeyFile})
+            exec ${datadogPkg}/bin/agent run -c /etc/datadog-agent/datadog.yaml
+          '';
+          serviceConfig.PermissionsStartOnly = true;
+        };
+
+        dd-jmxfetch = lib.mkIf (lib.hasAttr "jmx" cfg.checks) (makeService {
+          description = "Datadog JMX Fetcher";
+          path = [
+            datadogPkg
+            pkgs.python
+            pkgs.sysstat
+            pkgs.procps
+            pkgs.jdk
+          ];
+          serviceConfig.ExecStart = "${datadogPkg}/bin/dd-jmxfetch";
+        });
+
+        datadog-process-agent = lib.mkIf cfg.enableLiveProcessCollection (makeService {
+          description = "Datadog Live Process Agent";
+          path = [ ];
+          script = ''
+            export DD_API_KEY=$(head -n 1 ${cfg.apiKeyFile})
+            ${cfg.processAgentPackage}/bin/process-agent --config /etc/datadog-agent/datadog.yaml
+          '';
+        });
+
+        datadog-trace-agent = lib.mkIf cfg.enableTraceAgent (makeService {
+          description = "Datadog Trace Agent";
+          path = [ ];
+          script = ''
+            export DD_API_KEY=$(head -n 1 ${cfg.apiKeyFile})
+            ${datadogPkg}/bin/trace-agent --config /etc/datadog-agent/datadog.yaml
+          '';
+        });
+
+      };
+
+    environment.etc = etcfiles;
+  };
+}

nixos/modules/services/monitoring/do-agent.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.do-agent;
+
+in
+{
+  options.services.do-agent = {
+    enable = lib.mkEnableOption "do-agent, the DigitalOcean droplet metrics agent";
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.packages = [ pkgs.do-agent ];
+
+    systemd.services.do-agent = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = [
+          ""
+          "${pkgs.do-agent}/bin/do-agent --syslog"
+        ];
+        DynamicUser = true;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/fluent-bit.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+let
+  cfg = config.services.fluent-bit;
+
+  yamlFormat = pkgs.formats.yaml { };
+in
+{
+  options.services.fluent-bit = {
+    enable = lib.mkEnableOption "Fluent Bit";
+    package = lib.mkPackageOption pkgs "fluent-bit" { };
+    configurationFile = lib.mkOption {
+      type = lib.types.path;
+      default = yamlFormat.generate "fluent-bit.yaml" cfg.settings;
+      defaultText = lib.literalExpression ''yamlFormat.generate "fluent-bit.yaml" cfg.settings'';
+      description = ''
+        Fluent Bit configuration. See
+        <https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml>
+        for supported values.
+
+        {option}`configurationFile` takes precedence over {option}`settings`.
+
+        Note: Restricted evaluation blocks access to paths outside the Nix store.
+        This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
+        As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
+        `systemctl restart fluent-bit.service` must be used instead.
+      '';
+      example = "/etc/fluent-bit/fluent-bit.yaml";
+    };
+    settings = lib.mkOption {
+      type = yamlFormat.type;
+      default = { };
+      description = ''
+        See {option}`configurationFile`.
+
+        {option}`configurationFile` takes precedence over {option}`settings`.
+      '';
+      example = {
+        service = {
+          grace = 30;
+        };
+        pipeline = {
+          inputs = [
+            {
+              name = "systemd";
+              systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
+            }
+          ];
+          outputs = [
+            {
+              name = "file";
+              path = "/var/log/fluent-bit";
+              file = "fluent-bit.out";
+            }
+          ];
+        };
+      };
+    };
+    # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section.
+    graceLimit = lib.mkOption {
+      type = lib.types.nullOr (
+        lib.types.oneOf [
+          lib.types.ints.positive
+          lib.types.str
+        ]
+      );
+      default = null;
+      description = ''
+        The grace time limit. Sets the systemd unit's `TimeoutStopSec`.
+
+        The `service.grace` option in the Fluent Bit configuration should be ≤ this option.
+      '';
+      example = 30;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # See https://github.com/fluent/fluent-bit/blob/v3.2.6/init/systemd.in.
+    systemd.services.fluent-bit = {
+      description = "Fluent Bit";
+      after = [ "network.target" ];
+      requires = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        DynamicUser = true;
+        # See https://nixos.org/manual/nixos/stable#sec-logging.
+        SupplementaryGroups = "systemd-journal";
+        ExecStart = utils.escapeSystemdExecArgs [
+          (lib.getExe cfg.package)
+          "--config"
+          cfg.configurationFile
+        ];
+        Restart = "always";
+        TimeoutStopSec = lib.mkIf (cfg.graceLimit != null) cfg.graceLimit;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/fusion-inventory.nix

@@ -0,0 +1,65 @@
+# Fusion Inventory daemon.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.fusionInventory;
+
+  configFile = pkgs.writeText "fusion_inventory.conf" ''
+    server = ${lib.concatStringsSep ", " cfg.servers}
+
+    logger = stderr
+
+    ${cfg.extraConfig}
+  '';
+
+in
+{
+
+  ###### interface
+
+  options = {
+
+    services.fusionInventory = {
+
+      enable = lib.mkEnableOption "Fusion Inventory Agent";
+
+      servers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = ''
+          The urls of the OCS/GLPI servers to connect to.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Configuration that is injected verbatim into the configuration file.
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkIf cfg.enable {
+
+    users.users.fusion-inventory = {
+      description = "FusionInventory user";
+      isSystemUser = true;
+    };
+
+    systemd.services.fusion-inventory = {
+      description = "Fusion Inventory Agent";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        ExecStart = "${pkgs.fusionInventory}/bin/fusioninventory-agent --conf-file=${configFile} --daemon --no-fork";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/gatus.nix

@@ -0,0 +1,137 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+  cfg = config.services.gatus;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+  inherit (lib)
+    getExe
+    literalExpression
+    maintainers
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    ;
+
+  inherit (lib.types)
+    bool
+    int
+    nullOr
+    path
+    submodule
+    ;
+in
+{
+  options.services.gatus = {
+    enable = mkEnableOption "Gatus";
+
+    package = mkPackageOption pkgs "gatus" { };
+
+    configFile = mkOption {
+      type = path;
+      default = settingsFormat.generate "gatus.yaml" cfg.settings;
+      defaultText = literalExpression ''
+        let settingsFormat = pkgs.formats.yaml { }; in settingsFormat.generate "gatus.yaml" cfg.settings;
+      '';
+      description = ''
+        Path to the Gatus configuration file.
+        Overrides any configuration made using the `settings` option.
+      '';
+    };
+
+    environmentFile = mkOption {
+      type = nullOr path;
+      default = null;
+      description = ''
+        File to load as environment file.
+        Environmental variables from this file can be interpolated in the configuration file using `''${VARIABLE}`.
+        This is useful to avoid putting secrets into the nix store.
+      '';
+    };
+
+    settings = mkOption {
+      type = submodule {
+        freeformType = settingsFormat.type;
+        options = {
+          web.port = mkOption {
+            type = int;
+            default = 8080;
+            description = ''
+              The TCP port to serve the Gatus service at.
+            '';
+          };
+        };
+      };
+
+      default = { };
+
+      example = literalExpression ''
+        {
+          web.port = 8080;
+          endpoints = [{
+            name = "website";
+            url = "https://twin.sh/health";
+            interval = "5m";
+            conditions = [
+              "[STATUS] == 200"
+              "[BODY].status == UP"
+              "[RESPONSE_TIME] < 300"
+            ];
+          }];
+        }
+      '';
+
+      description = ''
+        Configuration for Gatus.
+        Supported options can be found at the [docs](https://gatus.io/docs).
+      '';
+    };
+
+    openFirewall = mkOption {
+      type = bool;
+      default = false;
+      description = ''
+        Whether to open the firewall for the Gatus web interface.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.gatus = {
+      description = "Automated developer-oriented status page";
+      after = [ "network-online.target" ];
+      requires = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        DynamicUser = true;
+        User = "gatus";
+        Group = "gatus";
+        Type = "simple";
+        Restart = "on-failure";
+        ExecStart = getExe cfg.package;
+        StateDirectory = "gatus";
+        SyslogIdentifier = "gatus";
+        EnvironmentFile = lib.optional (cfg.environmentFile != null) cfg.environmentFile;
+        # see https://github.com/prometheus-community/pro-bing#linux
+        AmbientCapabilities = "CAP_NET_RAW";
+        CapabilityBoundingSet = "CAP_NET_RAW";
+        NoNewPrivileges = true;
+      };
+
+      environment = {
+        GATUS_CONFIG_PATH = cfg.configFile;
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = lib.optionals cfg.openFirewall [ cfg.settings.web.port ];
+  };
+
+  meta.maintainers = with maintainers; [ pizzapim ];
+}

nixos/modules/services/monitoring/gitwatch.nix

@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  inherit (lib)
+    maintainers
+    mapAttrs'
+    mkEnableOption
+    mkOption
+    nameValuePair
+    optionalString
+    types
+    ;
+  mkSystemdService =
+    name: cfg:
+    nameValuePair "gitwatch-${name}" (
+      let
+        getvar = flag: var: optionalString (cfg."${var}" != null) "${flag} ${cfg."${var}"}";
+        branch = getvar "-b" "branch";
+        remote = getvar "-r" "remote";
+      in
+      rec {
+        inherit (cfg) enable;
+        after = [ "network-online.target" ];
+        wants = after;
+        wantedBy = [ "multi-user.target" ];
+        description = "gitwatch for ${name}";
+        path = with pkgs; [
+          gitwatch
+          git
+          openssh
+        ];
+        script = ''
+          if [ -n "${cfg.remote}" ] && ! [ -d "${cfg.path}" ]; then
+            git clone ${branch} "${cfg.remote}" "${cfg.path}"
+          fi
+          gitwatch ${remote} ${branch} ${cfg.path}
+        '';
+        serviceConfig.User = cfg.user;
+      }
+    );
+in
+{
+  options.services.gitwatch = mkOption {
+    description = ''
+      A set of git repositories to watch for. See
+      [gitwatch](https://github.com/gitwatch/gitwatch) for more.
+    '';
+    default = { };
+    example = {
+      my-repo = {
+        enable = true;
+        user = "user";
+        path = "/home/user/watched-project";
+        remote = "git@github.com:me/my-project.git";
+      };
+      disabled-repo = {
+        enable = false;
+        user = "user";
+        path = "/home/user/disabled-project";
+        remote = "git@github.com:me/my-old-project.git";
+        branch = "autobranch";
+      };
+    };
+    type =
+      with types;
+      attrsOf (submodule {
+        options = {
+          enable = mkEnableOption "watching for repo";
+          path = mkOption {
+            description = "The path to repo in local machine";
+            type = str;
+          };
+          user = mkOption {
+            description = "The name of services's user";
+            type = str;
+            default = "root";
+          };
+          remote = mkOption {
+            description = "Optional url of remote repository";
+            type = nullOr str;
+            default = null;
+          };
+          branch = mkOption {
+            description = "Optional branch in remote repository";
+            type = nullOr str;
+            default = null;
+          };
+        };
+      });
+  };
+  config.systemd.services = mapAttrs' mkSystemdService config.services.gitwatch;
+  meta.maintainers = with maintainers; [ shved ];
+}

nixos/modules/services/monitoring/glances.md

@@ -0,0 +1,20 @@
+# Glances {#module-serives-glances}
+
+Glances an Eye on your system. A top/htop alternative for GNU/Linux, BSD, Mac OS
+and Windows operating systems.
+
+Visit [the Glances project page](https://github.com/nicolargo/glances) to learn
+more about it.
+
+# Quickstart {#module-serives-glances-quickstart}
+
+Use the following configuration to start a public instance of Glances locally:
+
+```nix
+{
+  services.glances = {
+    enable = true;
+    openFirewall = true;
+  };
+}
+```

nixos/modules/services/monitoring/glances.nix

@@ -0,0 +1,111 @@
+{
+  pkgs,
+  config,
+  lib,
+  utils,
+  ...
+}:
+let
+  cfg = config.services.glances;
+
+  inherit (lib)
+    getExe
+    maintainers
+    mkEnableOption
+    mkOption
+    mkIf
+    mkPackageOption
+    ;
+
+  inherit (lib.types)
+    bool
+    listOf
+    port
+    str
+    ;
+
+  inherit (utils)
+    escapeSystemdExecArgs
+    ;
+
+in
+{
+  options.services.glances = {
+    enable = mkEnableOption "Glances";
+
+    package = mkPackageOption pkgs "glances" { };
+
+    port = mkOption {
+      description = "Port the server will isten on.";
+      type = port;
+      default = 61208;
+    };
+
+    openFirewall = mkOption {
+      description = "Open port in the firewall for glances.";
+      type = bool;
+      default = false;
+    };
+
+    extraArgs = mkOption {
+      type = listOf str;
+      default = [ "--webserver" ];
+      example = [
+        "--webserver"
+        "--disable-webui"
+      ];
+      description = ''
+        Extra command-line arguments to pass to glances.
+
+        See <https://glances.readthedocs.io/en/latest/cmds.html> for all available options.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services."glances" = {
+      description = "Glances";
+      documentation = [ "man:glances(1)" ];
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig = {
+        Type = "simple";
+        DynamicUser = true;
+        ExecStart = "${getExe cfg.package} --port ${toString cfg.port} ${escapeSystemdExecArgs cfg.extraArgs}";
+        Restart = "on-failure";
+
+        NoNewPrivileges = true;
+        ProtectSystem = "full";
+        ProtectHome = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+        MemoryDenyWriteExecute = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_NETLINK"
+          "AF_UNIX"
+        ];
+        LockPersonality = true;
+        RestrictRealtime = true;
+        ProtectClock = true;
+        ReadWritePaths = [ "/var/log" ];
+        CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+        AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+        SystemCallFilter = [ "@system-service" ];
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ cfg.port ];
+  };
+
+  meta.maintainers = with maintainers; [ claha ];
+}

nixos/modules/services/monitoring/glpi-agent.nix

@@ -0,0 +1,138 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.glpiAgent;
+
+  settingsType =
+    with lib.types;
+    attrsOf (oneOf [
+      bool
+      int
+      str
+      (listOf str)
+    ]);
+
+  formatValue =
+    v:
+    if lib.isBool v then
+      if v then "1" else "0"
+    else if lib.isList v then
+      lib.concatStringsSep "," v
+    else
+      toString v;
+
+  configContent = lib.concatStringsSep "\n" (
+    lib.mapAttrsToList (k: v: "${k} = ${formatValue v}") cfg.settings
+  );
+
+  configFile = pkgs.writeText "agent.cfg" configContent;
+
+in
+{
+  options = {
+    services.glpiAgent = {
+      enable = lib.mkEnableOption "GLPI Agent";
+
+      package = lib.mkPackageOption pkgs "glpi-agent" { };
+
+      settings = lib.mkOption {
+        type = settingsType;
+        default = { };
+        description = ''
+          GLPI Agent configuration options.
+          See <https://glpi-agent.readthedocs.io/en/latest/configuration.html> for all available options.
+
+          The 'server' option is mandatory and must point to your GLPI server.
+        '';
+        example = lib.literalExpression ''
+          {
+            server = [ "https://glpi.example.com/inventory" ];
+            delaytime = 3600;
+            tag = "production";
+            logger = [ "stderr" "file" ];
+            debug = 1;
+            "no-category" = [ "printer" "software" ];
+          }
+        '';
+      };
+
+      stateDir = lib.mkOption {
+        type = lib.types.str;
+        default = "/var/lib/glpi-agent";
+        description = "Directory where GLPI Agent stores its state.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.settings ? server;
+        message = "GLPI Agent requires a server to be configured in services.glpiAgent.settings.server";
+      }
+    ];
+
+    systemd.services.glpi-agent = {
+      description = "GLPI Agent";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      serviceConfig = {
+        ExecStart = lib.escapeShellArgs [
+          "${lib.getExe cfg.package}"
+          "--conf-file"
+          "${configFile}"
+          "--vardir"
+          "${cfg.stateDir}"
+          "--daemon"
+          "--no-fork"
+        ];
+
+        DynamicUser = true;
+        StateDirectory = "glpi-agent";
+        CapabilityBoundingSet = [ "CAP_SYS_ADMIN" ];
+        AmbientCapabilities = [ "CAP_SYS_ADMIN" ];
+
+        LimitCORE = 0;
+        LimitNOFILE = 65535;
+        LockPersonality = true;
+        MemorySwapMax = 0;
+        MemoryZSwapMax = 0;
+        PrivateTmp = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        Restart = "on-failure";
+        RestartSec = "10s";
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+          "AF_NETLINK"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "@resources"
+          "~@privileged"
+        ];
+        NoNewPrivileges = true;
+        UMask = "0077";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/goss.md

@@ -0,0 +1,44 @@
+# Goss {#module-services-goss}
+
+[goss](https://goss.rocks/) is a YAML based serverspec alternative tool
+for validating a server's configuration.
+
+## Basic Usage {#module-services-goss-basic-usage}
+
+A minimal configuration looks like this:
+
+```nix
+{
+  services.goss = {
+    enable = true;
+
+    environment = {
+      GOSS_FMT = "json";
+      GOSS_LOGLEVEL = "TRACE";
+    };
+
+    settings = {
+      addr."tcp://localhost:8080" = {
+        reachable = true;
+        local-address = "127.0.0.1";
+      };
+      command."check-goss-version" = {
+        exec = "${lib.getExe pkgs.goss} --version";
+        exit-status = 0;
+      };
+      dns.localhost.resolvable = true;
+      file."/nix" = {
+        filetype = "directory";
+        exists = true;
+      };
+      group.root.exists = true;
+      kernel-param."kernel.ostype".value = "Linux";
+      service.goss = {
+        enabled = true;
+        running = true;
+      };
+      user.root.exists = true;
+    };
+  };
+}
+```

nixos/modules/services/monitoring/goss.nix

@@ -0,0 +1,93 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.goss;
+
+  settingsFormat = pkgs.formats.yaml { };
+  configFile = settingsFormat.generate "goss.yaml" cfg.settings;
+
+in
+{
+  meta = {
+    doc = ./goss.md;
+    maintainers = [ lib.maintainers.anthonyroussel ];
+  };
+
+  options = {
+    services.goss = {
+      enable = lib.mkEnableOption "Goss daemon";
+
+      package = lib.mkPackageOption pkgs "goss" { };
+
+      environment = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        default = { };
+        example = {
+          GOSS_FMT = "json";
+          GOSS_LOGLEVEL = "FATAL";
+          GOSS_LISTEN = ":8080";
+        };
+        description = ''
+          Environment variables to set for the goss service.
+
+          See <https://github.com/goss-org/goss/blob/master/docs/manual.md>
+        '';
+      };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule { freeformType = settingsFormat.type; };
+        default = { };
+        example = {
+          addr."tcp://localhost:8080" = {
+            reachable = true;
+            local-address = "127.0.0.1";
+          };
+          service.goss = {
+            enabled = true;
+            running = true;
+          };
+        };
+        description = ''
+          The global options in `config` file in yaml format.
+
+          Refer to <https://github.com/goss-org/goss/blob/master/docs/goss-json-schema.yaml> for schema.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    systemd.services.goss = {
+      description = "Goss - Quick and Easy server validation";
+      unitConfig.Documentation = "https://github.com/goss-org/goss/blob/master/docs/manual.md";
+
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      wants = [ "network-online.target" ];
+
+      environment = {
+        GOSS_FILE = configFile;
+      }
+      // cfg.environment;
+
+      reloadTriggers = [ configFile ];
+
+      serviceConfig = {
+        DynamicUser = true;
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        ExecStart = "${cfg.package}/bin/goss serve";
+        Group = "goss";
+        Restart = "on-failure";
+        RestartSec = 5;
+        User = "goss";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/grafana-image-renderer.nix

@@ -0,0 +1,161 @@
+{
+  lib,
+  pkgs,
+  config,
+  ...
+}:
+let
+  cfg = config.services.grafana-image-renderer;
+
+  format = pkgs.formats.json { };
+
+  configFile = format.generate "grafana-image-renderer-config.json" cfg.settings;
+in
+{
+  options.services.grafana-image-renderer = {
+    enable = lib.mkEnableOption "grafana-image-renderer";
+
+    chromium = lib.mkOption {
+      type = lib.types.package;
+      description = ''
+        The chromium to use for image rendering.
+      '';
+    };
+
+    verbose = lib.mkEnableOption "verbosity for the service";
+
+    provisionGrafana = lib.mkEnableOption "Grafana configuration for grafana-image-renderer";
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = format.type;
+
+        options = {
+          service = {
+            port = lib.mkOption {
+              type = lib.types.port;
+              default = 8081;
+              description = ''
+                The TCP port to use for the rendering server.
+              '';
+            };
+            logging.level = lib.mkOption {
+              type = lib.types.enum [
+                "error"
+                "warning"
+                "info"
+                "debug"
+              ];
+              default = "info";
+              description = ''
+                The log-level of the {file}`grafana-image-renderer.service`-unit.
+              '';
+            };
+          };
+          rendering = {
+            width = lib.mkOption {
+              default = 1000;
+              type = lib.types.ints.positive;
+              description = ''
+                Width of the PNG used to display the alerting graph.
+              '';
+            };
+            height = lib.mkOption {
+              default = 500;
+              type = lib.types.ints.positive;
+              description = ''
+                Height of the PNG used to display the alerting graph.
+              '';
+            };
+            mode = lib.mkOption {
+              default = "default";
+              type = lib.types.enum [
+                "default"
+                "reusable"
+                "clustered"
+              ];
+              description = ''
+                Rendering mode of `grafana-image-renderer`:
+
+                - `default:` Creates on browser-instance
+                  per rendering request.
+                - `reusable:` One browser instance
+                  will be started and reused for each rendering request.
+                - `clustered:` allows to precisely
+                  configure how many browser-instances are supposed to be used. The values
+                  for that mode can be declared in `rendering.clustering`.
+              '';
+            };
+            args = lib.mkOption {
+              type = lib.types.listOf lib.types.str;
+              default = [ "--no-sandbox" ];
+              description = ''
+                List of CLI flags passed to `chromium`.
+              '';
+            };
+          };
+        };
+      };
+
+      default = { };
+
+      description = ''
+        Configuration attributes for `grafana-image-renderer`.
+
+        See <https://github.com/grafana/grafana-image-renderer/blob/ce1f81438e5f69c7fd7c73ce08bab624c4c92e25/default.json>
+        for supported values.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.provisionGrafana -> config.services.grafana.enable;
+        message = ''
+          To provision a Grafana instance to use grafana-image-renderer,
+          `services.grafana.enable` must be set to `true`!
+        '';
+      }
+    ];
+
+    services.grafana.settings.rendering = lib.mkIf cfg.provisionGrafana {
+      server_url = "http://localhost:${toString cfg.settings.service.port}/render";
+      callback_url = "http://${config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
+    };
+
+    services.grafana-image-renderer.chromium = lib.mkDefault pkgs.chromium;
+
+    services.grafana-image-renderer.settings = {
+      rendering = lib.mapAttrs (lib.const lib.mkDefault) {
+        chromeBin = "${cfg.chromium}/bin/chromium";
+        verboseLogging = cfg.verbose;
+        timezone = config.time.timeZone;
+      };
+
+      service = {
+        logging.level = lib.mkIf cfg.verbose (lib.mkDefault "debug");
+        metrics.enabled = lib.mkDefault false;
+      };
+    };
+
+    systemd.services.grafana-image-renderer = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      description = "Grafana backend plugin that handles rendering of panels & dashboards to PNGs using headless browser (Chromium/Chrome)";
+
+      environment = {
+        PUPPETEER_SKIP_CHROMIUM_DOWNLOAD = "true";
+      };
+
+      serviceConfig = {
+        DynamicUser = true;
+        PrivateTmp = true;
+        ExecStart = "${pkgs.grafana-image-renderer}/bin/grafana-image-renderer server --config=${configFile}";
+        Restart = "always";
+      };
+    };
+  };
+
+  meta.maintainers = with lib.maintainers; [ ma27 ];
+}

nixos/modules/services/monitoring/grafana-reporter.nix

@@ -0,0 +1,75 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.grafana_reporter;
+
+in
+{
+  options.services.grafana_reporter = {
+    enable = lib.mkEnableOption "grafana_reporter";
+
+    grafana = {
+      protocol = lib.mkOption {
+        description = "Grafana protocol.";
+        default = "http";
+        type = lib.types.enum [
+          "http"
+          "https"
+        ];
+      };
+      addr = lib.mkOption {
+        description = "Grafana address.";
+        default = "127.0.0.1";
+        type = lib.types.str;
+      };
+      port = lib.mkOption {
+        description = "Grafana port.";
+        default = 3000;
+        type = lib.types.port;
+      };
+
+    };
+    addr = lib.mkOption {
+      description = "Listening address.";
+      default = "127.0.0.1";
+      type = lib.types.str;
+    };
+
+    port = lib.mkOption {
+      description = "Listening port.";
+      default = 8686;
+      type = lib.types.port;
+    };
+
+    templateDir = lib.mkOption {
+      description = "Optional template directory to use custom tex templates";
+      default = pkgs.grafana_reporter;
+      defaultText = lib.literalExpression "pkgs.grafana_reporter";
+      type = lib.types.either lib.types.str lib.types.path;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.grafana_reporter = {
+      description = "Grafana Reporter Service Daemon";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      serviceConfig =
+        let
+          args = lib.concatStringsSep " " [
+            "-proto ${cfg.grafana.protocol}://"
+            "-ip ${cfg.grafana.addr}:${toString cfg.grafana.port}"
+            "-port :${toString cfg.port}"
+            "-templates ${cfg.templateDir}"
+          ];
+        in
+        {
+          ExecStart = "${pkgs.grafana-reporter}/bin/grafana-reporter ${args}";
+        };
+    };
+  };
+}

nixos/modules/services/monitoring/grafana-to-ntfy.nix

@@ -0,0 +1,121 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.grafana-to-ntfy;
+in
+{
+  options = {
+    services.grafana-to-ntfy = {
+      enable = lib.mkEnableOption "Grafana-to-ntfy (ntfy.sh) alerts channel";
+
+      package = lib.mkPackageOption pkgs "grafana-to-ntfy" { };
+
+      settings = {
+        ntfyUrl = lib.mkOption {
+          type = lib.types.str;
+          description = "The URL to the ntfy-sh topic.";
+          example = "https://push.example.com/grafana";
+        };
+
+        ntfyBAuthUser = lib.mkOption {
+          type = lib.types.nullOr lib.types.str;
+          description = ''
+            The ntfy-sh user to use for authenticating with the ntfy-sh instance.
+            Setting this option is required when using a ntfy-sh instance with access control enabled.
+          '';
+          default = null;
+          example = "grafana";
+        };
+
+        ntfyBAuthPass = lib.mkOption {
+          type = lib.types.path;
+          description = ''
+            The path to the password for the specified ntfy-sh user.
+            Setting this option is required when using a ntfy-sh instance with access control enabled.
+          '';
+          default = null;
+        };
+
+        bauthUser = lib.mkOption {
+          type = lib.types.str;
+          description = ''
+            The user that you will authenticate with in the Grafana webhook settings.
+            You can set this to whatever you like, as this is not the same as the ntfy-sh user.
+          '';
+          default = "admin";
+        };
+
+        bauthPass = lib.mkOption {
+          type = lib.types.path;
+          description = "The path to the password you will use in the Grafana webhook settings.";
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.grafana-to-ntfy = {
+      wantedBy = [ "multi-user.target" ];
+
+      script = ''
+        export BAUTH_PASS=$(${lib.getExe' config.systemd.package "systemd-creds"} cat BAUTH_PASS_FILE)
+        ${lib.optionalString (cfg.settings.ntfyBAuthPass != null) ''
+          export NTFY_BAUTH_PASS=$(${lib.getExe' config.systemd.package "systemd-creds"} cat NTFY_BAUTH_PASS_FILE)
+        ''}
+        exec ${lib.getExe cfg.package}
+      '';
+
+      environment = {
+        NTFY_URL = cfg.settings.ntfyUrl;
+        BAUTH_USER = cfg.settings.bauthUser;
+      }
+      // lib.optionalAttrs (cfg.settings.ntfyBAuthUser != null) {
+        NTFY_BAUTH_USER = cfg.settings.ntfyBAuthUser;
+      };
+
+      serviceConfig = {
+        LoadCredential = [
+          "BAUTH_PASS_FILE:${cfg.settings.bauthPass}"
+        ]
+        ++ lib.optional (
+          cfg.settings.ntfyBAuthPass != null
+        ) "NTFY_BAUTH_PASS_FILE:${cfg.settings.ntfyBAuthPass}";
+
+        DynamicUser = true;
+        CapabilityBoundingSet = [ "" ];
+        DeviceAllow = "";
+        LockPersonality = true;
+        PrivateDevices = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        MemoryDenyWriteExecute = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+        UMask = "0077";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/graphite.nix

@@ -0,0 +1,471 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.graphite;
+  opt = options.services.graphite;
+  writeTextOrNull = f: t: lib.mapNullable (pkgs.writeTextDir f) t;
+
+  dataDir = cfg.dataDir;
+  staticDir = cfg.dataDir + "/static";
+
+  graphiteLocalSettingsDir =
+    pkgs.runCommand "graphite_local_settings"
+      {
+        inherit graphiteLocalSettings;
+        preferLocalBuild = true;
+      }
+      ''
+        mkdir -p $out
+        ln -s $graphiteLocalSettings $out/graphite_local_settings.py
+      '';
+
+  graphiteLocalSettings = pkgs.writeText "graphite_local_settings.py" (
+    "STATIC_ROOT = '${staticDir}'\n"
+    + lib.optionalString (config.time.timeZone != null) "TIME_ZONE = '${config.time.timeZone}'\n"
+    + cfg.web.extraConfig
+  );
+
+  seyrenConfig = {
+    SEYREN_URL = cfg.seyren.seyrenUrl;
+    MONGO_URL = cfg.seyren.mongoUrl;
+    GRAPHITE_URL = cfg.seyren.graphiteUrl;
+  }
+  // cfg.seyren.extraConfig;
+
+  configDir = pkgs.buildEnv {
+    name = "graphite-config";
+    paths = lib.lists.filter (el: el != null) [
+      (writeTextOrNull "carbon.conf" cfg.carbon.config)
+      (writeTextOrNull "storage-aggregation.conf" cfg.carbon.storageAggregation)
+      (writeTextOrNull "storage-schemas.conf" cfg.carbon.storageSchemas)
+      (writeTextOrNull "blacklist.conf" cfg.carbon.blacklist)
+      (writeTextOrNull "whitelist.conf" cfg.carbon.whitelist)
+      (writeTextOrNull "rewrite-rules.conf" cfg.carbon.rewriteRules)
+      (writeTextOrNull "relay-rules.conf" cfg.carbon.relayRules)
+      (writeTextOrNull "aggregation-rules.conf" cfg.carbon.aggregationRules)
+    ];
+  };
+
+  carbonOpts = name: ''
+    --nodaemon --syslog --prefix=${name} --pidfile /run/${name}/${name}.pid ${name}
+  '';
+
+  carbonEnv = {
+    PYTHONPATH =
+      let
+        cenv = pkgs.python3.buildEnv.override {
+          extraLibs = [ pkgs.python3Packages.carbon ];
+        };
+      in
+      "${cenv}/${pkgs.python3.sitePackages}";
+    GRAPHITE_ROOT = dataDir;
+    GRAPHITE_CONF_DIR = configDir;
+    GRAPHITE_STORAGE_DIR = dataDir;
+  };
+
+in
+{
+
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "graphite" "api" ] "")
+    (lib.mkRemovedOptionModule [ "services" "graphite" "beacon" ] "")
+    (lib.mkRemovedOptionModule [ "services" "graphite" "pager" ] "")
+  ];
+
+  ###### interface
+
+  options.services.graphite = {
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/db/graphite";
+      description = ''
+        Data directory for graphite.
+      '';
+    };
+
+    web = {
+      enable = lib.mkOption {
+        description = "Whether to enable graphite web frontend.";
+        default = false;
+        type = lib.types.bool;
+      };
+
+      listenAddress = lib.mkOption {
+        description = "Graphite web frontend listen address.";
+        default = "127.0.0.1";
+        type = lib.types.str;
+      };
+
+      port = lib.mkOption {
+        description = "Graphite web frontend port.";
+        default = 8080;
+        type = lib.types.port;
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = ''
+          Graphite webapp settings. See:
+          <https://graphite.readthedocs.io/en/latest/config-local-settings.html>
+        '';
+      };
+    };
+
+    carbon = {
+      config = lib.mkOption {
+        description = "Content of carbon configuration file.";
+        default = ''
+          [cache]
+          # Listen on localhost by default for security reasons
+          UDP_RECEIVER_INTERFACE = 127.0.0.1
+          PICKLE_RECEIVER_INTERFACE = 127.0.0.1
+          LINE_RECEIVER_INTERFACE = 127.0.0.1
+          CACHE_QUERY_INTERFACE = 127.0.0.1
+          # Do not log every update
+          LOG_UPDATES = False
+          LOG_CACHE_HITS = False
+        '';
+        type = lib.types.str;
+      };
+
+      enableCache = lib.mkOption {
+        description = "Whether to enable carbon cache, the graphite storage daemon.";
+        default = false;
+        type = lib.types.bool;
+      };
+
+      storageAggregation = lib.mkOption {
+        description = "Defines how to aggregate data to lower-precision retentions.";
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = ''
+          [all_min]
+          pattern = \.min$
+          xFilesFactor = 0.1
+          aggregationMethod = min
+        '';
+      };
+
+      storageSchemas = lib.mkOption {
+        description = "Defines retention rates for storing metrics.";
+        default = "";
+        type = lib.types.nullOr lib.types.str;
+        example = ''
+          [apache_busyWorkers]
+          pattern = ^servers\.www.*\.workers\.busyWorkers$
+          retentions = 15s:7d,1m:21d,15m:5y
+        '';
+      };
+
+      blacklist = lib.mkOption {
+        description = "Any metrics received which match one of the expressions will be dropped.";
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = "^some\\.noisy\\.metric\\.prefix\\..*";
+      };
+
+      whitelist = lib.mkOption {
+        description = "Only metrics received which match one of the expressions will be persisted.";
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = ".*";
+      };
+
+      rewriteRules = lib.mkOption {
+        description = ''
+          Regular expression patterns that can be used to rewrite metric names
+          in a search and replace fashion.
+        '';
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = ''
+          [post]
+          _sum$ =
+          _avg$ =
+        '';
+      };
+
+      enableRelay = lib.mkOption {
+        description = "Whether to enable carbon relay, the carbon replication and sharding service.";
+        default = false;
+        type = lib.types.bool;
+      };
+
+      relayRules = lib.mkOption {
+        description = "Relay rules are used to send certain metrics to a certain backend.";
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = ''
+          [example]
+          pattern = ^mydata\.foo\..+
+          servers = 10.1.2.3, 10.1.2.4:2004, myserver.mydomain.com
+        '';
+      };
+
+      enableAggregator = lib.mkOption {
+        description = "Whether to enable carbon aggregator, the carbon buffering service.";
+        default = false;
+        type = lib.types.bool;
+      };
+
+      aggregationRules = lib.mkOption {
+        description = "Defines if and how received metrics will be aggregated.";
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        example = ''
+          <env>.applications.<app>.all.requests (60) = sum <env>.applications.<app>.*.requests
+          <env>.applications.<app>.all.latency (60) = avg <env>.applications.<app>.*.latency
+        '';
+      };
+    };
+
+    seyren = {
+      enable = lib.mkOption {
+        description = "Whether to enable seyren service.";
+        default = false;
+        type = lib.types.bool;
+      };
+
+      port = lib.mkOption {
+        description = "Seyren listening port.";
+        default = 8081;
+        type = lib.types.port;
+      };
+
+      seyrenUrl = lib.mkOption {
+        default = "http://localhost:${toString cfg.seyren.port}/";
+        defaultText = lib.literalExpression ''"http://localhost:''${toString config.${opt.seyren.port}}/"'';
+        description = "Host where seyren is accessible.";
+        type = lib.types.str;
+      };
+
+      graphiteUrl = lib.mkOption {
+        default = "http://${cfg.web.listenAddress}:${toString cfg.web.port}";
+        defaultText = lib.literalExpression ''"http://''${config.${opt.web.listenAddress}}:''${toString config.${opt.web.port}}"'';
+        description = "Host where graphite service runs.";
+        type = lib.types.str;
+      };
+
+      mongoUrl = lib.mkOption {
+        default = "mongodb://${config.services.mongodb.bind_ip}:27017/seyren";
+        defaultText = lib.literalExpression ''"mongodb://''${config.services.mongodb.bind_ip}:27017/seyren"'';
+        description = "Mongodb connection string.";
+        type = lib.types.str;
+      };
+
+      extraConfig = lib.mkOption {
+        default = { };
+        description = ''
+          Extra seyren configuration. See
+          <https://github.com/scobal/seyren#config>
+        '';
+        type = lib.types.attrsOf lib.types.str;
+        example = lib.literalExpression ''
+          {
+            GRAPHITE_USERNAME = "user";
+            GRAPHITE_PASSWORD = "pass";
+          }
+        '';
+      };
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.carbon.enableCache {
+      systemd.services.carbonCache =
+        let
+          name = "carbon-cache";
+        in
+        {
+          description = "Graphite Data Storage Backend";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          environment = carbonEnv;
+          serviceConfig = {
+            Slice = "system-graphite.slice";
+            RuntimeDirectory = name;
+            ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
+            User = "graphite";
+            Group = "graphite";
+            PermissionsStartOnly = true;
+            PIDFile = "/run/${name}/${name}.pid";
+          };
+          preStart = ''
+            install -dm0700 -o graphite -g graphite ${cfg.dataDir}
+            install -dm0700 -o graphite -g graphite ${cfg.dataDir}/whisper
+          '';
+        };
+    })
+
+    (lib.mkIf cfg.carbon.enableAggregator {
+      systemd.services.carbonAggregator =
+        let
+          name = "carbon-aggregator";
+        in
+        {
+          enable = cfg.carbon.enableAggregator;
+          description = "Carbon Data Aggregator";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          environment = carbonEnv;
+          serviceConfig = {
+            Slice = "system-graphite.slice";
+            RuntimeDirectory = name;
+            ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
+            User = "graphite";
+            Group = "graphite";
+            PIDFile = "/run/${name}/${name}.pid";
+          };
+        };
+    })
+
+    (lib.mkIf cfg.carbon.enableRelay {
+      systemd.services.carbonRelay =
+        let
+          name = "carbon-relay";
+        in
+        {
+          description = "Carbon Data Relay";
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          environment = carbonEnv;
+          serviceConfig = {
+            Slice = "system-graphite.slice";
+            RuntimeDirectory = name;
+            ExecStart = "${lib.getExe' pkgs.python3Packages.twisted "twistd"} ${carbonOpts name}";
+            User = "graphite";
+            Group = "graphite";
+            PIDFile = "/run/${name}/${name}.pid";
+          };
+        };
+    })
+
+    (lib.mkIf (cfg.carbon.enableCache || cfg.carbon.enableAggregator || cfg.carbon.enableRelay) {
+      environment.systemPackages = [
+        pkgs.python3Packages.carbon
+      ];
+    })
+
+    (lib.mkIf cfg.web.enable {
+      systemd.services.graphiteWeb = {
+        description = "Graphite Web Interface";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+        path = [ pkgs.perl ];
+        environment = {
+          PYTHONPATH =
+            let
+              penv = pkgs.python3.buildEnv.override {
+                extraLibs = [
+                  pkgs.python3Packages.graphite-web
+                ];
+              };
+              penvPack = "${penv}/${pkgs.python3.sitePackages}";
+            in
+            lib.concatStringsSep ":" [
+              "${graphiteLocalSettingsDir}"
+              "${penvPack}"
+              # explicitly adding pycairo in path because it cannot be imported via buildEnv
+              "${pkgs.python3Packages.pycairo}/${pkgs.python3.sitePackages}"
+            ];
+          DJANGO_SETTINGS_MODULE = "graphite.settings";
+          GRAPHITE_SETTINGS_MODULE = "graphite_local_settings";
+          GRAPHITE_CONF_DIR = configDir;
+          GRAPHITE_STORAGE_DIR = dataDir;
+          LD_LIBRARY_PATH = "${pkgs.cairo.out}/lib";
+        };
+        serviceConfig = {
+          ExecStart = ''
+            ${lib.getExe pkgs.python3Packages.waitress-django} \
+              --host=${cfg.web.listenAddress} --port=${toString cfg.web.port}
+          '';
+          User = "graphite";
+          Group = "graphite";
+          PermissionsStartOnly = true;
+          Slice = "system-graphite.slice";
+        };
+        preStart = ''
+          if ! test -e ${dataDir}/db-created; then
+            mkdir -p ${dataDir}/{whisper/,log/webapp/}
+            chmod 0700 ${dataDir}/{whisper/,log/webapp/}
+
+            ${lib.getExe' pkgs.python3Packages.django "django-admin"} migrate --noinput
+
+            chown -R graphite:graphite ${dataDir}
+
+            touch ${dataDir}/db-created
+          fi
+
+          # Only collect static files when graphite_web changes.
+          if ! [ "${dataDir}/current_graphite_web" -ef "${pkgs.python3Packages.graphite-web}" ]; then
+            mkdir -p ${staticDir}
+            ${lib.getExe' pkgs.python3Packages.django "django-admin"} collectstatic  --noinput --clear
+            chown -R graphite:graphite ${staticDir}
+            ln -sfT "${pkgs.python3Packages.graphite-web}" "${dataDir}/current_graphite_web"
+          fi
+        '';
+      };
+
+      environment.systemPackages = [ pkgs.python3Packages.graphite-web ];
+    })
+
+    (lib.mkIf cfg.seyren.enable {
+      systemd.services.seyren = {
+        description = "Graphite Alerting Dashboard";
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "network.target"
+          "mongodb.service"
+        ];
+        environment = seyrenConfig;
+        serviceConfig = {
+          ExecStart = "${lib.getExe pkgs.seyren} -httpPort ${toString cfg.seyren.port}";
+          WorkingDirectory = dataDir;
+          User = "graphite";
+          Group = "graphite";
+          Slice = "system-graphite.slice";
+        };
+        preStart = ''
+          if ! test -e ${dataDir}/db-created; then
+            mkdir -p ${dataDir}
+            chown graphite:graphite ${dataDir}
+          fi
+        '';
+      };
+
+      services.mongodb.enable = lib.mkDefault true;
+    })
+
+    (lib.mkIf
+      (
+        cfg.carbon.enableCache
+        || cfg.carbon.enableAggregator
+        || cfg.carbon.enableRelay
+        || cfg.web.enable
+        || cfg.seyren.enable
+      )
+      {
+        systemd.slices.system-graphite = {
+          description = "Graphite Graphing System Slice";
+          documentation = [ "https://graphite.readthedocs.io/en/latest/overview.html" ];
+        };
+
+        users.users.graphite = {
+          uid = config.ids.uids.graphite;
+          group = "graphite";
+          description = "Graphite daemon user";
+          home = dataDir;
+        };
+        users.groups.graphite.gid = config.ids.gids.graphite;
+      }
+    )
+  ];
+}

nixos/modules/services/monitoring/hdaps.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hdapsd;
+  hdapsd = [ pkgs.hdapsd ];
+in
+{
+  options = {
+    services.hdapsd.enable = lib.mkEnableOption ''
+      Hard Drive Active Protection System Daemon,
+      devices are detected and managed automatically by udev and systemd
+    '';
+  };
+
+  config = lib.mkIf cfg.enable {
+    boot.kernelModules = [ "hdapsd" ];
+    services.udev.packages = hdapsd;
+    systemd.packages = hdapsd;
+  };
+}

nixos/modules/services/monitoring/heapster.nix

@@ -0,0 +1,56 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.heapster;
+in
+{
+  options.services.heapster = {
+    enable = lib.mkEnableOption "Heapster monitoring";
+
+    source = lib.mkOption {
+      description = "Heapster metric source";
+      example = "kubernetes:https://kubernetes.default";
+      type = lib.types.str;
+    };
+
+    sink = lib.mkOption {
+      description = "Heapster metic sink";
+      example = "influxdb:http://localhost:8086";
+      type = lib.types.str;
+    };
+
+    extraOpts = lib.mkOption {
+      description = "Heapster extra options";
+      default = "";
+      type = lib.types.separatedString " ";
+    };
+
+    package = lib.mkPackageOption pkgs "heapster" { };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.heapster = {
+      wantedBy = [ "multi-user.target" ];
+      after = [
+        "cadvisor.service"
+        "kube-apiserver.service"
+      ];
+
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/heapster --source=${cfg.source} --sink=${cfg.sink} ${cfg.extraOpts}";
+        User = "heapster";
+      };
+    };
+
+    users.users.heapster = {
+      isSystemUser = true;
+      group = "heapster";
+      description = "Heapster user";
+    };
+    users.groups.heapster = { };
+  };
+}

nixos/modules/services/monitoring/incron.nix

@@ -0,0 +1,105 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+
+  cfg = config.services.incron;
+
+in
+
+{
+  options = {
+
+    services.incron = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable the incron daemon.
+
+          Note that commands run under incrontab only support common Nix profiles for the {env}`PATH` provided variable.
+        '';
+      };
+
+      allow = lib.mkOption {
+        type = lib.types.nullOr (lib.types.listOf lib.types.str);
+        default = null;
+        description = ''
+          Users allowed to use incrontab.
+
+          If empty then no user will be allowed to have their own incrontab.
+          If `null` then will defer to {option}`deny`.
+          If both {option}`allow` and {option}`deny` are null
+          then all users will be allowed to have their own incrontab.
+        '';
+      };
+
+      deny = lib.mkOption {
+        type = lib.types.nullOr (lib.types.listOf lib.types.str);
+        default = null;
+        description = "Users forbidden from using incrontab.";
+      };
+
+      systab = lib.mkOption {
+        type = lib.types.lines;
+        default = "";
+        description = "The system incrontab contents.";
+        example = ''
+          /var/mail IN_CLOSE_WRITE abc $@/$#
+          /tmp IN_ALL_EVENTS efg $@/$# $&
+        '';
+      };
+
+      extraPackages = lib.mkOption {
+        type = lib.types.listOf lib.types.package;
+        default = [ ];
+        example = lib.literalExpression "[ pkgs.rsync ]";
+        description = "Extra packages available to the system incrontab.";
+      };
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    warnings = lib.optional (
+      cfg.allow != null && cfg.deny != null
+    ) "If `services.incron.allow` is set then `services.incron.deny` will be ignored.";
+
+    environment.systemPackages = [ pkgs.incron ];
+
+    security.wrappers.incrontab = {
+      setuid = true;
+      owner = "root";
+      group = "root";
+      source = "${pkgs.incron}/bin/incrontab";
+    };
+
+    # incron won't read symlinks
+    environment.etc."incron.d/system" = {
+      mode = "0444";
+      text = cfg.systab;
+    };
+    environment.etc."incron.allow" = lib.mkIf (cfg.allow != null) {
+      text = lib.concatStringsSep "\n" cfg.allow;
+    };
+    environment.etc."incron.deny" = lib.mkIf (cfg.deny != null) {
+      text = lib.concatStringsSep "\n" cfg.deny;
+    };
+
+    systemd.services.incron = {
+      description = "File System Events Scheduler";
+      wantedBy = [ "multi-user.target" ];
+      path = cfg.extraPackages;
+      serviceConfig.PIDFile = "/run/incrond.pid";
+      serviceConfig.ExecStartPre = "${pkgs.coreutils}/bin/mkdir -m 710 -p /var/spool/incron";
+      serviceConfig.ExecStart = "${pkgs.incron}/bin/incrond --foreground";
+    };
+  };
+
+}

nixos/modules/services/monitoring/kapacitor.nix

@@ -0,0 +1,190 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.kapacitor;
+
+  kapacitorConf = pkgs.writeTextFile {
+    name = "kapacitord.conf";
+    text = ''
+      hostname="${config.networking.hostName}"
+      data_dir="${cfg.dataDir}"
+
+      [http]
+        bind-address = "${cfg.bind}:${toString cfg.port}"
+        log-enabled = false
+        auth-enabled = false
+
+      [task]
+        dir = "${cfg.dataDir}/tasks"
+        snapshot-interval = "${cfg.taskSnapshotInterval}"
+
+      [replay]
+        dir = "${cfg.dataDir}/replay"
+
+      [storage]
+        boltdb = "${cfg.dataDir}/kapacitor.db"
+
+      ${lib.optionalString (cfg.loadDirectory != null) ''
+        [load]
+          enabled = true
+          dir = "${cfg.loadDirectory}"
+      ''}
+
+      ${lib.optionalString (cfg.defaultDatabase.enable) ''
+        [[influxdb]]
+          name = "default"
+          enabled = true
+          default = true
+          urls = [ "${cfg.defaultDatabase.url}" ]
+          username = "${cfg.defaultDatabase.username}"
+          password = "${cfg.defaultDatabase.password}"
+      ''}
+
+      ${lib.optionalString (cfg.alerta.enable) ''
+        [alerta]
+          enabled = true
+          url = "${cfg.alerta.url}"
+          token = "${cfg.alerta.token}"
+          environment = "${cfg.alerta.environment}"
+          origin = "${cfg.alerta.origin}"
+      ''}
+
+      ${cfg.extraConfig}
+    '';
+  };
+in
+{
+  options.services.kapacitor = {
+    enable = lib.mkEnableOption "kapacitor";
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/kapacitor";
+      description = "Location where Kapacitor stores its state";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 9092;
+      description = "Port of Kapacitor";
+    };
+
+    bind = lib.mkOption {
+      type = lib.types.str;
+      default = "";
+      example = "0.0.0.0";
+      description = "Address to bind to. The default is to bind to all addresses";
+    };
+
+    extraConfig = lib.mkOption {
+      description = "These lines go into kapacitord.conf verbatim.";
+      default = "";
+      type = lib.types.lines;
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "kapacitor";
+      description = "User account under which Kapacitor runs";
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "kapacitor";
+      description = "Group under which Kapacitor runs";
+    };
+
+    taskSnapshotInterval = lib.mkOption {
+      type = lib.types.str;
+      description = "Specifies how often to snapshot the task state  (in InfluxDB time units)";
+      default = "1m0s";
+    };
+
+    loadDirectory = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "Directory where to load services from, such as tasks, templates and handlers (or null to disable service loading on startup)";
+      default = null;
+    };
+
+    defaultDatabase = {
+      enable = lib.mkEnableOption "kapacitor.defaultDatabase";
+
+      url = lib.mkOption {
+        description = "The URL to an InfluxDB server that serves as the default database";
+        example = "http://localhost:8086";
+        type = lib.types.str;
+      };
+
+      username = lib.mkOption {
+        description = "The username to connect to the remote InfluxDB server";
+        type = lib.types.str;
+      };
+
+      password = lib.mkOption {
+        description = "The password to connect to the remote InfluxDB server";
+        type = lib.types.str;
+      };
+    };
+
+    alerta = {
+      enable = lib.mkEnableOption "kapacitor alerta integration";
+
+      url = lib.mkOption {
+        description = "The URL to the Alerta REST API";
+        default = "http://localhost:5000";
+        type = lib.types.str;
+      };
+
+      token = lib.mkOption {
+        description = "Default Alerta authentication token";
+        type = lib.types.str;
+        default = "";
+      };
+
+      environment = lib.mkOption {
+        description = "Default Alerta environment";
+        type = lib.types.str;
+        default = "Production";
+      };
+
+      origin = lib.mkOption {
+        description = "Default origin of alert";
+        type = lib.types.str;
+        default = "kapacitor";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ pkgs.kapacitor ];
+
+    systemd.tmpfiles.settings."10-kapacitor".${cfg.dataDir}.d = {
+      inherit (cfg) user group;
+    };
+
+    systemd.services.kapacitor = {
+      description = "Kapacitor Real-Time Stream Processing Engine";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "networking.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.kapacitor}/bin/kapacitord -config ${kapacitorConf}";
+        User = "kapacitor";
+        Group = "kapacitor";
+      };
+    };
+
+    users.users.kapacitor = {
+      uid = config.ids.uids.kapacitor;
+      description = "Kapacitor user";
+      home = cfg.dataDir;
+    };
+
+    users.groups.kapacitor = {
+      gid = config.ids.gids.kapacitor;
+    };
+  };
+}

nixos/modules/services/monitoring/karma.nix

@@ -0,0 +1,125 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.karma;
+  yaml = pkgs.formats.yaml { };
+in
+{
+  options.services.karma = {
+    enable = lib.mkEnableOption "the Karma dashboard service";
+
+    package = lib.mkPackageOption pkgs "karma" { };
+
+    configFile = lib.mkOption {
+      type = lib.types.path;
+      default = yaml.generate "karma.yaml" cfg.settings;
+      defaultText = "A configuration file generated from the provided nix attributes settings option.";
+      description = ''
+        A YAML config file which can be used to configure karma instead of the nix-generated file.
+      '';
+      example = "/etc/karma/karma.conf";
+    };
+
+    environment = lib.mkOption {
+      type = with lib.types; attrsOf str;
+      default = { };
+      description = ''
+        Additional environment variables to provide to karma.
+      '';
+      example = {
+        ALERTMANAGER_URI = "https://alertmanager.example.com";
+        ALERTMANAGER_NAME = "single";
+      };
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to open ports in the firewall needed for karma to function.
+      '';
+    };
+
+    extraOptions = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [ ];
+      description = ''
+        Extra command line options.
+      '';
+      example = [
+        "--alertmanager.timeout 10s"
+      ];
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = yaml.type;
+
+        options.listen = {
+          address = lib.mkOption {
+            type = lib.types.str;
+            default = "127.0.0.1";
+            description = ''
+              Hostname or IP to listen on.
+            '';
+            example = "[::]";
+          };
+
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = 8080;
+            description = ''
+              HTTP port to listen on.
+            '';
+            example = 8182;
+          };
+        };
+      };
+      default = {
+        listen = {
+          address = "127.0.0.1";
+        };
+      };
+      description = ''
+        Karma dashboard configuration as nix attributes.
+
+        Reference: <https://github.com/prymitive/karma/blob/main/docs/CONFIGURATION.md>
+      '';
+      example = {
+        listen = {
+          address = "192.168.1.4";
+          port = "8000";
+          prefix = "/dashboard";
+        };
+        alertmanager = {
+          interval = "15s";
+          servers = [
+            {
+              name = "prod";
+              uri = "http://alertmanager.example.com";
+            }
+          ];
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.karma = {
+      description = "Alert dashboard for Prometheus Alertmanager";
+      wantedBy = [ "multi-user.target" ];
+      environment = cfg.environment;
+      serviceConfig = {
+        Type = "simple";
+        DynamicUser = true;
+        Restart = "on-failure";
+        ExecStart = "${pkgs.karma}/bin/karma --config.file ${cfg.configFile} ${lib.concatStringsSep " " cfg.extraOptions}";
+      };
+    };
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.settings.listen.port ];
+  };
+}

nixos/modules/services/monitoring/kthxbye.nix

@@ -0,0 +1,162 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.kthxbye;
+in
+
+{
+  options.services.kthxbye = {
+    enable = lib.mkEnableOption "kthxbye alert acknowledgement management daemon";
+
+    package = lib.mkPackageOption pkgs "kthxbye" { };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to open ports in the firewall needed for the daemon to function.
+      '';
+    };
+
+    extraOptions = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [ ];
+      description = ''
+        Extra command line options.
+
+        Documentation can be found [here](https://github.com/prymitive/kthxbye/blob/main/README.md).
+      '';
+      example = lib.literalExpression ''
+        [
+          "-extend-with-prefix 'ACK!'"
+        ];
+      '';
+    };
+
+    alertmanager = {
+      timeout = lib.mkOption {
+        type = lib.types.str;
+        default = "1m0s";
+        description = ''
+          Alertmanager request timeout duration in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+        '';
+        example = "30s";
+      };
+      uri = lib.mkOption {
+        type = lib.types.str;
+        default = "http://localhost:9093";
+        description = ''
+          Alertmanager URI to use.
+        '';
+        example = "https://alertmanager.example.com";
+      };
+    };
+
+    extendBy = lib.mkOption {
+      type = lib.types.str;
+      default = "15m0s";
+      description = ''
+        Extend silences by adding DURATION seconds.
+
+        DURATION should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "6h0m0s";
+    };
+
+    extendIfExpiringIn = lib.mkOption {
+      type = lib.types.str;
+      default = "5m0s";
+      description = ''
+        Extend silences that are about to expire in the next DURATION seconds.
+
+        DURATION should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "1m0s";
+    };
+
+    extendWithPrefix = lib.mkOption {
+      type = lib.types.str;
+      default = "ACK!";
+      description = ''
+        Extend silences with comment starting with PREFIX string.
+      '';
+      example = "!perma-silence";
+    };
+
+    interval = lib.mkOption {
+      type = lib.types.str;
+      default = "45s";
+      description = ''
+        Silence check interval duration in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "30s";
+    };
+
+    listenAddress = lib.mkOption {
+      type = lib.types.str;
+      default = "0.0.0.0";
+      description = ''
+        The address to listen on for HTTP requests.
+      '';
+      example = "127.0.0.1";
+    };
+
+    port = lib.mkOption {
+      type = lib.types.port;
+      default = 8080;
+      description = ''
+        The port to listen on for HTTP requests.
+      '';
+    };
+
+    logJSON = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Format logged messages as JSON.
+      '';
+    };
+
+    maxDuration = lib.mkOption {
+      type = with lib.types; nullOr str;
+      default = null;
+      description = ''
+        Maximum duration of a silence, it won't be extended anymore after reaching it.
+
+        Duration should be provided in the [time.Duration](https://pkg.go.dev/time#ParseDuration) format.
+      '';
+      example = "30d";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.kthxbye = {
+      description = "kthxbye Alertmanager ack management daemon";
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        ${cfg.package}/bin/kthxbye \
+          -alertmanager.timeout ${cfg.alertmanager.timeout} \
+          -alertmanager.uri ${cfg.alertmanager.uri} \
+          -extend-by ${cfg.extendBy} \
+          -extend-if-expiring-in ${cfg.extendIfExpiringIn} \
+          -extend-with-prefix ${cfg.extendWithPrefix} \
+          -interval ${cfg.interval} \
+          -listen ${cfg.listenAddress}:${toString cfg.port} \
+          ${lib.optionalString cfg.logJSON "-log-json"} \
+          ${lib.optionalString (cfg.maxDuration != null) "-max-duration ${cfg.maxDuration}"} \
+          ${lib.concatStringsSep " " cfg.extraOptions}
+      '';
+      serviceConfig = {
+        Type = "simple";
+        DynamicUser = true;
+        Restart = "on-failure";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.port ];
+  };
+}

nixos/modules/services/monitoring/librenms.nix

@@ -0,0 +1,764 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.librenms;
+  settingsFormat = pkgs.formats.json { };
+  configJson = settingsFormat.generate "librenms-config.json" cfg.settings;
+
+  package = cfg.package.override {
+    logDir = cfg.logDir;
+    dataDir = cfg.dataDir;
+  };
+
+  phpOptions = ''
+    log_errors = on
+    post_max_size = 100M
+    upload_max_filesize = 100M
+    memory_limit = ${toString cfg.settings.php_memory_limit}M
+    date.timezone = "${config.time.timeZone}"
+  '';
+  phpIni =
+    pkgs.runCommand "php.ini"
+      {
+        inherit (package) phpPackage;
+        inherit phpOptions;
+        preferLocalBuild = true;
+        passAsFile = [ "phpOptions" ];
+      }
+      ''
+        cat $phpPackage/etc/php.ini $phpOptionsPath > $out
+      '';
+
+  artisanWrapper = pkgs.writeShellScriptBin "librenms-artisan" ''
+    cd ${package}
+    sudo=exec
+    if [[ "$USER" != ${cfg.user} ]]; then
+      sudo='exec /run/wrappers/bin/sudo -u ${cfg.user}'
+    fi
+    $sudo ${package}/artisan "$@"
+  '';
+
+  lnmsWrapper = pkgs.writeShellScriptBin "lnms" ''
+    cd ${package}
+    sudo=exec
+    if [[ "$USER" != ${cfg.user} ]]; then
+    sudo='exec /run/wrappers/bin/sudo -u ${cfg.user}'
+    fi
+    $sudo ${package}/lnms "$@"
+  '';
+
+  configFile = pkgs.writeText "config.php" ''
+    <?php
+    $new_config = json_decode(file_get_contents("${cfg.dataDir}/config.json"), true);
+    $config = ($config == null) ? $new_config : array_merge($config, $new_config);
+
+    ${lib.optionalString (cfg.extraConfig != null) cfg.extraConfig}
+  '';
+
+in
+{
+  options.services.librenms = with lib; {
+    enable = mkEnableOption "LibreNMS network monitoring system";
+
+    package = lib.mkPackageOption pkgs "librenms" { };
+
+    finalPackage = lib.mkOption {
+      type = lib.types.package;
+      readOnly = true;
+      default = package;
+      defaultText = lib.literalExpression "package";
+      description = ''
+        The final package used by the module. This is the package that has all overrides.
+      '';
+    };
+
+    user = mkOption {
+      type = types.str;
+      default = "librenms";
+      description = ''
+        Name of the LibreNMS user.
+      '';
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "librenms";
+      description = ''
+        Name of the LibreNMS group.
+      '';
+    };
+
+    hostname = mkOption {
+      type = types.str;
+      default = config.networking.fqdnOrHostName;
+      defaultText = literalExpression "config.networking.fqdnOrHostName";
+      description = ''
+        The hostname to serve LibreNMS on.
+      '';
+    };
+
+    pollerThreads = mkOption {
+      type = types.int;
+      default = 16;
+      description = ''
+        Amount of threads of the cron-poller.
+      '';
+    };
+
+    enableOneMinutePolling = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enables the [1-Minute Polling](https://docs.librenms.org/Support/1-Minute-Polling/).
+        Changing this option will automatically convert your existing rrd files.
+      '';
+    };
+
+    enableLocalBilling = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Enable billing Cron-Jobs on the local instance. Enabled by default, but you may disable it
+        on some nodes within a distributed poller setup. See [the docs](https://docs.librenms.org/Extensions/Distributed-Poller/#discovery)
+        for more informations about billing with distributed pollers.
+      '';
+    };
+
+    useDistributedPollers = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enables [distributed pollers](https://docs.librenms.org/Extensions/Distributed-Poller/)
+        for this LibreNMS instance. This will enable a local `rrdcached` and `memcached` server.
+
+        To use this feature, make sure to configure your firewall that the distributed pollers
+        can reach the local `mysql`, `rrdcached` and `memcached` ports.
+      '';
+    };
+
+    distributedPoller = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Configure this LibreNMS instance as a [distributed poller](https://docs.librenms.org/Extensions/Distributed-Poller/).
+          This will disable all web features and just configure the poller features.
+          Use the `mysql` database of your main LibreNMS instance in the database settings.
+        '';
+      };
+
+      name = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = ''
+          Custom name of this poller.
+        '';
+      };
+
+      group = mkOption {
+        type = types.str;
+        default = "0";
+        example = "1,2";
+        description = ''
+          Group(s) of this poller.
+        '';
+      };
+
+      distributedBilling = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Enable distributed billing on this poller.
+
+          Note: according to [the docs](https://docs.librenms.org/Extensions/Distributed-Poller/#discovery),
+          billing should only be calculated on a single node per poller group. You can disable billing on
+          some nodes with the `services.librenms.enableLocalBilling` option.
+        '';
+      };
+
+      memcachedHost = mkOption {
+        type = types.str;
+        description = ''
+          Hostname or IP of the `memcached` server.
+        '';
+      };
+
+      memcachedPort = mkOption {
+        type = types.port;
+        default = 11211;
+        description = ''
+          Port of the `memcached` server.
+        '';
+      };
+
+      rrdcachedHost = mkOption {
+        type = types.str;
+        description = ''
+          Hostname or IP of the `rrdcached` server.
+        '';
+      };
+
+      rrdcachedPort = mkOption {
+        type = types.port;
+        default = 42217;
+        description = ''
+          Port of the `memcached` server.
+        '';
+      };
+    };
+
+    poolConfig = mkOption {
+      type =
+        with types;
+        attrsOf (oneOf [
+          str
+          int
+          bool
+        ]);
+      default = {
+        "pm" = "dynamic";
+        "pm.max_children" = 32;
+        "pm.start_servers" = 2;
+        "pm.min_spare_servers" = 2;
+        "pm.max_spare_servers" = 4;
+        "pm.max_requests" = 500;
+      };
+      description = ''
+        Options for the LibreNMS PHP pool. See the documentation on `php-fpm.conf`
+        for details on configuration directives.
+      '';
+    };
+
+    nginx = mkOption {
+      type = types.submodule (
+        recursiveUpdate (import ../web-servers/nginx/vhost-options.nix { inherit config lib; }) { }
+      );
+      default = { };
+      example = literalExpression ''
+        {
+          serverAliases = [
+            "librenms.''${config.networking.domain}"
+          ];
+          # To enable encryption and let let's encrypt take care of certificate
+          forceSSL = true;
+          enableACME = true;
+          # To set the LibreNMS virtualHost as the default virtualHost;
+          default = true;
+        }
+      '';
+      description = ''
+        With this option, you can customize the nginx virtualHost settings.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/librenms";
+      description = ''
+        Path of the LibreNMS state directory.
+      '';
+    };
+
+    logDir = mkOption {
+      type = types.path;
+      default = "/var/log/librenms";
+      description = ''
+        Path of the LibreNMS logging directory.
+      '';
+    };
+
+    database = {
+      createLocally = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether to create a local database automatically.
+        '';
+      };
+
+      host = mkOption {
+        default = "localhost";
+        description = ''
+          Hostname or IP of the MySQL/MariaDB server.
+          Ignored if 'socket' is defined.
+        '';
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 3306;
+        description = ''
+          Port of the MySQL/MariaDB server.
+          Ignored if 'socket' is defined.
+        '';
+      };
+
+      database = mkOption {
+        type = types.str;
+        default = "librenms";
+        description = ''
+          Name of the database on the MySQL/MariaDB server.
+        '';
+      };
+
+      username = mkOption {
+        type = types.str;
+        default = "librenms";
+        description = ''
+          Name of the user on the MySQL/MariaDB server.
+          Ignored if 'socket' is defined.
+        '';
+      };
+
+      passwordFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        example = "/run/secrets/mysql.pass";
+        description = ''
+          A file containing the password for the user of the MySQL/MariaDB server.
+          Must be readable for the LibreNMS user.
+          Ignored if 'socket' is defined, mandatory otherwise.
+        '';
+      };
+
+      socket = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = "/run/mysqld/mysqld.sock";
+        description = ''
+          A unix socket to mysql, accessible by the librenms user.
+          Useful when mysql is on the localhost.
+        '';
+      };
+    };
+
+    environmentFile = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        File containing env-vars to be substituted into the final config. Useful for secrets.
+        Does not apply to settings defined in `extraConfig`.
+      '';
+    };
+
+    settings = mkOption {
+      type = types.submodule {
+        freeformType = settingsFormat.type;
+        options = { };
+      };
+      description = ''
+        Attrset of the LibreNMS configuration.
+        See <https://docs.librenms.org/Support/Configuration/> for reference.
+        All possible options are listed [here](https://github.com/librenms/librenms/blob/master/resources/definitions/config_definitions.json).
+        See <https://docs.librenms.org/Extensions/Authentication/> for setting other authentication methods.
+      '';
+      default = { };
+      example = {
+        base_url = "/librenms/";
+        top_devices = true;
+        top_ports = false;
+      };
+    };
+
+    extraConfig = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        Additional config for LibreNMS that will be appended to the `config.php`. See
+        <https://github.com/librenms/librenms/blob/master/misc/config_definitions.json>
+        for possible options. Useful if you want to use PHP-Functions in your config.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.time.timeZone != null;
+        message = "You must set `time.timeZone` to use the LibreNMS module.";
+      }
+      {
+        assertion = cfg.database.createLocally -> cfg.database.host == "localhost";
+        message = "The database host must be \"localhost\" if services.librenms.database.createLocally is set to true.";
+      }
+      {
+        assertion = !(cfg.useDistributedPollers && cfg.distributedPoller.enable);
+        message = "The LibreNMS instance can't be a distributed poller and a full instance at the same time.";
+      }
+    ];
+
+    users.users.${cfg.user} = {
+      group = "${cfg.group}";
+      isSystemUser = true;
+    };
+
+    users.groups.${cfg.group} = { };
+
+    services.librenms.settings = {
+      # basic configs
+      "user" = cfg.user;
+      "own_hostname" = cfg.hostname;
+      "base_url" = lib.mkDefault "/";
+      "auth_mechanism" = lib.mkDefault "mysql";
+
+      # disable auto update function (won't work with NixOS)
+      "update" = false;
+
+      # enable fast ping by default
+      "ping_rrd_step" = 60;
+
+      # set default memory limit to 1G
+      "php_memory_limit" = lib.mkDefault 1024;
+
+      # one minute polling
+      "rrd.step" = if cfg.enableOneMinutePolling then 60 else 300;
+      "rrd.heartbeat" = if cfg.enableOneMinutePolling then 120 else 600;
+    }
+    // (lib.optionalAttrs cfg.distributedPoller.enable {
+      "distributed_poller" = true;
+      "distributed_poller_name" = lib.mkIf (
+        cfg.distributedPoller.name != null
+      ) cfg.distributedPoller.name;
+      "distributed_poller_group" = cfg.distributedPoller.group;
+      "distributed_billing" = cfg.distributedPoller.distributedBilling;
+      "distributed_poller_memcached_host" = cfg.distributedPoller.memcachedHost;
+      "distributed_poller_memcached_port" = cfg.distributedPoller.memcachedPort;
+      "rrdcached" =
+        "${cfg.distributedPoller.rrdcachedHost}:${toString cfg.distributedPoller.rrdcachedPort}";
+    })
+    // (lib.optionalAttrs cfg.useDistributedPollers {
+      "distributed_poller" = true;
+      # still enable a local poller with distributed polling
+      "distributed_poller_group" = lib.mkDefault "0";
+      "distributed_billing" = lib.mkDefault true;
+      "distributed_poller_memcached_host" = "localhost";
+      "distributed_poller_memcached_port" = 11211;
+      "rrdcached" = "localhost:42217";
+    });
+
+    services.memcached = lib.mkIf cfg.useDistributedPollers {
+      enable = true;
+      listen = "0.0.0.0";
+    };
+
+    systemd.services.rrdcached = lib.mkIf cfg.useDistributedPollers {
+      description = "rrdcached";
+      after = [ "librenms-setup.service" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = "forking";
+        User = cfg.user;
+        Group = cfg.group;
+        LimitNOFILE = 16384;
+        RuntimeDirectory = "rrdcached";
+        PidFile = "/run/rrdcached/rrdcached.pid";
+        # rrdcached params from https://docs.librenms.org/Extensions/Distributed-Poller/#config-sample
+        ExecStart = "${pkgs.rrdtool}/bin/rrdcached -l 0:42217 -R -j ${cfg.dataDir}/rrdcached-journal/ -F -b ${cfg.dataDir}/rrd -B -w 1800 -z 900 -p /run/rrdcached/rrdcached.pid";
+      };
+    };
+
+    services.mysql = lib.mkIf cfg.database.createLocally {
+      enable = true;
+      package = lib.mkDefault pkgs.mariadb;
+      settings.mysqld = {
+        innodb_file_per_table = 1;
+        lower_case_table_names = 0;
+      }
+      // (lib.optionalAttrs cfg.useDistributedPollers {
+        bind-address = "0.0.0.0";
+      });
+      ensureDatabases = [ cfg.database.database ];
+      ensureUsers = [
+        {
+          name = cfg.database.username;
+          ensurePermissions = {
+            "${cfg.database.database}.*" = "ALL PRIVILEGES";
+          };
+        }
+      ];
+      initialScript = lib.mkIf cfg.useDistributedPollers (
+        pkgs.writeText "mysql-librenms-init" ''
+          CREATE USER IF NOT EXISTS '${cfg.database.username}'@'%';
+          GRANT ALL PRIVILEGES ON ${cfg.database.database}.* TO '${cfg.database.username}'@'%';
+        ''
+      );
+    };
+
+    services.nginx = lib.mkIf (!cfg.distributedPoller.enable) {
+      enable = true;
+      virtualHosts."${cfg.hostname}" = lib.mkMerge [
+        cfg.nginx
+        {
+          root = lib.mkForce "${package}/html";
+          locations."/" = {
+            index = "index.php";
+            tryFiles = "$uri $uri/ /index.php?$query_string";
+          };
+          locations."~ .php$".extraConfig = ''
+            fastcgi_pass unix:${config.services.phpfpm.pools."librenms".socket};
+            fastcgi_split_path_info ^(.+\.php)(/.+)$;
+          '';
+        }
+      ];
+    };
+
+    services.phpfpm.pools.librenms = lib.mkIf (!cfg.distributedPoller.enable) {
+      user = cfg.user;
+      group = cfg.group;
+      inherit (package) phpPackage;
+      inherit phpOptions;
+      settings = {
+        "listen.mode" = "0660";
+        "listen.owner" = config.services.nginx.user;
+        "listen.group" = config.services.nginx.group;
+      }
+      // cfg.poolConfig;
+    };
+
+    systemd.services.librenms-scheduler = {
+      description = "LibreNMS Scheduler";
+      path = [ pkgs.unixtools.whereis ];
+      serviceConfig = {
+        Type = "oneshot";
+        WorkingDirectory = package;
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStart = "${artisanWrapper}/bin/librenms-artisan schedule:run";
+      };
+    };
+
+    systemd.timers.librenms-scheduler = {
+      description = "LibreNMS Scheduler";
+      wantedBy = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "minutely";
+        AccuracySec = "1second";
+      };
+    };
+
+    systemd.services.librenms-setup = {
+      description = "Preparation tasks for LibreNMS";
+      before = [ "phpfpm-librenms.service" ];
+      after = [
+        "systemd-tmpfiles-setup.service"
+        "network.target"
+      ]
+      ++ (lib.optional (cfg.database.host == "localhost") "mysql.service");
+      wantedBy = [ "multi-user.target" ];
+      restartTriggers = [
+        package
+        configFile
+      ];
+      path = [
+        pkgs.mariadb
+        pkgs.unixtools.whereis
+        pkgs.gnused
+      ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        EnvironmentFile = lib.mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+        User = cfg.user;
+        Group = cfg.group;
+        ExecStartPre = lib.mkIf cfg.database.createLocally [
+          "!${
+            pkgs.writeShellScript "librenms-db-init" (
+              if !isNull cfg.database.socket then
+                ''
+                  echo "ALTER USER '${cfg.database.username}'@'localhost' IDENTIFIED VIA unix_socket;" | ${pkgs.mariadb}/bin/mysql --socket='${cfg.database.socket}'
+                  ${lib.optionalString cfg.useDistributedPollers ''
+                    echo "ALTER USER '${cfg.database.username}'@'%' IDENTIFIED VIA unix_socket;" | ${pkgs.mariadb}/bin/mysql --socket='${cfg.database.socket}'
+                  ''}
+                ''
+              else
+                ''
+                  DB_PASSWORD=$(cat ${cfg.database.passwordFile} | tr -d '\n')
+                  echo "ALTER USER '${cfg.database.username}'@'localhost' IDENTIFIED BY '$DB_PASSWORD';" | ${pkgs.mariadb}/bin/mysql
+                  ${lib.optionalString cfg.useDistributedPollers ''
+                    echo "ALTER USER '${cfg.database.username}'@'%' IDENTIFIED BY '$DB_PASSWORD';" | ${pkgs.mariadb}/bin/mysql
+                  ''}
+                ''
+            )
+          }"
+        ];
+      };
+      script = ''
+        set -euo pipefail
+
+        # config setup
+        ln -sf ${configFile} ${cfg.dataDir}/config.php
+        ${pkgs.envsubst}/bin/envsubst -i ${configJson} -o ${cfg.dataDir}/config.json
+        export PHPRC=${phpIni}
+
+        INIT=false
+        if [[ ! -s ${cfg.dataDir}/.env ]]; then
+          INIT=true
+          # init .env file
+          echo "APP_KEY=" > ${cfg.dataDir}/.env
+          ${artisanWrapper}/bin/librenms-artisan key:generate --ansi
+          ${artisanWrapper}/bin/librenms-artisan webpush:vapid
+          echo "" >> ${cfg.dataDir}/.env
+          echo -n "NODE_ID=" >> ${cfg.dataDir}/.env
+          ${package.phpPackage}/bin/php -r "echo uniqid();" >> ${cfg.dataDir}/.env
+          echo "" >> ${cfg.dataDir}/.env
+        else
+          # .env file already exists --> only update database and cache config
+          ${pkgs.gnused}/bin/sed -i /^DB_/d ${cfg.dataDir}/.env
+          ${pkgs.gnused}/bin/sed -i /^CACHE_DRIVER/d ${cfg.dataDir}/.env
+        fi
+        ${lib.optionalString (cfg.useDistributedPollers || cfg.distributedPoller.enable) ''
+          echo "CACHE_DRIVER=memcached" >> ${cfg.dataDir}/.env
+        ''}
+        echo "DB_DATABASE=${cfg.database.database}" >> ${cfg.dataDir}/.env
+      ''
+      + (
+        if !isNull cfg.database.socket then
+          ''
+            # use socket connection
+            echo "DB_SOCKET=${cfg.database.socket}" >> ${cfg.dataDir}/.env
+            echo "DB_PASSWORD=null" >> ${cfg.dataDir}/.env
+          ''
+        else
+          ''
+            # use TCP connection
+            echo "DB_HOST=${cfg.database.host}" >> ${cfg.dataDir}/.env
+            echo "DB_PORT=${toString cfg.database.port}" >> ${cfg.dataDir}/.env
+            echo "DB_USERNAME=${cfg.database.username}" >> ${cfg.dataDir}/.env
+            echo -n "DB_PASSWORD=" >> ${cfg.dataDir}/.env
+            cat ${cfg.database.passwordFile} >> ${cfg.dataDir}/.env
+          ''
+      )
+      + ''
+        # clear cache if package has changed (cache may contain cached paths
+        # to the old package)
+        OLD_PACKAGE=$(cat ${cfg.dataDir}/package)
+        if [[ $OLD_PACKAGE != "${package}" ]]; then
+          rm -r ${cfg.dataDir}/cache/*
+        fi
+
+        # convert rrd files when the oneMinutePolling option is changed
+        OLD_ENABLED=$(cat ${cfg.dataDir}/one_minute_enabled)
+        if [[ $OLD_ENABLED != "${lib.boolToString cfg.enableOneMinutePolling}" ]]; then
+          ${package}/scripts/rrdstep.php -h all
+          echo "${lib.boolToString cfg.enableOneMinutePolling}" > ${cfg.dataDir}/one_minute_enabled
+        fi
+
+        # migrate db if package version has changed
+        # not necessary for every package change
+        OLD_VERSION=$(cat ${cfg.dataDir}/version)
+        if [[ $OLD_VERSION != "${package.version}" ]]; then
+          ${artisanWrapper}/bin/librenms-artisan migrate --force --no-interaction
+          echo "${package.version}" > ${cfg.dataDir}/version
+        fi
+
+        if [[ $INIT == "true" ]]; then
+          ${artisanWrapper}/bin/librenms-artisan db:seed --force --no-interaction
+        fi
+
+        # regenerate cache if package has changed
+        if [[ $OLD_PACKAGE != "${package}" ]]; then
+          ${artisanWrapper}/bin/librenms-artisan view:clear
+          ${artisanWrapper}/bin/librenms-artisan optimize:clear
+          ${artisanWrapper}/bin/librenms-artisan view:cache
+          ${artisanWrapper}/bin/librenms-artisan optimize
+          echo "${package}" > ${cfg.dataDir}/package
+        fi
+      '';
+    };
+
+    programs.mtr.enable = true;
+
+    services.logrotate = {
+      enable = true;
+      settings."${cfg.logDir}/librenms.log" = {
+        su = "${cfg.user} ${cfg.group}";
+        create = "0640 ${cfg.user} ${cfg.group}";
+        rotate = 6;
+        frequency = "weekly";
+        compress = true;
+        delaycompress = true;
+        missingok = true;
+        notifempty = true;
+      };
+    };
+
+    services.cron = {
+      enable = true;
+      systemCronJobs =
+        let
+          env = "PHPRC=${phpIni}";
+        in
+        [
+          # based on crontab provided by LibreNMS
+          "33 */6 * * * ${cfg.user} ${env} ${package}/cronic ${package}/discovery-wrapper.py 1"
+          "*/5 * * * * ${cfg.user} ${env} ${package}/discovery.php -h new >> /dev/null 2>&1"
+
+          "${
+            if cfg.enableOneMinutePolling then "*" else "*/5"
+          } * * * * ${cfg.user} ${env} ${package}/cronic ${package}/poller-wrapper.py ${toString cfg.pollerThreads}"
+          "* * * * * ${cfg.user} ${env} ${package}/alerts.php >> /dev/null 2>&1"
+
+          "*/5 * * * * ${cfg.user} ${env} ${package}/check-services.php >> /dev/null 2>&1"
+
+          # extra: fast ping
+          "* * * * * ${cfg.user} ${env} ${package}/ping.php >> /dev/null 2>&1"
+
+          # daily.sh tasks are split to exclude update
+          "19 0 * * * ${cfg.user} ${env} ${package}/daily.sh cleanup >> /dev/null 2>&1"
+          "19 0 * * * ${cfg.user} ${env} ${package}/daily.sh notifications >> /dev/null 2>&1"
+          "19 0 * * * ${cfg.user} ${env} ${package}/daily.sh peeringdb >> /dev/null 2>&1"
+          "19 0 * * * ${cfg.user} ${env} ${package}/daily.sh mac_oui >> /dev/null 2>&1"
+        ]
+        ++ lib.optionals cfg.enableLocalBilling [
+          "*/5 * * * * ${cfg.user} ${env} ${package}/poll-billing.php >> /dev/null 2>&1"
+          "01 * * * * ${cfg.user} ${env} ${package}/billing-calculate.php >> /dev/null 2>&1"
+        ];
+    };
+
+    security.wrappers = {
+      fping = {
+        setuid = true;
+        owner = "root";
+        group = "root";
+        source = "${pkgs.fping}/bin/fping";
+      };
+    };
+
+    environment.systemPackages = [
+      artisanWrapper
+      lnmsWrapper
+    ];
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.logDir}                               0750 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.logDir}/librenms.log                  0640 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}                              0750 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.dataDir}/.env                         0600 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.dataDir}/version                      0600 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.dataDir}/package                      0600 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.dataDir}/one_minute_enabled           0600 ${cfg.user} ${cfg.group} - -"
+      "f ${cfg.dataDir}/config.json                  0600 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage                      0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/app                  0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/debugbar             0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/framework            0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/framework/cache      0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/framework/sessions   0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/framework/views      0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/storage/logs                 0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/rrd                          0700 ${cfg.user} ${cfg.group} - -"
+      "d ${cfg.dataDir}/cache                        0700 ${cfg.user} ${cfg.group} - -"
+    ]
+    ++ lib.optionals cfg.useDistributedPollers [
+      "d ${cfg.dataDir}/rrdcached-journal            0700 ${cfg.user} ${cfg.group} - -"
+    ];
+
+  };
+
+  meta.maintainers = with lib.maintainers; [ netali ] ++ lib.teams.wdz.members;
+}

nixos/modules/services/monitoring/loki.nix

@@ -0,0 +1,157 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    escapeShellArgs
+    mkEnableOption
+    mkIf
+    mkOption
+    types
+    ;
+
+  cfg = config.services.loki;
+
+  prettyJSON =
+    conf:
+    pkgs.runCommand "loki-config.json" { } ''
+      echo '${builtins.toJSON conf}' | ${pkgs.jq}/bin/jq 'del(._module)' > $out
+    '';
+
+in
+{
+  options.services.loki = {
+    enable = mkEnableOption "Grafana Loki";
+
+    user = mkOption {
+      type = types.str;
+      default = "loki";
+      description = ''
+        User under which the Loki service runs.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "grafana-loki" { };
+
+    group = mkOption {
+      type = types.str;
+      default = "loki";
+      description = ''
+        Group under which the Loki service runs.
+      '';
+    };
+
+    dataDir = mkOption {
+      type = types.path;
+      default = "/var/lib/loki";
+      description = ''
+        Specify the data directory for Loki.
+      '';
+    };
+
+    configuration = mkOption {
+      type = (pkgs.formats.json { }).type;
+      default = { };
+      description = ''
+        Specify the configuration for Loki in Nix.
+
+        See [documentation of Grafana Loki](https://grafana.com/docs/loki/latest/configure/) for all available options.
+
+        Cannot be specified together with {option}`services.loki.configFile`.
+      '';
+    };
+
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Specify a configuration file that Loki should use.
+
+        Cannot be specified together with {option}`services.loki.configuration`.
+      '';
+    };
+
+    extraFlags = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "--server.http-listen-port=3101" ];
+      description = ''
+        Specify a list of additional command line flags,
+        which get escaped and are then passed to Loki.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = (
+          (cfg.configuration == { } -> cfg.configFile != null)
+          && (cfg.configFile != null -> cfg.configuration == { })
+        );
+        message = ''
+          Please specify either
+          'services.loki.configuration' or
+          'services.loki.configFile'.
+        '';
+      }
+    ];
+
+    environment.systemPackages = [ cfg.package ]; # logcli
+
+    users.groups.${cfg.group} = { };
+    users.users.${cfg.user} = {
+      description = "Loki Service User";
+      group = cfg.group;
+      home = cfg.dataDir;
+      createHome = true;
+      isSystemUser = true;
+    };
+
+    systemd.services.loki = {
+      description = "Loki Service Daemon";
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig =
+        let
+          conf =
+            if cfg.configFile == null then
+              # Config validation may fail when using extraFlags = [ "-config.expand-env=true" ].
+              # To work around this, we simply skip it when extraFlags is not empty.
+              if cfg.extraFlags == [ ] then
+                validateConfig (prettyJSON cfg.configuration)
+              else
+                prettyJSON cfg.configuration
+            else
+              cfg.configFile;
+          validateConfig =
+            file:
+            pkgs.runCommand "validate-loki-conf"
+              {
+                nativeBuildInputs = [ cfg.package ];
+              }
+              ''
+                loki -verify-config -config.file "${file}"
+                ln -s "${file}" "$out"
+              '';
+        in
+        {
+          ExecStart = "${cfg.package}/bin/loki --config.file=${conf} ${escapeShellArgs cfg.extraFlags}";
+          User = cfg.user;
+          Restart = "always";
+          PrivateTmp = true;
+          ProtectHome = true;
+          ProtectSystem = "full";
+          DevicePolicy = "closed";
+          NoNewPrivileges = true;
+          WorkingDirectory = cfg.dataDir;
+        };
+    };
+  };
+}

nixos/modules/services/monitoring/longview.nix

@@ -0,0 +1,183 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.longview;
+
+  runDir = "/run/longview";
+  configsDir = "${runDir}/longview.d";
+
+in
+{
+  options = {
+
+    services.longview = {
+
+      enable = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          If enabled, system metrics will be sent to Linode LongView.
+        '';
+      };
+
+      apiKey = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "01234567-89AB-CDEF-0123456789ABCDEF";
+        description = ''
+          Longview API key. To get this, look in Longview settings which
+          are found at <https://manager.linode.com/longview/>.
+
+          Warning: this secret is stored in the world-readable Nix store!
+          Use {option}`apiKeyFile` instead.
+        '';
+      };
+
+      apiKeyFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        example = "/run/keys/longview-api-key";
+        description = ''
+          A file containing the Longview API key.
+          To get this, look in Longview settings which
+          are found at <https://manager.linode.com/longview/>.
+
+          {option}`apiKeyFile` takes precedence over {option}`apiKey`.
+        '';
+      };
+
+      apacheStatusUrl = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "http://127.0.0.1/server-status";
+        description = ''
+          The Apache status page URL. If provided, Longview will
+          gather statistics from this location. This requires Apache
+          mod_status to be loaded and enabled.
+        '';
+      };
+
+      nginxStatusUrl = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        example = "http://127.0.0.1/nginx_status";
+        description = ''
+          The Nginx status page URL. Longview will gather statistics
+          from this URL. This requires the Nginx stub_status module to
+          be enabled and configured at the given location.
+        '';
+      };
+
+      mysqlUser = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = ''
+          The user for connecting to the MySQL database. If provided,
+          Longview will connect to MySQL and collect statistics about
+          queries, etc. This user does not need to have been granted
+          any extra privileges.
+        '';
+      };
+
+      mysqlPassword = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = ''
+          The password corresponding to {option}`mysqlUser`.
+          Warning: this is stored in cleartext in the Nix store!
+          Use {option}`mysqlPasswordFile` instead.
+        '';
+      };
+
+      mysqlPasswordFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        example = "/run/keys/dbpassword";
+        description = ''
+          A file containing the password corresponding to {option}`mysqlUser`.
+        '';
+      };
+
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.longview = {
+      description = "Longview Metrics Collection";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig.Type = "forking";
+      serviceConfig.ExecStop = "-${pkgs.coreutils}/bin/kill -TERM $MAINPID";
+      serviceConfig.ExecReload = "-${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      serviceConfig.PIDFile = "${runDir}/longview.pid";
+      serviceConfig.ExecStart = "${pkgs.longview}/bin/longview";
+      preStart = ''
+        umask 077
+        mkdir -p ${configsDir}
+      ''
+      + (lib.optionalString (cfg.apiKeyFile != null) ''
+        cp --no-preserve=all "${cfg.apiKeyFile}" ${runDir}/longview.key
+      '')
+      + (lib.optionalString (cfg.apacheStatusUrl != "") ''
+        cat > ${configsDir}/Apache.conf <<EOF
+        location ${cfg.apacheStatusUrl}?auto
+        EOF
+      '')
+      + (lib.optionalString (cfg.mysqlUser != "" && cfg.mysqlPasswordFile != null) ''
+        cat > ${configsDir}/MySQL.conf <<EOF
+        username ${cfg.mysqlUser}
+        password `head -n1 "${cfg.mysqlPasswordFile}"`
+        EOF
+      '')
+      + (lib.optionalString (cfg.nginxStatusUrl != "") ''
+        cat > ${configsDir}/Nginx.conf <<EOF
+        location ${cfg.nginxStatusUrl}
+        EOF
+      '');
+    };
+
+    warnings =
+      let
+        warn =
+          k: lib.optional (cfg.${k} != "") "config.services.longview.${k} is insecure. Use ${k}File instead.";
+      in
+      lib.concatMap warn [
+        "apiKey"
+        "mysqlPassword"
+      ];
+
+    assertions = [
+      {
+        assertion = cfg.apiKeyFile != null;
+        message = "Longview needs an API key configured";
+      }
+    ];
+
+    # Create API key file if not configured.
+    services.longview.apiKeyFile = lib.mkIf (cfg.apiKey != "") (
+      lib.mkDefault (
+        toString (
+          pkgs.writeTextFile {
+            name = "longview.key";
+            text = cfg.apiKey;
+          }
+        )
+      )
+    );
+
+    # Create MySQL password file if not configured.
+    services.longview.mysqlPasswordFile = lib.mkDefault (
+      toString (
+        pkgs.writeTextFile {
+          name = "mysql-password-file";
+          text = cfg.mysqlPassword;
+        }
+      )
+    );
+  };
+}

nixos/modules/services/monitoring/mackerel-agent.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.mackerel-agent;
+  settingsFmt = pkgs.formats.toml { };
+in
+{
+  options.services.mackerel-agent = {
+    enable = lib.mkEnableOption "mackerel.io agent";
+
+    # the upstream package runs as root, but doesn't seem to be strictly
+    # necessary for basic functionality
+    runAsRoot = lib.mkEnableOption "running as root";
+
+    autoRetirement = lib.mkEnableOption ''
+      retiring the host upon OS shutdown
+    '';
+
+    apiKeyFile = lib.mkOption {
+      type = lib.types.path;
+      example = "/run/keys/mackerel-api-key";
+      description = ''
+        Path to file containing the Mackerel API key. The file should contain a
+        single line of the following form:
+
+        `apikey = "EXAMPLE_API_KEY"`
+      '';
+    };
+
+    settings = lib.mkOption {
+      description = ''
+        Options for mackerel-agent.conf.
+
+        Documentation:
+        <https://mackerel.io/docs/entry/spec/agent>
+      '';
+
+      default = { };
+      example = {
+        verbose = false;
+        silent = false;
+      };
+
+      type = lib.types.submodule {
+        freeformType = settingsFmt.type;
+
+        options.host_status = {
+          on_start = lib.mkOption {
+            type = lib.types.enum [
+              "working"
+              "standby"
+              "maintenance"
+              "poweroff"
+            ];
+            description = "Host status after agent startup.";
+            default = "working";
+          };
+          on_stop = lib.mkOption {
+            type = lib.types.enum [
+              "working"
+              "standby"
+              "maintenance"
+              "poweroff"
+            ];
+            description = "Host status after agent shutdown.";
+            default = "poweroff";
+          };
+        };
+
+        options.diagnostic = lib.mkEnableOption "collecting memory usage for the agent itself";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = with pkgs; [ mackerel-agent ];
+
+    environment.etc = {
+      "mackerel-agent/mackerel-agent.conf".source =
+        settingsFmt.generate "mackerel-agent.conf" cfg.settings;
+      "mackerel-agent/conf.d/api-key.conf".source = cfg.apiKeyFile;
+    };
+
+    services.mackerel-agent.settings = {
+      root = lib.mkDefault "/var/lib/mackerel-agent";
+      pidfile = lib.mkDefault "/run/mackerel-agent/mackerel-agent.pid";
+
+      # conf.d stores the symlink to cfg.apiKeyFile
+      include = lib.mkDefault "/etc/mackerel-agent/conf.d/*.conf";
+    };
+
+    # upstream service file in https://github.com/mackerelio/mackerel-agent/blob/master/packaging/rpm/src/mackerel-agent.service
+    systemd.services.mackerel-agent = {
+      description = "mackerel.io agent";
+      wants = [ "network-online.target" ];
+      after = [
+        "network-online.target"
+        "nss-lookup.target"
+      ];
+      wantedBy = [ "multi-user.target" ];
+      environment = {
+        MACKEREL_PLUGIN_WORKDIR = lib.mkDefault "%C/mackerel-agent";
+      };
+      serviceConfig = {
+        DynamicUser = !cfg.runAsRoot;
+        PrivateTmp = lib.mkDefault true;
+        CacheDirectory = "mackerel-agent";
+        ConfigurationDirectory = "mackerel-agent";
+        RuntimeDirectory = "mackerel-agent";
+        StateDirectory = "mackerel-agent";
+        ExecStart = "${pkgs.mackerel-agent}/bin/mackerel-agent supervise";
+        ExecStopPost = lib.mkIf cfg.autoRetirement "${pkgs.mackerel-agent}/bin/mackerel-agent retire -force";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+        LimitNOFILE = lib.mkDefault 65536;
+        LimitNPROC = lib.mkDefault 65536;
+      };
+      restartTriggers = [
+        config.environment.etc."mackerel-agent/mackerel-agent.conf".source
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/metricbeat.nix

@@ -0,0 +1,168 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    attrValues
+    literalExpression
+    mkEnableOption
+    mkPackageOption
+    mkIf
+    mkOption
+    types
+    ;
+  cfg = config.services.metricbeat;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+in
+{
+  options = {
+
+    services.metricbeat = {
+
+      enable = mkEnableOption "metricbeat";
+
+      package = mkPackageOption pkgs "metricbeat" {
+        example = "metricbeat7";
+      };
+
+      modules = mkOption {
+        description = ''
+          Metricbeat modules are responsible for reading metrics from the various sources.
+
+          This is like `services.metricbeat.settings.metricbeat.modules`,
+          but structured as an attribute set. This has the benefit that multiple
+          NixOS modules can contribute settings to a single metricbeat module.
+
+          A module can be specified multiple times by choosing a different `<name>`
+          for each, but setting [](#opt-services.metricbeat.modules._name_.module) to the same value.
+
+          See <https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html>.
+        '';
+        default = { };
+        type = types.attrsOf (
+          types.submodule (
+            { name, ... }:
+            {
+              freeformType = settingsFormat.type;
+              options = {
+                module = mkOption {
+                  type = types.str;
+                  default = name;
+                  description = ''
+                    The name of the module.
+
+                    Look for the value after `module:` on the individual
+                    module pages linked from <https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html>.
+                  '';
+                };
+              };
+            }
+          )
+        );
+        example = {
+          system = {
+            metricsets = [
+              "cpu"
+              "load"
+              "memory"
+              "network"
+              "process"
+              "process_summary"
+              "uptime"
+              "socket_summary"
+            ];
+            enabled = true;
+            period = "10s";
+            processes = [ ".*" ];
+            cpu.metrics = [
+              "percentages"
+              "normalized_percentages"
+            ];
+            core.metrics = [ "percentages" ];
+          };
+        };
+      };
+
+      settings = mkOption {
+        type = types.submodule {
+          freeformType = settingsFormat.type;
+          options = {
+
+            name = mkOption {
+              type = types.str;
+              default = "";
+              description = ''
+                Name of the beat. Defaults to the hostname.
+                See <https://www.elastic.co/guide/en/beats/metricbeat/current/configuration-general-options.html#_name>.
+              '';
+            };
+
+            tags = mkOption {
+              type = types.listOf types.str;
+              default = [ ];
+              description = ''
+                Tags to place on the shipped metrics.
+                See <https://www.elastic.co/guide/en/beats/metricbeat/current/configuration-general-options.html#_tags_2>.
+              '';
+            };
+
+            metricbeat.modules = mkOption {
+              type = types.listOf settingsFormat.type;
+              default = [ ];
+              internal = true;
+              description = ''
+                The metric collecting modules. Use [](#opt-services.metricbeat.modules) instead.
+
+                See <https://www.elastic.co/guide/en/beats/metricbeat/current/metricbeat-modules.html>.
+              '';
+            };
+          };
+        };
+        default = { };
+        description = ''
+          Configuration for metricbeat. See <https://www.elastic.co/guide/en/beats/metricbeat/current/configuring-howto-metricbeat.html> for supported values.
+        '';
+      };
+
+    };
+  };
+
+  config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        # empty modules would cause a failure at runtime
+        assertion = cfg.settings.metricbeat.modules != [ ];
+        message = "services.metricbeat: You must configure one or more modules.";
+      }
+    ];
+
+    services.metricbeat.settings.metricbeat.modules = attrValues cfg.modules;
+
+    systemd.services.metricbeat = {
+      description = "metricbeat metrics shipper";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/metricbeat \
+            -c ${settingsFormat.generate "metricbeat.yml" cfg.settings} \
+            --path.data $STATE_DIRECTORY \
+            --path.logs $LOGS_DIRECTORY \
+            ;
+        '';
+        Restart = "always";
+        DynamicUser = true;
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+        StateDirectory = "metricbeat";
+        LogsDirectory = "metricbeat";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/mimir.nix

@@ -0,0 +1,97 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    escapeShellArgs
+    mkEnableOption
+    mkPackageOption
+    mkIf
+    mkOption
+    types
+    ;
+
+  cfg = config.services.mimir;
+
+  settingsFormat = pkgs.formats.yaml { };
+in
+{
+  options.services.mimir = {
+    enable = mkEnableOption "mimir";
+
+    configuration = mkOption {
+      type = (pkgs.formats.json { }).type;
+      default = { };
+      description = ''
+        Specify the configuration for Mimir in Nix.
+      '';
+    };
+
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Specify a configuration file that Mimir should use.
+      '';
+    };
+
+    package = mkPackageOption pkgs "mimir" { };
+
+    extraFlags = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "--config.expand-env=true" ];
+      description = ''
+        Specify a list of additional command line flags,
+        which get escaped and are then passed to Mimir.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    # for mimirtool
+    environment.systemPackages = [ cfg.package ];
+
+    assertions = [
+      {
+        assertion = (
+          (cfg.configuration == { } -> cfg.configFile != null)
+          && (cfg.configFile != null -> cfg.configuration == { })
+        );
+        message = ''
+          Please specify either
+          'services.mimir.configuration' or
+          'services.mimir.configFile'.
+        '';
+      }
+    ];
+
+    systemd.services.mimir = {
+      description = "mimir Service Daemon";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig =
+        let
+          conf =
+            if cfg.configFile == null then
+              settingsFormat.generate "config.yaml" cfg.configuration
+            else
+              cfg.configFile;
+        in
+        {
+          ExecStart = "${cfg.package}/bin/mimir --config.file=${conf} ${escapeShellArgs cfg.extraFlags}";
+          DynamicUser = true;
+          Restart = "always";
+          ProtectSystem = "full";
+          DevicePolicy = "closed";
+          NoNewPrivileges = true;
+          WorkingDirectory = "/var/lib/mimir";
+          StateDirectory = "mimir";
+        };
+    };
+  };
+}

nixos/modules/services/monitoring/monit.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.monit;
+in
+
+{
+  options.services.monit = {
+
+    enable = lib.mkEnableOption "Monit";
+
+    config = lib.mkOption {
+      type = lib.types.lines;
+      default = "";
+      description = "monitrc content";
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    environment.systemPackages = [ pkgs.monit ];
+
+    environment.etc.monitrc = {
+      text = cfg.config;
+      mode = "0400";
+    };
+
+    systemd.services.monit = {
+      description = "Pro-active monitoring utility for unix systems";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${pkgs.monit}/bin/monit -I -c /etc/monitrc";
+        ExecStop = "${pkgs.monit}/bin/monit -c /etc/monitrc quit";
+        ExecReload = "${pkgs.monit}/bin/monit -c /etc/monitrc reload";
+        KillMode = "process";
+        Restart = "always";
+      };
+      restartTriggers = [ config.environment.etc.monitrc.source ];
+    };
+
+  };
+
+  meta.maintainers = with lib.maintainers; [ ryantm ];
+}

nixos/modules/services/monitoring/munin.nix

@@ -0,0 +1,432 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+# TODO: support munin-async
+# TODO: LWP/Pg perl libs aren't recognized
+# TODO: support fastcgi
+# https://guide.munin-monitoring.org/en/latest/example/webserver/apache-cgi.html
+# spawn-fcgi -s /run/munin/fastcgi-graph.sock -U www-data   -u munin -g munin /usr/lib/munin/cgi/munin-cgi-graph
+# spawn-fcgi -s /run/munin/fastcgi-html.sock  -U www-data   -u munin -g munin /usr/lib/munin/cgi/munin-cgi-html
+# https://paste.sh/vofcctHP#-KbDSXVeWoifYncZmLfZzgum
+# nginx https://munin.readthedocs.org/en/latest/example/webserver/nginx.html
+let
+  nodeCfg = config.services.munin-node;
+  cronCfg = config.services.munin-cron;
+
+  muninConf = pkgs.writeText "munin.conf" ''
+    dbdir     /var/lib/munin
+    htmldir   /var/www/munin
+    logdir    /var/log/munin
+    rundir    /run/munin
+
+    ${lib.optionalString (cronCfg.extraCSS != "") "staticdir ${customStaticDir}"}
+
+    ${cronCfg.extraGlobalConfig}
+
+    ${cronCfg.hosts}
+  '';
+
+  nodeConf = pkgs.writeText "munin-node.conf" ''
+    log_level 3
+    log_file Sys::Syslog
+    port 4949
+    host *
+    background 0
+    user root
+    group root
+    host_name ${config.networking.hostName}
+    setsid 0
+
+    # wrapped plugins by makeWrapper being with dots
+    ignore_file ^\.
+
+    allow ^::1$
+    allow ^127\.0\.0\.1$
+
+    ${nodeCfg.extraConfig}
+  '';
+
+  pluginConf = pkgs.writeText "munin-plugin-conf" ''
+    [hddtemp_smartctl]
+    user root
+    group root
+
+    [meminfo]
+    user root
+    group root
+
+    [ipmi*]
+    user root
+    group root
+
+    [munin*]
+    env.UPDATE_STATSFILE /var/lib/munin/munin-update.stats
+
+    ${nodeCfg.extraPluginConfig}
+  '';
+
+  pluginConfDir = pkgs.stdenv.mkDerivation {
+    name = "munin-plugin-conf.d";
+    buildCommand = ''
+      mkdir $out
+      ln -s ${pluginConf} $out/nixos-config
+    '';
+  };
+
+  # Copy one Munin plugin into the Nix store with a specific name.
+  # This is suitable for use with plugins going directly into /etc/munin/plugins,
+  # i.e. munin.extraPlugins.
+  internOnePlugin = { name, path }: "cp -a '${path}' '${name}'";
+
+  # Copy an entire tree of Munin plugins into a single directory in the Nix
+  # store, with no renaming. The output is suitable for use with
+  # munin-node-configure --suggest, i.e. munin.extraAutoPlugins.
+  # Note that this flattens the input; this is intentional, as
+  # munin-node-configure won't recurse into subdirectories.
+  internManyPlugins = path: "find '${path}' -type f -perm /a+x -exec cp -a -t . '{}' '+'";
+
+  # Use the appropriate intern-fn to copy the plugins into the store and patch
+  # them afterwards in an attempt to get them to run on NixOS.
+  # This is a bit hairy because we can't just fix shebangs; lots of munin plugins
+  # hardcode paths like /sbin/mount rather than trusting $PATH, so we have to
+  # look for and update those throughout the script. At the same time, if the
+  # plugin comes from a package that is already nixified, we don't want to
+  # rewrite paths like /nix/store/foo/sbin/mount.
+  # For now we make the simplifying assumption that no file will contain lines
+  # which mix store paths and FHS paths, and thus run our substitution only on
+  # lines which do not contain store paths.
+  internAndFixPlugins =
+    name: intern-fn: paths:
+    pkgs.runCommand name { } ''
+      mkdir -p "$out"
+      cd "$out"
+      ${lib.concatStringsSep "\n" (map intern-fn paths)}
+      chmod -R u+w .
+      ${pkgs.findutils}/bin/find . -type f -exec ${pkgs.gnused}/bin/sed -E -i "
+        \%''${NIX_STORE}/%! s,(/usr)?/s?bin/,/run/current-system/sw/bin/,g
+      " '{}' '+'
+    '';
+
+  # TODO: write a derivation for munin-contrib, so that for contrib plugins
+  # you can just refer to them by name rather than needing to include a copy
+  # of munin-contrib in your nixos configuration.
+  extraPluginDir = internAndFixPlugins "munin-extra-plugins.d" internOnePlugin (
+    lib.attrsets.mapAttrsToList (k: v: {
+      name = k;
+      path = v;
+    }) nodeCfg.extraPlugins
+  );
+
+  extraAutoPluginDir =
+    internAndFixPlugins "munin-extra-auto-plugins.d" internManyPlugins
+      nodeCfg.extraAutoPlugins;
+
+  customStaticDir = pkgs.runCommand "munin-custom-static-data" { } ''
+    cp -a "${pkgs.munin}/etc/opt/munin/static" "$out"
+    cd "$out"
+    chmod -R u+w .
+    echo "${cronCfg.extraCSS}" >> style.css
+    echo "${cronCfg.extraCSS}" >> style-new.css
+  '';
+in
+
+{
+
+  options = {
+
+    services.munin-node = {
+
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Enable Munin Node agent. Munin node listens on 0.0.0.0 and
+          by default accepts connections only from 127.0.0.1 for security reasons.
+
+          See <https://guide.munin-monitoring.org/en/latest/architecture/index.html>.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          {file}`munin-node.conf` extra configuration. See
+          <https://guide.munin-monitoring.org/en/latest/reference/munin-node.conf.html>
+        '';
+      };
+
+      extraPluginConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          {file}`plugin-conf.d` extra plugin configuration. See
+          <https://guide.munin-monitoring.org/en/latest/plugin/use.html>
+        '';
+        example = ''
+          [fail2ban_*]
+          user root
+        '';
+      };
+
+      extraPlugins = lib.mkOption {
+        default = { };
+        type = with lib.types; attrsOf path;
+        description = ''
+          Additional Munin plugins to activate. Keys are the name of the plugin
+          symlink, values are the path to the underlying plugin script. You
+          can use the same plugin script multiple times (e.g. for wildcard
+          plugins).
+
+          Note that these plugins do not participate in autoconfiguration. If
+          you want to autoconfigure additional plugins, use
+          {option}`services.munin-node.extraAutoPlugins`.
+
+          Plugins enabled in this manner take precedence over autoconfigured
+          plugins.
+
+          Plugins will be copied into the Nix store, and it will attempt to
+          modify them to run properly by fixing hardcoded references to
+          `/bin`, `/usr/bin`,
+          `/sbin`, and `/usr/sbin`.
+        '';
+        example = lib.literalExpression ''
+          {
+            zfs_usage_bigpool = /src/munin-contrib/plugins/zfs/zfs_usage_;
+            zfs_usage_smallpool = /src/munin-contrib/plugins/zfs/zfs_usage_;
+            zfs_list = /src/munin-contrib/plugins/zfs/zfs_list;
+          };
+        '';
+      };
+
+      extraAutoPlugins = lib.mkOption {
+        default = [ ];
+        type = with lib.types; listOf path;
+        description = ''
+          Additional Munin plugins to autoconfigure, using
+          `munin-node-configure --suggest`. These should be
+          the actual paths to the plugin files (or directories containing them),
+          not just their names.
+
+          If you want to manually enable individual plugins instead, use
+          {option}`services.munin-node.extraPlugins`.
+
+          Note that only plugins that have the 'autoconfig' capability will do
+          anything if listed here, since plugins that cannot autoconfigure
+          won't be automatically enabled by
+          `munin-node-configure`.
+
+          Plugins will be copied into the Nix store, and it will attempt to
+          modify them to run properly by fixing hardcoded references to
+          `/bin`, `/usr/bin`,
+          `/sbin`, and `/usr/sbin`.
+        '';
+        example = lib.literalExpression ''
+          [
+            /src/munin-contrib/plugins/zfs
+            /src/munin-contrib/plugins/ssh
+          ];
+        '';
+      };
+
+      disabledPlugins = lib.mkOption {
+        # TODO: figure out why Munin isn't writing the log file and fix it.
+        # In the meantime this at least suppresses a useless graph full of
+        # NaNs in the output.
+        default = [ "munin_stats" ];
+        type = with lib.types; listOf str;
+        description = ''
+          Munin plugins to disable, even if
+          `munin-node-configure --suggest` tries to enable
+          them. To disable a wildcard plugin, use an actual wildcard, as in
+          the example.
+
+          munin_stats is disabled by default as it tries to read
+          `/var/log/munin/munin-update.log` for timing
+          information, and the NixOS build of Munin does not write this file.
+        '';
+        example = [
+          "diskstats"
+          "zfs_usage_*"
+        ];
+      };
+    };
+
+    services.munin-cron = {
+
+      enable = lib.mkOption {
+        default = false;
+        type = lib.types.bool;
+        description = ''
+          Enable munin-cron. Takes care of all heavy lifting to collect data from
+          nodes and draws graphs to html. Runs munin-update, munin-limits,
+          munin-graphs and munin-html in that order.
+
+          HTML output is in {file}`/var/www/munin/`, configure your
+          favourite webserver to serve static files.
+        '';
+      };
+
+      extraGlobalConfig = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          {file}`munin.conf` extra global configuration.
+          See <https://guide.munin-monitoring.org/en/latest/reference/munin.conf.html>.
+          Useful to setup notifications, see
+          <https://guide.munin-monitoring.org/en/latest/tutorial/alert.html>
+        '';
+        example = ''
+          contact.email.command mail -s "Munin notification for ''${var:host}" someone@example.com
+        '';
+      };
+
+      hosts = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Definitions of hosts of nodes to collect data from. Needs at least one
+          host for cron to succeed. See
+          <https://guide.munin-monitoring.org/en/latest/reference/munin.conf.html>
+        '';
+        example = lib.literalExpression ''
+          '''
+            [''${config.networking.hostName}]
+            address localhost
+          '''
+        '';
+      };
+
+      extraCSS = lib.mkOption {
+        default = "";
+        type = lib.types.lines;
+        description = ''
+          Custom styling for the HTML that munin-cron generates. This will be
+          appended to the CSS files used by munin-cron and will thus take
+          precedence over the builtin styles.
+        '';
+        example = ''
+          /* A simple dark theme. */
+          html, body { background: #222222; }
+          #header, #footer { background: #333333; }
+          img.i, img.iwarn, img.icrit, img.iunkn {
+            filter: invert(100%) hue-rotate(-30deg);
+          }
+        '';
+      };
+
+    };
+
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf (nodeCfg.enable || cronCfg.enable) {
+
+      environment.systemPackages = [ pkgs.munin ];
+
+      users.users.munin = {
+        description = "Munin monitoring user";
+        group = "munin";
+        uid = config.ids.uids.munin;
+        home = "/var/lib/munin";
+      };
+
+      users.groups.munin = {
+        gid = config.ids.gids.munin;
+      };
+
+    })
+    (lib.mkIf nodeCfg.enable {
+
+      systemd.services.munin-node = {
+        description = "Munin Node";
+        after = [ "network.target" ];
+        wantedBy = [ "multi-user.target" ];
+        path = with pkgs; [
+          munin
+          smartmontools
+          "/run/current-system/sw"
+          "/run/wrappers"
+        ];
+        environment.MUNIN_LIBDIR = "${pkgs.munin}/lib";
+        environment.MUNIN_PLUGSTATE = "/run/munin";
+        environment.MUNIN_LOGDIR = "/var/log/munin";
+        preStart = ''
+          echo "Updating munin plugins..."
+
+          mkdir -p /etc/munin/plugins
+          rm -rf /etc/munin/plugins/*
+
+          # Autoconfigure builtin plugins
+          ${pkgs.munin}/bin/munin-node-configure --suggest --shell --families contrib,auto,manual --config ${nodeConf} --libdir=${pkgs.munin}/lib/plugins --servicedir=/etc/munin/plugins --sconfdir=${pluginConfDir} 2>/dev/null | ${pkgs.bash}/bin/bash
+
+          # Autoconfigure extra plugins
+          ${pkgs.munin}/bin/munin-node-configure --suggest --shell --families contrib,auto,manual --config ${nodeConf} --libdir=${extraAutoPluginDir} --servicedir=/etc/munin/plugins --sconfdir=${pluginConfDir} 2>/dev/null | ${pkgs.bash}/bin/bash
+
+          ${lib.optionalString (nodeCfg.extraPlugins != { }) ''
+            # Link in manually enabled plugins
+            ln -f -s -t /etc/munin/plugins ${extraPluginDir}/*
+          ''}
+
+          ${lib.optionalString (nodeCfg.disabledPlugins != [ ]) ''
+            # Disable plugins
+            cd /etc/munin/plugins
+            rm -f ${toString nodeCfg.disabledPlugins}
+          ''}
+        '';
+        serviceConfig = {
+          ExecStart = "${pkgs.munin}/sbin/munin-node --config ${nodeConf} --servicedir /etc/munin/plugins/ --sconfdir=${pluginConfDir}";
+        };
+      };
+
+      # munin_stats plugin breaks as of 2.0.33 when this doesn't exist
+      systemd.tmpfiles.settings."10-munin"."/run/munin".d = {
+        mode = "0755";
+        user = "munin";
+        group = "munin";
+      };
+
+    })
+    (lib.mkIf cronCfg.enable {
+
+      # Munin is hardcoded to use DejaVu Mono and the graphs come out wrong if
+      # it's not available.
+      fonts.packages = [ pkgs.dejavu_fonts ];
+
+      systemd.timers.munin-cron = {
+        description = "batch Munin master programs";
+        wantedBy = [ "timers.target" ];
+        timerConfig.OnCalendar = "*:0/5";
+      };
+
+      systemd.services.munin-cron = {
+        description = "batch Munin master programs";
+        unitConfig.Documentation = "man:munin-cron(8)";
+
+        serviceConfig = {
+          Type = "oneshot";
+          User = "munin";
+          ExecStart = "${pkgs.munin}/bin/munin-cron --config ${muninConf}";
+        };
+      };
+
+      systemd.tmpfiles.settings."20-munin" =
+        let
+          defaultConfig = {
+            mode = "0755";
+            user = "munin";
+            group = "munin";
+          };
+        in
+        {
+          "/run/munin".d = defaultConfig;
+          "/var/log/munin".d = defaultConfig;
+          "/var/www/munin".d = defaultConfig;
+          "/var/lib/munin".d = defaultConfig;
+        };
+    })
+  ];
+}

nixos/modules/services/monitoring/nagios.nix

@@ -0,0 +1,224 @@
+# Nagios system/network monitoring daemon.
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.nagios;
+
+  nagiosState = "/var/lib/nagios";
+  nagiosLogDir = "/var/log/nagios";
+  urlPath = "/nagios";
+
+  nagiosObjectDefs = cfg.objectDefs;
+
+  nagiosObjectDefsDir = pkgs.runCommand "nagios-objects" {
+    inherit nagiosObjectDefs;
+    preferLocalBuild = true;
+  } "mkdir -p $out; ln -s $nagiosObjectDefs $out/";
+
+  nagiosCfgFile =
+    let
+      default = {
+        log_file = "${nagiosLogDir}/current";
+        log_archive_path = "${nagiosLogDir}/archive";
+        status_file = "${nagiosState}/status.dat";
+        object_cache_file = "${nagiosState}/objects.cache";
+        temp_file = "${nagiosState}/nagios.tmp";
+        lock_file = "/run/nagios.lock";
+        state_retention_file = "${nagiosState}/retention.dat";
+        query_socket = "${nagiosState}/nagios.qh";
+        check_result_path = "${nagiosState}";
+        command_file = "${nagiosState}/nagios.cmd";
+        cfg_dir = "${nagiosObjectDefsDir}";
+        nagios_user = "nagios";
+        nagios_group = "nagios";
+        illegal_macro_output_chars = "`~$&|'\"<>";
+        retain_state_information = "1";
+      };
+      lines = lib.mapAttrsToList (key: value: "${key}=${value}") (default // cfg.extraConfig);
+      content = lib.concatStringsSep "\n" lines;
+      file = pkgs.writeText "nagios.cfg" content;
+      validated = pkgs.runCommand "nagios-checked.cfg" { preferLocalBuild = true; } ''
+        cp ${file} nagios.cfg
+        # nagios checks the existence of /var/lib/nagios, but
+        # it does not exist in the build sandbox, so we fake it
+        mkdir lib
+        lib=$(readlink -f lib)
+        sed -i s@=${nagiosState}@=$lib@ nagios.cfg
+        ${pkgs.nagios}/bin/nagios -v nagios.cfg && cp ${file} $out
+      '';
+      defaultCfgFile = if cfg.validateConfig then validated else file;
+    in
+    if cfg.mainConfigFile == null then defaultCfgFile else cfg.mainConfigFile;
+
+  # Plain configuration for the Nagios web-interface with no
+  # authentication.
+  nagiosCGICfgFile = pkgs.writeText "nagios.cgi.conf" ''
+    main_config_file=${cfg.mainConfigFile}
+    use_authentication=0
+    url_html_path=${urlPath}
+  '';
+
+  extraHttpdConfig = ''
+    ScriptAlias ${urlPath}/cgi-bin ${pkgs.nagios}/sbin
+
+    <Directory "${pkgs.nagios}/sbin">
+      Options ExecCGI
+      Require all granted
+      SetEnv NAGIOS_CGI_CONFIG ${cfg.cgiConfigFile}
+    </Directory>
+
+    Alias ${urlPath} ${pkgs.nagios}/share
+
+    <Directory "${pkgs.nagios}/share">
+      Options None
+      Require all granted
+    </Directory>
+  '';
+
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "services"
+      "nagios"
+      "urlPath"
+    ] "The urlPath option has been removed as it is hard coded to /nagios in the nagios package.")
+  ];
+
+  meta.maintainers = with lib.maintainers; [ symphorien ];
+
+  options = {
+    services.nagios = {
+      enable = lib.mkEnableOption ''[Nagios](https://www.nagios.org/) to monitor your system or network'';
+
+      objectDefs = lib.mkOption {
+        description = ''
+          A list of Nagios object configuration files that must define
+          the hosts, host groups, services and contacts for the
+          network that you want Nagios to monitor.
+        '';
+        type = lib.types.listOf lib.types.path;
+        example = lib.literalExpression "[ ./objects.cfg ]";
+      };
+
+      plugins = lib.mkOption {
+        type = lib.types.listOf lib.types.package;
+        default = with pkgs; [
+          monitoring-plugins
+          msmtp
+          mailutils
+        ];
+        defaultText = lib.literalExpression "[pkgs.monitoring-plugins pkgs.msmtp pkgs.mailutils]";
+        description = ''
+          Packages to be added to the Nagios {env}`PATH`.
+          Typically used to add plugins, but can be anything.
+        '';
+      };
+
+      mainConfigFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.package;
+        default = null;
+        description = ''
+          If non-null, overrides the main configuration file of Nagios.
+        '';
+      };
+
+      extraConfig = lib.mkOption {
+        type = lib.types.attrsOf lib.types.str;
+        example = {
+          debug_level = "-1";
+          debug_file = "/var/log/nagios/debug.log";
+        };
+        default = { };
+        description = "Configuration to add to /etc/nagios.cfg";
+      };
+
+      validateConfig = lib.mkOption {
+        type = lib.types.bool;
+        default = pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform;
+        defaultText = lib.literalExpression "pkgs.stdenv.hostPlatform == pkgs.stdenv.buildPlatform";
+        description = "if true, the syntax of the nagios configuration file is checked at build time";
+      };
+
+      cgiConfigFile = lib.mkOption {
+        type = lib.types.package;
+        default = nagiosCGICfgFile;
+        defaultText = lib.literalExpression "nagiosCGICfgFile";
+        description = ''
+          Derivation for the configuration file of Nagios CGI scripts
+          that can be used in web servers for running the Nagios web interface.
+        '';
+      };
+
+      enableWebInterface = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to enable the Nagios web interface.  You should also
+          enable Apache ({option}`services.httpd.enable`).
+        '';
+      };
+
+      virtualHost = lib.mkOption {
+        type = lib.types.submodule (import ../web-servers/apache-httpd/vhost-options.nix);
+        example = lib.literalExpression ''
+          { hostName = "example.org";
+            adminAddr = "webmaster@example.org";
+            enableSSL = true;
+            sslServerCert = "/var/lib/acme/example.org/full.pem";
+            sslServerKey = "/var/lib/acme/example.org/key.pem";
+          }
+        '';
+        description = ''
+          Apache configuration can be done by adapting {option}`services.httpd.virtualHosts`.
+          See [](#opt-services.httpd.virtualHosts) for further information.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    users.users.nagios = {
+      description = "Nagios user";
+      uid = config.ids.uids.nagios;
+      home = nagiosState;
+      group = "nagios";
+    };
+
+    users.groups.nagios = { };
+
+    # This isn't needed, it's just so that the user can type "nagiostats
+    # -c /etc/nagios.cfg".
+    environment.etc."nagios.cfg".source = nagiosCfgFile;
+
+    environment.systemPackages = [ pkgs.nagios ];
+    systemd.services.nagios = {
+      description = "Nagios monitoring daemon";
+      path = [ pkgs.nagios ] ++ cfg.plugins;
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      restartTriggers = [ nagiosCfgFile ];
+
+      serviceConfig = {
+        User = "nagios";
+        Group = "nagios";
+        Restart = "always";
+        RestartSec = 2;
+        LogsDirectory = "nagios";
+        StateDirectory = "nagios";
+        ExecStart = "${pkgs.nagios}/bin/nagios /etc/nagios.cfg";
+      };
+    };
+
+    services.httpd.virtualHosts = lib.optionalAttrs cfg.enableWebInterface {
+      ${cfg.virtualHost.hostName} = lib.mkMerge [
+        cfg.virtualHost
+        { extraConfig = extraHttpdConfig; }
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/netdata.nix

@@ -0,0 +1,524 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.netdata;
+
+  wrappedPlugins = pkgs.runCommand "wrapped-plugins" { preferLocalBuild = true; } ''
+    mkdir -p $out/libexec/netdata/plugins.d
+    ln -s /run/wrappers/bin/apps.plugin $out/libexec/netdata/plugins.d/apps.plugin
+    ln -s /run/wrappers/bin/cgroup-network $out/libexec/netdata/plugins.d/cgroup-network
+    ln -s /run/wrappers/bin/perf.plugin $out/libexec/netdata/plugins.d/perf.plugin
+    ln -s /run/wrappers/bin/slabinfo.plugin $out/libexec/netdata/plugins.d/slabinfo.plugin
+    ln -s /run/wrappers/bin/freeipmi.plugin $out/libexec/netdata/plugins.d/freeipmi.plugin
+    ln -s /run/wrappers/bin/systemd-journal.plugin $out/libexec/netdata/plugins.d/systemd-journal.plugin
+    ln -s /run/wrappers/bin/logs-management.plugin $out/libexec/netdata/plugins.d/logs-management.plugin
+    ln -s /run/wrappers/bin/network-viewer.plugin $out/libexec/netdata/plugins.d/network-viewer.plugin
+    ln -s /run/wrappers/bin/debugfs.plugin $out/libexec/netdata/plugins.d/debugfs.plugin
+  '';
+
+  plugins = [
+    "${cfg.package}/libexec/netdata/plugins.d"
+    "${wrappedPlugins}/libexec/netdata/plugins.d"
+  ]
+  ++ cfg.extraPluginPaths;
+
+  configDirectory = pkgs.runCommand "netdata-config-d" { } ''
+    mkdir $out
+    ${lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (path: file: ''
+        mkdir -p "$out/$(dirname ${path})"
+        ${if path == "apps_groups.conf" then "cp" else "ln -s"} "${file}" "$out/${path}"
+      '') cfg.configDir
+    )}
+  '';
+
+  localConfig = {
+    global = {
+      "config directory" = "/etc/netdata/conf.d";
+      "plugins directory" = lib.concatStringsSep " " plugins;
+    };
+    web = {
+      "web files owner" = "root";
+      "web files group" = "root";
+    };
+    "plugin:cgroups" = {
+      "script to get cgroup network interfaces" =
+        "${wrappedPlugins}/libexec/netdata/plugins.d/cgroup-network";
+      "use unified cgroups" = "yes";
+    };
+  };
+  mkConfig = lib.generators.toINI { } (lib.recursiveUpdate localConfig cfg.config);
+  configFile = pkgs.writeText "netdata.conf" (
+    if cfg.configText != null then cfg.configText else mkConfig
+  );
+
+  defaultUser = "netdata";
+
+  isThereAnyWireGuardTunnels =
+    config.networking.wireguard.enable
+    || lib.any (
+      c: lib.hasAttrByPath [ "netdevConfig" "Kind" ] c && c.netdevConfig.Kind == "wireguard"
+    ) (builtins.attrValues config.systemd.network.netdevs);
+
+  extraNdsudoPathsEnv = pkgs.buildEnv {
+    name = "netdata-ndsudo-env";
+    paths = cfg.extraNdsudoPackages;
+    pathsToLink = [ "/bin" ];
+  };
+
+in
+{
+  options = {
+    services.netdata = {
+      enable = lib.mkEnableOption "netdata";
+
+      package = lib.mkPackageOption pkgs "netdata" { };
+
+      user = lib.mkOption {
+        type = lib.types.str;
+        default = "netdata";
+        description = "User account under which netdata runs.";
+      };
+
+      group = lib.mkOption {
+        type = lib.types.str;
+        default = "netdata";
+        description = "Group under which netdata runs.";
+      };
+
+      configText = lib.mkOption {
+        type = lib.types.nullOr lib.types.lines;
+        description = "Verbatim netdata.conf, cannot be combined with config.";
+        default = null;
+        example = ''
+          [global]
+          debug log = syslog
+          access log = syslog
+          error log = syslog
+        '';
+      };
+
+      python = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Whether to enable python-based plugins
+          '';
+        };
+        recommendedPythonPackages = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = ''
+            Whether to enable a set of recommended Python plugins
+            by installing extra Python packages.
+          '';
+        };
+        extraPackages = lib.mkOption {
+          type = lib.types.functionTo (lib.types.listOf lib.types.package);
+          default = ps: [ ];
+          defaultText = lib.literalExpression "ps: []";
+          example = lib.literalExpression ''
+            ps: [
+              ps.psycopg2
+              ps.docker
+              ps.dnspython
+            ]
+          '';
+          description = ''
+            Extra python packages available at runtime
+            to enable additional python plugins.
+          '';
+        };
+      };
+
+      extraPluginPaths = lib.mkOption {
+        type = lib.types.listOf lib.types.path;
+        default = [ ];
+        example = lib.literalExpression ''
+          [ "/path/to/plugins.d" ]
+        '';
+        description = ''
+          Extra paths to add to the netdata global "plugins directory"
+          option.  Useful for when you want to include your own
+          collection scripts.
+
+          Details about writing a custom netdata plugin are available at:
+          <https://docs.netdata.cloud/collectors/plugins.d/>
+
+          Cannot be combined with configText.
+        '';
+      };
+
+      extraNdsudoPackages = lib.mkOption {
+        type = lib.types.listOf lib.types.package;
+        default = [ ];
+        description = ''
+          Extra packages to add to `PATH` to make available to `ndsudo`.
+          ::: {.warning}
+          `ndsudo` has SUID privileges, be careful what packages you list here.
+          :::
+
+          ::: {.note}
+          `cfg.package` must be built with `withNdsudo = true`
+          :::
+        '';
+        example = ''
+          [
+            pkgs.smartmontools
+            pkgs.nvme-cli
+          ]
+        '';
+      };
+
+      config = lib.mkOption {
+        type = lib.types.attrsOf lib.types.attrs;
+        default = { };
+        description = "netdata.conf configuration as nix attributes. cannot be combined with configText.";
+        example = lib.literalExpression ''
+          global = {
+            "debug log" = "syslog";
+            "access log" = "syslog";
+            "error log" = "syslog";
+          };
+        '';
+      };
+
+      configDir = lib.mkOption {
+        type = lib.types.attrsOf lib.types.path;
+        default = { };
+        description = ''
+          Complete netdata config directory except netdata.conf.
+          The default configuration is merged with changes
+          defined in this option.
+          Each top-level attribute denotes a path in the configuration
+          directory as in environment.etc.
+          Its value is the absolute path and must be readable by netdata.
+          Cannot be combined with configText.
+        '';
+        example = lib.literalExpression ''
+          "health_alarm_notify.conf" = pkgs.writeText "health_alarm_notify.conf" '''
+            sendmail="/path/to/sendmail"
+          ''';
+          "health.d" = "/run/secrets/netdata/health.d";
+        '';
+      };
+
+      claimTokenFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        description = ''
+          If set, automatically registers the agent using the given claim token
+          file.
+        '';
+      };
+
+      enableAnalyticsReporting = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Enable reporting of anonymous usage statistics to Netdata Inc. via either
+          Google Analytics (in versions prior to 1.29.4), or Netdata Inc.'s
+          self-hosted PostHog (in versions 1.29.4 and later).
+          See: <https://learn.netdata.cloud/docs/agent/anonymous-statistics>
+        '';
+      };
+
+      deadlineBeforeStopSec = lib.mkOption {
+        type = lib.types.int;
+        default = 120;
+        description = ''
+          In order to detect when netdata is misbehaving, we run a concurrent task pinging netdata (wait-for-netdata-up)
+          in the systemd unit.
+
+          If after a while, this task does not succeed, we stop the unit and mark it as failed.
+
+          You can control this deadline in seconds with this option, it's useful to bump it
+          if you have (1) a lot of data (2) doing upgrades (3) have low IOPS/throughput.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.config != { } -> cfg.configText == null;
+        message = "Cannot specify both config and configText";
+      }
+    ];
+
+    # Includes a set of recommended Python plugins in exchange of imperfect disk consumption.
+    services.netdata.python.extraPackages = lib.mkIf cfg.python.recommendedPythonPackages (ps: [
+      ps.requests
+      ps.pandas
+      ps.numpy
+      ps.psycopg2
+      ps.python-ldap
+      ps.netdata-pandas
+    ]);
+
+    services.netdata.configDir.".opt-out-from-anonymous-statistics" = lib.mkIf (
+      !cfg.enableAnalyticsReporting
+    ) (pkgs.writeText ".opt-out-from-anonymous-statistics" "");
+    environment.etc."netdata/netdata.conf".source = configFile;
+    environment.etc."netdata/conf.d".source = configDirectory;
+
+    systemd.tmpfiles.settings = lib.mkIf cfg.package.withNdsudo {
+      "95-netdata-ndsudo" = {
+        "/var/lib/netdata/ndsudo" = {
+          "d" = {
+            mode = "0550";
+            user = cfg.user;
+            group = cfg.group;
+          };
+        };
+
+        "/var/lib/netdata/ndsudo/ndsudo" = {
+          "L+" = {
+            argument = "/run/wrappers/bin/ndsudo";
+          };
+        };
+
+        "/var/lib/netdata/ndsudo/runtime-dependencies" = {
+          "L+" = {
+            argument = "${extraNdsudoPathsEnv}/bin";
+          };
+        };
+      };
+    };
+
+    systemd.services.netdata = {
+      description = "Real time performance monitoring";
+      after = [
+        "network.target"
+        "suid-sgid-wrappers.service"
+      ];
+      # No wrapper means no "useful" netdata.
+      requires = [ "suid-sgid-wrappers.service" ];
+      wantedBy = [ "multi-user.target" ];
+      path =
+        (with pkgs; [
+          curl
+          gawk
+          iproute2
+          which
+          procps
+          bash
+          nvme-cli # for go.d
+          iw # for charts.d
+          apcupsd # for charts.d
+          # TODO: firehol # for FireQoS -- this requires more NixOS module support.
+          util-linux # provides logger command; required for syslog health alarms
+        ])
+        ++ lib.optional cfg.python.enable (pkgs.python3.withPackages cfg.python.extraPackages)
+        ++ lib.optional config.virtualisation.libvirtd.enable config.virtualisation.libvirtd.package
+        ++ lib.optional config.virtualisation.docker.enable config.virtualisation.docker.package
+        ++ lib.optionals config.virtualisation.podman.enable [
+          pkgs.jq
+          config.virtualisation.podman.package
+        ]
+        ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package;
+      environment = {
+        PYTHONPATH = "${cfg.package}/libexec/netdata/python.d/python_modules";
+        NETDATA_PIPENAME = "/run/netdata/ipc";
+      }
+      // lib.optionalAttrs (!cfg.enableAnalyticsReporting) {
+        DO_NOT_TRACK = "1";
+      };
+      restartTriggers = [
+        config.environment.etc."netdata/netdata.conf".source
+        config.environment.etc."netdata/conf.d".source
+      ];
+      serviceConfig = {
+        ExecStart = "${cfg.package}/bin/netdata -P /run/netdata/netdata.pid -D -c /etc/netdata/netdata.conf";
+        ExecReload = "${pkgs.util-linux}/bin/kill -s HUP -s USR1 -s USR2 $MAINPID";
+        ExecStartPost = pkgs.writeShellScript "wait-for-netdata-up" ''
+          while [ "$(${cfg.package}/bin/netdatacli ping)" != pong ]; do sleep 0.5; done
+        '';
+
+        TimeoutStopSec = cfg.deadlineBeforeStopSec;
+        Restart = "on-failure";
+        # User and group
+        User = cfg.user;
+        Group = cfg.group;
+        # Performance
+        LimitNOFILE = "30000";
+        # Runtime directory and mode
+        RuntimeDirectory = "netdata";
+        RuntimeDirectoryMode = "0750";
+        # State directory and mode
+        StateDirectory = "netdata";
+        StateDirectoryMode = "0750";
+        # Cache directory and mode
+        CacheDirectory = "netdata";
+        CacheDirectoryMode = "0750";
+        # Logs directory and mode
+        LogsDirectory = "netdata";
+        LogsDirectoryMode = "0750";
+        # Configuration directory and mode
+        ConfigurationDirectory = "netdata";
+        ConfigurationDirectoryMode = "0755";
+        # AmbientCapabilities
+        AmbientCapabilities = lib.optional isThereAnyWireGuardTunnels "CAP_NET_ADMIN";
+        # Capabilities
+        CapabilityBoundingSet = [
+          "CAP_DAC_OVERRIDE" # is required for freeipmi and slabinfo plugins
+          "CAP_DAC_READ_SEARCH" # is required for apps and systemd-journal plugin
+          "CAP_NET_RAW" # is required for fping app
+          "CAP_PERFMON" # is required for perf plugin
+          "CAP_SETPCAP" # is required for apps, perf and slabinfo plugins
+          "CAP_SETUID" # is required for cgroups and cgroups-network plugins
+          "CAP_SYSLOG" # is required for systemd-journal plugin
+          "CAP_SYS_ADMIN" # is required for perf plugin
+          "CAP_SYS_CHROOT" # is required for cgroups plugin
+          "CAP_SYS_PTRACE" # is required for apps plugin
+          "CAP_SYS_RESOURCE" # is required for ebpf plugin
+        ]
+        ++ lib.optionals cfg.package.withIpmi [
+          "CAP_FOWNER"
+          "CAP_SYS_RAWIO"
+        ]
+        ++ lib.optional isThereAnyWireGuardTunnels "CAP_NET_ADMIN";
+        # Sandboxing
+        ProtectSystem = "full";
+        ProtectHome = "read-only";
+        PrivateTmp = true;
+        ProtectControlGroups = true;
+        PrivateMounts = true;
+      }
+      // (lib.optionalAttrs (cfg.claimTokenFile != null) {
+        LoadCredential = [
+          "netdata_claim_token:${cfg.claimTokenFile}"
+        ];
+
+        ExecStartPre = pkgs.writeShellScript "netdata-claim" ''
+          set -euo pipefail
+
+          if [[ -f /var/lib/netdata/cloud.d/claimed_id ]]; then
+            # Already registered
+            exit
+          fi
+
+          exec ${cfg.package}/bin/netdata-claim.sh \
+            -token="$(< "$CREDENTIALS_DIRECTORY/netdata_claim_token")" \
+            -url=https://app.netdata.cloud \
+            -daemon-not-running
+        '';
+      });
+    };
+
+    security.wrappers = {
+      "apps.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/apps.plugin.org";
+        capabilities = "cap_dac_read_search,cap_sys_ptrace+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+
+      "debugfs.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/debugfs.plugin.org";
+        capabilities = "cap_dac_read_search+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+
+      "cgroup-network" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/cgroup-network.org";
+        capabilities = "cap_setuid+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+
+      "perf.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/perf.plugin.org";
+        capabilities = "cap_sys_admin+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+
+      "slabinfo.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/slabinfo.plugin.org";
+        capabilities = "cap_dac_override+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+
+    }
+    // lib.optionalAttrs (cfg.package.withIpmi) {
+      "freeipmi.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/freeipmi.plugin.org";
+        capabilities = "cap_dac_override,cap_fowner,cap_sys_rawio+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+    }
+    // lib.optionalAttrs (cfg.package.withNetworkViewer) {
+      "network-viewer.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/network-viewer.plugin.org";
+        capabilities = "cap_sys_admin,cap_dac_read_search,cap_sys_ptrace+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+    }
+    // lib.optionalAttrs (cfg.package.withNdsudo) {
+      "ndsudo" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/ndsudo.org";
+        setuid = true;
+        owner = "root";
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+    }
+    // lib.optionalAttrs (cfg.package.withSystemdJournal) {
+      "systemd-journal.plugin" = {
+        source = "${cfg.package}/libexec/netdata/plugins.d/systemd-journal.plugin.org";
+        capabilities = "cap_dac_read_search,cap_syslog+ep";
+        owner = cfg.user;
+        group = cfg.group;
+        permissions = "u+rx,g+x,o-rwx";
+      };
+    };
+
+    security.pam.loginLimits = [
+      {
+        domain = "netdata";
+        type = "soft";
+        item = "nofile";
+        value = "10000";
+      }
+      {
+        domain = "netdata";
+        type = "hard";
+        item = "nofile";
+        value = "30000";
+      }
+    ];
+
+    users.users = lib.optionalAttrs (cfg.user == defaultUser) {
+      ${defaultUser} = {
+        group = defaultUser;
+        isSystemUser = true;
+        extraGroups =
+          lib.optional config.virtualisation.docker.enable "docker"
+          ++ lib.optional config.virtualisation.podman.enable "podman";
+      };
+    };
+
+    users.groups = lib.optionalAttrs (cfg.group == defaultUser) {
+      ${defaultUser} = { };
+    };
+
+  };
+}

nixos/modules/services/monitoring/nezha-agent.nix

@@ -0,0 +1,296 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.nezha-agent;
+
+  # nezha-agent uses yaml as the configuration file format.
+  # Since we need to use jq to update the content, so here we generate json
+  settingsFormat = pkgs.formats.json { };
+  configFile = settingsFormat.generate "config.json" cfg.settings;
+in
+{
+  meta = {
+    maintainers = with lib.maintainers; [ moraxyc ];
+  };
+  options = {
+    services.nezha-agent = {
+      enable = lib.mkEnableOption "Agent of Nezha Monitoring";
+
+      package = lib.mkPackageOption pkgs "nezha-agent" { };
+
+      debug = lib.mkEnableOption "verbose log";
+
+      settings = lib.mkOption {
+        description = ''
+          Generate to {file}`config.json` as a Nix attribute set.
+          Check the [guide](https://nezha.wiki/en_US/guide/agent.html)
+          for possible options.
+        '';
+        type = lib.types.submodule {
+          freeformType = settingsFormat.type;
+
+          options = {
+            disable_command_execute = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Disable executing the command from dashboard.
+              '';
+            };
+            disable_nat = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Disable NAT penetration.
+              '';
+            };
+            disable_send_query = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Disable sending TCP/ICMP/HTTP requests.
+              '';
+            };
+            gpu = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Enable GPU monitoring.
+              '';
+            };
+            tls = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Enable SSL/TLS encryption.
+              '';
+            };
+            temperature = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Enable temperature monitoring.
+              '';
+            };
+            use_ipv6_country_code = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Use ipv6 countrycode to report location.
+              '';
+            };
+            skip_connection_count = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Do not monitor the number of connections.
+              '';
+            };
+            skip_procs_count = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Do not monitor the number of processes.
+              '';
+            };
+            report_delay = lib.mkOption {
+              type = lib.types.ints.between 1 4;
+              default = 3;
+              description = ''
+                The interval between system status reportings.
+                The value must be an integer from 1 to 4.
+              '';
+            };
+            server = lib.mkOption {
+              type = lib.types.str;
+              example = "127.0.0.1:8008";
+              description = ''
+                Address to the dashboard.
+              '';
+            };
+            uuid = lib.mkOption {
+              type = with lib.types; nullOr str;
+              # pre-defined uuid of Dns in RFC 4122
+              example = "6ba7b810-9dad-11d1-80b4-00c04fd430c8";
+              default = null;
+              description = ''
+                Must be set to a unique identifier, preferably a UUID according to
+                RFC 4122. UUIDs can be generated with `uuidgen` command, found in
+                the `util-linux` package.
+
+                Set {option}`services.nezha-agent.genUuid` to true to generate uuid
+                from {option}`networking.fqdn` automatically.
+              '';
+            };
+          };
+        };
+      };
+
+      genUuid = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Whether to generate uuid from fqdn automatically.
+          Please note that changes in hostname/domain will result in different uuid.
+        '';
+      };
+
+      clientSecretFile = lib.mkOption {
+        type = with lib.types; nullOr path;
+        default = null;
+        description = ''
+          Path to the file contained the client_secret of the dashboard.
+        '';
+      };
+    };
+  };
+
+  imports = with lib; [
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "disableCommandExecute" ]
+      [ "services" "nezha-agent" "settings" "disable_command_execute" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "disableNat" ]
+      [ "services" "nezha-agent" "settings" "disable_nat" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "disableSendQuery" ]
+      [ "services" "nezha-agent" "settings" "disable_send_query" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "gpu" ]
+      [ "services" "nezha-agent" "settings" "gpu" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "tls" ]
+      [ "services" "nezha-agent" "settings" "tls" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "temperature" ]
+      [ "services" "nezha-agent" "settings" "temperature" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "useIPv6CountryCode" ]
+      [ "services" "nezha-agent" "settings" "use_ipv6_country_code" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "skipConnection" ]
+      [ "services" "nezha-agent" "settings" "skip_connection_count" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "skipProcess" ]
+      [ "services" "nezha-agent" "settings" "skip_procs_count" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "reportDelay" ]
+      [ "services" "nezha-agent" "settings" "report_delay" ]
+    )
+    (mkRenamedOptionModule
+      [ "services" "nezha-agent" "server" ]
+      [ "services" "nezha-agent" "settings" "server" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "nezha-agent" "extraFlags" ] ''
+      Use `services.nezha-agent.settings` instead.
+
+      Nezha-agent v1 is no longer configured via command line flags.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "nezha-agent" "passwordFile" ] ''
+      Use `services.nezha-agent.clientSecretFile` instead.
+
+      Nezha-agent v1 uses the client secret from the dashboard to connect.
+    '')
+  ];
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.settings.uuid == null -> cfg.genUuid;
+        message = "Please set `service.nezha-agent.settings.uuid` while `genUuid` is false.";
+      }
+      {
+        assertion = cfg.settings.uuid != null -> !cfg.genUuid;
+        message = "When `service.nezha-agent.genUuid = true`, `settings.uuid` cannot be set.";
+      }
+    ];
+
+    services.nezha-agent.settings = {
+      debug = cfg.debug;
+      # Automatic updates should never be enabled in NixOS.
+      disable_auto_update = true;
+      disable_force_update = true;
+    };
+
+    systemd.services.nezha-agent = {
+      serviceConfig = {
+        Restart = "on-failure";
+        StateDirectory = "nezha-agent";
+        RuntimeDirectory = "nezha-agent";
+        WorkingDirectory = "/var/lib/nezha-agent";
+        ReadWritePaths = "/var/lib/nezha-agent";
+
+        LoadCredential = lib.optionalString (
+          cfg.clientSecretFile != null
+        ) "client-secret:${cfg.clientSecretFile}";
+
+        # Hardening
+        ProcSubset = "all"; # Needed to get host information
+        DynamicUser = true;
+        RemoveIPC = true;
+        LockPersonality = true;
+        ProtectClock = true;
+        MemoryDenyWriteExecute = true;
+        PrivateUsers = true;
+        ProtectHostname = true;
+        RestrictSUIDSGID = true;
+        AmbientCapabilities = [ ];
+        CapabilityBoundingSet = "";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        SystemCallArchitectures = "native";
+        UMask = "0066";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+        ];
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        PrivateDevices = "yes";
+      };
+      environment.HOME = "/var/lib/nezha-agent";
+      enableStrictShellChecks = true;
+      startLimitIntervalSec = 10;
+      startLimitBurst = 3;
+      script = ''
+        cp "${configFile}" "''${RUNTIME_DIRECTORY}"/config.json
+        ${lib.optionalString (cfg.clientSecretFile != null) ''
+          ${lib.getExe pkgs.jq} --arg client_secret "$(<"''${CREDENTIALS_DIRECTORY}"/client-secret)" \
+            '. + { client_secret: $client_secret }' < "''${RUNTIME_DIRECTORY}"/config.json > "''${RUNTIME_DIRECTORY}"/config.json.tmp
+          mv "''${RUNTIME_DIRECTORY}"/config.json.tmp "''${RUNTIME_DIRECTORY}"/config.json
+        ''}
+        ${lib.optionalString cfg.genUuid ''
+          ${lib.getExe pkgs.jq} --arg uuid "$(${lib.getExe' pkgs.util-linux "uuidgen"} --md5 -n @dns -N "${config.networking.fqdn}")" \
+            '. + { uuid: $uuid }' < "''${RUNTIME_DIRECTORY}"/config.json > "''${RUNTIME_DIRECTORY}"/config.json.tmp
+          mv "''${RUNTIME_DIRECTORY}"/config.json.tmp "''${RUNTIME_DIRECTORY}"/config.json
+        ''}
+        ${lib.getExe cfg.package} --config "''${RUNTIME_DIRECTORY}"/config.json
+      '';
+      wantedBy = [ "multi-user.target" ];
+    };
+  };
+}

nixos/modules/services/monitoring/ocsinventory-agent.md

@@ -0,0 +1,33 @@
+# OCS Inventory Agent {#module-services-ocsinventory-agent}
+
+[OCS Inventory NG](https://ocsinventory-ng.org/) or Open Computers and Software inventory
+is an application designed to help IT administrator to keep track of the hardware and software
+configurations of computers that are installed on their network.
+
+OCS Inventory collects information about the hardware and software of networked machines
+through the **OCS Inventory Agent** program.
+
+This NixOS module enables you to install and configure this agent so that it sends information from your computer to the OCS Inventory server.
+
+For more technical information about OCS Inventory Agent, refer to [the Wiki documentation](https://wiki.ocsinventory-ng.org/03.Basic-documentation/Setting-up-the-UNIX-agent-manually-on-client-computers/).
+
+
+## Basic Usage {#module-services-ocsinventory-agent-basic-usage}
+
+A minimal configuration looks like this:
+
+```nix
+{
+  services.ocsinventory-agent = {
+    enable = true;
+    settings = {
+      server = "https://ocsinventory.localhost:8080/ocsinventory";
+      tag = "01234567890123";
+    };
+  };
+}
+```
+
+This configuration will periodically run the ocsinventory-agent SystemD service.
+
+The OCS Inventory Agent will inventory the computer and then sends the results to the specified OCS Inventory Server.

nixos/modules/services/monitoring/ocsinventory-agent.nix

@@ -0,0 +1,140 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.ocsinventory-agent;
+
+  settingsFormat = pkgs.formats.keyValue {
+    mkKeyValue = lib.generators.mkKeyValueDefault { } "=";
+  };
+
+in
+{
+  meta = {
+    doc = ./ocsinventory-agent.md;
+    maintainers = with lib.maintainers; [ anthonyroussel ];
+  };
+
+  options = {
+    services.ocsinventory-agent = {
+      enable = lib.mkEnableOption "OCS Inventory Agent";
+
+      package = lib.mkPackageOption pkgs "ocsinventory-agent" { };
+
+      settings = lib.mkOption {
+        type = lib.types.submodule {
+          freeformType = settingsFormat.type.nestedTypes.elemType;
+
+          options = {
+            server = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              example = "https://ocsinventory.localhost:8080/ocsinventory";
+              default = null;
+              description = ''
+                The URI of the OCS Inventory server where to send the inventory file.
+
+                This option is ignored if {option}`services.ocsinventory-agent.settings.local` is set.
+              '';
+            };
+
+            local = lib.mkOption {
+              type = lib.types.nullOr lib.types.path;
+              example = "/var/lib/ocsinventory-agent/reports";
+              default = null;
+              description = ''
+                If specified, the OCS Inventory Agent will run in offline mode
+                and the resulting inventory file will be stored in the specified path.
+              '';
+            };
+
+            ca = lib.mkOption {
+              type = lib.types.path;
+              default = config.security.pki.caBundle;
+              defaultText = lib.literalExpression "config.security.pki.caBundle";
+              description = ''
+                Path to CA certificates file in PEM format, for server
+                SSL certificate validation.
+              '';
+            };
+
+            tag = lib.mkOption {
+              type = lib.types.nullOr lib.types.str;
+              default = null;
+              example = "01234567890123";
+              description = "Tag for the generated inventory.";
+            };
+
+            debug = lib.mkEnableOption "debug mode";
+          };
+        };
+        default = { };
+        example = {
+          debug = true;
+          server = "https://ocsinventory.localhost:8080/ocsinventory";
+          tag = "01234567890123";
+        };
+        description = ''
+          Configuration for /etc/ocsinventory-agent/ocsinventory-agent.cfg.
+
+          Refer to
+          {manpage}`ocsinventory-agent(1)` for available options.
+        '';
+      };
+
+      interval = lib.mkOption {
+        type = lib.types.str;
+        default = "daily";
+        example = "06:00";
+        description = ''
+          How often we run the ocsinventory-agent service. Runs by default every daily.
+
+          The format is described in
+          {manpage}`systemd.time(7)`.
+        '';
+      };
+    };
+  };
+
+  config =
+    let
+      configFile = settingsFormat.generate "ocsinventory-agent.cfg" cfg.settings;
+
+    in
+    lib.mkIf cfg.enable {
+      # Path of the configuration file is hard-coded and cannot be changed
+      # https://github.com/OCSInventory-NG/UnixAgent/blob/v2.10.0/lib/Ocsinventory/Agent/Config.pm#L78
+      #
+      environment.etc."ocsinventory-agent/ocsinventory-agent.cfg".source = configFile;
+
+      systemd.services.ocsinventory-agent = {
+        description = "OCS Inventory Agent service";
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+
+        reloadTriggers = [ configFile ];
+
+        serviceConfig = {
+          ExecStart = lib.getExe cfg.package;
+          ConfigurationDirectory = "ocsinventory-agent";
+          StateDirectory = "ocsinventory-agent";
+        };
+      };
+
+      systemd.timers.ocsinventory-agent = {
+        description = "Launch OCS Inventory Agent regularly";
+        wantedBy = [ "timers.target" ];
+
+        timerConfig = {
+          OnCalendar = cfg.interval;
+          AccuracySec = "1h";
+          RandomizedDelaySec = 240;
+          Persistent = true;
+          Unit = "ocsinventory-agent.service";
+        };
+      };
+    };
+}

nixos/modules/services/monitoring/opentelemetry-collector.nix

@@ -0,0 +1,88 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkEnableOption
+    mkPackageOption
+    mkIf
+    mkOption
+    types
+    getExe
+    ;
+
+  cfg = config.services.opentelemetry-collector;
+  opentelemetry-collector = cfg.package;
+
+  settingsFormat = pkgs.formats.yaml { };
+in
+{
+  options.services.opentelemetry-collector = {
+    enable = mkEnableOption "Opentelemetry Collector";
+
+    package = mkPackageOption pkgs "opentelemetry-collector" { };
+
+    settings = mkOption {
+      type = settingsFormat.type;
+      default = { };
+      description = ''
+        Specify the configuration for Opentelemetry Collector in Nix.
+
+        See <https://opentelemetry.io/docs/collector/configuration/> for available options.
+      '';
+    };
+
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Specify a path to a configuration file that Opentelemetry Collector should use.
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = ((cfg.settings == { }) != (cfg.configFile == null));
+        message = ''
+          Please specify a configuration for Opentelemetry Collector with either
+          'services.opentelemetry-collector.settings' or
+          'services.opentelemetry-collector.configFile'.
+        '';
+      }
+    ];
+
+    systemd.services.opentelemetry-collector = {
+      description = "Opentelemetry Collector Service Daemon";
+      wantedBy = [ "multi-user.target" ];
+
+      serviceConfig =
+        let
+          conf =
+            if cfg.configFile == null then
+              settingsFormat.generate "config.yaml" cfg.settings
+            else
+              cfg.configFile;
+        in
+        {
+          ExecStart = "${getExe opentelemetry-collector} --config=file:${conf}";
+          DynamicUser = true;
+          Restart = "always";
+          ProtectSystem = "full";
+          DevicePolicy = "closed";
+          NoNewPrivileges = true;
+          WorkingDirectory = "%S/opentelemetry-collector";
+          StateDirectory = "opentelemetry-collector";
+          SupplementaryGroups = [
+            # allow to read the systemd journal for opentelemetry-collector
+            "systemd-journal"
+          ];
+        };
+    };
+  };
+}

nixos/modules/services/monitoring/osquery.nix

@@ -0,0 +1,125 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.osquery;
+  dirname =
+    path:
+    with lib.strings;
+    with lib.lists;
+    concatStringsSep "/" (init (splitString "/" (normalizePath path)));
+
+  # conf is the osquery configuration file used when the --config_plugin=filesystem.
+  # filesystem is the osquery default value for the config_plugin flag.
+  conf = pkgs.writeText "osquery.conf" (builtins.toJSON cfg.settings);
+
+  # flagfile is the file containing osquery command line flags to be
+  # provided to the application using the special --flagfile option.
+  flagfile = pkgs.writeText "osquery.flags" (
+    lib.concatStringsSep "\n" (
+      lib.mapAttrsToList (name: value: "--${name}=${value}")
+        # Use the conf derivation if not otherwise specified.
+        ({ config_path = conf; } // cfg.flags)
+    )
+  );
+
+  osqueryi = pkgs.runCommand "osqueryi" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
+    mkdir -p $out/bin
+    makeWrapper ${pkgs.osquery}/bin/osqueryi $out/bin/osqueryi \
+      --add-flags "--flagfile ${flagfile} --disable-database"
+  '';
+in
+{
+  options.services.osquery = {
+    enable = lib.mkEnableOption "osqueryd daemon";
+
+    settings = lib.mkOption {
+      default = { };
+      description = ''
+        Configuration to be written to the osqueryd JSON configuration file.
+        To understand the configuration format, refer to <https://osquery.readthedocs.io/en/stable/deployment/configuration/#configuration-components>.
+      '';
+      example = {
+        options.utc = false;
+      };
+      type = lib.types.attrs;
+    };
+
+    flags = lib.mkOption {
+      default = { };
+      description = ''
+        Attribute set of flag names and values to be written to the osqueryd flagfile.
+        For more information, refer to <https://osquery.readthedocs.io/en/stable/installation/cli-flags>.
+      '';
+      example = {
+        config_refresh = "10";
+      };
+      type =
+        with lib.types;
+        submodule {
+          freeformType = attrsOf str;
+          options = {
+            database_path = lib.mkOption {
+              default = "/var/lib/osquery/osquery.db";
+              readOnly = true;
+              description = ''
+                Path used for the database file.
+
+                ::: {.note}
+                If left as the default value, this directory will be automatically created before the
+                service starts, otherwise you are responsible for ensuring the directory exists with
+                the appropriate ownership and permissions.
+              '';
+              type = path;
+            };
+            logger_path = lib.mkOption {
+              default = "/var/log/osquery";
+              readOnly = true;
+              description = ''
+                Base directory used for logging.
+
+                ::: {.note}
+                If left as the default value, this directory will be automatically created before the
+                service starts, otherwise you are responsible for ensuring the directory exists with
+                the appropriate ownership and permissions.
+              '';
+              type = path;
+            };
+            pidfile = lib.mkOption {
+              default = "/run/osquery/osqueryd.pid";
+              readOnly = true;
+              description = "Path used for pid file.";
+              type = path;
+            };
+          };
+        };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ osqueryi ];
+    systemd.services.osqueryd = {
+      after = [
+        "network.target"
+        "syslog.service"
+      ];
+      description = "The osquery daemon";
+      serviceConfig = {
+        ExecStart = "${pkgs.osquery}/bin/osqueryd --flagfile ${flagfile}";
+        PIDFile = cfg.flags.pidfile;
+        LogsDirectory = lib.mkIf (cfg.flags.logger_path == "/var/log/osquery") [ "osquery" ];
+        StateDirectory = lib.mkIf (cfg.flags.database_path == "/var/lib/osquery/osquery.db") [ "osquery" ];
+        Restart = "always";
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+    systemd.tmpfiles.settings."10-osquery".${dirname (cfg.flags.pidfile)}.d = {
+      user = "root";
+      group = "root";
+      mode = "0755";
+    };
+  };
+}

nixos/modules/services/monitoring/parsedmarc.md

@@ -0,0 +1,118 @@
+# parsedmarc {#module-services-parsedmarc}
+[parsedmarc](https://domainaware.github.io/parsedmarc/) is a service
+which parses incoming [DMARC](https://dmarc.org/) reports and stores
+or sends them to a downstream service for further analysis. In
+combination with Elasticsearch, Grafana and the included Grafana
+dashboard, it provides a handy overview of DMARC reports over time.
+
+## Basic usage {#module-services-parsedmarc-basic-usage}
+A very minimal setup which reads incoming reports from an external
+email address and saves them to a local Elasticsearch instance looks
+like this:
+
+```nix
+{
+  services.parsedmarc = {
+    enable = true;
+    settings.imap = {
+      host = "imap.example.com";
+      user = "alice@example.com";
+      password = "/path/to/imap_password_file";
+    };
+    provision.geoIp = false; # Not recommended!
+  };
+}
+```
+
+Note that GeoIP provisioning is disabled in the example for
+simplicity, but should be turned on for fully functional reports.
+
+## Local mail {#module-services-parsedmarc-local-mail}
+Instead of watching an external inbox, a local inbox can be
+automatically provisioned. The recipient's name is by default set to
+`dmarc`, but can be configured in
+[services.parsedmarc.provision.localMail.recipientName](options.html#opt-services.parsedmarc.provision.localMail.recipientName). You
+need to add an MX record pointing to the host. More concretely: for
+the example to work, an MX record needs to be set up for
+`monitoring.example.com` and the complete email address that should be
+configured in the domain's dmarc policy is
+`dmarc@monitoring.example.com`.
+
+```nix
+{
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      localMail = {
+        enable = true;
+        hostname = monitoring.example.com;
+      };
+      geoIp = false; # Not recommended!
+    };
+  };
+}
+```
+
+## Grafana and GeoIP {#module-services-parsedmarc-grafana-geoip}
+The reports can be visualized and summarized with parsedmarc's
+official Grafana dashboard. For all views to work, and for the data to
+be complete, GeoIP databases are also required. The following example
+shows a basic deployment where the provisioned Elasticsearch instance
+is automatically added as a Grafana datasource, and the dashboard is
+added to Grafana as well.
+
+```nix
+{
+  services.parsedmarc = {
+    enable = true;
+    provision = {
+      localMail = {
+        enable = true;
+        hostname = url;
+      };
+      grafana = {
+        datasource = true;
+        dashboard = true;
+      };
+    };
+  };
+
+  # Not required, but recommended for full functionality
+  services.geoipupdate = {
+    settings = {
+      AccountID = 0;
+      LicenseKey = "/path/to/license_key_file";
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    addr = "0.0.0.0";
+    domain = url;
+    rootUrl = "https://" + url;
+    protocol = "socket";
+    security = {
+      adminUser = "admin";
+      adminPasswordFile = "/path/to/admin_password_file";
+      secretKeyFile = "/path/to/secret_key_file";
+    };
+  };
+
+  services.nginx = {
+    enable = true;
+    recommendedTlsSettings = true;
+    recommendedOptimisation = true;
+    recommendedGzipSettings = true;
+    recommendedProxySettings = true;
+    upstreams.grafana.servers."unix:/${config.services.grafana.socket}" = { };
+    virtualHosts.${url} = {
+      root = config.services.grafana.staticRootPath;
+      enableACME = true;
+      forceSSL = true;
+      locations."/".tryFiles = "$uri @grafana";
+      locations."@grafana".proxyPass = "http://grafana";
+    };
+  };
+  users.users.nginx.extraGroups = [ "grafana" ];
+}
+```

nixos/modules/services/monitoring/parsedmarc.nix

@@ -0,0 +1,612 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.parsedmarc;
+  opt = options.services.parsedmarc;
+  isSecret = v: isAttrs v && v ? _secret && isString v._secret;
+  ini = pkgs.formats.ini {
+    mkKeyValue = lib.flip lib.generators.mkKeyValueDefault "=" {
+      mkValueString =
+        v:
+        if isInt v then
+          toString v
+        else if isString v then
+          v
+        else if true == v then
+          "True"
+        else if false == v then
+          "False"
+        else if isSecret v then
+          hashString "sha256" v._secret
+        else
+          throw "unsupported type ${typeOf v}: ${(lib.generators.toPretty { }) v}";
+    };
+  };
+  inherit (builtins)
+    elem
+    isAttrs
+    isString
+    isInt
+    isList
+    typeOf
+    hashString
+    ;
+in
+{
+  options.services.parsedmarc = {
+
+    enable = lib.mkEnableOption ''
+      parsedmarc, a DMARC report monitoring service
+    '';
+
+    provision = {
+      localMail = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          description = ''
+            Whether Postfix and Dovecot should be set up to receive
+            mail locally. parsedmarc will be configured to watch the
+            local inbox as the automatically created user specified in
+            [](#opt-services.parsedmarc.provision.localMail.recipientName)
+          '';
+        };
+
+        recipientName = lib.mkOption {
+          type = lib.types.str;
+          default = "dmarc";
+          description = ''
+            The DMARC mail recipient name, i.e. the name part of the
+            email address which receives DMARC reports.
+
+            A local user with this name will be set up and assigned a
+            randomized password on service start.
+          '';
+        };
+
+        hostname = lib.mkOption {
+          type = lib.types.str;
+          default = config.networking.fqdn;
+          defaultText = lib.literalExpression "config.networking.fqdn";
+          example = "monitoring.example.com";
+          description = ''
+            The hostname to use when configuring Postfix.
+
+            Should correspond to the host's fully qualified domain
+            name and the domain part of the email address which
+            receives DMARC reports. You also have to set up an MX record
+            pointing to this domain name.
+          '';
+        };
+      };
+
+      geoIp = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether to enable and configure the [geoipupdate](#opt-services.geoipupdate.enable)
+          service to automatically fetch GeoIP databases. Not crucial,
+          but recommended for full functionality.
+
+          To finish the setup, you need to manually set the [](#opt-services.geoipupdate.settings.AccountID) and
+          [](#opt-services.geoipupdate.settings.LicenseKey)
+          options.
+        '';
+      };
+
+      elasticsearch = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Whether to set up and use a local instance of Elasticsearch.
+        '';
+      };
+
+      grafana = {
+        datasource = lib.mkOption {
+          type = lib.types.bool;
+          default = cfg.provision.elasticsearch && config.services.grafana.enable;
+          defaultText = lib.literalExpression ''
+            config.${opt.provision.elasticsearch} && config.${options.services.grafana.enable}
+          '';
+          apply = x: x && cfg.provision.elasticsearch;
+          description = ''
+            Whether the automatically provisioned Elasticsearch
+            instance should be added as a grafana datasource. Has no
+            effect unless
+            [](#opt-services.parsedmarc.provision.elasticsearch)
+            is also enabled.
+          '';
+        };
+
+        dashboard = lib.mkOption {
+          type = lib.types.bool;
+          default = config.services.grafana.enable;
+          defaultText = lib.literalExpression "config.services.grafana.enable";
+          description = ''
+            Whether the official parsedmarc grafana dashboard should
+            be provisioned to the local grafana instance.
+          '';
+        };
+      };
+    };
+
+    settings = lib.mkOption {
+      example = lib.literalExpression ''
+        {
+          imap = {
+            host = "imap.example.com";
+            user = "alice@example.com";
+            password = { _secret = "/run/keys/imap_password" };
+          };
+          mailbox = {
+            watch = true;
+            batch_size = 30;
+          };
+          splunk_hec = {
+            url = "https://splunkhec.example.com";
+            token = { _secret = "/run/keys/splunk_token" };
+            index = "email";
+          };
+        }
+      '';
+      description = ''
+        Configuration parameters to set in
+        {file}`parsedmarc.ini`. For a full list of
+        available parameters, see
+        <https://domainaware.github.io/parsedmarc/#configuration-file>.
+
+        Settings containing secret data should be set to an attribute
+        set containing the attribute `_secret` - a
+        string pointing to a file containing the value the option
+        should be set to. See the example to get a better picture of
+        this: in the resulting {file}`parsedmarc.ini`
+        file, the `splunk_hec.token` key will be set
+        to the contents of the
+        {file}`/run/keys/splunk_token` file.
+      '';
+
+      type = lib.types.submodule {
+        freeformType = ini.type;
+
+        options = {
+          general = {
+            save_aggregate = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Save aggregate report data to Elasticsearch and/or Splunk.
+              '';
+            };
+
+            save_forensic = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Save forensic report data to Elasticsearch and/or Splunk.
+              '';
+            };
+          };
+
+          mailbox = {
+            watch = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Use the IMAP IDLE command to process messages as they arrive.
+              '';
+            };
+
+            delete = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Delete messages after processing them, instead of archiving them.
+              '';
+            };
+          };
+
+          imap = {
+            host = lib.mkOption {
+              type = lib.types.str;
+              default = "localhost";
+              description = ''
+                The IMAP server hostname or IP address.
+              '';
+            };
+
+            port = lib.mkOption {
+              type = lib.types.port;
+              default = 993;
+              description = ''
+                The IMAP server port.
+              '';
+            };
+
+            ssl = lib.mkOption {
+              type = lib.types.bool;
+              default = true;
+              description = ''
+                Use an encrypted SSL/TLS connection.
+              '';
+            };
+
+            user = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                The IMAP server username.
+              '';
+            };
+
+            password = lib.mkOption {
+              type = with lib.types; nullOr (either path (attrsOf path));
+              default = null;
+              description = ''
+                The IMAP server password.
+
+                Always handled as a secret whether the value is
+                wrapped in a `{ _secret = ...; }`
+                attrset or not (refer to [](#opt-services.parsedmarc.settings) for
+                details).
+              '';
+              apply = x: if isAttrs x || x == null then x else { _secret = x; };
+            };
+          };
+
+          smtp = {
+            host = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                The SMTP server hostname or IP address.
+              '';
+            };
+
+            port = lib.mkOption {
+              type = with lib.types; nullOr port;
+              default = null;
+              description = ''
+                The SMTP server port.
+              '';
+            };
+
+            ssl = lib.mkOption {
+              type = with lib.types; nullOr bool;
+              default = null;
+              description = ''
+                Use an encrypted SSL/TLS connection.
+              '';
+            };
+
+            user = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                The SMTP server username.
+              '';
+            };
+
+            password = lib.mkOption {
+              type = with lib.types; nullOr (either path (attrsOf path));
+              default = null;
+              description = ''
+                The SMTP server password.
+
+                Always handled as a secret whether the value is
+                wrapped in a `{ _secret = ...; }`
+                attrset or not (refer to [](#opt-services.parsedmarc.settings) for
+                details).
+              '';
+              apply = x: if isAttrs x || x == null then x else { _secret = x; };
+            };
+
+            from = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                The `From` address to use for the
+                outgoing mail.
+              '';
+            };
+
+            to = lib.mkOption {
+              type = with lib.types; nullOr (listOf str);
+              default = null;
+              description = ''
+                The addresses to send outgoing mail to.
+              '';
+              apply = x: if x == [ ] || x == null then null else lib.concatStringsSep "," x;
+            };
+          };
+
+          elasticsearch = {
+            hosts = lib.mkOption {
+              default = [ ];
+              type = with lib.types; listOf str;
+              apply = x: if x == [ ] then null else lib.concatStringsSep "," x;
+              description = ''
+                A list of Elasticsearch hosts to push parsed reports
+                to.
+              '';
+            };
+
+            user = lib.mkOption {
+              type = with lib.types; nullOr str;
+              default = null;
+              description = ''
+                Username to use when connecting to Elasticsearch, if
+                required.
+              '';
+            };
+
+            password = lib.mkOption {
+              type = with lib.types; nullOr (either path (attrsOf path));
+              default = null;
+              description = ''
+                The password to use when connecting to Elasticsearch,
+                if required.
+
+                Always handled as a secret whether the value is
+                wrapped in a `{ _secret = ...; }`
+                attrset or not (refer to [](#opt-services.parsedmarc.settings) for
+                details).
+              '';
+              apply = x: if isAttrs x || x == null then x else { _secret = x; };
+            };
+
+            ssl = lib.mkOption {
+              type = lib.types.bool;
+              default = false;
+              description = ''
+                Whether to use an encrypted SSL/TLS connection.
+              '';
+            };
+
+            cert_path = lib.mkOption {
+              type = lib.types.path;
+              default = config.security.pki.caBundle;
+              defaultText = lib.literalExpression "config.security.pki.caBundle";
+              description = ''
+                The path to a TLS certificate bundle used to verify
+                the server's certificate.
+              '';
+            };
+          };
+        };
+
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    warnings =
+      let
+        deprecationWarning =
+          optname:
+          "Starting in 8.0.0, the `${optname}` option has been moved from the `services.parsedmarc.settings.imap`"
+          + "configuration section to the `services.parsedmarc.settings.mailbox` configuration section.";
+        hasImapOpt = lib.flip builtins.hasAttr cfg.settings.imap;
+        movedOptions = [
+          "reports_folder"
+          "archive_folder"
+          "watch"
+          "delete"
+          "test"
+          "batch_size"
+        ];
+      in
+      builtins.map deprecationWarning (builtins.filter hasImapOpt movedOptions);
+
+    services.elasticsearch.enable = lib.mkDefault cfg.provision.elasticsearch;
+
+    services.geoipupdate = lib.mkIf cfg.provision.geoIp {
+      enable = true;
+      settings = {
+        EditionIDs = [
+          "GeoLite2-ASN"
+          "GeoLite2-City"
+          "GeoLite2-Country"
+        ];
+        DatabaseDirectory = "/var/lib/GeoIP";
+      };
+    };
+
+    services.dovecot2 = lib.mkIf cfg.provision.localMail.enable {
+      enable = true;
+      protocols = [ "imap" ];
+    };
+
+    services.postfix = lib.mkIf cfg.provision.localMail.enable {
+      enable = true;
+      settings.main = {
+        myhostname = cfg.provision.localMail.hostname;
+        myorigin = cfg.provision.localMail.hostname;
+        mydestination = cfg.provision.localMail.hostname;
+      };
+    };
+
+    services.grafana = {
+      declarativePlugins =
+        with pkgs.grafanaPlugins;
+        lib.mkIf cfg.provision.grafana.dashboard [
+          grafana-worldmap-panel
+          grafana-piechart-panel
+        ];
+
+      provision = {
+        enable = cfg.provision.grafana.datasource || cfg.provision.grafana.dashboard;
+        datasources.settings.datasources =
+          let
+            esVersion = lib.getVersion config.services.elasticsearch.package;
+          in
+          lib.mkIf cfg.provision.grafana.datasource [
+            {
+              name = "dmarc-ag";
+              type = "elasticsearch";
+              access = "proxy";
+              url = "http://localhost:9200";
+              jsonData = {
+                timeField = "date_range";
+                inherit esVersion;
+              };
+            }
+            {
+              name = "dmarc-fo";
+              type = "elasticsearch";
+              access = "proxy";
+              url = "http://localhost:9200";
+              jsonData = {
+                timeField = "date_range";
+                inherit esVersion;
+              };
+            }
+          ];
+        dashboards.settings.providers = lib.mkIf cfg.provision.grafana.dashboard [
+          {
+            name = "parsedmarc";
+            options.path = "${pkgs.parsedmarc.dashboard}";
+          }
+        ];
+      };
+    };
+
+    services.parsedmarc.settings = lib.mkMerge [
+      (lib.mkIf cfg.provision.elasticsearch {
+        elasticsearch = {
+          hosts = [ "http://localhost:9200" ];
+          ssl = false;
+        };
+      })
+      (lib.mkIf cfg.provision.localMail.enable {
+        imap = {
+          host = "localhost";
+          port = 143;
+          ssl = false;
+          user = cfg.provision.localMail.recipientName;
+          password = "${pkgs.writeText "imap-password" "@imap-password@"}";
+        };
+        mailbox = {
+          watch = true;
+        };
+      })
+    ];
+
+    systemd.services.parsedmarc =
+      let
+        # Remove any empty attributes from the config, i.e. empty
+        # lists, empty attrsets and null. This makes it possible to
+        # list interesting options in `settings` without them always
+        # ending up in the resulting config.
+        filteredConfig = lib.converge (lib.filterAttrsRecursive (
+          _: v:
+          !elem v [
+            null
+            [ ]
+            { }
+          ]
+        )) cfg.settings;
+
+        # Extract secrets (attributes set to an attrset with a
+        # "_secret" key) from the settings and generate the commands
+        # to run to perform the secret replacements.
+        secretPaths = lib.catAttrs "_secret" (lib.collect isSecret filteredConfig);
+        parsedmarcConfig = ini.generate "parsedmarc.ini" filteredConfig;
+        mkSecretReplacement = file: ''
+          replace-secret ${
+            lib.escapeShellArgs [
+              (hashString "sha256" file)
+              file
+              "/run/parsedmarc/parsedmarc.ini"
+            ]
+          }
+        '';
+        secretReplacements = lib.concatMapStrings mkSecretReplacement secretPaths;
+      in
+      {
+        wantedBy = [ "multi-user.target" ];
+        after = [
+          "postfix.service"
+          "dovecot2.service"
+          "elasticsearch.service"
+        ];
+        path = with pkgs; [
+          replace-secret
+          openssl
+          shadow
+        ];
+        serviceConfig = {
+          ExecStartPre =
+            let
+              startPreFullPrivileges = ''
+                set -o errexit -o pipefail -o nounset -o errtrace
+                shopt -s inherit_errexit
+
+                umask u=rwx,g=,o=
+                cp ${parsedmarcConfig} /run/parsedmarc/parsedmarc.ini
+                chown parsedmarc:parsedmarc /run/parsedmarc/parsedmarc.ini
+                ${secretReplacements}
+              ''
+              + lib.optionalString cfg.provision.localMail.enable ''
+                openssl rand -hex 64 >/run/parsedmarc/dmarc_user_passwd
+                replace-secret '@imap-password@' '/run/parsedmarc/dmarc_user_passwd' /run/parsedmarc/parsedmarc.ini
+                echo "Setting new randomized password for user '${cfg.provision.localMail.recipientName}'."
+                cat <(echo -n "${cfg.provision.localMail.recipientName}:") /run/parsedmarc/dmarc_user_passwd | chpasswd
+              '';
+            in
+            "+${pkgs.writeShellScript "parsedmarc-start-pre-full-privileges" startPreFullPrivileges}";
+          Type = "simple";
+          User = "parsedmarc";
+          Group = "parsedmarc";
+          DynamicUser = true;
+          RuntimeDirectory = "parsedmarc";
+          RuntimeDirectoryMode = "0700";
+          CapabilityBoundingSet = "";
+          PrivateDevices = true;
+          PrivateMounts = true;
+          PrivateUsers = true;
+          ProtectClock = true;
+          ProtectControlGroups = true;
+          ProtectHome = true;
+          ProtectHostname = true;
+          ProtectKernelLogs = true;
+          ProtectKernelModules = true;
+          ProtectKernelTunables = true;
+          ProtectProc = "invisible";
+          ProcSubset = "pid";
+          SystemCallFilter = [
+            "@system-service"
+            "~@privileged"
+            "~@resources"
+          ];
+          RestrictAddressFamilies = [
+            "AF_UNIX"
+            "AF_INET"
+            "AF_INET6"
+          ];
+          RestrictRealtime = true;
+          RestrictNamespaces = true;
+          MemoryDenyWriteExecute = true;
+          LockPersonality = true;
+          SystemCallArchitectures = "native";
+          ExecStart = "${lib.getExe pkgs.parsedmarc} -c /run/parsedmarc/parsedmarc.ini";
+        };
+      };
+
+    users.users.${cfg.provision.localMail.recipientName} = lib.mkIf cfg.provision.localMail.enable {
+      isNormalUser = true;
+      description = "DMARC mail recipient";
+    };
+  };
+
+  meta.doc = ./parsedmarc.md;
+  meta.maintainers = [ lib.maintainers.talyz ];
+}

nixos/modules/services/monitoring/pgscv.nix

@@ -0,0 +1,75 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    mkEnableOption
+    mkIf
+    mkOption
+    mkPackageOption
+    types
+    ;
+
+  cfg = config.services.pgscv;
+
+  settingsFormat = pkgs.formats.yaml { };
+  configFile = settingsFormat.generate "config.yaml" cfg.settings;
+in
+{
+  options.services.pgscv = {
+    enable = mkEnableOption "pgSCV, a PostgreSQL ecosystem metrics collector";
+
+    package = mkPackageOption pkgs "pgscv" { };
+
+    logLevel = mkOption {
+      type = types.enum [
+        "debug"
+        "info"
+        "warn"
+        "error"
+      ];
+      default = "info";
+      description = "Log level for pgSCV.";
+    };
+
+    settings = mkOption {
+      type = settingsFormat.type;
+      default = { };
+      description = ''
+        Configuration for pgSCV, in YAML format.
+
+        See [configuration reference](https://github.com/cherts/pgscv/wiki/Configuration-settings-reference).
+      '';
+    };
+  };
+
+  config = mkIf cfg.enable {
+    systemd.services.pgscv = {
+      description = "pgSCV - PostgreSQL ecosystem metrics collector";
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+      path = [ pkgs.glibc ]; # shells out to getconf
+
+      serviceConfig = {
+        User = "postgres";
+        Group = "postgres";
+        ExecStart = utils.escapeSystemdExecArgs [
+          (lib.getExe cfg.package)
+          "--log-level=${cfg.logLevel}"
+          "--config-file=${configFile}"
+        ];
+        KillMode = "control-group";
+        TimeoutSec = 5;
+        Restart = "on-failure";
+        RestartSec = 10;
+        OOMScoreAdjust = 1000;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/alertmanager-gotify-bridge.nix

@@ -0,0 +1,194 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.alertmanagerGotify;
+  pkg = cfg.package;
+  inherit (lib)
+    mkEnableOption
+    mkOption
+    types
+    mkIf
+    mkPackageOption
+    optionalString
+    ;
+in
+{
+  meta.maintainers = with lib.maintainers; [ juli0604 ];
+  options.services.prometheus.alertmanagerGotify = {
+    enable = mkEnableOption "alertmagager-gotify";
+    package = mkPackageOption pkgs "alertmanager-gotify-bridge" { };
+    bindAddress = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      description = "The address the server will listen on (bind address).";
+    };
+    defaultPriority = mkOption {
+      type = types.int;
+      default = 5;
+      description = "The default priority for messages sent to gotify.";
+    };
+    debug = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Enables extended logs for debugging purposes. Should be disabled in productive mode.";
+    };
+    dispatchErrors = mkOption {
+      type = types.bool;
+      default = false;
+      description = "When enabled, alerts will be tried to dispatch with an error message regarding faulty templating or missing fields to help debugging.";
+    };
+    extendedDetails = mkOption {
+      type = types.bool;
+      default = false;
+      description = "When enabled, alerts are presented in HTML format and include colorized status (FIR|RES), alert start time, and a link to the generator of the alert.";
+    };
+    messageAnnotation = mkOption {
+      type = types.str;
+      description = "Annotation holding the alert message.";
+    };
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Opens the bridge port in the firewall.";
+    };
+    port = mkOption {
+      type = types.port;
+      default = 8080;
+      description = "The local port the bridge is listening on.";
+    };
+    priorityAnnotation = mkOption {
+      type = types.str;
+      default = "priority";
+      description = "Annotation holding the priority of the alert.";
+    };
+    timeout = mkOption {
+      type = types.ints.positive;
+      default = 5;
+      description = "The time between sending a message and the timeout.";
+    };
+    titleAnnotation = mkOption {
+      type = types.str;
+      default = "summary";
+      description = "Annotation holding the title of the alert";
+    };
+    webhookPath = mkOption {
+      type = types.str;
+      default = "/gotify_webhook";
+      description = "The URL path to handle requests on.";
+    };
+    environmentFile = mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        File containing additional config environment variables for alertmanager-gotify-bridge.
+        This is especially for secrets like GOTIFY_TOKEN and AUTH_PASSWORD.
+      '';
+    };
+    gotifyEndpoint = {
+      host = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = "The hostname or ip your gotify endpoint is running.";
+      };
+      port = mkOption {
+        type = types.port;
+        default = 443;
+        description = "The port your gotify endpoint is running.";
+      };
+      tls = mkOption {
+        type = types.bool;
+        default = true;
+        description = "If your gotify endpoint uses https, leave this option set to default";
+      };
+    };
+    metrics = {
+      username = mkOption {
+        type = types.str;
+        description = "The username used to access your metrics.";
+      };
+      namespace = mkOption {
+        type = types.str;
+        default = "alertmanager-gotify-bridge";
+        description = "The namescape of the metrics.";
+      };
+      path = mkOption {
+        type = types.str;
+        default = "/metrics";
+        description = "The path under which the metrics will be exposed.";
+      };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    users = {
+      groups.alertmanager-gotify = { };
+      users.alertmanager-gotify = {
+        group = "alertmanager-gotify";
+        isSystemUser = true;
+      };
+    };
+
+    networking.firewall = mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
+
+    systemd.services.alertmanager-gotify-bridge = {
+      description = "A bridge between Prometheus AlertManager and a Gotify server";
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        ExecStart = "${lib.getExe pkg} ${optionalString cfg.debug "--debug"}";
+        EnvironmentFile = lib.mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+        User = "alertmanager-gotify";
+        Group = "alertmanager-gotify";
+
+        #hardening
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateIPC = true;
+        DevicePolicy = "closed";
+        ProtectSystem = "strict";
+        ProtectHome = "read-only";
+        ProtectControlGroups = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectKernelTunables = true;
+        ProtectHostname = true;
+        ProtectProc = true;
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        MemoryDenyWriteExecute = true;
+        LockPersonality = true;
+        ProcSubset = "pid";
+        SystemCallArchitectures = "native";
+        RemoveIPC = true;
+
+      };
+      environment = {
+        BIND_ADDRESS = cfg.bindAddress;
+        DEFAULT_PRIORITY = toString cfg.defaultPriority;
+        DISPATCH_ERRORS = toString cfg.dispatchErrors;
+        EXTENDED_DETAILS = toString cfg.extendedDetails;
+        MESSAGE_ANNOTATION = cfg.messageAnnotation;
+        PORT = toString cfg.port;
+        PRIORITY_ANNOTATION = cfg.priorityAnnotation;
+        TIMEOUT = "${toString cfg.timeout}s";
+        TITLE_ANNOTATION = cfg.titleAnnotation;
+        WEBHOOK_PATH = cfg.webhookPath;
+        GOTIFY_ENDPOINT = "${
+          if cfg.gotifyEndpoint.tls then "https://" else "http://"
+        }${toString cfg.gotifyEndpoint.host}:${toString cfg.gotifyEndpoint.port}/message";
+        AUTH_USERNAME = cfg.metrics.username;
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/alertmanager-irc-relay.nix

@@ -0,0 +1,108 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.alertmanagerIrcRelay;
+
+  configFormat = pkgs.formats.yaml { };
+  configFile = configFormat.generate "alertmanager-irc-relay.yml" cfg.settings;
+in
+{
+  options.services.prometheus.alertmanagerIrcRelay = {
+    enable = lib.mkEnableOption "Alertmanager IRC Relay";
+
+    package = lib.mkPackageOption pkgs "alertmanager-irc-relay" { };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Extra command line options to pass to alertmanager-irc-relay.";
+    };
+
+    settings = lib.mkOption {
+      type = configFormat.type;
+      example = lib.literalExpression ''
+        {
+          http_host = "localhost";
+          http_port = 8000;
+
+          irc_host = "irc.example.com";
+          irc_port = 7000;
+          irc_nickname = "myalertbot";
+
+          irc_channels = [
+            { name = "#mychannel"; }
+          ];
+        }
+      '';
+      description = ''
+        Configuration for Alertmanager IRC Relay as a Nix attribute set.
+        For a reference, check out the
+        [example configuration](https://github.com/google/alertmanager-irc-relay#configuring-and-running-the-bot)
+        and the
+        [source code](https://github.com/google/alertmanager-irc-relay/blob/master/config.go).
+
+        Note: The webhook's URL MUST point to the IRC channel where the message
+        should be posted. For `#mychannel` from the example, this would be
+        `http://localhost:8080/mychannel`.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.alertmanager-irc-relay = {
+      description = "Alertmanager IRC Relay";
+
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/alertmanager-irc-relay \
+          -config ${configFile} \
+          ${lib.escapeShellArgs cfg.extraFlags}
+        '';
+
+        DynamicUser = true;
+        NoNewPrivileges = true;
+
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateIPC = true;
+
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+
+        SystemCallFilter = [
+          "@system-service"
+          "~@cpu-emulation"
+          "~@privileged"
+          "~@reboot"
+          "~@setuid"
+          "~@swap"
+        ];
+      };
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.oxzi ];
+}

nixos/modules/services/monitoring/prometheus/alertmanager-ntfy.nix

@@ -0,0 +1,206 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.alertmanager-ntfy;
+
+  settingsFormat = pkgs.formats.yaml { };
+  settingsFile = settingsFormat.generate "settings.yml" cfg.settings;
+
+  configsArg = lib.concatStringsSep "," (
+    [ settingsFile ] ++ lib.imap0 (i: _: "%d/config-${toString i}.yml") cfg.extraConfigFiles
+  );
+in
+
+{
+  meta.maintainers = with lib.maintainers; [ defelo ];
+
+  options.services.prometheus.alertmanager-ntfy = {
+    enable = lib.mkEnableOption "alertmanager-ntfy";
+
+    package = lib.mkPackageOption pkgs "alertmanager-ntfy" { };
+
+    settings = lib.mkOption {
+      description = ''
+        Configuration of alertmanager-ntfy.
+        See <https://github.com/alexbakker/alertmanager-ntfy> for more information.
+      '';
+      default = { };
+
+      type = lib.types.submodule {
+        freeformType = settingsFormat.type;
+
+        options = {
+          http.addr = lib.mkOption {
+            type = lib.types.str;
+            description = "The address to listen on.";
+            default = "127.0.0.1:8000";
+            example = ":8000";
+          };
+
+          ntfy = {
+            baseurl = lib.mkOption {
+              type = lib.types.str;
+              description = "The base URL of the ntfy.sh instance.";
+              example = "https://ntfy.sh";
+            };
+
+            notification = {
+              topic = lib.mkOption {
+                type = lib.types.str;
+                description = ''
+                  __Note:__ when using ntfy.sh and other public instances
+                  it is recommended to set this option to an empty string and set the actual topic via
+                  [](#opt-services.prometheus.alertmanager-ntfy.extraConfigFiles) since
+                  the `topic` in `ntfy.sh` is essentially a password.
+
+                  The topic to which alerts should be published.
+                  Can either be a hardcoded string or a gval expression that evaluates to a string.
+                '';
+                example = "alertmanager";
+              };
+
+              priority = lib.mkOption {
+                type = lib.types.str;
+                description = ''
+                  The ntfy.sh message priority (see <https://docs.ntfy.sh/publish/#message-priority> for more information).
+                  Can either be a hardcoded string or a gval expression that evaluates to a string.
+                '';
+                default = ''status == "firing" ? "high" : "default"'';
+              };
+
+              tags = lib.mkOption {
+                type = lib.types.listOf (
+                  lib.types.submodule {
+                    options = {
+                      tag = lib.mkOption {
+                        type = lib.types.str;
+                        description = ''
+                          The tag to add.
+                          See <https://docs.ntfy.sh/emojis> for a list of all supported emojis.
+                        '';
+                        example = "rotating_light";
+                      };
+
+                      condition = lib.mkOption {
+                        type = lib.types.nullOr lib.types.str;
+                        description = ''
+                          The condition under which this tag should be added.
+                          Tags with no condition are always included.
+                        '';
+                        default = null;
+                        example = ''status == "firing"'';
+                      };
+                    };
+                  }
+                );
+                description = ''
+                  Tags to add to ntfy.sh messages.
+                  See <https://docs.ntfy.sh/publish/#tags-emojis> for more information.
+                '';
+                default = [
+                  {
+                    tag = "green_circle";
+                    condition = ''status == "resolved"'';
+                  }
+                  {
+                    tag = "red_circle";
+                    condition = ''status == "firing"'';
+                  }
+                ];
+              };
+
+              templates = {
+                title = lib.mkOption {
+                  type = lib.types.str;
+                  description = "The ntfy.sh message title template.";
+                  default = ''
+                    {{ if eq .Status "resolved" }}Resolved: {{ end }}{{ index .Annotations "summary" }}
+                  '';
+                };
+
+                description = lib.mkOption {
+                  type = lib.types.str;
+                  description = "The ntfy.sh message description template.";
+                  default = ''
+                    {{ index .Annotations "description" }}
+                  '';
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+
+    extraConfigFiles = lib.mkOption {
+      type = lib.types.listOf lib.types.path;
+      default = [ ];
+      example = [ "/run/secrets/alertmanager-ntfy.yml" ];
+      description = ''
+        Config files to merge into the settings defined in [](#opt-services.prometheus.alertmanager-ntfy.settings).
+        This is useful to avoid putting secrets into the Nix store.
+        See <https://github.com/alexbakker/alertmanager-ntfy> for more information.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.alertmanager-ntfy = {
+      wantedBy = [ "multi-user.target" ];
+
+      wants = [ "network-online.target" ];
+      after = [ "network-online.target" ];
+
+      serviceConfig = {
+        User = "alertmanager-ntfy";
+        Group = "alertmanager-ntfy";
+        DynamicUser = true;
+
+        LoadCredential = lib.imap0 (i: path: "config-${toString i}.yml:${path}") cfg.extraConfigFiles;
+
+        ExecStart = "${lib.getExe cfg.package} --configs ${configsArg}";
+
+        Restart = "always";
+        RestartSec = 5;
+
+        # Hardening
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = [ "" ];
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        RemoveIPC = true;
+        RestrictAddressFamilies = [ "AF_INET AF_INET6" ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+          "~@resources"
+        ];
+        UMask = "0077";
+      };
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/alertmanager-webhook-logger.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.alertmanagerWebhookLogger;
+in
+{
+  options.services.prometheus.alertmanagerWebhookLogger = {
+    enable = lib.mkEnableOption "Alertmanager Webhook Logger";
+
+    package = lib.mkPackageOption pkgs "alertmanager-webhook-logger" { };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = "Extra command line options to pass to alertmanager-webhook-logger.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.alertmanager-webhook-logger = {
+      description = "Alertmanager Webhook Logger";
+
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/alertmanager-webhook-logger \
+          ${lib.escapeShellArgs cfg.extraFlags}
+        '';
+
+        CapabilityBoundingSet = [ "" ];
+        DeviceAllow = [ "" ];
+        DynamicUser = true;
+        NoNewPrivileges = true;
+
+        MemoryDenyWriteExecute = true;
+
+        LockPersonality = true;
+
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ProtectHome = "tmpfs";
+
+        PrivateTmp = true;
+        PrivateDevices = true;
+        PrivateIPC = true;
+
+        ProcSubset = "pid";
+
+        ProtectHostname = true;
+        ProtectClock = true;
+        ProtectKernelTunables = true;
+        ProtectKernelModules = true;
+        ProtectKernelLogs = true;
+        ProtectControlGroups = true;
+
+        Restart = "on-failure";
+
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+
+        SystemCallFilter = [
+          "@system-service"
+          "~@cpu-emulation"
+          "~@privileged"
+          "~@reboot"
+          "~@setuid"
+          "~@swap"
+        ];
+      };
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.jpds ];
+}

nixos/modules/services/monitoring/prometheus/alertmanager.nix

@@ -0,0 +1,269 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.prometheus.alertmanager;
+  mkConfigFile = pkgs.writeText "alertmanager.yml" (builtins.toJSON cfg.configuration);
+
+  checkedConfig =
+    file:
+    if cfg.checkConfig then
+      pkgs.runCommand "checked-config" { nativeBuildInputs = [ cfg.package ]; } ''
+        ln -s ${file} $out
+        amtool check-config $out
+      ''
+    else
+      file;
+
+  alertmanagerYml =
+    let
+      yml =
+        if cfg.configText != null then pkgs.writeText "alertmanager.yml" cfg.configText else mkConfigFile;
+    in
+    checkedConfig yml;
+
+  cmdlineArgs =
+    cfg.extraFlags
+    ++ [
+      "--config.file /tmp/alert-manager-substituted.yaml"
+      "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
+      "--log.level ${cfg.logLevel}"
+      "--storage.path /var/lib/alertmanager"
+      (toString (map (peer: "--cluster.peer ${peer}:9094") cfg.clusterPeers))
+    ]
+    ++ (lib.optional (cfg.webExternalUrl != null) "--web.external-url ${cfg.webExternalUrl}")
+    ++ (lib.optional (cfg.logFormat != null) "--log.format ${cfg.logFormat}");
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "user" ]
+      "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a user setting."
+    )
+    (lib.mkRemovedOptionModule [ "services" "prometheus" "alertmanager" "group" ]
+      "The alertmanager service is now using systemd's DynamicUser mechanism which obviates a group setting."
+    )
+    (lib.mkRemovedOptionModule [ "services" "prometheus" "alertmanagerURL" ] ''
+      Due to incompatibility, the alertmanagerURL option has been removed,
+      please use 'services.prometheus.alertmanagers' instead.
+    '')
+  ];
+
+  options = {
+    services.prometheus.alertmanager = {
+      enable = lib.mkEnableOption "Prometheus Alertmanager";
+
+      package = lib.mkPackageOption pkgs "prometheus-alertmanager" { };
+
+      configuration = lib.mkOption {
+        type = lib.types.nullOr lib.types.attrs;
+        default = null;
+        description = ''
+          Alertmanager configuration as nix attribute set.
+
+          The contents of the resulting config file are processed using envsubst.
+          `$` needs to be escaped as `$$` to be preserved.
+        '';
+      };
+
+      configText = lib.mkOption {
+        type = lib.types.nullOr lib.types.lines;
+        default = null;
+        description = ''
+          Alertmanager configuration as YAML text. If non-null, this option
+          defines the text that is written to alertmanager.yml. If null, the
+          contents of alertmanager.yml is generated from the structured config
+          options.
+
+          The contents of the resulting config file are processed using envsubst.
+          `$` needs to be escaped as `$$` to be preserved.
+        '';
+      };
+
+      checkConfig = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Check configuration with `amtool check-config`. The call to `amtool` is
+          subject to sandboxing by Nix.
+
+          If you use credentials stored in external files
+          (`environmentFile`, etc),
+          they will not be visible to `amtool`
+          and it will report errors, despite a correct configuration.
+        '';
+      };
+
+      logFormat = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        description = ''
+          If set use a syslog logger or JSON logging.
+        '';
+      };
+
+      logLevel = lib.mkOption {
+        type = lib.types.enum [
+          "debug"
+          "info"
+          "warn"
+          "error"
+          "fatal"
+        ];
+        default = "warn";
+        description = ''
+          Only log messages with the given severity or above.
+        '';
+      };
+
+      webExternalUrl = lib.mkOption {
+        type = lib.types.nullOr lib.types.str;
+        default = null;
+        description = ''
+          The URL under which Alertmanager is externally reachable (for example, if Alertmanager is served via a reverse proxy).
+          Used for generating relative and absolute links back to Alertmanager itself.
+          If the URL has a path portion, it will be used to prefix all HTTP endoints served by Alertmanager.
+          If omitted, relevant URL components will be derived automatically.
+        '';
+      };
+
+      listenAddress = lib.mkOption {
+        type = lib.types.str;
+        default = "";
+        description = ''
+          Address to listen on for the web interface and API. Empty string will listen on all interfaces.
+          "localhost" will listen on 127.0.0.1 (but not ::1).
+        '';
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 9093;
+        description = ''
+          Port to listen on for the web interface and API.
+        '';
+      };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open port in firewall for incoming connections.
+        '';
+      };
+
+      clusterPeers = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = ''
+          Initial peers for HA cluster.
+        '';
+      };
+
+      extraFlags = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        default = [ ];
+        description = ''
+          Extra commandline options when launching the Alertmanager.
+        '';
+      };
+
+      environmentFile = lib.mkOption {
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+        example = "/root/alertmanager.env";
+        description = ''
+          File to load as environment file. Environment variables
+          from this file will be interpolated into the config file
+          using envsubst with this syntax:
+          `$ENVIRONMENT ''${VARIABLE}`
+        '';
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.enable {
+      assertions = lib.singleton {
+        assertion = cfg.configuration != null || cfg.configText != null;
+        message =
+          "Can not enable alertmanager without a configuration. "
+          + "Set either the `configuration` or `configText` attribute.";
+      };
+    })
+    (lib.mkIf cfg.enable {
+      networking.firewall.allowedTCPPorts = lib.optional cfg.openFirewall cfg.port;
+
+      systemd.services.alertmanager = {
+        wantedBy = [ "multi-user.target" ];
+        wants = [ "network-online.target" ];
+        after = [ "network-online.target" ];
+        preStart = ''
+          ${lib.getBin pkgs.envsubst}/bin/envsubst -o "/tmp/alert-manager-substituted.yaml" \
+                                                   -i "${alertmanagerYml}"
+        '';
+        serviceConfig = {
+          ExecStart =
+            "${cfg.package}/bin/alertmanager"
+            + lib.optionalString (lib.length cmdlineArgs != 0) (
+              " \\\n  " + lib.concatStringsSep " \\\n  " cmdlineArgs
+            );
+          ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+
+          EnvironmentFile = lib.mkIf (cfg.environmentFile != null) cfg.environmentFile;
+
+          CapabilityBoundingSet = [ "" ];
+          DeviceAllow = [ "" ];
+          DynamicUser = true;
+          NoNewPrivileges = true;
+
+          MemoryDenyWriteExecute = true;
+
+          LockPersonality = true;
+
+          ProtectProc = "invisible";
+          ProtectSystem = "strict";
+          ProtectHome = "tmpfs";
+
+          PrivateTmp = true;
+          PrivateDevices = true;
+          PrivateIPC = true;
+
+          ProcSubset = "pid";
+
+          ProtectHostname = true;
+          ProtectClock = true;
+          ProtectKernelTunables = true;
+          ProtectKernelModules = true;
+          ProtectKernelLogs = true;
+          ProtectControlGroups = true;
+
+          Restart = "always";
+
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+            "AF_NETLINK"
+          ];
+          RestrictNamespaces = true;
+          RestrictRealtime = true;
+          RestrictSUIDSGID = true;
+
+          StateDirectory = "alertmanager";
+          SystemCallFilter = [
+            "@system-service"
+            "~@cpu-emulation"
+            "~@privileged"
+            "~@reboot"
+            "~@setuid"
+            "~@swap"
+          ];
+
+          WorkingDirectory = "/tmp";
+        };
+      };
+    })
+  ];
+}

nixos/modules/services/monitoring/prometheus/exporters.md

@@ -0,0 +1,192 @@
+# Prometheus exporters {#module-services-prometheus-exporters}
+
+Prometheus exporters provide metrics for the
+[prometheus monitoring system](https://prometheus.io).
+
+## Configuration {#module-services-prometheus-exporters-configuration}
+
+One of the most common exporters is the
+[node exporter](https://github.com/prometheus/node_exporter),
+it provides hardware and OS metrics from the host it's
+running on. The exporter could be configured as follows:
+```nix
+{
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = 9100;
+    enabledCollectors = [
+      "logind"
+      "systemd"
+    ];
+    disabledCollectors = [ "textfile" ];
+    openFirewall = true;
+    firewallFilter = "-i br0 -p tcp -m tcp --dport 9100";
+  };
+}
+```
+It should now serve all metrics from the collectors that are explicitly
+enabled and the ones that are
+[enabled by default](https://github.com/prometheus/node_exporter#enabled-by-default),
+via http under `/metrics`. In this
+example the firewall should just allow incoming connections to the
+exporter's port on the bridge interface `br0` (this would
+have to be configured separately of course). For more information about
+configuration see `man configuration.nix` or search through
+the [available options](https://nixos.org/nixos/options.html#prometheus.exporters).
+
+Prometheus can now be configured to consume the metrics produced by the exporter:
+```nix
+{
+  services.prometheus = {
+    # ...
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [
+          {
+            targets = [
+              "localhost:${toString config.services.prometheus.exporters.node.port}"
+            ];
+          }
+        ];
+      }
+    ];
+
+    # ...
+  };
+}
+```
+
+## Adding a new exporter {#module-services-prometheus-exporters-new-exporter}
+
+To add a new exporter, it has to be packaged first (see
+`nixpkgs/pkgs/servers/monitoring/prometheus/` for
+examples), then a module can be added. The postfix exporter is used in this
+example:
+
+  - Some default options for all exporters are provided by
+    `nixpkgs/nixos/modules/services/monitoring/prometheus/exporters.nix`:
+
+      - `enable`
+      - `port`
+      - `listenAddress`
+      - `extraFlags`
+      - `openFirewall`
+      - `firewallFilter`
+      - `firewallRules`
+      - `user`
+      - `group`
+  - As there is already a package available, the module can now be added. This
+    is accomplished by adding a new file to the
+    `nixos/modules/services/monitoring/prometheus/exporters/`
+    directory, which will be called postfix.nix and contains all exporter
+    specific options and configuration:
+    ```nix
+    # nixpkgs/nixos/modules/services/prometheus/exporters/postfix.nix
+    {
+      config,
+      lib,
+      pkgs,
+      options,
+    }:
+    let
+      # for convenience we define cfg here
+      cfg = config.services.prometheus.exporters.postfix;
+    in
+    {
+      port = 9154; # The postfix exporter listens on this port by default
+
+      # `extraOpts` is an attribute set which contains additional options
+      # (and optional overrides for default options).
+      # Note that this attribute is optional.
+      extraOpts = {
+        telemetryPath = lib.mkOption {
+          type = lib.types.str;
+          default = "/metrics";
+          description = ''
+            Path under which to expose metrics.
+          '';
+        };
+        logfilePath = lib.mkOption {
+          type = lib.types.path;
+          default = /var/log/postfix_exporter_input.log;
+          example = /var/log/mail.log;
+          description = ''
+            Path where Postfix writes log entries.
+            This file will be truncated by this exporter!
+          '';
+        };
+        showqPath = lib.mkOption {
+          type = lib.types.path;
+          default = /var/spool/postfix/public/showq;
+          example = /var/lib/postfix/queue/public/showq;
+          description = ''
+            Path at which Postfix places its showq socket.
+          '';
+        };
+      };
+
+      # `serviceOpts` is an attribute set which contains configuration
+      # for the exporter's systemd service. One of
+      # `serviceOpts.script` and `serviceOpts.serviceConfig.ExecStart`
+      # has to be specified here. This will be merged with the default
+      # service configuration.
+      # Note that by default 'DynamicUser' is 'true'.
+      serviceOpts = {
+        serviceConfig = {
+          DynamicUser = false;
+          ExecStart = ''
+            ${pkgs.prometheus-postfix-exporter}/bin/postfix_exporter \
+              --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+              --web.telemetry-path ${cfg.telemetryPath} \
+              ${lib.concatStringsSep " \\\n  " cfg.extraFlags}
+          '';
+        };
+      };
+    }
+    ```
+  - This should already be enough for the postfix exporter. Additionally one
+    could now add assertions and conditional default values. This can be done
+    in the 'meta-module' that combines all exporter definitions and generates
+    the submodules:
+    `nixpkgs/nixos/modules/services/prometheus/exporters.nix`
+
+## Updating an exporter module {#module-services-prometheus-exporters-update-exporter-module}
+
+Should an exporter option change at some point, it is possible to add
+information about the change to the exporter definition similar to
+`nixpkgs/nixos/modules/rename.nix`:
+```nix
+{
+  config,
+  lib,
+  pkgs,
+  options,
+}:
+
+let
+  cfg = config.services.prometheus.exporters.nginx;
+in
+{
+  port = 9113;
+  extraOpts = {
+    # additional module options
+    # ...
+  };
+  serviceOpts = {
+    # service configuration
+    # ...
+  };
+  imports = [
+    # 'services.prometheus.exporters.nginx.telemetryEndpoint' -> 'services.prometheus.exporters.nginx.telemetryPath'
+    (lib.mkRenamedOptionModule [ "telemetryEndpoint" ] [ "telemetryPath" ])
+
+    # removed option 'services.prometheus.exporters.nginx.insecure'
+    (lib.mkRemovedOptionModule [ "insecure" ] ''
+      This option was replaced by 'prometheus.exporters.nginx.sslVerify' which defaults to true.
+    '')
+    ({ options.warnings = options.warnings; })
+  ];
+}
+```

nixos/modules/services/monitoring/prometheus/exporters.nix

@@ -0,0 +1,597 @@
+{
+  config,
+  pkgs,
+  lib,
+  options,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    concatStrings
+    foldl
+    foldl'
+    genAttrs
+    literalExpression
+    maintainers
+    mapAttrs
+    mapAttrsToList
+    mkDefault
+    mkEnableOption
+    mkIf
+    mkMerge
+    mkOption
+    optional
+    types
+    mkOptionDefault
+    flip
+    attrNames
+    xor
+    ;
+
+  cfg = config.services.prometheus.exporters;
+
+  # each attribute in `exporterOpts` is expected to have specified:
+  #   - port        (types.int):   port on which the exporter listens
+  #   - serviceOpts (types.attrs): config that is merged with the
+  #                                default definition of the exporter's
+  #                                systemd service
+  #   - extraOpts   (types.attrs): extra configuration options to
+  #                                configure the exporter with, which
+  #                                are appended to the default options
+  #
+  #  Note that `extraOpts` is optional, but a script for the exporter's
+  #  systemd service must be provided by specifying either
+  #  `serviceOpts.script` or `serviceOpts.serviceConfig.ExecStart`
+
+  exporterOpts =
+    (genAttrs
+      [
+        "apcupsd"
+        "artifactory"
+        "bind"
+        "bird"
+        "bitcoin"
+        "blackbox"
+        "borgmatic"
+        "buildkite-agent"
+        "ecoflow"
+        "chrony"
+        "collectd"
+        "deluge"
+        "dmarc"
+        "dnsmasq"
+        "dnssec"
+        "domain"
+        "dovecot"
+        "ebpf"
+        "fastly"
+        "flow"
+        "fritz"
+        "fritzbox"
+        "frr"
+        "graphite"
+        "idrac"
+        "imap-mailstat"
+        "influxdb"
+        "ipmi"
+        "jitsi"
+        "json"
+        "junos-czerwonk"
+        "kafka"
+        "kea"
+        "keylight"
+        "klipper"
+        "knot"
+        "libvirt"
+        "lnd"
+        "mail"
+        "mailman3"
+        "mikrotik"
+        "modemmanager"
+        "mongodb"
+        "mqtt"
+        "mysqld"
+        "nats"
+        "nextcloud"
+        "nginx"
+        "nginxlog"
+        "node"
+        "node-cert"
+        "nut"
+        "nvidia-gpu"
+        "pgbouncer"
+        "php-fpm"
+        "pihole"
+        "ping"
+        "postfix"
+        "postgres"
+        "process"
+        "pve"
+        "py-air-control"
+        "rasdaemon"
+        "redis"
+        "restic"
+        "rspamd"
+        "rtl_433"
+        "sabnzbd"
+        "scaphandre"
+        "script"
+        "shelly"
+        "smartctl"
+        "smokeping"
+        "snmp"
+        "sql"
+        "statsd"
+        "storagebox"
+        "surfboard"
+        "systemd"
+        "tibber"
+        "unbound"
+        "unpoller"
+        "v2ray"
+        "varnish"
+        "wireguard"
+        "zfs"
+      ]
+      (
+        name:
+        import (./. + "/exporters/${name}.nix") {
+          inherit
+            config
+            lib
+            pkgs
+            options
+            utils
+            ;
+        }
+      )
+    )
+    // (mapAttrs
+      (
+        name: params:
+        import (./. + "/exporters/${params.name}.nix") {
+          inherit
+            config
+            lib
+            pkgs
+            options
+            utils
+            ;
+          type = params.type;
+        }
+      )
+      {
+        exportarr-bazarr = {
+          name = "exportarr";
+          type = "bazarr";
+        };
+        exportarr-lidarr = {
+          name = "exportarr";
+          type = "lidarr";
+        };
+        exportarr-prowlarr = {
+          name = "exportarr";
+          type = "prowlarr";
+        };
+        exportarr-radarr = {
+          name = "exportarr";
+          type = "radarr";
+        };
+        exportarr-readarr = {
+          name = "exportarr";
+          type = "readarr";
+        };
+        exportarr-sonarr = {
+          name = "exportarr";
+          type = "sonarr";
+        };
+      }
+    );
+
+  mkExporterOpts = (
+    { name, port }:
+    {
+      enable = mkEnableOption "the prometheus ${name} exporter";
+      port = mkOption {
+        type = types.port;
+        default = port;
+        description = ''
+          Port to listen on.
+        '';
+      };
+      listenAddress = mkOption {
+        type = types.str;
+        default = "0.0.0.0";
+        description = ''
+          Address to listen on.
+        '';
+      };
+      extraFlags = mkOption {
+        type = types.listOf types.str;
+        default = [ ];
+        description = ''
+          Extra commandline options to pass to the ${name} exporter.
+        '';
+      };
+      openFirewall = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Open port in firewall for incoming connections.
+        '';
+      };
+      firewallFilter = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        example = literalExpression ''
+          "-i eth0 -p tcp -m tcp --dport ${toString port}"
+        '';
+        description = ''
+          Specify a filter for iptables to use when
+          {option}`services.prometheus.exporters.${name}.openFirewall`
+          is true. It is used as `ip46tables -I nixos-fw firewallFilter -j nixos-fw-accept`.
+        '';
+      };
+      firewallRules = mkOption {
+        type = types.nullOr types.lines;
+        default = null;
+        example = literalExpression ''
+          iifname "eth0" tcp dport ${toString port} counter accept
+        '';
+        description = ''
+          Specify rules for nftables to add to the input chain
+          when {option}`services.prometheus.exporters.${name}.openFirewall` is true.
+        '';
+      };
+      user = mkOption {
+        type = types.str;
+        default = "${name}-exporter";
+        description = ''
+          User name under which the ${name} exporter shall be run.
+        '';
+      };
+      group = mkOption {
+        type = types.str;
+        default = "${name}-exporter";
+        description = ''
+          Group under which the ${name} exporter shall be run.
+        '';
+      };
+    }
+  );
+
+  mkSubModule =
+    {
+      name,
+      port,
+      extraOpts,
+      imports,
+    }:
+    {
+      ${name} = mkOption {
+        type = types.submodule [
+          {
+            inherit imports;
+            options = (
+              mkExporterOpts {
+                inherit name port;
+              }
+              // extraOpts
+            );
+          }
+          (
+            { config, ... }:
+            mkIf config.openFirewall {
+              firewallFilter = mkDefault "-p tcp -m tcp --dport ${toString config.port}";
+              firewallRules = mkDefault ''tcp dport ${toString config.port} accept comment "${name}-exporter"'';
+            }
+          )
+        ];
+        internal = true;
+        default = { };
+      };
+    };
+
+  mkSubModules = (
+    foldl' (a: b: a // b) { } (
+      mapAttrsToList (
+        name: opts:
+        mkSubModule {
+          inherit name;
+          inherit (opts) port;
+          extraOpts = opts.extraOpts or { };
+          imports = opts.imports or [ ];
+        }
+      ) exporterOpts
+    )
+  );
+
+  mkExporterConf =
+    {
+      name,
+      conf,
+      serviceOpts,
+    }:
+    let
+      enableDynamicUser = serviceOpts.serviceConfig.DynamicUser or true;
+      nftables = config.networking.nftables.enable;
+    in
+    mkIf conf.enable {
+      warnings = conf.warnings or [ ];
+      assertions = conf.assertions or [ ];
+      users.users."${name}-exporter" = (
+        mkIf (conf.user == "${name}-exporter" && !enableDynamicUser) {
+          description = "Prometheus ${name} exporter service user";
+          isSystemUser = true;
+          inherit (conf) group;
+        }
+      );
+      users.groups = mkMerge [
+        (mkIf (conf.group == "${name}-exporter" && !enableDynamicUser) {
+          "${name}-exporter" = { };
+        })
+        (mkIf (name == "smartctl") {
+          "smartctl-exporter-access" = { };
+        })
+      ];
+      services.udev.extraRules = mkIf (name == "smartctl") ''
+        ACTION=="add", SUBSYSTEM=="nvme", KERNEL=="nvme[0-9]*", RUN+="${pkgs.acl}/bin/setfacl -m g:smartctl-exporter-access:rw /dev/$kernel"
+      '';
+      networking.firewall.extraCommands = mkIf (conf.openFirewall && !nftables) (concatStrings [
+        "ip46tables -A nixos-fw ${conf.firewallFilter} "
+        "-m comment --comment ${name}-exporter -j nixos-fw-accept"
+      ]);
+      networking.firewall.extraInputRules = mkIf (conf.openFirewall && nftables) conf.firewallRules;
+      systemd.services."prometheus-${name}-exporter" = mkMerge [
+        {
+          wantedBy = [ "multi-user.target" ];
+          after = [ "network.target" ];
+          serviceConfig.Restart = mkDefault "always";
+          serviceConfig.PrivateTmp = mkDefault true;
+          serviceConfig.WorkingDirectory = mkDefault /tmp;
+          serviceConfig.DynamicUser = mkDefault enableDynamicUser;
+          serviceConfig.User = mkDefault conf.user;
+          serviceConfig.Group = conf.group;
+          # Hardening
+          serviceConfig.CapabilityBoundingSet = mkDefault [ "" ];
+          serviceConfig.DeviceAllow = [ "" ];
+          serviceConfig.LockPersonality = true;
+          serviceConfig.MemoryDenyWriteExecute = true;
+          serviceConfig.NoNewPrivileges = true;
+          serviceConfig.PrivateDevices = mkDefault true;
+          serviceConfig.ProtectClock = mkDefault true;
+          serviceConfig.ProtectControlGroups = true;
+          serviceConfig.ProtectHome = true;
+          serviceConfig.ProtectHostname = true;
+          serviceConfig.ProtectKernelLogs = true;
+          serviceConfig.ProtectKernelModules = true;
+          serviceConfig.ProtectKernelTunables = true;
+          serviceConfig.ProtectSystem = mkDefault "strict";
+          serviceConfig.RemoveIPC = true;
+          serviceConfig.RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+          ];
+          serviceConfig.RestrictNamespaces = true;
+          serviceConfig.RestrictRealtime = true;
+          serviceConfig.RestrictSUIDSGID = true;
+          serviceConfig.SystemCallArchitectures = "native";
+          serviceConfig.UMask = "0077";
+        }
+        serviceOpts
+      ];
+    };
+in
+{
+
+  options.services.prometheus.exporters = mkOption {
+    type = types.submodule {
+      options = mkSubModules;
+      imports = [
+        ../../../misc/assertions.nix
+        (lib.mkRenamedOptionModule [ "unifi-poller" ] [ "unpoller" ])
+        (lib.mkRemovedOptionModule [ "minio" ] ''
+          The Minio exporter has been removed, as it was broken and unmaintained.
+          See the 24.11 release notes for more information.
+        '')
+        (lib.mkRemovedOptionModule [ "tor" ] ''
+          The Tor exporter has been removed, as it was broken and unmaintained.
+        '')
+      ];
+    };
+    description = "Prometheus exporter configuration";
+    default = { };
+    example = literalExpression ''
+      {
+        node = {
+          enable = true;
+          enabledCollectors = [ "systemd" ];
+        };
+        varnish.enable = true;
+      }
+    '';
+  };
+
+  config = mkMerge (
+    [
+      {
+        assertions = [
+          {
+            assertion =
+              cfg.ipmi.enable -> (cfg.ipmi.configFile != null) -> (!(lib.hasPrefix "/tmp/" cfg.ipmi.configFile));
+            message = ''
+              Config file specified in `services.prometheus.exporters.ipmi.configFile' must
+                not reside within /tmp - it won't be visible to the systemd service.
+            '';
+          }
+          {
+            assertion =
+              cfg.ipmi.enable
+              -> (cfg.ipmi.webConfigFile != null)
+              -> (!(lib.hasPrefix "/tmp/" cfg.ipmi.webConfigFile));
+            message = ''
+              Config file specified in `services.prometheus.exporters.ipmi.webConfigFile' must
+                not reside within /tmp - it won't be visible to the systemd service.
+            '';
+          }
+          {
+            assertion =
+              cfg.restic.enable -> ((cfg.restic.repository == null) != (cfg.restic.repositoryFile == null));
+            message = ''
+              Please specify either 'services.prometheus.exporters.restic.repository'
+                or 'services.prometheus.exporters.restic.repositoryFile'.
+            '';
+          }
+          {
+            assertion =
+              cfg.snmp.enable -> ((cfg.snmp.configurationPath == null) != (cfg.snmp.configuration == null));
+            message = ''
+              Please ensure you have either `services.prometheus.exporters.snmp.configuration'
+                or `services.prometheus.exporters.snmp.configurationPath' set!
+            '';
+          }
+          {
+            assertion =
+              cfg.mikrotik.enable -> ((cfg.mikrotik.configFile == null) != (cfg.mikrotik.configuration == null));
+            message = ''
+              Please specify either `services.prometheus.exporters.mikrotik.configuration'
+                or `services.prometheus.exporters.mikrotik.configFile'.
+            '';
+          }
+          {
+            assertion = cfg.mail.enable -> ((cfg.mail.configFile == null) != (cfg.mail.configuration == null));
+            message = ''
+              Please specify either 'services.prometheus.exporters.mail.configuration'
+                or 'services.prometheus.exporters.mail.configFile'.
+            '';
+          }
+          {
+            assertion = cfg.mysqld.runAsLocalSuperUser -> config.services.mysql.enable;
+            message = ''
+              The exporter is configured to run as 'services.mysql.user', but
+                'services.mysql.enable' is set to false.
+            '';
+          }
+          {
+            assertion =
+              cfg.nextcloud.enable -> ((cfg.nextcloud.passwordFile == null) != (cfg.nextcloud.tokenFile == null));
+            message = ''
+              Please specify either 'services.prometheus.exporters.nextcloud.passwordFile' or
+                'services.prometheus.exporters.nextcloud.tokenFile'
+            '';
+          }
+          {
+            assertion = cfg.sql.enable -> ((cfg.sql.configFile == null) != (cfg.sql.configuration == null));
+            message = ''
+              Please specify either 'services.prometheus.exporters.sql.configuration' or
+                'services.prometheus.exporters.sql.configFile'
+            '';
+          }
+          {
+            assertion = cfg.scaphandre.enable -> (pkgs.stdenv.targetPlatform.isx86_64 == true);
+            message = ''
+              Scaphandre only support x86_64 architectures.
+            '';
+          }
+          {
+            assertion =
+              cfg.scaphandre.enable
+              -> ((lib.kernel.whenHelpers pkgs.linux.version).whenOlder "5.11" true).condition == false;
+            message = ''
+              Scaphandre requires a kernel version newer than '5.11', '${pkgs.linux.version}' given.
+            '';
+          }
+          {
+            assertion = cfg.scaphandre.enable -> (builtins.elem "intel_rapl_common" config.boot.kernelModules);
+            message = ''
+              Scaphandre needs 'intel_rapl_common' kernel module to be enabled. Please add it in 'boot.kernelModules'.
+            '';
+          }
+          {
+            assertion =
+              cfg.idrac.enable -> ((cfg.idrac.configurationPath == null) != (cfg.idrac.configuration == null));
+            message = ''
+              Please ensure you have either `services.prometheus.exporters.idrac.configuration'
+                or `services.prometheus.exporters.idrac.configurationPath' set!
+            '';
+          }
+          {
+            assertion =
+              cfg.deluge.enable
+              -> ((cfg.deluge.delugePassword == null) != (cfg.deluge.delugePasswordFile == null));
+            message = ''
+              Please ensure you have either `services.prometheus.exporters.deluge.delugePassword'
+                or `services.prometheus.exporters.deluge.delugePasswordFile' set!
+            '';
+          }
+          {
+            assertion =
+              cfg.pgbouncer.enable
+              -> (xor (cfg.pgbouncer.connectionEnvFile == null) (cfg.pgbouncer.connectionString == null));
+            message = ''
+              Options `services.prometheus.exporters.pgbouncer.connectionEnvFile` and
+              `services.prometheus.exporters.pgbouncer.connectionString` are mutually exclusive!
+            '';
+          }
+        ]
+        ++ (flip map (attrNames exporterOpts) (exporter: {
+          assertion = cfg.${exporter}.firewallFilter != null -> cfg.${exporter}.openFirewall;
+          message = ''
+            The `firewallFilter'-option of exporter ${exporter} doesn't have any effect unless
+            `openFirewall' is set to `true'!
+          '';
+        }))
+        ++ config.services.prometheus.exporters.assertions;
+        warnings = [
+          (mkIf
+            (
+              config.services.prometheus.exporters.idrac.enable
+              && config.services.prometheus.exporters.idrac.configurationPath != null
+            )
+            ''
+              Configuration file in `services.prometheus.exporters.idrac.configurationPath` may override
+              `services.prometheus.exporters.idrac.listenAddress` and/or `services.prometheus.exporters.idrac.port`.
+              Consider using `services.prometheus.exporters.idrac.configuration` instead.
+            ''
+          )
+        ]
+        ++ config.services.prometheus.exporters.warnings;
+      }
+    ]
+    ++ [
+      (mkIf config.services.prometheus.exporters.rtl_433.enable {
+        hardware.rtl-sdr.enable = mkDefault true;
+      })
+    ]
+    ++ [
+      (mkIf config.services.postfix.enable {
+        services.prometheus.exporters.postfix.group = mkDefault config.services.postfix.setgidGroup;
+      })
+    ]
+    ++ [
+      (mkIf config.services.prometheus.exporters.deluge.enable {
+        system.activationScripts = {
+          deluge-exported.text = ''
+            mkdir -p /etc/deluge-exporter
+            echo "DELUGE_PASSWORD=$(cat ${config.services.prometheus.exporters.deluge.delugePasswordFile})" > /etc/deluge-exporter/password
+          '';
+        };
+      })
+    ]
+    ++ (mapAttrsToList (
+      name: conf:
+      mkExporterConf {
+        inherit name;
+        inherit (conf) serviceOpts;
+        conf = cfg.${name};
+      }
+    ) exporterOpts)
+  );
+
+  meta = {
+    doc = ./exporters.md;
+    maintainers = [ ];
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/apcupsd.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.apcupsd;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9162;
+  extraOpts = {
+    apcupsdAddress = mkOption {
+      type = types.str;
+      default = ":3551";
+      description = ''
+        Address of the apcupsd Network Information Server (NIS).
+      '';
+    };
+
+    apcupsdNetwork = mkOption {
+      type = types.enum [
+        "tcp"
+        "tcp4"
+        "tcp6"
+      ];
+      default = "tcp";
+      description = ''
+        Network of the apcupsd Network Information Server (NIS): one of "tcp", "tcp4", or "tcp6".
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-apcupsd-exporter}/bin/apcupsd_exporter \
+          -telemetry.addr ${cfg.listenAddress}:${toString cfg.port} \
+          -apcupsd.addr ${cfg.apcupsdAddress} \
+          -apcupsd.network ${cfg.apcupsdNetwork} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/artifactory.nix

@@ -0,0 +1,64 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.artifactory;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9531;
+  extraOpts = {
+    scrapeUri = mkOption {
+      type = types.str;
+      default = "http://localhost:8081/artifactory";
+      description = ''
+        URI on which to scrape JFrog Artifactory.
+      '';
+    };
+
+    artiUsername = mkOption {
+      type = types.str;
+      description = ''
+        Username for authentication against JFrog Artifactory API.
+      '';
+    };
+
+    artiPassword = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        Password for authentication against JFrog Artifactory API.
+        One of the password or access token needs to be set.
+      '';
+    };
+
+    artiAccessToken = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        Access token for authentication against JFrog Artifactory API.
+        One of the password or access token needs to be set.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-artifactory-exporter}/bin/artifactory_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --artifactory.scrape-uri ${cfg.scrapeUri} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+      Environment = [
+        "ARTI_USERNAME=${cfg.artiUsername}"
+        "ARTI_PASSWORD=${cfg.artiPassword}"
+        "ARTI_ACCESS_TOKEN=${cfg.artiAccessToken}"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/bind.nix

@@ -0,0 +1,72 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.bind;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9119;
+  extraOpts = {
+    bindURI = mkOption {
+      type = types.str;
+      default = "http://localhost:8053/";
+      description = ''
+        HTTP XML API address of an Bind server.
+      '';
+    };
+    bindTimeout = mkOption {
+      type = types.str;
+      default = "10s";
+      description = ''
+        Timeout for trying to get stats from Bind.
+      '';
+    };
+    bindVersion = mkOption {
+      type = types.enum [
+        "xml.v2"
+        "xml.v3"
+        "auto"
+      ];
+      default = "auto";
+      description = ''
+        BIND statistics version. Can be detected automatically.
+      '';
+    };
+    bindGroups = mkOption {
+      type = types.listOf (
+        types.enum [
+          "server"
+          "view"
+          "tasks"
+        ]
+      );
+      default = [
+        "server"
+        "view"
+      ];
+      description = ''
+        List of statistics to collect. Available: [server, view, tasks]
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-bind-exporter}/bin/bind_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --bind.pid-file /var/run/named/named.pid \
+          --bind.timeout ${toString cfg.bindTimeout} \
+          --bind.stats-url ${cfg.bindURI} \
+          --bind.stats-version ${cfg.bindVersion} \
+          --bind.stats-groups ${concatStringsSep "," cfg.bindGroups} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/bird.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.bird;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    singleton
+    ;
+in
+{
+  port = 9324;
+  extraOpts = {
+    birdVersion = mkOption {
+      type = types.enum [
+        1
+        2
+      ];
+      default = 2;
+      description = ''
+        Specifies whether BIRD1 or BIRD2 is in use.
+      '';
+    };
+    birdSocket = mkOption {
+      type = types.path;
+      default = "/run/bird/bird.ctl";
+      description = ''
+        Path to BIRD2 (or BIRD1 v4) socket.
+      '';
+    };
+    newMetricFormat = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Enable the new more-generic metric format.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      SupplementaryGroups = "bird";
+      ExecStart = ''
+        ${pkgs.prometheus-bird-exporter}/bin/bird_exporter \
+          -web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          -bird.socket ${cfg.birdSocket} \
+          -bird.v2=${if cfg.birdVersion == 2 then "true" else "false"} \
+          -format.new=${if cfg.newMetricFormat then "true" else "false"} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+      RestrictAddressFamilies = [
+        # Need AF_UNIX to collect data
+        "AF_UNIX"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/bitcoin.nix

@@ -0,0 +1,93 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.bitcoin;
+  inherit (lib) mkOption types;
+in
+{
+  port = 9332;
+  extraOpts = {
+    package = lib.mkPackageOption pkgs "prometheus-bitcoin-exporter" { };
+
+    rpcUser = mkOption {
+      type = types.str;
+      default = "bitcoinrpc";
+      description = ''
+        RPC user name.
+      '';
+    };
+
+    rpcPasswordFile = mkOption {
+      type = types.path;
+      description = ''
+        File containing RPC password.
+      '';
+    };
+
+    rpcScheme = mkOption {
+      type = types.enum [
+        "http"
+        "https"
+      ];
+      default = "http";
+      description = ''
+        Whether to connect to bitcoind over http or https.
+      '';
+    };
+
+    rpcHost = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = ''
+        RPC host.
+      '';
+    };
+
+    rpcPort = mkOption {
+      type = types.port;
+      default = 8332;
+      description = ''
+        RPC port number.
+      '';
+    };
+
+    refreshSeconds = mkOption {
+      type = types.ints.unsigned;
+      default = 300;
+      description = ''
+        How often to ask bitcoind for metrics.
+      '';
+    };
+
+    extraEnv = mkOption {
+      type = types.attrsOf types.str;
+      default = { };
+      description = ''
+        Extra environment variables for the exporter.
+      '';
+    };
+  };
+  serviceOpts = {
+    script = ''
+      BITCOIN_RPC_PASSWORD=$(cat ${cfg.rpcPasswordFile})
+      export BITCOIN_RPC_PASSWORD
+      exec ${cfg.package}/bin/bitcoind-monitor.py
+    '';
+
+    environment = {
+      BITCOIN_RPC_USER = cfg.rpcUser;
+      BITCOIN_RPC_SCHEME = cfg.rpcScheme;
+      BITCOIN_RPC_HOST = cfg.rpcHost;
+      BITCOIN_RPC_PORT = toString cfg.rpcPort;
+      METRICS_ADDR = cfg.listenAddress;
+      METRICS_PORT = toString cfg.port;
+      REFRESH_SECONDS = toString cfg.refreshSeconds;
+    }
+    // cfg.extraEnv;
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/blackbox.nix

@@ -0,0 +1,88 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  logPrefix = "services.prometheus.exporter.blackbox";
+  cfg = config.services.prometheus.exporters.blackbox;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    escapeShellArg
+    ;
+
+  # This ensures that we can deal with string paths, path types and
+  # store-path strings with context.
+  coerceConfigFile =
+    file:
+    if (builtins.isPath file) || (lib.isStorePath file) then
+      file
+    else
+      (
+        lib.warn ''
+          ${logPrefix}: configuration file "${file}" is being copied to the nix-store.
+          If you would like to avoid that, please set enableConfigCheck to false.
+        '' /.
+        + file
+      );
+  checkConfigLocation =
+    file:
+    if lib.hasPrefix "/tmp/" file then
+      throw "${logPrefix}: configuration file must not reside within /tmp - it won't be visible to the systemd service."
+    else
+      file;
+  checkConfig =
+    file:
+    pkgs.runCommand "checked-blackbox-exporter.conf"
+      {
+        preferLocalBuild = true;
+        nativeBuildInputs = [ pkgs.buildPackages.prometheus-blackbox-exporter ];
+      }
+      ''
+        ln -s ${coerceConfigFile file} $out
+        blackbox_exporter --config.check --config.file $out
+      '';
+in
+{
+  port = 9115;
+  extraOpts = {
+    configFile = mkOption {
+      type = types.path;
+      description = ''
+        Path to configuration file.
+      '';
+    };
+    enableConfigCheck = mkOption {
+      type = types.bool;
+      default = true;
+      description = ''
+        Whether to run a correctness check for the configuration file. This depends
+        on the configuration file residing in the nix-store. Paths passed as string will
+        be copied to the store.
+      '';
+    };
+  };
+
+  serviceOpts =
+    let
+      adjustedConfigFile =
+        if cfg.enableConfigCheck then checkConfig cfg.configFile else checkConfigLocation cfg.configFile;
+    in
+    {
+      serviceConfig = {
+        AmbientCapabilities = [ "CAP_NET_RAW" ]; # for ping probes
+        ExecStart = ''
+          ${pkgs.prometheus-blackbox-exporter}/bin/blackbox_exporter \
+            --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+            --config.file ${escapeShellArg adjustedConfigFile} \
+            ${concatStringsSep " \\\n  " cfg.extraFlags}
+        '';
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      };
+    };
+}

nixos/modules/services/monitoring/prometheus/exporters/borgmatic.nix

@@ -0,0 +1,34 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.borgmatic;
+in
+{
+  port = 9996;
+  extraOpts.configFile = lib.mkOption {
+    type = lib.types.path;
+    default = "/etc/borgmatic/config.yaml";
+    description = ''
+      The path to the borgmatic config file
+    '';
+  };
+
+  serviceOpts = {
+    serviceConfig = {
+      DynamicUser = false;
+      ProtectSystem = false;
+      ProtectHome = lib.mkForce false;
+      ExecStart = ''
+        ${pkgs.prometheus-borgmatic-exporter}/bin/borgmatic-exporter run \
+          --port ${toString cfg.port} \
+          --config ${toString cfg.configFile} \
+          ${lib.concatMapStringsSep " " (f: lib.escapeShellArg f) cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/buildkite-agent.nix

@@ -0,0 +1,75 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.buildkite-agent;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    optionalString
+    literalExpression
+    ;
+in
+{
+  port = 9876;
+  extraOpts = {
+    tokenPath = mkOption {
+      type = types.nullOr types.path;
+      apply = final: if final == null then null else toString final;
+      description = ''
+        The token from your Buildkite "Agents" page.
+
+        A run-time path to the token file, which is supposed to be provisioned
+        outside of Nix store.
+      '';
+    };
+    interval = mkOption {
+      type = types.str;
+      default = "30s";
+      example = "1min";
+      description = ''
+        How often to update metrics.
+      '';
+    };
+    endpoint = mkOption {
+      type = types.str;
+      default = "https://agent.buildkite.com/v3";
+      description = ''
+        The Buildkite Agent API endpoint.
+      '';
+    };
+    queues = mkOption {
+      type = with types; nullOr (listOf str);
+      default = null;
+      example = literalExpression ''[ "my-queue1" "my-queue2" ]'';
+      description = ''
+        Which specific queues to process.
+      '';
+    };
+  };
+  serviceOpts = {
+    script =
+      let
+        queues = concatStringsSep " " (map (q: "-queue ${q}") cfg.queues);
+      in
+      ''
+        export BUILDKITE_AGENT_TOKEN="$(cat ${toString cfg.tokenPath})"
+        exec ${pkgs.buildkite-agent-metrics}/bin/buildkite-agent-metrics \
+          -backend prometheus \
+          -interval ${cfg.interval} \
+          -endpoint ${cfg.endpoint} \
+          ${optionalString (cfg.queues != null) queues} \
+          -prometheus-addr "${cfg.listenAddress}:${toString cfg.port}" ${concatStringsSep " " cfg.extraFlags}
+      '';
+    serviceConfig = {
+      DynamicUser = false;
+      RuntimeDirectory = "buildkite-agent-metrics";
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/chrony.nix

@@ -0,0 +1,97 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.exporters.chrony;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    concatMapStringsSep
+    ;
+in
+{
+  port = 9123;
+  extraOpts = {
+    chronyServerAddress = mkOption {
+      type = types.str;
+      default = "unix:///run/chrony/chronyd.sock";
+      example = [ "192.82.0.1:323" ];
+      description = ''
+        ChronyServerAddress of the chrony server side command port. (Not enabled by default.)
+        Defaults to the local unix socket.
+      '';
+    };
+    user = mkOption {
+      type = types.str;
+      default = "chrony";
+      description = ''
+        User name under which the chrony exporter shall be run.
+        This allows the exporter to talk to chrony using a unix socket, which is owned by chrony.
+        The exporter startup with the default user chrony will fail without local chrony instance.
+      '';
+    };
+    group = mkOption {
+      type = types.str;
+      default = "chrony";
+      description = ''
+        Group under which the chrony exporter shall be run.
+        This allows the exporter to talk to chrony using a unix socket, which is owned by chrony group.
+        The service startup with the default group chrony will fail without local chrony instance.
+      '';
+    };
+    enabledCollectors = mkOption {
+      type = types.listOf types.str;
+      default = [
+        "tracking"
+        "sources"
+        "sources.with-ntpdata"
+        "serverstats"
+        "dns-lookups"
+      ];
+      example = [ "dns-lookups" ];
+      description = ''
+        Collectors to enable.
+        Currently all collectors are enabled by default.
+      '';
+    };
+    disabledCollectors = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "sources.with-ntpdata" ];
+      description = ''
+        Collectors to disable which are enabled by default.
+        Disable sources.with-ntpdata for network scraper. Option requires unix socket.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      ProtectClock = true;
+      ProtectSystem = "strict";
+      Restart = "on-failure";
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+        "AF_UNIX"
+      ];
+      RestrictNamespaces = true;
+      RestrictRealtime = true;
+      ExecStart = ''
+        ${lib.getExe pkgs.prometheus-chrony-exporter} \
+          ${concatMapStringsSep " " (x: "--collector." + x) cfg.enabledCollectors} \
+          ${concatMapStringsSep " " (x: "--no-collector." + x) cfg.disabledCollectors} \
+          --chrony.address ${cfg.chronyServerAddress} \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          ${concatStringsSep " " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/collectd.nix

@@ -0,0 +1,104 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.collectd;
+  inherit (lib)
+    mkOption
+    mkEnableOption
+    types
+    optionalString
+    concatStringsSep
+    escapeShellArg
+    ;
+in
+{
+  port = 9103;
+  extraOpts = {
+    collectdBinary = {
+      enable = mkEnableOption "collectd binary protocol receiver";
+
+      authFile = mkOption {
+        default = null;
+        type = types.nullOr types.path;
+        description = "File mapping user names to pre-shared keys (passwords).";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 25826;
+        description = "Network address on which to accept collectd binary network packets.";
+      };
+
+      listenAddress = mkOption {
+        type = types.str;
+        default = "0.0.0.0";
+        description = ''
+          Address to listen on for binary network packets.
+        '';
+      };
+
+      securityLevel = mkOption {
+        type = types.enum [
+          "None"
+          "Sign"
+          "Encrypt"
+        ];
+        default = "None";
+        description = ''
+          Minimum required security level for accepted packets.
+        '';
+      };
+    };
+
+    logFormat = mkOption {
+      type = types.enum [
+        "logfmt"
+        "json"
+      ];
+      default = "logfmt";
+      example = "json";
+      description = ''
+        Set the log format.
+      '';
+    };
+
+    logLevel = mkOption {
+      type = types.enum [
+        "debug"
+        "info"
+        "warn"
+        "error"
+        "fatal"
+      ];
+      default = "info";
+      description = ''
+        Only log messages with the given severity or above.
+      '';
+    };
+  };
+  serviceOpts =
+    let
+      collectSettingsArgs = optionalString (cfg.collectdBinary.enable) ''
+        --collectd.listen-address ${cfg.collectdBinary.listenAddress}:${toString cfg.collectdBinary.port} \
+        --collectd.security-level ${cfg.collectdBinary.securityLevel} \
+      '';
+    in
+    {
+      serviceConfig = {
+        ExecStart = ''
+          ${pkgs.prometheus-collectd-exporter}/bin/collectd_exporter \
+            --log.format ${escapeShellArg cfg.logFormat} \
+            --log.level ${cfg.logLevel} \
+            --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+            ${collectSettingsArgs} \
+            ${concatStringsSep " \\\n  " cfg.extraFlags}
+        '';
+      };
+    };
+}

nixos/modules/services/monitoring/prometheus/exporters/deluge.nix

@@ -0,0 +1,94 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.deluge;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9354;
+
+  extraOpts = {
+    delugeHost = mkOption {
+      type = types.str;
+      default = "localhost";
+      description = ''
+        Hostname where deluge server is running.
+      '';
+    };
+
+    delugePort = mkOption {
+      type = types.port;
+      default = 58846;
+      description = ''
+        Port where deluge server is listening.
+      '';
+    };
+
+    delugeUser = mkOption {
+      type = types.str;
+      default = "localclient";
+      description = ''
+        User to connect to deluge server.
+      '';
+    };
+
+    delugePassword = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        Password to connect to deluge server.
+
+        This stores the password unencrypted in the nix store and is thus considered unsafe. Prefer
+        using the delugePasswordFile option.
+      '';
+    };
+
+    delugePasswordFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        File containing the password to connect to deluge server.
+      '';
+    };
+
+    exportPerTorrentMetrics = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable per-torrent metrics.
+
+        This may significantly increase the number of time series depending on the number of
+        torrents in your Deluge instance.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-deluge-exporter}/bin/deluge-exporter
+      '';
+      Environment = [
+        "LISTEN_PORT=${toString cfg.port}"
+        "LISTEN_ADDRESS=${toString cfg.listenAddress}"
+
+        "DELUGE_HOST=${cfg.delugeHost}"
+        "DELUGE_USER=${cfg.delugeUser}"
+        "DELUGE_PORT=${toString cfg.delugePort}"
+      ]
+      ++ lib.optionals (cfg.delugePassword != null) [
+        "DELUGE_PASSWORD=${cfg.delugePassword}"
+      ]
+      ++ lib.optionals cfg.exportPerTorrentMetrics [
+        "PER_TORRENT_METRICS=1"
+      ];
+      EnvironmentFile = lib.optionalString (
+        cfg.delugePasswordFile != null
+      ) "/etc/deluge-exporter/password";
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/dmarc.nix

@@ -0,0 +1,129 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.dmarc;
+  inherit (lib) mkOption types optionalString;
+
+  json = builtins.toJSON {
+    inherit (cfg) folders port;
+    listen_addr = cfg.listenAddress;
+    storage_path = "$STATE_DIRECTORY";
+    imap = (builtins.removeAttrs cfg.imap [ "passwordFile" ]) // {
+      password = "$IMAP_PASSWORD";
+      use_ssl = true;
+    };
+    poll_interval_seconds = cfg.pollIntervalSeconds;
+    deduplication_max_seconds = cfg.deduplicationMaxSeconds;
+    logging = {
+      version = 1;
+      disable_existing_loggers = false;
+    };
+  };
+in
+{
+  port = 9797;
+  extraOpts = {
+    imap = {
+      host = mkOption {
+        type = types.str;
+        default = "localhost";
+        description = ''
+          Hostname of IMAP server to connect to.
+        '';
+      };
+      port = mkOption {
+        type = types.port;
+        default = 993;
+        description = ''
+          Port of the IMAP server to connect to.
+        '';
+      };
+      username = mkOption {
+        type = types.str;
+        example = "postmaster@example.org";
+        description = ''
+          Login username for the IMAP connection.
+        '';
+      };
+      passwordFile = mkOption {
+        type = types.str;
+        example = "/run/secrets/dovecot_pw";
+        description = ''
+          File containing the login password for the IMAP connection.
+        '';
+      };
+    };
+    folders = {
+      inbox = mkOption {
+        type = types.str;
+        default = "INBOX";
+        description = ''
+          IMAP mailbox that is checked for incoming DMARC aggregate reports
+        '';
+      };
+      done = mkOption {
+        type = types.str;
+        default = "Archive";
+        description = ''
+          IMAP mailbox that successfully processed reports are moved to.
+        '';
+      };
+      error = mkOption {
+        type = types.str;
+        default = "Invalid";
+        description = ''
+          IMAP mailbox that emails are moved to that could not be processed.
+        '';
+      };
+    };
+    pollIntervalSeconds = mkOption {
+      type = types.ints.unsigned;
+      default = 60;
+      description = ''
+        How often to poll the IMAP server in seconds.
+      '';
+    };
+    deduplicationMaxSeconds = mkOption {
+      type = types.ints.unsigned;
+      default = 604800;
+      defaultText = "7 days (in seconds)";
+      description = ''
+        How long individual report IDs will be remembered to avoid
+        counting double delivered reports twice.
+      '';
+    };
+    debug = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Whether to declare enable `--debug`.
+      '';
+    };
+  };
+  serviceOpts = {
+    path = with pkgs; [
+      envsubst
+      coreutils
+    ];
+    serviceConfig = {
+      StateDirectory = "prometheus-dmarc-exporter";
+      WorkingDirectory = "/var/lib/prometheus-dmarc-exporter";
+      ExecStart = "${pkgs.writeShellScript "setup-cfg" ''
+        export IMAP_PASSWORD="$(<${cfg.imap.passwordFile})"
+        envsubst \
+          -i ${pkgs.writeText "dmarc-exporter.json.template" json} \
+          -o ''${STATE_DIRECTORY}/dmarc-exporter.json
+
+        exec ${pkgs.dmarc-metrics-exporter}/bin/dmarc-metrics-exporter \
+          --configuration /var/lib/prometheus-dmarc-exporter/dmarc-exporter.json \
+          ${optionalString cfg.debug "--debug"}
+      ''}";
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/dnsmasq.nix

@@ -0,0 +1,48 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.dnsmasq;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    escapeShellArg
+    ;
+in
+{
+  port = 9153;
+  extraOpts = {
+    dnsmasqListenAddress = mkOption {
+      type = types.str;
+      default = "localhost:53";
+      description = ''
+        Address on which dnsmasq listens.
+      '';
+    };
+    leasesPath = mkOption {
+      type = types.path;
+      default = "/var/lib/dnsmasq/dnsmasq.leases";
+      example = "/var/lib/misc/dnsmasq.leases";
+      description = ''
+        Path to the `dnsmasq.leases` file.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-dnsmasq-exporter}/bin/dnsmasq_exporter \
+          --listen ${cfg.listenAddress}:${toString cfg.port} \
+          --dnsmasq ${cfg.dnsmasqListenAddress} \
+          --leases_path ${escapeShellArg cfg.leasesPath} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/dnssec.nix

@@ -0,0 +1,101 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.exporters.dnssec;
+  configFormat = pkgs.formats.toml { };
+  configFile = configFormat.generate "dnssec-checks.toml" cfg.configuration;
+in
+{
+  port = 9204;
+  extraOpts = {
+    configuration = lib.mkOption {
+      type = lib.types.nullOr lib.types.attrs;
+      default = null;
+      description = ''
+        dnssec exporter configuration as nix attribute set.
+
+        See <https://github.com/chrj/prometheus-dnssec-exporter/blob/master/README.md>
+        for the description of the configuration file format.
+      '';
+      example = lib.literalExpression ''
+        {
+          records = [
+            {
+              zone = "ietf.org";
+              record = "@";
+              type = "SOA";
+            }
+            {
+              zone = "verisigninc.com";
+              record = "@";
+              type = "SOA";
+            }
+          ];
+        }
+      '';
+    };
+
+    listenAddress = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        Listen address as host IP and port definition.
+      '';
+      example = ":9204";
+    };
+
+    resolvers = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        DNSSEC capable resolver to be used for the check.
+      '';
+      example = [ "0.0.0.0:53" ];
+    };
+
+    timeout = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      description = ''
+        DNS request timeout duration.
+      '';
+      example = "10s";
+    };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Extra commandline options when launching Prometheus.
+      '';
+    };
+  };
+
+  serviceOpts = {
+    serviceConfig =
+      let
+        startScript = pkgs.writeShellScriptBin "prometheus-dnssec-exporter-start" "${lib.concatStringsSep
+          " "
+          (
+            [ "${pkgs.prometheus-dnssec-exporter}/bin/prometheus-dnssec-exporter" ]
+            ++ lib.optionals (cfg.configuration != null) [ "-config ${configFile}" ]
+            ++ lib.optionals (cfg.listenAddress != null) [
+              "-listen-address ${lib.escapeShellArg cfg.listenAddress}"
+            ]
+            ++ lib.optionals (cfg.resolvers != [ ]) [
+              "-resolvers ${lib.escapeShellArg (lib.concatStringsSep "," cfg.resolvers)}"
+            ]
+            ++ lib.optionals (cfg.timeout != null) [ "-timeout ${lib.escapeShellArg cfg.timeout}" ]
+            ++ cfg.extraFlags
+          )
+        }";
+      in
+      {
+        ExecStart = lib.getExe startScript;
+      };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/domain.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.domain;
+  inherit (lib) concatStringsSep;
+in
+{
+  port = 9222;
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-domain-exporter}/bin/domain_exporter \
+          --bind ${cfg.listenAddress}:${toString cfg.port} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/dovecot.nix

@@ -0,0 +1,105 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.dovecot;
+  inherit (lib)
+    mkOption
+    types
+    escapeShellArg
+    concatStringsSep
+    ;
+in
+{
+  port = 9166;
+  extraOpts = {
+    telemetryPath = mkOption {
+      type = types.str;
+      default = "/metrics";
+      description = ''
+        Path under which to expose metrics.
+      '';
+    };
+    socketPath = mkOption {
+      type = types.path;
+      default = "/var/run/dovecot/stats";
+      example = "/var/run/dovecot2/old-stats";
+      description = ''
+        Path under which the stats socket is placed.
+        The user/group under which the exporter runs,
+        should be able to access the socket in order
+        to scrape the metrics successfully.
+
+        Please keep in mind that the stats module has changed in
+        [Dovecot 2.3+](https://wiki2.dovecot.org/Upgrading/2.3) which
+        is not [compatible with this exporter](https://github.com/kumina/dovecot_exporter/issues/8).
+
+        The following extra config has to be passed to Dovecot to ensure that recent versions
+        work with this exporter:
+        ```
+        {
+          services.prometheus.exporters.dovecot.enable = true;
+          services.prometheus.exporters.dovecot.socketPath = "/var/run/dovecot2/old-stats";
+          services.dovecot2.mailPlugins.globally.enable = [ "old_stats" ];
+          services.dovecot2.extraConfig = '''
+            service old-stats {
+              unix_listener old-stats {
+                user = dovecot-exporter
+                group = dovecot-exporter
+                mode = 0660
+              }
+              fifo_listener old-stats-mail {
+                mode = 0660
+                user = dovecot
+                group = dovecot
+              }
+              fifo_listener old-stats-user {
+                mode = 0660
+                user = dovecot
+                group = dovecot
+              }
+            }
+            plugin {
+              old_stats_refresh = 30 secs
+              old_stats_track_cmds = yes
+            }
+          ''';
+        }
+        ```
+      '';
+    };
+    scopes = mkOption {
+      type = types.listOf types.str;
+      default = [ "user" ];
+      example = [
+        "user"
+        "global"
+      ];
+      description = ''
+        Stats scopes to query.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      DynamicUser = false;
+      ExecStart = ''
+        ${lib.getExe pkgs.dovecot_exporter} \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --web.telemetry-path ${cfg.telemetryPath} \
+          --dovecot.socket-path ${escapeShellArg cfg.socketPath} \
+          --dovecot.scopes ${concatStringsSep "," cfg.scopes} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+      RestrictAddressFamilies = [
+        # Need AF_UNIX to collect data
+        "AF_UNIX"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/ebpf.nix

@@ -0,0 +1,49 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.ebpf;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    ;
+in
+{
+  port = 9435;
+  extraOpts = {
+    names = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "timers" ];
+      description = ''
+        List of eBPF programs to load
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      AmbientCapabilities = [
+        "CAP_BPF"
+        "CAP_DAC_READ_SEARCH"
+        "CAP_PERFMON"
+      ];
+      CapabilityBoundingSet = [
+        "CAP_BPF"
+        "CAP_DAC_READ_SEARCH"
+        "CAP_PERFMON"
+      ];
+      ExecStart = ''
+        ${pkgs.prometheus-ebpf-exporter}/bin/ebpf_exporter \
+        --config.dir=${pkgs.prometheus-ebpf-exporter}/examples \
+        --config.names=${concatStringsSep "," cfg.names} \
+        --web.listen-address ${cfg.listenAddress}:${toString cfg.port}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/ecoflow.nix

@@ -0,0 +1,151 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.prometheus.exporters.ecoflow;
+  inherit (lib) mkOption types;
+in
+{
+  port = 2112;
+  extraOpts = {
+    exporterType = mkOption {
+      type = types.enum [
+        "rest"
+        "mqtt"
+      ];
+      default = "rest";
+      example = "mqtt";
+      description = ''
+        The type of exporter you'd like to use.
+        Possible values: "rest" and "mqtt". Default value is "rest".
+        Choose "rest" for the ecoflow online cloud api use "rest" and define: accessKey, secretKey.
+        Choose "mqtt" for the lan realtime integration use "mqtt" and define: email, password, devices.
+      '';
+    };
+    ecoflowAccessKeyFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-access-key;
+      description = ''
+        Path to the file with your personal api access string from the Ecoflow development website <https://developer-eu.ecoflow.com>.
+        Do to share or commit your plaintext scecrets to a public repo use: agenix or soaps.
+      '';
+    };
+    ecoflowSecretKeyFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-secret-key;
+      description = ''
+        Path to the file with your personal api secret string from the Ecoflow development website <https://developer-eu.ecoflow.com>.
+        Do to share or commit your plaintext scecrets to a public repo use: agenix or soaps.
+      '';
+    };
+    ecoflowEmailFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-email;
+      description = ''
+        Path to the file with your personal ecoflow app login email address.
+        Do to share or commit your plaintext scecrets to a public repo use: agenix or soaps.
+      '';
+    };
+    ecoflowPasswordFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-password;
+      description = ''
+        Path to the file with your personal ecoflow app login email password.
+        Do to share or commit your plaintext passwords to a public repo use: agenix or soaps here!
+      '';
+    };
+    ecoflowDevicesFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-devices;
+      description = ''
+        File must contain one line, example: R3300000,R3400000,NC430000,....
+        The list of devices serial numbers separated by comma. For instance: SN1,SN2,SN3.
+        Instead of "devicesFile" you can specify "devicesPrettynamesFile" which will also work. You can specify both.
+        Do to share or commit your plaintext serial numbers to a public repo use: agenix or soaps.
+      '';
+    };
+    ecoflowDevicesPrettyNamesFile = mkOption {
+      type = types.path;
+      default = /etc/ecoflow-devices-pretty-names;
+      description = ''
+        File must contain one line, example: {"R3300000":"Delta 2","R3400000":"Delta Pro",...}
+        The key/value map of custom names for your devices. Key is a serial number, value is a device name you want
+        to see in Grafana Dashboard. It's helpful to see a meaningful name in Grafana dashboard instead of a serialnumber.
+        Do to share or commit your plaintext serial numbers to a public repo use: agenix or soaps.
+      '';
+    };
+    debug = mkOption {
+      type = types.str;
+      default = "0";
+      example = "1";
+      description = ''
+        Enable debug log messages. Disabled by default. Set to "1" to enable.
+      '';
+    };
+    prefix = mkOption {
+      type = types.str;
+      default = "ecoflow";
+      example = "ecoflow_privateSite";
+      description = ''
+        The prefix that will be added to all metrics. Default value is ecoflow.
+        For instance metric bms_bmsStatus.minCellTemp will be exported to prometheus as ecoflow.bms_bmsStatus.minCellTemp.
+        With default value "ecoflow" you can use Grafana Dashboard with ID 17812 without any changes.
+      '';
+    };
+    scrapingInterval = mkOption {
+      type = types.ints.positive;
+      default = 30;
+      example = 120;
+      description = ''
+        Scrapping interval in seconds. How often should the exporter execute requests to Ecoflow Rest API in order to get the data.
+        Default value is 30 seconds. Align this value with your prometheus scraper interval settings.
+      '';
+    };
+    mqttDeviceOfflineThreshold = mkOption {
+      type = types.ints.positive;
+      default = 60;
+      example = 120;
+      description = ''
+        The threshold in seconds which indicates how long we should wait for a metric message from MQTT broker.
+        Default value: 60 seconds. If we don't receive message within 60 seconds we consider that device is offline.
+        If we don't receive messages within the threshold for all devices, we'll try to reconnect to the MQTT broker.
+        There is a strange behavior that MQTT stop sends messages if you open Ecoflow mobile app and then close it).
+      '';
+    };
+  };
+  serviceOpts = {
+    environment = {
+      PROMETHEUS_ENABLED = "1";
+      EXPORTER_TYPE = cfg.exporterType;
+      DEBUG_ENABLED = cfg.debug;
+      METRIC_PREFIX = cfg.prefix;
+      SCRAPING_INTERVAL = toString cfg.scrapingInterval;
+      MQTT_DEVICE_OFFLINE_THRESHOLD_SECONDS = toString cfg.mqttDeviceOfflineThreshold;
+    };
+    script = ''
+      export ECOFLOW_ACCESS_KEY="$(cat ${toString cfg.ecoflowAccessKeyFile})"
+      export ECOFLOW_SECRET_KEY="$(cat ${toString cfg.ecoflowSecretKeyFile})"
+      export ECOFLOW_EMAIL="$(cat ${toString cfg.ecoflowEmailFile})"
+      export ECOFLOW_PASSWORD="$(cat ${toString cfg.ecoflowPasswordFile})"
+      export ECOFLOW_DEVICES="$(cat ${toString cfg.ecoflowDevicesFile})"
+      export ECOFLOW_DEVICES_PRETTY_NAMES="$(cat ${toString cfg.ecoflowDevicesPrettyNamesFile})"
+      exec ${lib.getExe pkgs.go-ecoflow-exporter}'';
+    serviceConfig = {
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+      MemoryDenyWriteExecute = true;
+      NoNewPrivileges = true;
+      ProtectSystem = "strict";
+      Restart = "on-failure";
+      RestrictAddressFamilies = [
+        "AF_INET"
+        "AF_INET6"
+      ];
+      RestrictNamespaces = true;
+      User = "prometheus"; # context needed to runtime access encrypted token and secrets
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/exportarr.nix

@@ -0,0 +1,63 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  type,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters."exportarr-${type}";
+  exportarrEnvironment = (lib.mapAttrs (_: toString) cfg.environment) // {
+    PORT = toString cfg.port;
+    URL = cfg.url;
+    API_KEY_FILE = lib.mkIf (cfg.apiKeyFile != null) "%d/api-key";
+  };
+in
+{
+  port = 9708;
+  extraOpts = {
+    url = lib.mkOption {
+      type = lib.types.str;
+      default = "http://127.0.0.1";
+      description = ''
+        The full URL to Sonarr, Radarr, or Lidarr.
+      '';
+    };
+
+    apiKeyFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = ''
+        File containing the api-key.
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "exportarr" { };
+
+    environment = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      default = { };
+      description = ''
+        See [the configuration guide](https://github.com/onedr0p/exportarr#configuration) for available options.
+      '';
+      example = {
+        PROWLARR__BACKFILL = true;
+      };
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      LoadCredential = lib.optionalString (cfg.apiKeyFile != null) "api-key:${cfg.apiKeyFile}";
+      ExecStart = ''${cfg.package}/bin/exportarr ${type} "$@"'';
+      ProcSubset = "pid";
+      ProtectProc = "invisible";
+      SystemCallFilter = [
+        "@system-service"
+        "~@privileged"
+      ];
+    };
+    environment = exportarrEnvironment;
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/fastly.nix

@@ -0,0 +1,59 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  inherit (lib)
+    getExe
+    mkOption
+    optionals
+    types
+    ;
+
+  inherit (utils) escapeSystemdExecArgs;
+
+  cfg = config.services.prometheus.exporters.fastly;
+in
+{
+  port = 9118;
+  extraOpts = with types; {
+    configFile = mkOption {
+      type = nullOr path;
+      default = null;
+      example = "./fastly-exporter-config.txt";
+      description = ''
+        Path to a fastly-exporter configuration file.
+        Example one can be generated with `fastly-exporter --config-file-example`.
+      '';
+    };
+
+    environmentFile = mkOption {
+      type = path;
+      description = ''
+        An environment file containg at least the FASTLY_API_TOKEN= environment
+        variable.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      EnvironmentFile = cfg.environmentFile;
+      ExecStart = escapeSystemdExecArgs (
+        [
+          (getExe pkgs.prometheus-fastly-exporter)
+          "-listen"
+          "${cfg.listenAddress}:${toString cfg.port}"
+        ]
+        ++ optionals (cfg.configFile != null) [
+          "--config-file"
+          cfg.configFile
+        ]
+        ++ cfg.extraFlags
+      );
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/flow.nix

@@ -0,0 +1,62 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.flow;
+  inherit (lib)
+    mkOption
+    types
+    literalExpression
+    concatStringsSep
+    optionalString
+    ;
+in
+{
+  port = 9590;
+  extraOpts = {
+    brokers = mkOption {
+      type = types.listOf types.str;
+      example = literalExpression ''[ "kafka.example.org:19092" ]'';
+      description = "List of Kafka brokers to connect to.";
+    };
+
+    asn = mkOption {
+      type = types.ints.positive;
+      example = 65542;
+      description = "The ASN being monitored.";
+    };
+
+    partitions = mkOption {
+      type = types.listOf types.int;
+      default = [ ];
+      description = ''
+        The number of the partitions to consume, none means all.
+      '';
+    };
+
+    topic = mkOption {
+      type = types.str;
+      example = "pmacct.acct";
+      description = "The Kafka topic to consume from.";
+    };
+  };
+
+  serviceOpts = {
+    serviceConfig = {
+      DynamicUser = true;
+      ExecStart = ''
+        ${pkgs.prometheus-flow-exporter}/bin/flow-exporter \
+          -asn ${toString cfg.asn} \
+          -topic ${cfg.topic} \
+          -brokers ${concatStringsSep "," cfg.brokers} \
+          ${optionalString (cfg.partitions != [ ]) "-partitions ${concatStringsSep "," cfg.partitions}"} \
+          -addr ${cfg.listenAddress}:${toString cfg.port} ${concatStringsSep " " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/fritz.nix

@@ -0,0 +1,115 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+let
+  inherit (lib) mkOption types;
+  cfg = config.services.prometheus.exporters.fritz;
+  yaml = pkgs.formats.yaml { };
+  configFile = yaml.generate "fritz-exporter.yaml" cfg.settings;
+in
+{
+  port = 9787;
+
+  extraOpts = {
+    settings = mkOption {
+      description = "Configuration settings for fritz-exporter.";
+      type = types.submodule {
+        freeformType = yaml.type;
+
+        options = {
+          # Pull existing port option into config file.
+          port = mkOption {
+            type = types.port;
+            default = cfg.port;
+            internal = true;
+            visible = false;
+          };
+          # Pull existing listen address option into config file.
+          listen_address = mkOption {
+            type = types.str;
+            default = cfg.listenAddress;
+            internal = true;
+            visible = false;
+          };
+          log_level = mkOption {
+            type = types.enum [
+              "DEBUG"
+              "INFO"
+              "WARNING"
+              "ERROR"
+              "CRITICAL"
+            ];
+            default = "INFO";
+            description = ''
+              Log level to use for the exporter.
+            '';
+          };
+          devices = mkOption {
+            default = [ ];
+            description = "Fritz!-devices to monitor using the exporter.";
+            type =
+              with types;
+              listOf (submodule {
+                freeformType = yaml.type;
+
+                options = {
+                  name = mkOption {
+                    type = types.str;
+                    default = "";
+                    description = ''
+                      Name to use for the device.
+                    '';
+                  };
+                  hostname = mkOption {
+                    type = types.str;
+                    default = "fritz.box";
+                    description = ''
+                      Hostname under which the target device is reachable.
+                    '';
+                  };
+                  username = mkOption {
+                    type = types.str;
+                    description = ''
+                      Username to authenticate with the target device.
+                    '';
+                  };
+                  password_file = mkOption {
+                    type = types.path;
+                    description = ''
+                      Path to a file which contains the password to authenticate with the target device.
+                      Needs to be readable by the user the exporter runs under.
+                    '';
+                  };
+                  host_info = mkOption {
+                    type = types.bool;
+                    description = ''
+                      Enable extended host info for this device. *Warning*: This will heavily increase scrape time.
+                    '';
+                    default = false;
+                  };
+                };
+              });
+          };
+        };
+      };
+    };
+  };
+
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = utils.escapeSystemdExecArgs (
+        [
+          (lib.getExe pkgs.fritz-exporter)
+          "--config"
+          configFile
+        ]
+        ++ cfg.extraFlags
+      );
+      DynamicUser = false;
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/fritzbox.nix

@@ -0,0 +1,43 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.fritzbox;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9133;
+  extraOpts = {
+    gatewayAddress = mkOption {
+      type = types.str;
+      default = "fritz.box";
+      description = ''
+        The hostname or IP of the FRITZ!Box.
+      '';
+    };
+
+    gatewayPort = mkOption {
+      type = types.port;
+      default = 49000;
+      description = ''
+        The port of the FRITZ!Box UPnP service.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-fritzbox-exporter}/bin/exporter \
+          -listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          -gateway-address ${cfg.gatewayAddress} \
+          -gateway-port ${toString cfg.gatewayPort} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/frr.nix

@@ -0,0 +1,66 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.frr;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    concatMapStringsSep
+    ;
+in
+{
+  port = 9342;
+  extraOpts = {
+    user = mkOption {
+      type = types.str;
+      default = "frr";
+      description = ''
+        User name under which the frr exporter shall be run.
+        The exporter talks to frr using a unix socket, which is owned by frr.
+      '';
+    };
+    group = mkOption {
+      type = types.str;
+      default = "frrtty";
+      description = ''
+        Group under which the frr exporter shall be run.
+        The exporter talks to frr using a unix socket, which is owned by frrtty group.
+      '';
+    };
+    enabledCollectors = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "vrrp" ];
+      description = ''
+        Collectors to enable. The collectors listed here are enabled in addition to the default ones.
+      '';
+    };
+    disabledCollectors = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      example = [ "bfd" ];
+      description = ''
+        Collectors to disable which are enabled by default.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      DynamicUser = false;
+      RuntimeDirectory = "prometheus-frr-exporter";
+      RestrictAddressFamilies = [ "AF_UNIX" ];
+      ExecStart = ''
+        ${lib.getExe pkgs.prometheus-frr-exporter} \
+          ${concatMapStringsSep " " (x: "--collector." + x) cfg.enabledCollectors} \
+          ${concatMapStringsSep " " (x: "--no-collector." + x) cfg.disabledCollectors} \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} ${concatStringsSep " " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/graphite.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.graphite;
+  format = pkgs.formats.yaml { };
+in
+{
+  port = 9108;
+  extraOpts = {
+    graphitePort = lib.mkOption {
+      type = lib.types.port;
+      default = 9109;
+      description = ''
+        Port to use for the graphite server.
+      '';
+    };
+    mappingSettings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = format.type;
+        options = { };
+      };
+      default = { };
+      description = ''
+        Mapping configuration for the exporter, see
+        <https://github.com/prometheus/graphite_exporter#yaml-config> for
+        available options.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-graphite-exporter}/bin/graphite_exporter \
+          --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          --graphite.listen-address ${cfg.listenAddress}:${toString cfg.graphitePort} \
+          --graphite.mapping-config ${format.generate "mapping.yml" cfg.mappingSettings} \
+          ${lib.concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/idrac.nix

@@ -0,0 +1,77 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.idrac;
+  inherit (lib) mkOption types;
+
+  configFile =
+    if cfg.configurationPath != null then
+      cfg.configurationPath
+    else
+      pkgs.writeText "idrac.yml" (builtins.toJSON cfg.configuration);
+in
+{
+  port = 9348;
+  extraOpts = {
+    configurationPath = mkOption {
+      type = with types; nullOr path;
+      default = null;
+      example = "/etc/prometheus-idrac-exporter/idrac.yml";
+      description = ''
+        Path to the service's config file. This path can either be a computed path in /nix/store or a path in the local filesystem.
+
+        The config file should NOT be stored in /nix/store as it will contain passwords and/or keys in plain text.
+
+        Mutually exclusive with `configuration` option.
+
+        Configuration reference: <https://github.com/mrlhansen/idrac_exporter/#configuration>
+      '';
+    };
+    configuration = mkOption {
+      type = types.nullOr types.attrs;
+      description = ''
+        Configuration for iDRAC exporter, as a nix attribute set.
+
+        Configuration reference: <https://github.com/mrlhansen/idrac_exporter/#configuration>
+
+        Mutually exclusive with `configurationPath` option.
+      '';
+      default = null;
+      example = {
+        timeout = 10;
+        retries = 1;
+        hosts = {
+          default = {
+            username = "username";
+            password = "password";
+          };
+        };
+        metrics = {
+          system = true;
+          sensors = true;
+          power = true;
+          sel = true;
+          storage = true;
+          memory = true;
+        };
+      };
+    };
+  };
+
+  serviceOpts = {
+    serviceConfig = {
+      LoadCredential = "configFile:${configFile}";
+      ExecStart = "${pkgs.prometheus-idrac-exporter}/bin/idrac_exporter -config %d/configFile";
+      Environment = [
+        "IDRAC_EXPORTER_LISTEN_ADDRESS=${cfg.listenAddress}"
+        "IDRAC_EXPORTER_LISTEN_PORT=${toString cfg.port}"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/imap-mailstat.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.imap-mailstat;
+  valueToString =
+    value:
+    if (builtins.typeOf value == "string") then
+      "\"${value}\""
+    else
+      (
+        if (builtins.typeOf value == "int") then
+          "${toString value}"
+        else
+          (
+            if (builtins.typeOf value == "bool") then
+              (if value then "true" else "false")
+            else
+              "XXX ${toString value}"
+          )
+      );
+  inherit (lib)
+    mkOption
+    types
+    concatStrings
+    concatStringsSep
+    attrValues
+    mapAttrs
+    optionalString
+    ;
+  createConfigFile =
+    accounts:
+    # unfortunately on toTOML yet
+    # https://github.com/NixOS/nix/issues/3929
+    pkgs.writeText "imap-mailstat-exporter.conf" ''
+      ${concatStrings (
+        attrValues (
+          mapAttrs (
+            name: config:
+            "[[Accounts]]\nname = \"${name}\"\n${
+              concatStrings (attrValues (mapAttrs (k: v: "${k} = ${valueToString v}\n") config))
+            }"
+          ) accounts
+        )
+      )}
+    '';
+  mkOpt =
+    type: description:
+    mkOption {
+      type = types.nullOr type;
+      default = null;
+      description = description;
+    };
+  accountOptions.options = {
+    mailaddress = mkOpt types.str "Your email address (at the moment used as login name)";
+    username = mkOpt types.str "If empty string mailaddress value is used";
+    password = mkOpt types.str "";
+    serveraddress = mkOpt types.str "mailserver name or address";
+    serverport = mkOpt types.int "imap port number (at the moment only tls connection is supported)";
+    starttls = mkOpt types.bool "set to true for using STARTTLS to start a TLS connection";
+  };
+in
+{
+  port = 8081;
+  extraOpts = {
+    oldestUnseenDate = mkOption {
+      type = types.bool;
+      default = false;
+      description = ''
+        Enable metric with timestamp of oldest unseen mail
+      '';
+    };
+    accounts = mkOption {
+      type = types.attrsOf (types.submodule accountOptions);
+      default = { };
+      description = ''
+        Accounts to monitor
+      '';
+    };
+    configurationFile = mkOption {
+      type = types.path;
+      example = "/path/to/config-file";
+      description = ''
+        File containing the configuration
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-imap-mailstat-exporter}/bin/imap-mailstat-exporter \
+          -config ${createConfigFile cfg.accounts} \
+          ${optionalString cfg.oldestUnseenDate "-oldestunseendate"} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/influxdb.nix

@@ -0,0 +1,39 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.influxdb;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9122;
+  extraOpts = {
+    sampleExpiry = mkOption {
+      type = types.str;
+      default = "5m";
+      example = "10m";
+      description = "How long a sample is valid for";
+    };
+    udpBindAddress = mkOption {
+      type = types.str;
+      default = ":9122";
+      example = "192.0.2.1:9122";
+      description = "Address on which to listen for udp packets";
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      RuntimeDirectory = "prometheus-influxdb-exporter";
+      ExecStart = ''
+        ${pkgs.prometheus-influxdb-exporter}/bin/influxdb_exporter \
+        --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+        --influxdb.sample-expiry ${cfg.sampleExpiry} ${concatStringsSep " " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/ipmi.nix

@@ -0,0 +1,65 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  logPrefix = "services.prometheus.exporter.ipmi";
+  cfg = config.services.prometheus.exporters.ipmi;
+  inherit (lib)
+    mkOption
+    types
+    concatStringsSep
+    optionals
+    escapeShellArg
+    ;
+in
+{
+  port = 9290;
+
+  extraOpts = {
+    configFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Path to configuration file.
+      '';
+    };
+
+    webConfigFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Path to configuration file that can enable TLS or authentication.
+      '';
+    };
+  };
+
+  serviceOpts.serviceConfig = {
+    ExecStart =
+      with cfg;
+      concatStringsSep " " (
+        [
+          "${pkgs.prometheus-ipmi-exporter}/bin/ipmi_exporter"
+          "--web.listen-address ${listenAddress}:${toString port}"
+        ]
+        ++ optionals (cfg.webConfigFile != null) [
+          "--web.config.file ${escapeShellArg cfg.webConfigFile}"
+        ]
+        ++ optionals (cfg.configFile != null) [
+          "--config.file ${escapeShellArg cfg.configFile}"
+        ]
+        ++ extraFlags
+      );
+
+    ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+    RestrictAddressFamilies = [
+      "AF_INET"
+      "AF_INET6"
+      "AF_UNIX"
+    ];
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/jitsi.nix

@@ -0,0 +1,50 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.jitsi;
+  inherit (lib)
+    mkOption
+    types
+    escapeShellArg
+    concatStringsSep
+    ;
+in
+{
+  port = 9700;
+  extraOpts = {
+    url = mkOption {
+      type = types.str;
+      default = "http://localhost:8080/colibri/stats";
+      description = ''
+        Jitsi Videobridge metrics URL to monitor.
+        This is usually /colibri/stats on port 8080 of the jitsi videobridge host.
+      '';
+    };
+    interval = mkOption {
+      type = types.str;
+      default = "30s";
+      example = "1min";
+      description = ''
+        How often to scrape new data
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-jitsi-exporter}/bin/jitsiexporter \
+          -url ${escapeShellArg cfg.url} \
+          -host ${cfg.listenAddress} \
+          -port ${toString cfg.port} \
+          -interval ${toString cfg.interval} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/json.nix

@@ -0,0 +1,57 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.json;
+  inherit (lib)
+    mkOption
+    types
+    escapeShellArg
+    concatStringsSep
+    mkRemovedOptionModule
+    ;
+in
+{
+  port = 7979;
+  extraOpts = {
+    configFile = mkOption {
+      type = types.path;
+      description = ''
+        Path to configuration file.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-json-exporter}/bin/json_exporter \
+          --config.file ${escapeShellArg cfg.configFile} \
+          --web.listen-address="${cfg.listenAddress}:${toString cfg.port}" \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+  imports = [
+    (mkRemovedOptionModule [ "url" ] ''
+      This option was removed. The URL of the endpoint serving JSON
+      must now be provided to the exporter by prometheus via the url
+      parameter `target'.
+
+      In prometheus a scrape URL would look like this:
+
+        http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint
+
+      For more information, take a look at the official documentation
+      (https://github.com/prometheus-community/json_exporter) of the json_exporter.
+    '')
+    {
+      options.warnings = options.warnings;
+      options.assertions = options.assertions;
+    }
+  ];
+}

nixos/modules/services/monitoring/prometheus/exporters/junos-czerwonk.nix

@@ -0,0 +1,86 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.junos-czerwonk;
+  inherit (lib)
+    mkOption
+    types
+    escapeShellArg
+    mkIf
+    concatStringsSep
+    ;
+
+  configFile =
+    if cfg.configuration != null then configurationFile else (escapeShellArg cfg.configurationFile);
+
+  configurationFile = pkgs.writeText "prometheus-junos-czerwonk-exporter.conf" (
+    builtins.toJSON (cfg.configuration)
+  );
+in
+{
+  port = 9326;
+  extraOpts = {
+    environmentFile = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      description = ''
+        File containing env-vars to be substituted into the exporter's config.
+      '';
+    };
+    configurationFile = mkOption {
+      type = types.nullOr types.path;
+      default = null;
+      description = ''
+        Specify the JunOS exporter configuration file to use.
+      '';
+    };
+    configuration = mkOption {
+      type = types.nullOr types.attrs;
+      default = null;
+      description = ''
+        JunOS exporter configuration as nix attribute set. Mutually exclusive with the `configurationFile` option.
+      '';
+      example = {
+        devices = [
+          {
+            host = "router1";
+            key_file = "/path/to/key";
+          }
+        ];
+      };
+    };
+    telemetryPath = mkOption {
+      type = types.str;
+      default = "/metrics";
+      description = ''
+        Path under which to expose metrics.
+      '';
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      DynamicUser = false;
+      EnvironmentFile = mkIf (cfg.environmentFile != null) [ cfg.environmentFile ];
+      RuntimeDirectory = "prometheus-junos-czerwonk-exporter";
+      ExecStartPre = [
+        "${pkgs.writeShellScript "subst-secrets-junos-czerwonk-exporter" ''
+          umask 0077
+          ${pkgs.envsubst}/bin/envsubst -i ${configFile} -o ''${RUNTIME_DIRECTORY}/junos-exporter.json
+        ''}"
+      ];
+      ExecStart = ''
+        ${pkgs.prometheus-junos-czerwonk-exporter}/bin/junos_exporter \
+          -web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+          -web.telemetry-path ${cfg.telemetryPath} \
+          -config.file ''${RUNTIME_DIRECTORY}/junos-exporter.json \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/kafka.nix

@@ -0,0 +1,56 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.kafka;
+  inherit (lib)
+    mkIf
+    mkOption
+    mkMerge
+    types
+    concatStringsSep
+    ;
+in
+{
+  port = 8080;
+
+  extraOpts = {
+    package = lib.mkPackageOption pkgs "kminion" { };
+
+    environmentFile = mkOption {
+      type = with types; nullOr path;
+      default = null;
+      description = ''
+        File containing the credentials to access the repository, in the
+        format of an EnvironmentFile as described by systemd.exec(5)
+      '';
+    };
+  };
+  serviceOpts = mkMerge (
+    [
+      {
+        serviceConfig = {
+          ExecStart = ''
+            ${lib.getExe cfg.package}
+          '';
+          EnvironmentFile = mkIf (cfg.environmentFile != null) cfg.environmentFile;
+          RestartSec = "5s";
+          RestrictAddressFamilies = [
+            "AF_INET"
+            "AF_INET6"
+          ];
+        };
+      }
+    ]
+    ++ [
+      (mkIf config.services.apache-kafka.enable {
+        after = [ "apache-kafka.service" ];
+        requires = [ "apache-kafka.service" ];
+      })
+    ]
+  );
+}

nixos/modules/services/monitoring/prometheus/exporters/kea.nix

@@ -0,0 +1,65 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.kea;
+  inherit (lib)
+    mkOption
+    types
+    mkRenamedOptionModule
+    literalExpression
+    ;
+in
+{
+  imports = [
+    (mkRenamedOptionModule [ "controlSocketPaths" ] [ "targets" ])
+  ];
+  port = 9547;
+  extraOpts = {
+    targets = mkOption {
+      type = types.listOf types.str;
+      example = literalExpression ''
+        [
+          "/run/kea/kea-dhcp4.socket"
+          "/run/kea/kea-dhcp6.socket"
+          "http://127.0.0.1:8547"
+        ]
+      '';
+      description = ''
+        Paths or URLs to the Kea control socket.
+      '';
+    };
+  };
+  serviceOpts = {
+    after = [
+      "kea-dhcp4-server.service"
+      "kea-dhcp6-server.service"
+    ];
+    serviceConfig = {
+      User = "kea";
+      DynamicUser = true;
+      ExecStart = utils.escapeSystemdExecArgs (
+        [
+          (lib.getExe pkgs.prometheus-kea-exporter)
+          "--address"
+          cfg.listenAddress
+          "--port"
+          cfg.port
+        ]
+        ++ cfg.extraFlags
+        ++ cfg.targets
+      );
+      RuntimeDirectory = "kea";
+      RuntimeDirectoryPreserve = true;
+      RestrictAddressFamilies = [
+        # Need AF_UNIX to collect data
+        "AF_UNIX"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/keylight.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.keylight;
+  inherit (lib) concatStringsSep;
+in
+{
+  port = 9288;
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-keylight-exporter}/bin/keylight_exporter \
+          -metrics.addr ${cfg.listenAddress}:${toString cfg.port} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/klipper.nix

@@ -0,0 +1,55 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.klipper;
+  inherit (lib)
+    mkOption
+    mkMerge
+    mkIf
+    types
+    concatStringsSep
+    any
+    optionalString
+    ;
+  moonraker = config.services.moonraker;
+in
+{
+  port = 9101;
+  extraOpts = {
+    package = lib.mkPackageOption pkgs "prometheus-klipper-exporter" { };
+
+    moonrakerApiKey = mkOption {
+      type = types.str;
+      default = "";
+      description = ''
+        API Key to authenticate with the Moonraker APIs.
+        Only needed if the host running the exporter is not a trusted client to Moonraker.
+      '';
+    };
+  };
+  serviceOpts = mkMerge (
+    [
+      {
+        serviceConfig = {
+          ExecStart = concatStringsSep " " [
+            "${cfg.package}/bin/prometheus-klipper-exporter"
+            (optionalString (cfg.moonrakerApiKey != "") "--moonraker.apikey ${cfg.moonrakerApiKey}")
+            "--web.listen-address ${cfg.listenAddress}:${toString cfg.port}"
+            "${concatStringsSep " " cfg.extraFlags}"
+          ];
+        };
+      }
+    ]
+    ++ [
+      (mkIf config.services.moonraker.enable {
+        after = [ "moonraker.service" ];
+        requires = [ "moonraker.service" ];
+      })
+    ]
+  );
+}

nixos/modules/services/monitoring/prometheus/exporters/knot.nix

@@ -0,0 +1,69 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.knot;
+  inherit (lib)
+    mkOption
+    types
+    literalExpression
+    concatStringsSep
+    ;
+in
+{
+  port = 9433;
+  extraOpts = {
+    knotLibraryPath = mkOption {
+      type = types.nullOr types.str;
+      default = null;
+      example = literalExpression ''"''${pkgs.knot-dns.out}/lib/libknot.so"'';
+      description = ''
+        Path to the library of `knot-dns`.
+      '';
+    };
+
+    knotSocketPath = mkOption {
+      type = types.str;
+      default = "/run/knot/knot.sock";
+      description = ''
+        Socket path of {manpage}`knotd(8)`.
+      '';
+    };
+
+    knotSocketTimeout = mkOption {
+      type = types.ints.positive;
+      default = 2000;
+      description = ''
+        Timeout in seconds.
+      '';
+    };
+  };
+  serviceOpts = {
+    path = with pkgs; [
+      procps
+    ];
+    serviceConfig = {
+      ExecStart = ''
+        ${pkgs.prometheus-knot-exporter}/bin/knot-exporter \
+          --web-listen-addr ${cfg.listenAddress} \
+          --web-listen-port ${toString cfg.port} \
+          --knot-socket-path ${cfg.knotSocketPath} \
+          --knot-socket-timeout ${toString cfg.knotSocketTimeout} \
+          ${lib.optionalString (cfg.knotLibraryPath != null) "--knot-library-path ${cfg.knotLibraryPath}"} \
+          ${concatStringsSep " \\\n  " cfg.extraFlags}
+      '';
+      SupplementaryGroups = [
+        "knot"
+      ];
+      RestrictAddressFamilies = [
+        # Need AF_UNIX to collect data
+        "AF_UNIX"
+      ];
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/libvirt.nix

@@ -0,0 +1,29 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.libvirt;
+in
+{
+  port = 9177;
+  extraOpts = {
+    libvirtUri = lib.mkOption {
+      type = lib.types.str;
+      default = "qemu:///system";
+      description = "Libvirt URI from which to extract metrics";
+    };
+  };
+  serviceOpts = {
+    serviceConfig = {
+      ExecStart = ''
+        ${lib.getExe pkgs.prometheus-libvirt-exporter} \
+        --web.listen-address ${cfg.listenAddress}:${toString cfg.port} \
+        --libvirt.uri ${cfg.libvirtUri} ${lib.concatStringsSep " " cfg.extraFlags}
+      '';
+    };
+  };
+}

nixos/modules/services/monitoring/prometheus/exporters/lnd.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  lib,
+  pkgs,
+  options,
+  ...
+}:
+
+let
+  cfg = config.services.prometheus.exporters.lnd;
+  inherit (lib) mkOption types concatStringsSep;
+in
+{
+  port = 9092;
+  extraOpts = {
+    lndHost = mkOption {
+      type = types.str;
+      default = "localhost:10009";
+      description = ''
+        lnd instance gRPC address:port.
+      '';
+    };
+
+    lndTlsPath = mkOption {
+      type = types.path;
+      description = ''
+        Path to lnd TLS certificate.
+      '';
+    };
+
+    lndMacaroonDir = mkOption {
+      type = types.path;
+      description = ''
+        Path to lnd macaroons.
+      '';
+    };
+  };
+  serviceOpts.serviceConfig = {
+    ExecStart = ''
+      ${pkgs.prometheus-lnd-exporter}/bin/lndmon \
+        --prometheus.listenaddr=${cfg.listenAddress}:${toString cfg.port} \
+        --prometheus.logdir=/var/log/prometheus-lnd-exporter \
+        --lnd.host=${cfg.lndHost} \
+        --lnd.tlspath=${cfg.lndTlsPath} \
+        --lnd.macaroondir=${cfg.lndMacaroonDir} \
+        ${concatStringsSep " \\\n  " cfg.extraFlags}
+    '';
+    LogsDirectory = "prometheus-lnd-exporter";
+    ReadOnlyPaths = [
+      cfg.lndTlsPath
+      cfg.lndMacaroonDir
+    ];
+  };
+}