push sheeet

nixos/modules/services/cluster/corosync/default.nix

@@ -0,0 +1,121 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.corosync;
+in
+{
+  # interface
+  options.services.corosync = {
+    enable = lib.mkEnableOption "corosync";
+
+    package = lib.mkPackageOption pkgs "corosync" { };
+
+    clusterName = lib.mkOption {
+      type = lib.types.str;
+      default = "nixcluster";
+      description = "Name of the corosync cluster.";
+    };
+
+    extraOptions = lib.mkOption {
+      type = with lib.types; listOf str;
+      default = [ ];
+      description = "Additional options with which to start corosync.";
+    };
+
+    nodelist = lib.mkOption {
+      description = "Corosync nodelist: all cluster members.";
+      default = [ ];
+      type =
+        with lib.types;
+        listOf (submodule {
+          options = {
+            nodeid = lib.mkOption {
+              type = int;
+              description = "Node ID number";
+            };
+            name = lib.mkOption {
+              type = str;
+              description = "Node name";
+            };
+            ring_addrs = lib.mkOption {
+              type = listOf str;
+              description = "List of addresses, one for each ring.";
+            };
+          };
+        });
+    };
+  };
+
+  # implementation
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [ cfg.package ];
+
+    environment.etc."corosync/corosync.conf".text = ''
+      totem {
+        version: 2
+        secauth: on
+        cluster_name: ${cfg.clusterName}
+        transport: knet
+      }
+
+      nodelist {
+        ${lib.concatMapStrings (
+          {
+            nodeid,
+            name,
+            ring_addrs,
+          }:
+          ''
+            node {
+              nodeid: ${toString nodeid}
+              name: ${name}
+              ${lib.concatStrings (
+                lib.imap0 (i: addr: ''
+                  ring${toString i}_addr: ${addr}
+                '') ring_addrs
+              )}
+            }
+          ''
+        ) cfg.nodelist}
+      }
+
+      quorum {
+        # only corosync_votequorum is supported
+        provider: corosync_votequorum
+        wait_for_all: 0
+        ${lib.optionalString (builtins.length cfg.nodelist < 3) ''
+          two_node: 1
+        ''}
+      }
+
+      logging {
+        to_syslog: yes
+      }
+    '';
+
+    environment.etc."corosync/uidgid.d/root".text = ''
+      # allow pacemaker connection by root
+      uidgid {
+        uid: 0
+        gid: 0
+      }
+    '';
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.corosync = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "corosync";
+        StateDirectoryMode = "0700";
+      };
+    };
+
+    environment.etc."sysconfig/corosync".text = lib.optionalString (cfg.extraOptions != [ ]) ''
+      COROSYNC_OPTIONS="${lib.escapeShellArgs cfg.extraOptions}"
+    '';
+  };
+}

nixos/modules/services/cluster/druid/default.nix

@@ -0,0 +1,299 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.druid;
+  inherit (lib)
+    concatStrings
+    concatStringsSep
+    mapAttrsToList
+    concatMap
+    attrByPath
+    mkIf
+    mkMerge
+    mkEnableOption
+    mkOption
+    types
+    mkPackageOption
+    ;
+
+  druidServiceOption = serviceName: {
+    enable = mkEnableOption serviceName;
+
+    restartIfChanged = mkOption {
+      type = types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on clusters running critical applications.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = false;
+    };
+
+    config = mkOption {
+      default = { };
+      type = types.attrsOf types.anything;
+      description = ''
+        (key=value) Configuration to be written to runtime.properties of the druid ${serviceName}
+        <https://druid.apache.org/docs/latest/configuration/index.html>
+      '';
+      example = {
+        "druid.plainTextPort" = "8082";
+        "druid.service" = "servicename";
+      };
+    };
+
+    jdk = mkPackageOption pkgs "JDK" { default = [ "jdk17_headless" ]; };
+
+    jvmArgs = mkOption {
+      type = types.str;
+      default = "";
+      description = "Arguments to pass to the JVM";
+    };
+
+    openFirewall = mkOption {
+      type = types.bool;
+      default = false;
+      description = "Open firewall ports for ${serviceName}.";
+    };
+
+    internalConfig = mkOption {
+      default = { };
+      type = types.attrsOf types.anything;
+      internal = true;
+      description = "Internal Option to add to runtime.properties for ${serviceName}.";
+    };
+  };
+
+  druidServiceConfig =
+    {
+      name,
+      serviceOptions ? cfg."${name}",
+      allowedTCPPorts ? [ ],
+      tmpDirs ? [ ],
+      extraConfig ? { },
+    }:
+    (mkIf serviceOptions.enable (mkMerge [
+      {
+        systemd = {
+          services."druid-${name}" = {
+            after = [ "network.target" ];
+
+            description = "Druid ${name}";
+
+            wantedBy = [ "multi-user.target" ];
+
+            inherit (serviceOptions) restartIfChanged;
+
+            path = [
+              cfg.package
+              serviceOptions.jdk
+            ];
+
+            script =
+              let
+                cfgFile =
+                  fileName: properties:
+                  pkgs.writeTextDir fileName (
+                    concatStringsSep "\n" (mapAttrsToList (n: v: "${n}=${toString v}") properties)
+                  );
+
+                commonConfigFile = cfgFile "common.runtime.properties" cfg.commonConfig;
+
+                configFile = cfgFile "runtime.properties" (serviceOptions.config // serviceOptions.internalConfig);
+
+                extraClassPath = concatStrings (map (path: ":" + path) cfg.extraClassPaths);
+
+                extraConfDir = concatStrings (map (dir: ":" + dir + "/*") cfg.extraConfDirs);
+              in
+              ''
+                run-java -Dlog4j.configurationFile=file:${cfg.log4j} \
+                  -Ddruid.extensions.directory=${cfg.package}/extensions \
+                  -Ddruid.extensions.hadoopDependenciesDir=${cfg.package}/hadoop-dependencies \
+                  -classpath  ${commonConfigFile}:${configFile}:${cfg.package}/lib/\*${extraClassPath}${extraConfDir} \
+                  ${serviceOptions.jvmArgs} \
+                  org.apache.druid.cli.Main server ${name}
+              '';
+
+            serviceConfig = {
+              User = "druid";
+              SyslogIdentifier = "druid-${name}";
+              Restart = "always";
+            };
+          };
+
+          tmpfiles.rules = concatMap (x: [ "d ${x} 0755 druid druid" ]) (cfg.commonTmpDirs ++ tmpDirs);
+        };
+        networking.firewall.allowedTCPPorts = mkIf (attrByPath [
+          "openFirewall"
+        ] false serviceOptions) allowedTCPPorts;
+
+        users = {
+          users.druid = {
+            description = "Druid user";
+            group = "druid";
+            isNormalUser = true;
+          };
+          groups.druid = { };
+        };
+      }
+      extraConfig
+    ]));
+in
+{
+  options.services.druid = {
+    package = mkPackageOption pkgs "apache-druid" { default = [ "druid" ]; };
+
+    commonConfig = mkOption {
+      default = { };
+
+      type = types.attrsOf types.anything;
+
+      description = "(key=value) Configuration to be written to common.runtime.properties";
+
+      example = {
+        "druid.zk.service.host" = "localhost:2181";
+        "druid.metadata.storage.type" = "mysql";
+        "druid.metadata.storage.connector.connectURI" = "jdbc:mysql://localhost:3306/druid";
+        "druid.extensions.loadList" = ''[ "mysql-metadata-storage" ]'';
+      };
+    };
+
+    commonTmpDirs = mkOption {
+      default = [ "/var/log/druid/requests" ];
+      type = types.listOf types.str;
+      description = "Common List of directories used by druid processes";
+    };
+
+    log4j = mkOption {
+      type = types.path;
+      description = "Log4j Configuration for the druid process";
+    };
+
+    extraClassPaths = mkOption {
+      default = [ ];
+      type = types.listOf types.str;
+      description = "Extra classpath to include in the jvm";
+    };
+
+    extraConfDirs = mkOption {
+      default = [ ];
+      type = types.listOf types.path;
+      description = "Extra Conf Dirs to include in the jvm";
+    };
+
+    overlord = druidServiceOption "Druid Overlord";
+
+    coordinator = druidServiceOption "Druid Coordinator";
+
+    broker = druidServiceOption "Druid Broker";
+
+    historical = (druidServiceOption "Druid Historical") // {
+      segmentLocations = mkOption {
+
+        default = null;
+
+        description = "Locations where the historical will store its data.";
+
+        type =
+          with types;
+          nullOr (
+            listOf (submodule {
+              options = {
+                path = mkOption {
+                  type = path;
+                  description = "the path to store the segments";
+                };
+
+                maxSize = mkOption {
+                  type = str;
+                  description = "Max size the druid historical can occupy";
+                };
+
+                freeSpacePercent = mkOption {
+                  type = float;
+                  default = 1.0;
+                  description = "Druid Historical will fail to write if it exceeds this value";
+                };
+              };
+            })
+          );
+
+      };
+    };
+
+    middleManager = druidServiceOption "Druid middleManager";
+    router = druidServiceOption "Druid Router";
+  };
+  config = mkMerge [
+    (druidServiceConfig rec {
+      name = "overlord";
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8090 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "coordinator";
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8081 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "broker";
+
+      tmpDirs = [ (attrByPath [ "druid.lookup.snapshotWorkingDir" ] "" cfg."${name}".config) ];
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8082 cfg."${name}".config) ];
+    })
+
+    (druidServiceConfig rec {
+      name = "historical";
+
+      tmpDirs = [
+        (attrByPath [ "druid.lookup.snapshotWorkingDir" ] "" cfg."${name}".config)
+      ]
+      ++ (map (x: x.path) cfg."${name}".segmentLocations);
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8083 cfg."${name}".config) ];
+
+      extraConfig.services.druid.historical.internalConfig."druid.segmentCache.locations" =
+        builtins.toJSON cfg.historical.segmentLocations;
+    })
+
+    (druidServiceConfig rec {
+      name = "middleManager";
+
+      tmpDirs = [
+        "/var/log/druid/indexer"
+      ]
+      ++ [ (attrByPath [ "druid.indexer.task.baseTaskDir" ] "" cfg."${name}".config) ];
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8091 cfg."${name}".config) ];
+
+      extraConfig = {
+        services.druid.middleManager.internalConfig = {
+          "druid.indexer.runner.javaCommand" = "${cfg.middleManager.jdk}/bin/java";
+          "druid.indexer.runner.javaOpts" =
+            (attrByPath [ "druid.indexer.runner.javaOpts" ] "" cfg.middleManager.config)
+            + " -Dlog4j.configurationFile=file:${cfg.log4j}";
+        };
+
+        networking.firewall.allowedTCPPortRanges = mkIf cfg.middleManager.openFirewall [
+          {
+            from = attrByPath [ "druid.indexer.runner.startPort" ] 8100 cfg.middleManager.config;
+            to = attrByPath [ "druid.indexer.runner.endPort" ] 65535 cfg.middleManager.config;
+          }
+        ];
+      };
+    })
+
+    (druidServiceConfig rec {
+      name = "router";
+
+      allowedTCPPorts = [ (attrByPath [ "druid.plaintextPort" ] 8888 cfg."${name}".config) ];
+    })
+  ];
+
+}

nixos/modules/services/cluster/hadoop/conf.nix

@@ -0,0 +1,58 @@
+{
+  cfg,
+  pkgs,
+  lib,
+}:
+let
+  propertyXml =
+    name: value:
+    lib.optionalString (value != null) ''
+      <property>
+        <name>${name}</name>
+        <value>${builtins.toString value}</value>
+      </property>
+    '';
+  siteXml =
+    fileName: properties:
+    pkgs.writeTextDir fileName ''
+      <?xml version="1.0" encoding="UTF-8" standalone="no"?>
+      <!-- generated by NixOS -->
+      <configuration>
+        ${builtins.concatStringsSep "\n" (pkgs.lib.mapAttrsToList propertyXml properties)}
+      </configuration>
+    '';
+  cfgLine = name: value: ''
+    ${name}=${builtins.toString value}
+  '';
+  cfgFile =
+    fileName: properties:
+    pkgs.writeTextDir fileName ''
+      # generated by NixOS
+      ${builtins.concatStringsSep "" (pkgs.lib.mapAttrsToList cfgLine properties)}
+    '';
+  userFunctions = ''
+    hadoop_verify_logdir() {
+      echo Skipping verification of log directory
+    }
+  '';
+  hadoopEnv = ''
+    export HADOOP_LOG_DIR=/tmp/hadoop/$USER
+  '';
+in
+pkgs.runCommand "hadoop-conf" { } (
+  with cfg;
+  ''
+    mkdir -p $out/
+    cp ${siteXml "core-site.xml" (coreSite // coreSiteInternal)}/* $out/
+    cp ${siteXml "hdfs-site.xml" (hdfsSiteDefault // hdfsSite // hdfsSiteInternal)}/* $out/
+    cp ${siteXml "hbase-site.xml" (hbaseSiteDefault // hbaseSite // hbaseSiteInternal)}/* $out/
+    cp ${siteXml "mapred-site.xml" (mapredSiteDefault // mapredSite)}/* $out/
+    cp ${siteXml "yarn-site.xml" (yarnSiteDefault // yarnSite // yarnSiteInternal)}/* $out/
+    cp ${siteXml "httpfs-site.xml" httpfsSite}/* $out/
+    cp ${cfgFile "container-executor.cfg" containerExecutorCfg}/* $out/
+    cp ${pkgs.writeTextDir "hadoop-user-functions.sh" userFunctions}/* $out/
+    cp ${pkgs.writeTextDir "hadoop-env.sh" hadoopEnv}/* $out/
+    cp ${log4jProperties} $out/log4j.properties
+    ${lib.concatMapStringsSep "\n" (dir: "cp -f -r ${dir}/* $out/") extraConfDirs}
+  ''
+)

nixos/modules/services/cluster/hadoop/default.nix

@@ -0,0 +1,232 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  opt = options.services.hadoop;
+in
+{
+  imports = [
+    ./yarn.nix
+    ./hdfs.nix
+    ./hbase.nix
+  ];
+
+  options.services.hadoop = {
+    coreSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "fs.defaultFS" = "hdfs://localhost";
+        }
+      '';
+      description = ''
+        Hadoop core-site.xml definition
+        <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/core-default.xml>
+      '';
+    };
+    coreSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to core-site.xml based on module options
+      '';
+    };
+
+    hdfsSiteDefault = lib.mkOption {
+      default = {
+        "dfs.namenode.rpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-address" = "0.0.0.0:9870";
+        "dfs.namenode.servicerpc-bind-host" = "0.0.0.0";
+        "dfs.namenode.http-bind-host" = "0.0.0.0";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for hdfs-site.xml
+      '';
+    };
+    hdfsSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "dfs.nameservices" = "namenode1";
+        }
+      '';
+      description = ''
+        Additional options and overrides for hdfs-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/hdfs-default.xml>
+      '';
+    };
+    hdfsSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hdfs-site.xml based on module options
+      '';
+    };
+
+    mapredSiteDefault = lib.mkOption {
+      default = {
+        "mapreduce.framework.name" = "yarn";
+        "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+        "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
+      };
+      defaultText = lib.literalExpression ''
+        {
+          "mapreduce.framework.name" = "yarn";
+          "yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+          "mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
+        }
+      '';
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for mapred-site.xml
+      '';
+    };
+    mapredSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "mapreduce.map.java.opts" = "-Xmx900m -XX:+UseParallelGC";
+        }
+      '';
+      description = ''
+        Additional options and overrides for mapred-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml>
+      '';
+    };
+
+    yarnSiteDefault = lib.mkOption {
+      default = {
+        "yarn.nodemanager.admin-env" = "PATH=$PATH";
+        "yarn.nodemanager.aux-services" = "mapreduce_shuffle";
+        "yarn.nodemanager.aux-services.mapreduce_shuffle.class" = "org.apache.hadoop.mapred.ShuffleHandler";
+        "yarn.nodemanager.bind-host" = "0.0.0.0";
+        "yarn.nodemanager.container-executor.class" =
+          "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor";
+        "yarn.nodemanager.env-whitelist" =
+          "JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,LANG,TZ";
+        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
+        "yarn.nodemanager.linux-container-executor.path" =
+          "/run/wrappers/yarn-nodemanager/bin/container-executor";
+        "yarn.nodemanager.log-dirs" = "/var/log/hadoop/yarn/nodemanager";
+        "yarn.resourcemanager.bind-host" = "0.0.0.0";
+        "yarn.resourcemanager.scheduler.class" =
+          "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for yarn-site.xml
+      '';
+    };
+    yarnSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "yarn.resourcemanager.hostname" = "''${config.networking.hostName}";
+        }
+      '';
+      description = ''
+        Additional options and overrides for yarn-site.xml
+        <https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-common/yarn-default.xml>
+      '';
+    };
+    yarnSiteInternal = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to yarn-site.xml based on module options
+      '';
+    };
+
+    httpfsSite = lib.mkOption {
+      default = { };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        {
+          "hadoop.http.max.threads" = 500;
+        }
+      '';
+      description = ''
+        Hadoop httpfs-site.xml definition
+        <https://hadoop.apache.org/docs/current/hadoop-hdfs-httpfs/httpfs-default.html>
+      '';
+    };
+
+    log4jProperties = lib.mkOption {
+      default = "${cfg.package}/etc/hadoop/log4j.properties";
+      defaultText = lib.literalExpression ''
+        "''${config.${opt.package}}/etc/hadoop/log4j.properties"
+      '';
+      type = lib.types.path;
+      example = lib.literalExpression ''
+        "''${pkgs.hadoop}/etc/hadoop/log4j.properties";
+      '';
+      description = "log4j.properties file added to HADOOP_CONF_DIR";
+    };
+
+    containerExecutorCfg = lib.mkOption {
+      default = {
+        # must be the same as yarn.nodemanager.linux-container-executor.group in yarnSite
+        "yarn.nodemanager.linux-container-executor.group" = "hadoop";
+        "min.user.id" = 1000;
+        "feature.terminal.enabled" = 1;
+        "feature.mount-cgroup.enabled" = 1;
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      example = lib.literalExpression ''
+        options.services.hadoop.containerExecutorCfg.default // {
+          "feature.terminal.enabled" = 0;
+        }
+      '';
+      description = ''
+        Yarn container-executor.cfg definition
+        <https://hadoop.apache.org/docs/r2.7.2/hadoop-yarn/hadoop-yarn-site/SecureContainer.html>
+      '';
+    };
+
+    extraConfDirs = lib.mkOption {
+      default = [ ];
+      type = lib.types.listOf lib.types.path;
+      example = lib.literalExpression ''
+        [
+          ./extraHDFSConfs
+          ./extraYARNConfs
+        ]
+      '';
+      description = "Directories containing additional config files to be added to HADOOP_CONF_DIR";
+    };
+
+    gatewayRole.enable = lib.mkEnableOption "gateway role for deploying hadoop configs";
+
+    package = lib.mkPackageOption pkgs "hadoop" { };
+  };
+
+  config = lib.mkIf cfg.gatewayRole.enable {
+    users.groups.hadoop = {
+      gid = config.ids.gids.hadoop;
+    };
+    environment = {
+      systemPackages = [ cfg.package ];
+      etc."hadoop-conf".source =
+        let
+          hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+        in
+        "${hadoopConf}";
+      variables.HADOOP_CONF_DIR = "/etc/hadoop-conf/";
+    };
+  };
+}

nixos/modules/services/cluster/hadoop/hbase.nix

@@ -0,0 +1,246 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  mkIfNotNull = x: lib.mkIf (x != null) x;
+  # generic hbase role options
+  hbaseRoleOption =
+    name: extraOpts:
+    {
+      enable = lib.mkEnableOption "HBase ${name}";
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open firewall ports for HBase ${name}.";
+      };
+
+      restartIfChanged = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Restart ${name} con config change.";
+      };
+
+      extraFlags = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        example = lib.literalExpression ''[ "--backup" ]'';
+        description = "Extra flags for the ${name} service.";
+      };
+
+      environment = lib.mkOption {
+        type = with lib.types; attrsOf str;
+        default = { };
+        example = lib.literalExpression ''
+          {
+            HBASE_MASTER_OPTS = "-Dcom.sun.management.jmxremote.ssl=true";
+          }
+        '';
+        description = "Environment variables passed to ${name}.";
+      };
+    }
+    // extraOpts;
+  # generic hbase role configs
+  hbaseRoleConfig =
+    name: ports:
+    (lib.mkIf cfg.hbase."${name}".enable {
+      services.hadoop.gatewayRole = {
+        enable = true;
+        enableHbaseCli = lib.mkDefault true;
+      };
+
+      systemd.services."hbase-${lib.toLower name}" = {
+        description = "HBase ${name}";
+        wantedBy = [ "multi-user.target" ];
+        path =
+          with cfg;
+          [ hbase.package ] ++ lib.optional (with cfg.hbase.master; enable && initHDFS) package;
+        preStart = lib.mkIf (with cfg.hbase.master; enable && initHDFS) (
+          lib.concatStringsSep "\n" (
+            map (x: "HADOOP_USER_NAME=hdfs hdfs --config /etc/hadoop-conf ${x}") [
+              "dfsadmin -safemode wait"
+              "dfs -mkdir -p ${cfg.hbase.rootdir}"
+              "dfs -chown hbase ${cfg.hbase.rootdir}"
+            ]
+          )
+        );
+
+        inherit (cfg.hbase."${name}") environment;
+        script = lib.concatStringsSep " " (
+          [
+            "hbase --config /etc/hadoop-conf/"
+            "${lib.toLower name} start"
+          ]
+          ++ cfg.hbase."${name}".extraFlags
+          ++ map (x: "--${lib.toLower x} ${toString cfg.hbase.${name}.${x}}") (
+            lib.filter (x: lib.hasAttr x cfg.hbase.${name}) [
+              "port"
+              "infoPort"
+            ]
+          )
+        );
+
+        serviceConfig = {
+          User = "hbase";
+          SyslogIdentifier = "hbase-${lib.toLower name}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.hbaseSiteInternal."hbase.rootdir" = cfg.hbase.rootdir;
+
+      networking = {
+        firewall.allowedTCPPorts = lib.mkIf cfg.hbase."${name}".openFirewall ports;
+        hosts = lib.mkIf (with cfg.hbase.regionServer; enable && overrideHosts) {
+          "127.0.0.2" = lib.mkForce [ ];
+          "::1" = lib.mkForce [ ];
+        };
+      };
+
+    });
+in
+{
+  options.services.hadoop = {
+
+    gatewayRole.enableHbaseCli = lib.mkEnableOption "HBase CLI tools";
+
+    hbaseSiteDefault = lib.mkOption {
+      default = {
+        "hbase.regionserver.ipc.address" = "0.0.0.0";
+        "hbase.master.ipc.address" = "0.0.0.0";
+        "hbase.master.info.bindAddress" = "0.0.0.0";
+        "hbase.regionserver.info.bindAddress" = "0.0.0.0";
+
+        "hbase.cluster.distributed" = "true";
+      };
+      type = lib.types.attrsOf lib.types.anything;
+      description = ''
+        Default options for hbase-site.xml
+      '';
+    };
+    hbaseSite = lib.mkOption {
+      default = { };
+      type = with lib.types; attrsOf anything;
+      example = lib.literalExpression ''
+        {
+          "hbase.hregion.max.filesize" = 20*1024*1024*1024;
+          "hbase.table.normalization.enabled" = "true";
+        }
+      '';
+      description = ''
+        Additional options and overrides for hbase-site.xml
+        <https://github.com/apache/hbase/blob/rel/2.4.11/hbase-common/src/main/resources/hbase-default.xml>
+      '';
+    };
+    hbaseSiteInternal = lib.mkOption {
+      default = { };
+      type = with lib.types; attrsOf anything;
+      internal = true;
+      description = ''
+        Internal option to add configs to hbase-site.xml based on module options
+      '';
+    };
+
+    hbase = {
+
+      package = lib.mkPackageOption pkgs "hbase" { };
+
+      rootdir = lib.mkOption {
+        description = ''
+          This option will set "hbase.rootdir" in hbase-site.xml and determine
+          the directory shared by region servers and into which HBase persists.
+          The URL should be 'fully-qualified' to include the filesystem scheme.
+          If a core-site.xml is provided, the FS scheme defaults to the value
+          of "fs.defaultFS".
+
+          Filesystems other than HDFS (like S3, QFS, Swift) are also supported.
+        '';
+        type = lib.types.str;
+        example = "hdfs://nameservice1/hbase";
+        default = "/hbase";
+      };
+      zookeeperQuorum = lib.mkOption {
+        description = ''
+          This option will set "hbase.zookeeper.quorum" in hbase-site.xml.
+          Comma separated list of servers in the ZooKeeper ensemble.
+        '';
+        type = with lib.types; nullOr commas;
+        example = "zk1.internal,zk2.internal,zk3.internal";
+        default = null;
+      };
+    }
+    // (
+      let
+        ports = port: infoPort: {
+          port = lib.mkOption {
+            type = lib.types.port;
+            default = port;
+            description = "RPC port";
+          };
+          infoPort = lib.mkOption {
+            type = lib.types.port;
+            default = infoPort;
+            description = "web UI port";
+          };
+        };
+      in
+      lib.mapAttrs hbaseRoleOption {
+        master.initHDFS = lib.mkEnableOption "initialization of the hbase directory on HDFS";
+        regionServer.overrideHosts = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = ''
+            Remove /etc/hosts entries for "127.0.0.2" and "::1" defined in nixos/modules/config/networking.nix
+            Regionservers must be able to resolve their hostnames to their IP addresses, through PTR records
+            or /etc/hosts entries.
+          '';
+        };
+        thrift = ports 9090 9095;
+        rest = ports 8080 8085;
+      }
+    );
+  };
+
+  config = lib.mkMerge (
+    [
+
+      (lib.mkIf cfg.gatewayRole.enable {
+
+        environment.systemPackages = lib.mkIf cfg.gatewayRole.enableHbaseCli [ cfg.hbase.package ];
+
+        services.hadoop.hbaseSiteInternal = with cfg.hbase; {
+          "hbase.zookeeper.quorum" = mkIfNotNull zookeeperQuorum;
+        };
+
+        users.users.hbase = {
+          description = "Hadoop HBase user";
+          group = "hadoop";
+          isSystemUser = true;
+        };
+      })
+    ]
+    ++ (lib.mapAttrsToList hbaseRoleConfig {
+      master = [
+        16000
+        16010
+      ];
+      regionServer = [
+        16020
+        16030
+      ];
+      thrift = with cfg.hbase.thrift; [
+        port
+        infoPort
+      ];
+      rest = with cfg.hbase.rest; [
+        port
+        infoPort
+      ];
+    })
+  );
+}

nixos/modules/services/cluster/hadoop/hdfs.nix

@@ -0,0 +1,237 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+
+  # Config files for hadoop services
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+
+  # Generator for HDFS service options
+  hadoopServiceOption =
+    {
+      serviceName,
+      firewallOption ? true,
+      extraOpts ? null,
+    }:
+    {
+      enable = lib.mkEnableOption serviceName;
+      restartIfChanged = lib.mkOption {
+        type = lib.types.bool;
+        description = ''
+          Automatically restart the service on config change.
+          This can be set to false to defer restarts on clusters running critical applications.
+          Please consider the security implications of inadvertently running an older version,
+          and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+        '';
+        default = false;
+      };
+      extraFlags = lib.mkOption {
+        type = with lib.types; listOf str;
+        default = [ ];
+        description = "Extra command line flags to pass to ${serviceName}";
+        example = [
+          "-Dcom.sun.management.jmxremote"
+          "-Dcom.sun.management.jmxremote.port=8010"
+        ];
+      };
+      extraEnv = lib.mkOption {
+        type = with lib.types; attrsOf str;
+        default = { };
+        description = "Extra environment variables for ${serviceName}";
+      };
+    }
+    // (lib.optionalAttrs firewallOption {
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open firewall ports for ${serviceName}.";
+      };
+    })
+    // (lib.optionalAttrs (extraOpts != null) extraOpts);
+
+  # Generator for HDFS service configs
+  hadoopServiceConfig =
+    {
+      name,
+      serviceOptions ? cfg.hdfs."${lib.toLower name}",
+      description ? "Hadoop HDFS ${name}",
+      User ? "hdfs",
+      allowedTCPPorts ? [ ],
+      preStart ? "",
+      environment ? { },
+      extraConfig ? { },
+    }:
+    (
+
+      lib.mkIf serviceOptions.enable (
+        lib.mkMerge [
+          {
+            systemd.services."hdfs-${lib.toLower name}" = {
+              inherit description preStart;
+              environment = environment // serviceOptions.extraEnv;
+              wantedBy = [ "multi-user.target" ];
+              inherit (serviceOptions) restartIfChanged;
+              serviceConfig = {
+                inherit User;
+                SyslogIdentifier = "hdfs-${lib.toLower name}";
+                ExecStart = "${cfg.package}/bin/hdfs --config ${hadoopConf} ${lib.toLower name} ${lib.escapeShellArgs serviceOptions.extraFlags}";
+                Restart = "always";
+              };
+            };
+
+            services.hadoop.gatewayRole.enable = true;
+
+            networking.firewall.allowedTCPPorts = lib.mkIf (
+              (builtins.hasAttr "openFirewall" serviceOptions) && serviceOptions.openFirewall
+            ) allowedTCPPorts;
+          }
+          extraConfig
+        ]
+      )
+    );
+
+in
+{
+  options.services.hadoop.hdfs = {
+
+    namenode = hadoopServiceOption { serviceName = "HDFS NameNode"; } // {
+      formatOnInit = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Format HDFS namenode on first start. This is useful for quickly spinning up
+          ephemeral HDFS clusters with a single namenode.
+          For HA clusters, initialization involves multiple steps across multiple nodes.
+          Follow this guide to initialize an HA cluster manually:
+          <https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HDFSHighAvailabilityWithQJM.html>
+        '';
+      };
+    };
+
+    datanode = hadoopServiceOption { serviceName = "HDFS DataNode"; } // {
+      dataDirs = lib.mkOption {
+        default = null;
+        description = "Tier and path definitions for datanode storage.";
+        type =
+          with lib.types;
+          nullOr (
+            listOf (submodule {
+              options = {
+                type = lib.mkOption {
+                  type = enum [
+                    "SSD"
+                    "DISK"
+                    "ARCHIVE"
+                    "RAM_DISK"
+                  ];
+                  description = ''
+                    Storage types ([SSD]/[DISK]/[ARCHIVE]/[RAM_DISK]) for HDFS storage policies.
+                  '';
+                };
+                path = lib.mkOption {
+                  type = path;
+                  example = [ "/var/lib/hadoop/hdfs/dn" ];
+                  description = "Determines where on the local filesystem a data node should store its blocks.";
+                };
+              };
+            })
+          );
+      };
+    };
+
+    journalnode = hadoopServiceOption { serviceName = "HDFS JournalNode"; };
+
+    zkfc = hadoopServiceOption {
+      serviceName = "HDFS ZooKeeper failover controller";
+      firewallOption = false;
+    };
+
+    httpfs = hadoopServiceOption { serviceName = "HDFS JournalNode"; } // {
+      tempPath = lib.mkOption {
+        type = lib.types.path;
+        default = "/tmp/hadoop/httpfs";
+        description = "HTTPFS_TEMP path used by HTTPFS";
+      };
+    };
+
+  };
+
+  config = lib.mkMerge [
+    (hadoopServiceConfig {
+      name = "NameNode";
+      allowedTCPPorts = [
+        9870 # namenode.http-address
+        8020 # namenode.rpc-address
+        8022 # namenode.servicerpc-address
+        8019 # dfs.ha.zkfc.port
+      ];
+      preStart = (
+        lib.mkIf cfg.hdfs.namenode.formatOnInit "${cfg.package}/bin/hdfs --config ${hadoopConf} namenode -format -nonInteractive || true"
+      );
+    })
+
+    (hadoopServiceConfig {
+      name = "DataNode";
+      # port numbers for datanode changed between hadoop 2 and 3
+      allowedTCPPorts =
+        if lib.versionAtLeast cfg.package.version "3" then
+          [
+            9864 # datanode.http.address
+            9866 # datanode.address
+            9867 # datanode.ipc.address
+          ]
+        else
+          [
+            50075 # datanode.http.address
+            50010 # datanode.address
+            50020 # datanode.ipc.address
+          ];
+      extraConfig.services.hadoop.hdfsSiteInternal."dfs.datanode.data.dir" = lib.mkIf (
+        cfg.hdfs.datanode.dataDirs != null
+      ) (lib.concatMapStringsSep "," (x: "[" + x.type + "]file://" + x.path) cfg.hdfs.datanode.dataDirs);
+    })
+
+    (hadoopServiceConfig {
+      name = "JournalNode";
+      allowedTCPPorts = [
+        8480 # dfs.journalnode.http-address
+        8485 # dfs.journalnode.rpc-address
+      ];
+    })
+
+    (hadoopServiceConfig {
+      name = "zkfc";
+      description = "Hadoop HDFS ZooKeeper failover controller";
+    })
+
+    (hadoopServiceConfig {
+      name = "HTTPFS";
+      environment.HTTPFS_TEMP = cfg.hdfs.httpfs.tempPath;
+      preStart = "mkdir -p $HTTPFS_TEMP";
+      User = "httpfs";
+      allowedTCPPorts = [
+        14000 # httpfs.http.port
+      ];
+    })
+
+    (lib.mkIf cfg.gatewayRole.enable {
+      users.users.hdfs = {
+        description = "Hadoop HDFS user";
+        group = "hadoop";
+        uid = config.ids.uids.hdfs;
+      };
+    })
+    (lib.mkIf cfg.hdfs.httpfs.enable {
+      users.users.httpfs = {
+        description = "Hadoop HTTPFS user";
+        group = "hadoop";
+        isSystemUser = true;
+      };
+    })
+
+  ];
+}

nixos/modules/services/cluster/hadoop/yarn.nix

@@ -0,0 +1,226 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.hadoop;
+  hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
+  restartIfChanged = lib.mkOption {
+    type = lib.types.bool;
+    description = ''
+      Automatically restart the service on config change.
+      This can be set to false to defer restarts on clusters running critical applications.
+      Please consider the security implications of inadvertently running an older version,
+      and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+    '';
+    default = false;
+  };
+  extraFlags = lib.mkOption {
+    type = with lib.types; listOf str;
+    default = [ ];
+    description = "Extra command line flags to pass to the service";
+    example = [
+      "-Dcom.sun.management.jmxremote"
+      "-Dcom.sun.management.jmxremote.port=8010"
+    ];
+  };
+  extraEnv = lib.mkOption {
+    type = with lib.types; attrsOf str;
+    default = { };
+    description = "Extra environment variables";
+  };
+in
+{
+  options.services.hadoop.yarn = {
+    resourcemanager = {
+      enable = lib.mkEnableOption "Hadoop YARN ResourceManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open firewall ports for resourcemanager
+        '';
+      };
+    };
+    nodemanager = {
+      enable = lib.mkEnableOption "Hadoop YARN NodeManager";
+      inherit restartIfChanged extraFlags extraEnv;
+
+      resource = {
+        cpuVCores = lib.mkOption {
+          description = "Number of vcores that can be allocated for containers.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationVCores = lib.mkOption {
+          description = "The maximum virtual CPU cores any container can be allocated.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        memoryMB = lib.mkOption {
+          description = "Amount of physical memory, in MB, that can be allocated for containers.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+        maximumAllocationMB = lib.mkOption {
+          description = "The maximum physical memory any container can be allocated.";
+          type = with lib.types; nullOr ints.positive;
+          default = null;
+        };
+      };
+
+      useCGroups = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Use cgroups to enforce resource limits on containers
+        '';
+      };
+
+      localDir = lib.mkOption {
+        description = "List of directories to store localized files in.";
+        type = with lib.types; nullOr (listOf path);
+        example = [ "/var/lib/hadoop/yarn/nm" ];
+        default = null;
+      };
+
+      addBinBash = lib.mkOption {
+        type = lib.types.bool;
+        default = true;
+        description = ''
+          Add /bin/bash. This is needed by the linux container executor's launch script.
+        '';
+      };
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = ''
+          Open firewall ports for nodemanager.
+          Because containers can listen on any ephemeral port, TCP ports 1024–65535 will be opened.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkMerge [
+    (lib.mkIf cfg.gatewayRole.enable {
+      users.users.yarn = {
+        description = "Hadoop YARN user";
+        group = "hadoop";
+        uid = config.ids.uids.yarn;
+      };
+    })
+
+    (lib.mkIf cfg.yarn.resourcemanager.enable {
+      systemd.services.yarn-resourcemanager = {
+        description = "Hadoop YARN ResourceManager";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.yarn.resourcemanager) restartIfChanged;
+        environment = cfg.yarn.resourcemanager.extraEnv;
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-resourcemanager";
+          ExecStart =
+            "${cfg.package}/bin/yarn --config ${hadoopConf} "
+            + " resourcemanager ${lib.escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.gatewayRole.enable = true;
+
+      networking.firewall.allowedTCPPorts = (
+        lib.mkIf cfg.yarn.resourcemanager.openFirewall [
+          8088 # resourcemanager.webapp.address
+          8030 # resourcemanager.scheduler.address
+          8031 # resourcemanager.resource-tracker.address
+          8032 # resourcemanager.address
+          8033 # resourcemanager.admin.address
+        ]
+      );
+    })
+
+    (lib.mkIf cfg.yarn.nodemanager.enable {
+      # Needed because yarn hardcodes /bin/bash in container start scripts
+      # These scripts can't be patched, they are generated at runtime
+      systemd.tmpfiles.rules = [
+        (lib.mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
+      ];
+
+      systemd.services.yarn-nodemanager = {
+        description = "Hadoop YARN NodeManager";
+        wantedBy = [ "multi-user.target" ];
+        inherit (cfg.yarn.nodemanager) restartIfChanged;
+        environment = cfg.yarn.nodemanager.extraEnv;
+
+        preStart = ''
+          # create log dir
+          mkdir -p /var/log/hadoop/yarn/nodemanager
+          chown yarn:hadoop /var/log/hadoop/yarn/nodemanager
+
+          # set up setuid container executor binary
+          umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
+          rm -rf /run/wrappers/yarn-nodemanager/ || true
+          mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
+          cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
+          chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
+          chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
+          cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
+        '';
+
+        serviceConfig = {
+          User = "yarn";
+          SyslogIdentifier = "yarn-nodemanager";
+          PermissionsStartOnly = true;
+          ExecStart =
+            "${cfg.package}/bin/yarn --config ${hadoopConf} "
+            + " nodemanager ${lib.escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
+          Restart = "always";
+        };
+      };
+
+      services.hadoop.gatewayRole.enable = true;
+
+      services.hadoop.yarnSiteInternal =
+        with cfg.yarn.nodemanager;
+        lib.mkMerge [
+          {
+            "yarn.nodemanager.local-dirs" = lib.mkIf (localDir != null) (concatStringsSep "," localDir);
+            "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores;
+            "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB;
+            "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores;
+            "yarn.nodemanager.resource.memory-mb" = resource.memoryMB;
+          }
+          (lib.mkIf useCGroups (
+            lib.warnIf (lib.versionOlder cfg.package.version "3.5.0")
+              ''
+                hadoop < 3.5.0 does not support cgroup v2
+                setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended
+                see: https://issues.apache.org/jira/browse/YARN-11669
+              ''
+              {
+                "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn";
+                "yarn.nodemanager.linux-container-executor.resources-handler.class" =
+                  "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler";
+                "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true";
+                "yarn.nodemanager.linux-container-executor.cgroups.mount-path" =
+                  "/run/wrappers/yarn-nodemanager/cgroup";
+              }
+          ))
+        ];
+
+      networking.firewall.allowedTCPPortRanges = [
+        (lib.mkIf (cfg.yarn.nodemanager.openFirewall) {
+          from = 1024;
+          to = 65535;
+        })
+      ];
+    })
+
+  ];
+}

nixos/modules/services/cluster/k3s/default.nix

@@ -0,0 +1,913 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.k3s;
+  removeOption =
+    config: instruction:
+    lib.mkRemovedOptionModule (
+      [
+        "services"
+        "k3s"
+      ]
+      ++ config
+    ) instruction;
+
+  manifestDir = "/var/lib/rancher/k3s/server/manifests";
+  chartDir = "/var/lib/rancher/k3s/server/static/charts";
+  imageDir = "/var/lib/rancher/k3s/agent/images";
+  containerdConfigTemplateFile = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl";
+  yamlFormat = pkgs.formats.yaml { };
+  yamlDocSeparator = builtins.toFile "yaml-doc-separator" "\n---\n";
+  # Manifests need a valid YAML suffix to be respected by k3s
+  mkManifestTarget =
+    name: if (lib.hasSuffix ".yaml" name || lib.hasSuffix ".yml" name) then name else name + ".yaml";
+  # Produces a list containing all duplicate manifest names
+  duplicateManifests = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
+    builtins.attrNames cfg.manifests
+  );
+  # Produces a list containing all duplicate chart names
+  duplicateCharts = lib.intersectLists (builtins.attrNames cfg.autoDeployCharts) (
+    builtins.attrNames cfg.charts
+  );
+
+  # Converts YAML -> JSON -> Nix
+  fromYaml =
+    path:
+    builtins.fromJSON (
+      builtins.readFile (
+        pkgs.runCommand "${path}-converted.json" { nativeBuildInputs = [ pkgs.yq-go ]; } ''
+          yq --no-colors --output-format json ${path} > $out
+        ''
+      )
+    );
+
+  # Replace prefixes and characters that are problematic in file names
+  cleanHelmChartName =
+    name:
+    let
+      woPrefix = lib.removePrefix "https://" (lib.removePrefix "oci://" name);
+    in
+    lib.replaceStrings
+      [
+        "/"
+        ":"
+      ]
+      [
+        "-"
+        "-"
+      ]
+      woPrefix;
+
+  # Fetch a Helm chart from a public registry. This only supports a basic Helm pull.
+  fetchHelm =
+    {
+      name,
+      repo,
+      version,
+      hash ? lib.fakeHash,
+    }:
+    let
+      isOci = lib.hasPrefix "oci://" repo;
+      pullCmd = if isOci then repo else "--repo ${repo} ${name}";
+      name' = if isOci then "${repo}-${version}" else "${repo}-${name}-${version}";
+    in
+    pkgs.runCommand (cleanHelmChartName "${name'}.tgz")
+      {
+        inherit (lib.fetchers.normalizeHash { } { inherit hash; }) outputHash outputHashAlgo;
+        impureEnvVars = lib.fetchers.proxyImpureEnvVars;
+        nativeBuildInputs = with pkgs; [
+          kubernetes-helm
+          cacert
+          # Helm requires HOME to refer to a writable dir
+          writableTmpDirAsHomeHook
+        ];
+      }
+      ''
+        helm pull ${pullCmd} --version ${version}
+        mv ./*.tgz $out
+      '';
+
+  # Returns the path to a YAML manifest file
+  mkExtraDeployManifest =
+    x:
+    # x is a derivation that provides a YAML file
+    if lib.isDerivation x then
+      x.outPath
+    # x is an attribute set that needs to be converted to a YAML file
+    else if builtins.isAttrs x then
+      (yamlFormat.generate "extra-deploy-chart-manifest" x)
+    # assume x is a path to a YAML file
+    else
+      x;
+
+  # Generate a HelmChart custom resource.
+  mkHelmChartCR =
+    name: value:
+    let
+      chartValues = if (lib.isPath value.values) then fromYaml value.values else value.values;
+      # use JSON for values as it's a subset of YAML and understood by the k3s Helm controller
+      valuesContent = builtins.toJSON chartValues;
+    in
+    # merge with extraFieldDefinitions to allow setting advanced values and overwrite generated
+    # values
+    lib.recursiveUpdate {
+      apiVersion = "helm.cattle.io/v1";
+      kind = "HelmChart";
+      metadata = {
+        inherit name;
+        namespace = "kube-system";
+      };
+      spec = {
+        inherit valuesContent;
+        inherit (value) targetNamespace createNamespace;
+        chart = "https://%{KUBERNETES_API}%/static/charts/${name}.tgz";
+      };
+    } value.extraFieldDefinitions;
+
+  # Generate a HelmChart custom resource together with extraDeploy manifests. This
+  # generates possibly a multi document YAML file that the auto deploy mechanism of k3s
+  # deploys.
+  mkAutoDeployChartManifest = name: value: {
+    # target is the final name of the link created for the manifest file
+    target = mkManifestTarget name;
+    inherit (value) enable package;
+    # source is a store path containing the complete manifest file
+    source = pkgs.concatText "auto-deploy-chart-${name}.yaml" (
+      [
+        (yamlFormat.generate "helm-chart-manifest-${name}.yaml" (mkHelmChartCR name value))
+      ]
+      # alternate the YAML doc separator (---) and extraDeploy manifests to create
+      # multi document YAMLs
+      ++ (lib.concatMap (x: [
+        yamlDocSeparator
+        (mkExtraDeployManifest x)
+      ]) value.extraDeploy)
+    );
+  };
+
+  autoDeployChartsModule = lib.types.submodule (
+    { config, ... }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          example = false;
+          description = ''
+            Whether to enable the installation of this Helm chart. Note that setting
+            this option to `false` will not uninstall the chart from the cluster, if
+            it was previously installed. Please use the the `--disable` flag or `.skip`
+            files to delete/disable Helm charts, as mentioned in the
+            [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
+          '';
+        };
+
+        repo = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "https://kubernetes.github.io/ingress-nginx";
+          description = ''
+            The repo of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        name = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "ingress-nginx";
+          description = ''
+            The name of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        version = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "4.7.0";
+          description = ''
+            The version of the Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        hash = lib.mkOption {
+          type = lib.types.str;
+          example = "sha256-ej+vpPNdiOoXsaj1jyRpWLisJgWo8EqX+Z5VbpSjsPA=";
+          default = "";
+          description = ''
+            The hash of the packaged Helm chart. Only has an effect if `package` is not set.
+            The Helm chart is fetched during build time and placed as a `.tgz` archive on the
+            filesystem.
+          '';
+        };
+
+        package = lib.mkOption {
+          type = with lib.types; either path package;
+          example = lib.literalExpression "../my-helm-chart.tgz";
+          description = ''
+            The packaged Helm chart. Overwrites the options `repo`, `name`, `version`
+            and `hash` in case of conflicts.
+          '';
+        };
+
+        targetNamespace = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          default = "default";
+          example = "kube-system";
+          description = "The namespace in which the Helm chart gets installed.";
+        };
+
+        createNamespace = lib.mkOption {
+          type = lib.types.bool;
+          default = false;
+          example = true;
+          description = "Whether to create the target namespace if not present.";
+        };
+
+        values = lib.mkOption {
+          type = with lib.types; either path attrs;
+          default = { };
+          example = {
+            replicaCount = 3;
+            hostName = "my-host";
+            server = {
+              name = "nginx";
+              port = 80;
+            };
+          };
+          description = ''
+            Override default chart values via Nix expressions. This is equivalent to setting
+            values in a `values.yaml` file.
+
+            WARNING: The values (including secrets!) specified here are exposed unencrypted
+            in the world-readable nix store.
+          '';
+        };
+
+        extraDeploy = lib.mkOption {
+          type = with lib.types; listOf (either path attrs);
+          default = [ ];
+          example = lib.literalExpression ''
+            [
+              ../manifests/my-extra-deployment.yaml
+              {
+                apiVersion = "v1";
+                kind = "Service";
+                metadata = {
+                  name = "app-service";
+                };
+                spec = {
+                  selector = {
+                    "app.kubernetes.io/name" = "MyApp";
+                  };
+                  ports = [
+                    {
+                      name = "name-of-service-port";
+                      protocol = "TCP";
+                      port = 80;
+                      targetPort = "http-web-svc";
+                    }
+                  ];
+                };
+              }
+            ];
+          '';
+          description = "List of extra Kubernetes manifests to deploy with this Helm chart.";
+        };
+
+        extraFieldDefinitions = lib.mkOption {
+          inherit (yamlFormat) type;
+          default = { };
+          example = {
+            spec = {
+              bootstrap = true;
+              helmVersion = "v2";
+              backOffLimit = 3;
+              jobImage = "custom-helm-controller:v0.0.1";
+            };
+          };
+          description = ''
+            Extra HelmChart field definitions that are merged with the rest of the HelmChart
+            custom resource. This can be used to set advanced fields or to overwrite
+            generated fields. See <https://docs.k3s.io/helm#helmchart-field-definitions>
+            for possible fields.
+          '';
+        };
+      };
+
+      config.package = lib.mkDefault (fetchHelm {
+        inherit (config)
+          repo
+          name
+          version
+          hash
+          ;
+      });
+    }
+  );
+
+  manifestModule = lib.types.submodule (
+    {
+      name,
+      config,
+      options,
+      ...
+    }:
+    {
+      options = {
+        enable = lib.mkOption {
+          type = lib.types.bool;
+          default = true;
+          description = "Whether this manifest file should be generated.";
+        };
+
+        target = lib.mkOption {
+          type = lib.types.nonEmptyStr;
+          example = "manifest.yaml";
+          description = ''
+            Name of the symlink (relative to {file}`${manifestDir}`).
+            Defaults to the attribute name.
+          '';
+        };
+
+        content = lib.mkOption {
+          type = with lib.types; nullOr (either attrs (listOf attrs));
+          default = null;
+          description = ''
+            Content of the manifest file. A single attribute set will
+            generate a single document YAML file. A list of attribute sets
+            will generate multiple documents separated by `---` in a single
+            YAML file.
+          '';
+        };
+
+        source = lib.mkOption {
+          type = lib.types.path;
+          example = lib.literalExpression "./manifests/app.yaml";
+          description = ''
+            Path of the source `.yaml` file.
+          '';
+        };
+      };
+
+      config = {
+        target = lib.mkDefault (mkManifestTarget name);
+        source = lib.mkIf (config.content != null) (
+          let
+            name' = "k3s-manifest-" + builtins.baseNameOf name;
+            docName = "k3s-manifest-doc-" + builtins.baseNameOf name;
+            mkSource =
+              value:
+              if builtins.isList value then
+                pkgs.concatText name' (
+                  lib.concatMap (x: [
+                    yamlDocSeparator
+                    (yamlFormat.generate docName x)
+                  ]) value
+                )
+              else
+                yamlFormat.generate name' value;
+          in
+          lib.mkDerivedConfig options.content mkSource
+        );
+      };
+    }
+  );
+in
+{
+  imports = [ (removeOption [ "docker" ] "k3s docker option is no longer supported.") ];
+
+  # interface
+  options.services.k3s = {
+    enable = lib.mkEnableOption "k3s";
+
+    package = lib.mkPackageOption pkgs "k3s" { };
+
+    role = lib.mkOption {
+      description = ''
+        Whether k3s should run as a server or agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - Starts by default as a standalone server using an embedded sqlite datastore.
+        - Configure `clusterInit = true` to switch over to embedded etcd datastore and enable HA mode.
+        - Configure `serverAddr` to join an already-initialized HA cluster.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
+      '';
+      default = "server";
+      type = lib.types.enum [
+        "server"
+        "agent"
+      ];
+    };
+
+    serverAddr = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The k3s server to connect to.
+
+        Servers and agents need to communicate each other. Read
+        [the networking docs](https://rancher.com/docs/k3s/latest/en/installation/installation-requirements/#networking)
+        to know how to configure the firewall.
+      '';
+      example = "https://10.0.0.10:6443";
+      default = "";
+    };
+
+    clusterInit = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Initialize HA cluster using an embedded etcd datastore.
+
+        If this option is `false` and `role` is `server`
+
+        On a server that was using the default embedded sqlite backend,
+        enabling this option will migrate to an embedded etcd DB.
+
+        If an HA cluster using the embedded etcd datastore was already initialized,
+        this option has no effect.
+
+        This option only makes sense in a server that is not connecting to another server.
+
+        If you are configuring an HA cluster with an embedded etcd,
+        the 1st server must have `clusterInit = true`
+        and other servers must connect to it using `serverAddr`.
+      '';
+    };
+
+    token = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        The k3s token to use when connecting to a server.
+
+        WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the tokenFile option instead.
+      '';
+      default = "";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing k3s token to use when connecting to the server.";
+      default = null;
+    };
+
+    extraFlags = lib.mkOption {
+      description = "Extra flags to pass to the k3s command.";
+      type = with lib.types; either str (listOf str);
+      default = [ ];
+      example = [
+        "--disable traefik"
+        "--cluster-cidr 10.24.0.0/16"
+      ];
+    };
+
+    disableAgent = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = "Only run the server. This option only makes sense for a server.";
+    };
+
+    environmentFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = ''
+        File path containing environment variables for configuring the k3s service in the format of an EnvironmentFile. See {manpage}`systemd.exec(5)`.
+      '';
+      default = null;
+    };
+
+    configPath = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+      description = "File path containing the k3s YAML config. This is useful when the config is generated (for example on boot).";
+    };
+
+    manifests = lib.mkOption {
+      type = lib.types.attrsOf manifestModule;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          deployment.source = ../manifests/deployment.yaml;
+          my-service = {
+            enable = false;
+            target = "app-service.yaml";
+            content = {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "app-service";
+              };
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
+              };
+            };
+          };
+
+          nginx.content = [
+            {
+              apiVersion = "v1";
+              kind = "Pod";
+              metadata = {
+                name = "nginx";
+                labels = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+              };
+              spec = {
+                containers = [
+                  {
+                    name = "nginx";
+                    image = "nginx:1.14.2";
+                    ports = [
+                      {
+                        containerPort = 80;
+                        name = "http-web-svc";
+                      }
+                    ];
+                  }
+                ];
+              };
+            }
+            {
+              apiVersion = "v1";
+              kind = "Service";
+              metadata = {
+                name = "nginx-service";
+              };
+              spec = {
+                selector = {
+                  "app.kubernetes.io/name" = "MyApp";
+                };
+                ports = [
+                  {
+                    name = "name-of-service-port";
+                    protocol = "TCP";
+                    port = 80;
+                    targetPort = "http-web-svc";
+                  }
+                ];
+              };
+            }
+          ];
+        };
+      '';
+      description = ''
+        Auto-deploying manifests that are linked to {file}`${manifestDir}` before k3s starts.
+        Note that deleting manifest files will not remove or otherwise modify the resources
+        it created. Please use the the `--disable` flag or `.skip` files to delete/disable AddOns,
+        as mentioned in the [docs](https://docs.k3s.io/installation/packaged-components#disabling-manifests).
+        This option only makes sense on server nodes (`role = server`).
+        Read the [auto-deploying manifests docs](https://docs.k3s.io/installation/packaged-components#auto-deploying-manifests-addons)
+        for further information.
+      '';
+    };
+
+    charts = lib.mkOption {
+      type = with lib.types; attrsOf (either path package);
+      default = { };
+      example = lib.literalExpression ''
+        nginx = ../charts/my-nginx-chart.tgz;
+        redis = ../charts/my-redis-chart.tgz;
+      '';
+      description = ''
+        Packaged Helm charts that are linked to {file}`${chartDir}` before k3s starts.
+        The attribute name will be used as the link target (relative to {file}`${chartDir}`).
+        The specified charts will only be placed on the file system and made available to the
+        Kubernetes APIServer from within the cluster. See the [](#opt-services.k3s.autoDeployCharts)
+        option and the [k3s Helm controller docs](https://docs.k3s.io/helm#using-the-helm-controller)
+        to deploy Helm charts. This option only makes sense on server nodes (`role = server`).
+      '';
+    };
+
+    containerdConfigTemplate = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      default = null;
+      example = lib.literalExpression ''
+        # Base K3s config
+        {{ template "base" . }}
+
+        # Add a custom runtime
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom"]
+          runtime_type = "io.containerd.runc.v2"
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes."custom".options]
+          BinaryName = "/path/to/custom-container-runtime"
+      '';
+      description = ''
+        Config template for containerd, to be placed at
+        `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl`.
+        See the K3s docs on [configuring containerd](https://docs.k3s.io/advanced#configuring-containerd).
+      '';
+    };
+
+    images = lib.mkOption {
+      type = with lib.types; listOf package;
+      default = [ ];
+      example = lib.literalExpression ''
+        [
+          (pkgs.dockerTools.pullImage {
+            imageName = "docker.io/bitnami/keycloak";
+            imageDigest = "sha256:714dfadc66a8e3adea6609bda350345bd3711657b7ef3cf2e8015b526bac2d6b";
+            hash = "sha256-IM2BLZ0EdKIZcRWOtuFY9TogZJXCpKtPZnMnPsGlq0Y=";
+            finalImageTag = "21.1.2-debian-11-r0";
+          })
+
+          config.services.k3s.package.airgap-images
+        ]
+      '';
+      description = ''
+        List of derivations that provide container images.
+        All images are linked to {file}`${imageDir}` before k3s starts and consequently imported
+        by the k3s agent. Consider importing the k3s airgap images archive of the k3s package in
+        use, if you want to pre-provision this node with all k3s container images. This option
+        only makes sense on nodes with an enabled agent.
+      '';
+    };
+
+    gracefulNodeShutdown = {
+      enable = lib.mkEnableOption ''
+        graceful node shutdowns where the kubelet attempts to detect
+        node system shutdown and terminates pods running on the node. See the
+        [documentation](https://kubernetes.io/docs/concepts/cluster-administration/node-shutdown/#graceful-node-shutdown)
+        for further information.
+      '';
+
+      shutdownGracePeriod = lib.mkOption {
+        type = lib.types.nonEmptyStr;
+        default = "30s";
+        example = "1m30s";
+        description = ''
+          Specifies the total duration that the node should delay the shutdown by. This is the total
+          grace period for pod termination for both regular and critical pods.
+        '';
+      };
+
+      shutdownGracePeriodCriticalPods = lib.mkOption {
+        type = lib.types.nonEmptyStr;
+        default = "10s";
+        example = "15s";
+        description = ''
+          Specifies the duration used to terminate critical pods during a node shutdown. This should be
+          less than `shutdownGracePeriod`.
+        '';
+      };
+    };
+
+    extraKubeletConfig = lib.mkOption {
+      type = with lib.types; attrsOf anything;
+      default = { };
+      example = {
+        podsPerCore = 3;
+        memoryThrottlingFactor = 0.69;
+        containerLogMaxSize = "5Mi";
+      };
+      description = ''
+        Extra configuration to add to the kubelet's configuration file. The subset of the kubelet's
+        configuration that can be configured via a file is defined by the
+        [KubeletConfiguration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/)
+        struct. See the
+        [documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
+        for further information.
+      '';
+    };
+
+    extraKubeProxyConfig = lib.mkOption {
+      type = with lib.types; attrsOf anything;
+      default = { };
+      example = {
+        mode = "nftables";
+        clientConnection.kubeconfig = "/var/lib/rancher/k3s/agent/kubeproxy.kubeconfig";
+      };
+      description = ''
+        Extra configuration to add to the kube-proxy's configuration file. The subset of the kube-proxy's
+        configuration that can be configured via a file is defined by the
+        [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/)
+        struct. Note that the kubeconfig param will be override by `clientConnection.kubeconfig`, so you must
+        set the `clientConnection.kubeconfig` if you want to use `extraKubeProxyConfig`.
+      '';
+    };
+
+    autoDeployCharts = lib.mkOption {
+      type = lib.types.attrsOf autoDeployChartsModule;
+      apply = lib.mapAttrs mkAutoDeployChartManifest;
+      default = { };
+      example = lib.literalExpression ''
+        {
+          harbor = {
+            name = "harbor";
+            repo = "https://helm.goharbor.io";
+            version = "1.14.0";
+            hash = "sha256-fMP7q1MIbvzPGS9My91vbQ1d3OJMjwc+o8YE/BXZaYU=";
+            values = {
+              existingSecretAdminPassword = "harbor-admin";
+              expose = {
+                tls = {
+                  enabled = true;
+                  certSource = "secret";
+                  secret.secretName = "my-tls-secret";
+                };
+                ingress = {
+                  hosts.core = "example.com";
+                  className = "nginx";
+                };
+              };
+            };
+          };
+          nginx = {
+            repo = "oci://registry-1.docker.io/bitnamicharts/nginx";
+            version = "20.0.0";
+            hash = "sha256-sy+tzB+i9jIl/tqOMzzuhVhTU4EZVsoSBtPznxF/36c=";
+          };
+          custom-chart = {
+            package = ../charts/my-chart.tgz;
+            values = ../values/my-values.yaml;
+            extraFieldDefinitions = {
+              spec.timeout = "60s";
+            };
+          };
+        }
+      '';
+      description = ''
+        Auto deploying Helm charts that are installed by the k3s Helm controller. Avoid to use
+        attribute names that are also used in the [](#opt-services.k3s.manifests) and
+        [](#opt-services.k3s.charts) options. Manifests with the same name will override
+        auto deploying charts with the same name. Similiarly, charts with the same name will
+        overwrite the Helm chart contained in auto deploying charts. This option only makes
+        sense on server nodes (`role = server`). See the
+        [k3s Helm documentation](https://docs.k3s.io/helm) for further information.
+      '';
+    };
+  };
+
+  # implementation
+
+  config = lib.mkIf cfg.enable {
+    warnings =
+      (lib.optional (cfg.role != "server" && cfg.manifests != { })
+        "k3s: Auto deploying manifests are only installed on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (cfg.role != "server" && cfg.charts != { })
+        "k3s: Helm charts are only made available to the cluster on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (cfg.role != "server" && cfg.autoDeployCharts != { })
+        "k3s: Auto deploying Helm charts are only installed on server nodes (role == server), they will be ignored by this node."
+      )
+      ++ (lib.optional (duplicateManifests != [ ])
+        "k3s: The following auto deploying charts are overriden by manifests of the same name: ${toString duplicateManifests}."
+      )
+      ++ (lib.optional (duplicateCharts != [ ])
+        "k3s: The following auto deploying charts are overriden by charts of the same name: ${toString duplicateCharts}."
+      )
+      ++ (lib.optional (
+        cfg.disableAgent && cfg.images != [ ]
+      ) "k3s: Images are only imported on nodes with an enabled agent, they will be ignored by this node")
+      ++ (lib.optional (
+        cfg.role == "agent" && cfg.configPath == null && cfg.serverAddr == ""
+      ) "k3s: serverAddr or configPath (with 'server' key) should be set if role is 'agent'")
+      ++ (lib.optional
+        (cfg.role == "agent" && cfg.configPath == null && cfg.tokenFile == null && cfg.token == "")
+        "k3s: Token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'"
+      );
+
+    assertions = [
+      {
+        assertion = cfg.role == "agent" -> !cfg.disableAgent;
+        message = "k3s: disableAgent must be false if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !cfg.clusterInit;
+        message = "k3s: clusterInit must be false if role is 'agent'";
+      }
+    ];
+
+    environment.systemPackages = [ config.services.k3s.package ];
+
+    # Use systemd-tmpfiles to activate k3s content
+    systemd.tmpfiles.settings."10-k3s" =
+      let
+        # Merge manifest with manifests generated from auto deploying charts, keep only enabled manifests
+        enabledManifests = lib.filterAttrs (_: v: v.enable) (cfg.autoDeployCharts // cfg.manifests);
+        # Merge charts with charts contained in enabled auto deploying charts
+        helmCharts =
+          (lib.concatMapAttrs (n: v: { ${n} = v.package; }) (
+            lib.filterAttrs (_: v: v.enable) cfg.autoDeployCharts
+          ))
+          // cfg.charts;
+        # Make a systemd-tmpfiles rule for a manifest
+        mkManifestRule = manifest: {
+          name = "${manifestDir}/${manifest.target}";
+          value = {
+            "L+".argument = "${manifest.source}";
+          };
+        };
+        # Ensure that all chart targets have a .tgz suffix
+        mkChartTarget = name: if (lib.hasSuffix ".tgz" name) then name else name + ".tgz";
+        # Make a systemd-tmpfiles rule for a chart
+        mkChartRule = target: source: {
+          name = "${chartDir}/${mkChartTarget target}";
+          value = {
+            "L+".argument = "${source}";
+          };
+        };
+        # Make a systemd-tmpfiles rule for a container image
+        mkImageRule = image: {
+          name = "${imageDir}/${image.name}";
+          value = {
+            "L+".argument = "${image}";
+          };
+        };
+      in
+      (lib.mapAttrs' (_: v: mkManifestRule v) enabledManifests)
+      // (lib.mapAttrs' (n: v: mkChartRule n v) helmCharts)
+      // (builtins.listToAttrs (map mkImageRule cfg.images))
+      // (lib.optionalAttrs (cfg.containerdConfigTemplate != null) {
+        ${containerdConfigTemplateFile} = {
+          "L+".argument = "${pkgs.writeText "config.toml.tmpl" cfg.containerdConfigTemplate}";
+        };
+      });
+
+    systemd.services.k3s =
+      let
+        kubeletParams =
+          (lib.optionalAttrs (cfg.gracefulNodeShutdown.enable) {
+            inherit (cfg.gracefulNodeShutdown) shutdownGracePeriod shutdownGracePeriodCriticalPods;
+          })
+          // cfg.extraKubeletConfig;
+        kubeletConfig = (pkgs.formats.yaml { }).generate "k3s-kubelet-config" (
+          {
+            apiVersion = "kubelet.config.k8s.io/v1beta1";
+            kind = "KubeletConfiguration";
+          }
+          // kubeletParams
+        );
+
+        kubeProxyConfig = (pkgs.formats.yaml { }).generate "k3s-kubeProxy-config" (
+          {
+            apiVersion = "kubeproxy.config.k8s.io/v1alpha1";
+            kind = "KubeProxyConfiguration";
+          }
+          // cfg.extraKubeProxyConfig
+        );
+      in
+      {
+        description = "k3s service";
+        after = [
+          "firewall.service"
+          "network-online.target"
+        ];
+        wants = [
+          "firewall.service"
+          "network-online.target"
+        ];
+        wantedBy = [ "multi-user.target" ];
+        path = lib.optional config.boot.zfs.enabled config.boot.zfs.package;
+        serviceConfig = {
+          # See: https://github.com/rancher/k3s/blob/dddbd16305284ae4bd14c0aade892412310d7edc/install.sh#L197
+          Type = if cfg.role == "agent" then "exec" else "notify";
+          KillMode = "process";
+          Delegate = "yes";
+          Restart = "always";
+          RestartSec = "5s";
+          LimitNOFILE = 1048576;
+          LimitNPROC = "infinity";
+          LimitCORE = "infinity";
+          TasksMax = "infinity";
+          EnvironmentFile = cfg.environmentFile;
+          ExecStart = lib.concatStringsSep " \\\n " (
+            [ "${cfg.package}/bin/k3s ${cfg.role}" ]
+            ++ (lib.optional cfg.clusterInit "--cluster-init")
+            ++ (lib.optional cfg.disableAgent "--disable-agent")
+            ++ (lib.optional (cfg.serverAddr != "") "--server ${cfg.serverAddr}")
+            ++ (lib.optional (cfg.token != "") "--token ${cfg.token}")
+            ++ (lib.optional (cfg.tokenFile != null) "--token-file ${cfg.tokenFile}")
+            ++ (lib.optional (cfg.configPath != null) "--config ${cfg.configPath}")
+            ++ (lib.optional (kubeletParams != { }) "--kubelet-arg=config=${kubeletConfig}")
+            ++ (lib.optional (cfg.extraKubeProxyConfig != { }) "--kube-proxy-arg=config=${kubeProxyConfig}")
+            ++ (lib.flatten cfg.extraFlags)
+          );
+        };
+      };
+  };
+
+  meta.maintainers = lib.teams.k3s.members;
+}

nixos/modules/services/cluster/kubernetes/addon-manager.nix

@@ -0,0 +1,184 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  cfg = top.addonManager;
+
+  isRBACEnabled = lib.elem "RBAC" top.apiserver.authorizationMode;
+
+  addons = pkgs.runCommand "kubernetes-addons" { } ''
+    mkdir -p $out
+    # since we are mounting the addons to the addon manager, they need to be copied
+    ${lib.concatMapStringsSep ";" (a: "cp -v ${a}/* $out/") (
+      lib.mapAttrsToList (name: addon: pkgs.writeTextDir "${name}.json" (builtins.toJSON addon)) (
+        cfg.addons
+      )
+    )}
+  '';
+in
+{
+  ###### interface
+  options.services.kubernetes.addonManager = with lib.types; {
+
+    bootstrapAddons = lib.mkOption {
+      description = ''
+        Bootstrap addons are like regular addons, but they are applied with cluster-admin rights.
+        They are applied at addon-manager startup only.
+      '';
+      default = { };
+      type = attrsOf attrs;
+      example = lib.literalExpression ''
+        {
+          "my-service" = {
+            "apiVersion" = "v1";
+            "kind" = "Service";
+            "metadata" = {
+              "name" = "my-service";
+              "namespace" = "default";
+            };
+            "spec" = { ... };
+          };
+        }
+      '';
+    };
+
+    addons = lib.mkOption {
+      description = "Kubernetes addons (any kind of Kubernetes resource can be an addon).";
+      default = { };
+      type = attrsOf (either attrs (listOf attrs));
+      example = lib.literalExpression ''
+        {
+          "my-service" = {
+            "apiVersion" = "v1";
+            "kind" = "Service";
+            "metadata" = {
+              "name" = "my-service";
+              "namespace" = "default";
+            };
+            "spec" = { ... };
+          };
+        }
+        // import <nixpkgs/nixos/modules/services/cluster/kubernetes/dns.nix> { cfg = config.services.kubernetes; };
+      '';
+    };
+
+    enable = lib.mkEnableOption "Kubernetes addon manager";
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    environment.etc."kubernetes/addons".source = "${addons}/";
+
+    systemd.services.kube-addon-manager = {
+      description = "Kubernetes addon manager";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      environment.ADDON_PATH = "/etc/kubernetes/addons/";
+      path = [ pkgs.gawk ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = "${top.package}/bin/kube-addons";
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+        Restart = "on-failure";
+        RestartSec = 10;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.addonManager.bootstrapAddons = lib.mkIf isRBACEnabled (
+      let
+        name = "system:kube-addon-manager";
+        namespace = "kube-system";
+      in
+      {
+
+        kube-addon-manager-r = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "Role";
+          metadata = {
+            inherit name namespace;
+          };
+          rules = [
+            {
+              apiGroups = [ "*" ];
+              resources = [ "*" ];
+              verbs = [ "*" ];
+            }
+          ];
+        };
+
+        kube-addon-manager-rb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "RoleBinding";
+          metadata = {
+            inherit name namespace;
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "Role";
+            inherit name;
+          };
+          subjects = [
+            {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "User";
+              inherit name;
+            }
+          ];
+        };
+
+        kube-addon-manager-cluster-lister-cr = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRole";
+          metadata = {
+            name = "${name}:cluster-lister";
+          };
+          rules = [
+            {
+              apiGroups = [ "*" ];
+              resources = [ "*" ];
+              verbs = [ "list" ];
+            }
+          ];
+        };
+
+        kube-addon-manager-cluster-lister-crb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRoleBinding";
+          metadata = {
+            name = "${name}:cluster-lister";
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "ClusterRole";
+            name = "${name}:cluster-lister";
+          };
+          subjects = [
+            {
+              kind = "User";
+              inherit name;
+            }
+          ];
+        };
+      }
+    );
+
+    services.kubernetes.pki.certs = {
+      addonManager = top.lib.mkCert {
+        name = "kube-addon-manager";
+        CN = "system:kube-addon-manager";
+        action = "systemctl restart kube-addon-manager.service";
+      };
+    };
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/addons/dns.nix

@@ -0,0 +1,401 @@
+{
+  config,
+  options,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  version = "1.10.1";
+  cfg = config.services.kubernetes.addons.dns;
+  ports = {
+    dns = 10053;
+    health = 10054;
+    metrics = 10055;
+  };
+in
+{
+  options.services.kubernetes.addons.dns = {
+    enable = lib.mkEnableOption "kubernetes dns addon";
+
+    clusterIp = lib.mkOption {
+      description = "Dns addon clusterIP";
+
+      # this default is also what kubernetes users
+      default =
+        (lib.concatStringsSep "." (
+          lib.take 3 (lib.splitString "." config.services.kubernetes.apiserver.serviceClusterIpRange)
+        ))
+        + ".254";
+      defaultText = lib.literalMD ''
+        The `x.y.z.254` IP of
+        `config.${options.services.kubernetes.apiserver.serviceClusterIpRange}`.
+      '';
+      type = lib.types.str;
+    };
+
+    clusterDomain = lib.mkOption {
+      description = "Dns cluster domain";
+      default = "cluster.local";
+      type = lib.types.str;
+    };
+
+    replicas = lib.mkOption {
+      description = "Number of DNS pod replicas to deploy in the cluster.";
+      default = 2;
+      type = lib.types.int;
+    };
+
+    reconcileMode = lib.mkOption {
+      description = ''
+        Controls the addon manager reconciliation mode for the DNS addon.
+
+        Setting reconcile mode to EnsureExists makes it possible to tailor DNS behavior by editing the coredns ConfigMap.
+
+        See: <https://github.com/kubernetes/kubernetes/blob/master/cluster/addons/addon-manager/README.md>.
+      '';
+      default = "Reconcile";
+      type = lib.types.enum [
+        "Reconcile"
+        "EnsureExists"
+      ];
+    };
+
+    coredns = lib.mkOption {
+      description = "Docker image to seed for the CoreDNS container.";
+      type = lib.types.attrs;
+      default = {
+        imageName = "coredns/coredns";
+        imageDigest = "sha256:a0ead06651cf580044aeb0a0feba63591858fb2e43ade8c9dea45a6a89ae7e5e";
+        finalImageTag = version;
+        sha256 = "0wg696920smmal7552a2zdhfncndn5kfammfa8bk8l7dz9bhk0y1";
+      };
+    };
+
+    corefile = lib.mkOption {
+      description = ''
+        Custom coredns corefile configuration.
+
+        See: <https://coredns.io/manual/toc/#configuration>.
+      '';
+      type = lib.types.str;
+      default = ''
+        .:${toString ports.dns} {
+          errors
+          health :${toString ports.health}
+          kubernetes ${cfg.clusterDomain} in-addr.arpa ip6.arpa {
+            pods insecure
+            fallthrough in-addr.arpa ip6.arpa
+          }
+          prometheus :${toString ports.metrics}
+          forward . /etc/resolv.conf
+          cache 30
+          loop
+          reload
+          loadbalance
+        }'';
+      defaultText = lib.literalExpression ''
+        '''
+          .:${toString ports.dns} {
+            errors
+            health :${toString ports.health}
+            kubernetes ''${config.services.kubernetes.addons.dns.clusterDomain} in-addr.arpa ip6.arpa {
+              pods insecure
+              fallthrough in-addr.arpa ip6.arpa
+            }
+            prometheus :${toString ports.metrics}
+            forward . /etc/resolv.conf
+            cache 30
+            loop
+            reload
+            loadbalance
+          }
+        '''
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.kubernetes.kubelet.seedDockerImages = lib.singleton (
+      pkgs.dockerTools.pullImage cfg.coredns
+    );
+
+    services.kubernetes.addonManager.bootstrapAddons = {
+      coredns-cr = {
+        apiVersion = "rbac.authorization.k8s.io/v1";
+        kind = "ClusterRole";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/bootstrapping" = "rbac-defaults";
+          };
+          name = "system:coredns";
+        };
+        rules = [
+          {
+            apiGroups = [ "" ];
+            resources = [
+              "endpoints"
+              "services"
+              "pods"
+              "namespaces"
+            ];
+            verbs = [
+              "list"
+              "watch"
+            ];
+          }
+          {
+            apiGroups = [ "" ];
+            resources = [ "nodes" ];
+            verbs = [ "get" ];
+          }
+          {
+            apiGroups = [ "discovery.k8s.io" ];
+            resources = [ "endpointslices" ];
+            verbs = [
+              "list"
+              "watch"
+            ];
+          }
+        ];
+      };
+
+      coredns-crb = {
+        apiVersion = "rbac.authorization.k8s.io/v1";
+        kind = "ClusterRoleBinding";
+        metadata = {
+          annotations = {
+            "rbac.authorization.kubernetes.io/autoupdate" = "true";
+          };
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/bootstrapping" = "rbac-defaults";
+          };
+          name = "system:coredns";
+        };
+        roleRef = {
+          apiGroup = "rbac.authorization.k8s.io";
+          kind = "ClusterRole";
+          name = "system:coredns";
+        };
+        subjects = [
+          {
+            kind = "ServiceAccount";
+            name = "coredns";
+            namespace = "kube-system";
+          }
+        ];
+      };
+    };
+
+    services.kubernetes.addonManager.addons = {
+      coredns-sa = {
+        apiVersion = "v1";
+        kind = "ServiceAccount";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+      };
+
+      coredns-cm = {
+        apiVersion = "v1";
+        kind = "ConfigMap";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+        data = {
+          Corefile = cfg.corefile;
+        };
+      };
+
+      coredns-deploy = {
+        apiVersion = "apps/v1";
+        kind = "Deployment";
+        metadata = {
+          labels = {
+            "addonmanager.kubernetes.io/mode" = cfg.reconcileMode;
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "CoreDNS";
+          };
+          name = "coredns";
+          namespace = "kube-system";
+        };
+        spec = {
+          replicas = cfg.replicas;
+          selector = {
+            matchLabels = {
+              k8s-app = "kube-dns";
+            };
+          };
+          strategy = {
+            rollingUpdate = {
+              maxUnavailable = 1;
+            };
+            type = "RollingUpdate";
+          };
+          template = {
+            metadata = {
+              labels = {
+                k8s-app = "kube-dns";
+              };
+            };
+            spec = {
+              containers = [
+                {
+                  args = [
+                    "-conf"
+                    "/etc/coredns/Corefile"
+                  ];
+                  image = with cfg.coredns; "${imageName}:${finalImageTag}";
+                  imagePullPolicy = "Never";
+                  livenessProbe = {
+                    failureThreshold = 5;
+                    httpGet = {
+                      path = "/health";
+                      port = ports.health;
+                      scheme = "HTTP";
+                    };
+                    initialDelaySeconds = 60;
+                    successThreshold = 1;
+                    timeoutSeconds = 5;
+                  };
+                  name = "coredns";
+                  ports = [
+                    {
+                      containerPort = ports.dns;
+                      name = "dns";
+                      protocol = "UDP";
+                    }
+                    {
+                      containerPort = ports.dns;
+                      name = "dns-tcp";
+                      protocol = "TCP";
+                    }
+                    {
+                      containerPort = ports.metrics;
+                      name = "metrics";
+                      protocol = "TCP";
+                    }
+                  ];
+                  resources = {
+                    limits = {
+                      memory = "170Mi";
+                    };
+                    requests = {
+                      cpu = "100m";
+                      memory = "70Mi";
+                    };
+                  };
+                  securityContext = {
+                    allowPrivilegeEscalation = false;
+                    capabilities = {
+                      drop = [ "all" ];
+                    };
+                    readOnlyRootFilesystem = true;
+                  };
+                  volumeMounts = [
+                    {
+                      mountPath = "/etc/coredns";
+                      name = "config-volume";
+                      readOnly = true;
+                    }
+                  ];
+                }
+              ];
+              dnsPolicy = "Default";
+              nodeSelector = {
+                "beta.kubernetes.io/os" = "linux";
+              };
+              serviceAccountName = "coredns";
+              tolerations = [
+                {
+                  effect = "NoSchedule";
+                  key = "node-role.kubernetes.io/master";
+                }
+                {
+                  key = "CriticalAddonsOnly";
+                  operator = "Exists";
+                }
+              ];
+              volumes = [
+                {
+                  configMap = {
+                    items = [
+                      {
+                        key = "Corefile";
+                        path = "Corefile";
+                      }
+                    ];
+                    name = "coredns";
+                  };
+                  name = "config-volume";
+                }
+              ];
+            };
+          };
+        };
+      };
+
+      coredns-svc = {
+        apiVersion = "v1";
+        kind = "Service";
+        metadata = {
+          annotations = {
+            "prometheus.io/port" = toString ports.metrics;
+            "prometheus.io/scrape" = "true";
+          };
+          labels = {
+            "addonmanager.kubernetes.io/mode" = "Reconcile";
+            k8s-app = "kube-dns";
+            "kubernetes.io/cluster-service" = "true";
+            "kubernetes.io/name" = "CoreDNS";
+          };
+          name = "kube-dns";
+          namespace = "kube-system";
+        };
+        spec = {
+          clusterIP = cfg.clusterIp;
+          ports = [
+            {
+              name = "dns";
+              port = 53;
+              targetPort = ports.dns;
+              protocol = "UDP";
+            }
+            {
+              name = "dns-tcp";
+              port = 53;
+              targetPort = ports.dns;
+              protocol = "TCP";
+            }
+          ];
+          selector = {
+            k8s-app = "kube-dns";
+          };
+        };
+      };
+    };
+
+    services.kubernetes.kubelet.clusterDns = lib.mkDefault [ cfg.clusterIp ];
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/apiserver.nix

@@ -0,0 +1,541 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.apiserver;
+
+  isRBACEnabled = lib.elem "RBAC" cfg.authorizationMode;
+
+  apiserverServiceIP = (
+    lib.concatStringsSep "." (lib.take 3 (lib.splitString "." cfg.serviceClusterIpRange)) + ".1"
+  );
+in
+{
+
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "apiserver" "admissionControl" ]
+      [ "services" "kubernetes" "apiserver" "enableAdmissionPlugins" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "apiserver" "address" ]
+      [ "services" "kubernetes" "apiserver" "bindAddress" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecureBindAddress" ] "")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "insecurePort" ] "")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "apiserver" "publicAddress" ] "")
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "servers" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "servers" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "keyFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "keyFile" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "certFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "certFile" ]
+    )
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "etcd" "caFile" ]
+      [ "services" "kubernetes" "apiserver" "etcd" "caFile" ]
+    )
+  ];
+
+  ###### interface
+  options.services.kubernetes.apiserver = with lib.types; {
+
+    advertiseAddress = lib.mkOption {
+      description = ''
+        Kubernetes apiserver IP address on which to advertise the apiserver
+        to members of the cluster. This address must be reachable by the rest
+        of the cluster.
+      '';
+      default = null;
+      type = nullOr str;
+    };
+
+    allowPrivileged = lib.mkOption {
+      description = "Whether to allow privileged containers on Kubernetes.";
+      default = false;
+      type = bool;
+    };
+
+    authorizationMode = lib.mkOption {
+      description = ''
+        Kubernetes apiserver authorization mode (AlwaysAllow/AlwaysDeny/ABAC/Webhook/RBAC/Node). See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
+      '';
+      default = [
+        "RBAC"
+        "Node"
+      ]; # Enabling RBAC by default, although kubernetes default is AllowAllow
+      type = listOf (enum [
+        "AlwaysAllow"
+        "AlwaysDeny"
+        "ABAC"
+        "Webhook"
+        "RBAC"
+        "Node"
+      ]);
+    };
+
+    authorizationPolicy = lib.mkOption {
+      description = ''
+        Kubernetes apiserver authorization policy file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authorization/>
+      '';
+      default = [ ];
+      type = listOf attrs;
+    };
+
+    basicAuthFile = lib.mkOption {
+      description = ''
+        Kubernetes apiserver basic authentication file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    bindAddress = lib.mkOption {
+      description = ''
+        The IP address on which to listen for the --secure-port port.
+        The associated interface(s) must be reachable by the rest
+        of the cluster, and by CLI/web clients.
+      '';
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    clientCaFile = lib.mkOption {
+      description = "Kubernetes apiserver CA file for client auth.";
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    disableAdmissionPlugins = lib.mkOption {
+      description = ''
+        Kubernetes admission control plugins to disable. See
+        <https://kubernetes.io/docs/admin/admission-controllers/>
+      '';
+      default = [ ];
+      type = listOf str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes apiserver";
+
+    enableAdmissionPlugins = lib.mkOption {
+      description = ''
+        Kubernetes admission control plugins to enable. See
+        <https://kubernetes.io/docs/admin/admission-controllers/>
+      '';
+      default = [
+        "NamespaceLifecycle"
+        "LimitRanger"
+        "ServiceAccount"
+        "ResourceQuota"
+        "DefaultStorageClass"
+        "DefaultTolerationSeconds"
+        "NodeRestriction"
+      ];
+      example = [
+        "NamespaceLifecycle"
+        "NamespaceExists"
+        "LimitRanger"
+        "SecurityContextDeny"
+        "ServiceAccount"
+        "ResourceQuota"
+        "PodSecurityPolicy"
+        "NodeRestriction"
+        "DefaultStorageClass"
+      ];
+      type = listOf str;
+    };
+
+    etcd = {
+      servers = lib.mkOption {
+        description = "List of etcd servers.";
+        default = [ "http://127.0.0.1:2379" ];
+        type = types.listOf types.str;
+      };
+
+      keyFile = lib.mkOption {
+        description = "Etcd key file.";
+        default = null;
+        type = types.nullOr types.path;
+      };
+
+      certFile = lib.mkOption {
+        description = "Etcd cert file.";
+        default = null;
+        type = types.nullOr types.path;
+      };
+
+      caFile = lib.mkOption {
+        description = "Etcd ca file.";
+        default = top.caFile;
+        defaultText = lib.literalExpression "config.${otop.caFile}";
+        type = types.nullOr types.path;
+      };
+    };
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes apiserver extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    extraSANs = lib.mkOption {
+      description = "Extra x509 Subject Alternative Names to be added to the kubernetes apiserver tls cert.";
+      default = [ ];
+      type = listOf str;
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeletClientCaFile = lib.mkOption {
+      description = "Path to a cert file for connecting to kubelet.";
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    kubeletClientCertFile = lib.mkOption {
+      description = "Client certificate to use for connections to kubelet.";
+      default = null;
+      type = nullOr path;
+    };
+
+    kubeletClientKeyFile = lib.mkOption {
+      description = "Key to use for connections to kubelet.";
+      default = null;
+      type = nullOr path;
+    };
+
+    preferredAddressTypes = lib.mkOption {
+      description = "List of the preferred NodeAddressTypes to use for kubelet connections.";
+      type = nullOr str;
+      default = null;
+    };
+
+    proxyClientCertFile = lib.mkOption {
+      description = "Client certificate to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
+    proxyClientKeyFile = lib.mkOption {
+      description = "Key to use for connections to proxy.";
+      default = null;
+      type = nullOr path;
+    };
+
+    runtimeConfig = lib.mkOption {
+      description = ''
+        Api runtime configuration. See
+        <https://kubernetes.io/docs/tasks/administer-cluster/cluster-management/>
+      '';
+      default = "authentication.k8s.io/v1beta1=true";
+      example = "api/all=false,api/v1=true";
+      type = str;
+    };
+
+    storageBackend = lib.mkOption {
+      description = ''
+        Kubernetes apiserver storage backend.
+      '';
+      default = "etcd3";
+      type = enum [
+        "etcd2"
+        "etcd3"
+      ];
+    };
+
+    securePort = lib.mkOption {
+      description = "Kubernetes apiserver secure port.";
+      default = 6443;
+      type = int;
+    };
+
+    apiAudiences = lib.mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "api,https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountIssuer = lib.mkOption {
+      description = ''
+        Kubernetes apiserver ServiceAccount issuer.
+      '';
+      default = "https://kubernetes.default.svc";
+      type = str;
+    };
+
+    serviceAccountSigningKeyFile = lib.mkOption {
+      description = ''
+        Path to the file that contains the current private key of the service
+        account token issuer. The issuer will sign issued ID tokens with this
+        private key.
+      '';
+      type = path;
+    };
+
+    serviceAccountKeyFile = lib.mkOption {
+      description = ''
+        File containing PEM-encoded x509 RSA or ECDSA private or public keys,
+        used to verify ServiceAccount tokens. The specified file can contain
+        multiple keys, and the flag can be specified multiple times with
+        different files. If unspecified, --tls-private-key-file is used.
+        Must be specified when --service-account-signing-key is provided
+      '';
+      type = path;
+    };
+
+    serviceClusterIpRange = lib.mkOption {
+      description = ''
+        A CIDR notation IP range from which to assign service cluster IPs.
+        This must not overlap with any IP ranges assigned to nodes for pods.
+      '';
+      default = "10.0.0.0/24";
+      type = str;
+    };
+
+    tlsCertFile = lib.mkOption {
+      description = "Kubernetes apiserver certificate file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = lib.mkOption {
+      description = "Kubernetes apiserver private key file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tokenAuthFile = lib.mkOption {
+      description = ''
+        Kubernetes apiserver token authentication file. See
+        <https://kubernetes.io/docs/reference/access-authn-authz/authentication>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+    webhookConfig = lib.mkOption {
+      description = ''
+        Kubernetes apiserver Webhook config file. It uses the kubeconfig file format.
+        See <https://kubernetes.io/docs/reference/access-authn-authz/webhook/>
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkMerge [
+
+    (lib.mkIf cfg.enable {
+      systemd.services.kube-apiserver = {
+        description = "Kubernetes APIServer Service";
+        wantedBy = [ "kubernetes.target" ];
+        after = [ "network.target" ];
+        serviceConfig = {
+          Slice = "kubernetes.slice";
+          ExecStart = ''
+            ${top.package}/bin/kube-apiserver \
+            --allow-privileged=${lib.boolToString cfg.allowPrivileged} \
+            --authorization-mode=${lib.concatStringsSep "," cfg.authorizationMode} \
+              ${lib.optionalString (lib.elem "ABAC" cfg.authorizationMode) "--authorization-policy-file=${pkgs.writeText "kube-auth-policy.jsonl" (lib.concatMapStringsSep "\n" (l: builtins.toJSON l) cfg.authorizationPolicy)}"} \
+              ${lib.optionalString (lib.elem "Webhook" cfg.authorizationMode) "--authorization-webhook-config-file=${cfg.webhookConfig}"} \
+            --bind-address=${cfg.bindAddress} \
+            ${lib.optionalString (cfg.advertiseAddress != null) "--advertise-address=${cfg.advertiseAddress}"} \
+            ${lib.optionalString (cfg.clientCaFile != null) "--client-ca-file=${cfg.clientCaFile}"} \
+            --disable-admission-plugins=${lib.concatStringsSep "," cfg.disableAdmissionPlugins} \
+            --enable-admission-plugins=${lib.concatStringsSep "," cfg.enableAdmissionPlugins} \
+            --etcd-servers=${lib.concatStringsSep "," cfg.etcd.servers} \
+            ${lib.optionalString (cfg.etcd.caFile != null) "--etcd-cafile=${cfg.etcd.caFile}"} \
+            ${lib.optionalString (cfg.etcd.certFile != null) "--etcd-certfile=${cfg.etcd.certFile}"} \
+            ${lib.optionalString (cfg.etcd.keyFile != null) "--etcd-keyfile=${cfg.etcd.keyFile}"} \
+            ${
+              lib.optionalString (cfg.featureGates != { })
+                "--feature-gates=${
+                  (lib.concatStringsSep "," (
+                    builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                  ))
+                }"
+            } \
+            ${lib.optionalString (cfg.basicAuthFile != null) "--basic-auth-file=${cfg.basicAuthFile}"} \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientCaFile != null
+              ) "--kubelet-certificate-authority=${cfg.kubeletClientCaFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientCertFile != null
+              ) "--kubelet-client-certificate=${cfg.kubeletClientCertFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.kubeletClientKeyFile != null
+              ) "--kubelet-client-key=${cfg.kubeletClientKeyFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.preferredAddressTypes != null
+              ) "--kubelet-preferred-address-types=${cfg.preferredAddressTypes}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.proxyClientCertFile != null
+              ) "--proxy-client-cert-file=${cfg.proxyClientCertFile}"
+            } \
+            ${
+              lib.optionalString (
+                cfg.proxyClientKeyFile != null
+              ) "--proxy-client-key-file=${cfg.proxyClientKeyFile}"
+            } \
+            ${lib.optionalString (cfg.runtimeConfig != "") "--runtime-config=${cfg.runtimeConfig}"} \
+            --secure-port=${toString cfg.securePort} \
+            --api-audiences=${toString cfg.apiAudiences} \
+            --service-account-issuer=${toString cfg.serviceAccountIssuer} \
+            --service-account-signing-key-file=${cfg.serviceAccountSigningKeyFile} \
+            --service-account-key-file=${cfg.serviceAccountKeyFile} \
+            --service-cluster-ip-range=${cfg.serviceClusterIpRange} \
+            --storage-backend=${cfg.storageBackend} \
+            ${lib.optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \
+            ${lib.optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"} \
+            ${lib.optionalString (cfg.tokenAuthFile != null) "--token-auth-file=${cfg.tokenAuthFile}"} \
+            ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+            ${cfg.extraOpts}
+          '';
+          WorkingDirectory = top.dataDir;
+          User = "kubernetes";
+          Group = "kubernetes";
+          AmbientCapabilities = "cap_net_bind_service";
+          Restart = "on-failure";
+          RestartSec = 5;
+        };
+
+        unitConfig = {
+          StartLimitIntervalSec = 0;
+        };
+      };
+
+      services.etcd = {
+        clientCertAuth = lib.mkDefault true;
+        peerClientCertAuth = lib.mkDefault true;
+        listenClientUrls = lib.mkDefault [ "https://0.0.0.0:2379" ];
+        listenPeerUrls = lib.mkDefault [ "https://0.0.0.0:2380" ];
+        advertiseClientUrls = lib.mkDefault [ "https://${top.masterAddress}:2379" ];
+        initialCluster = lib.mkDefault [ "${top.masterAddress}=https://${top.masterAddress}:2380" ];
+        name = lib.mkDefault top.masterAddress;
+        initialAdvertisePeerUrls = lib.mkDefault [ "https://${top.masterAddress}:2380" ];
+      };
+
+      services.kubernetes.addonManager.bootstrapAddons = lib.mkIf isRBACEnabled {
+
+        apiserver-kubelet-api-admin-crb = {
+          apiVersion = "rbac.authorization.k8s.io/v1";
+          kind = "ClusterRoleBinding";
+          metadata = {
+            name = "system:kube-apiserver:kubelet-api-admin";
+          };
+          roleRef = {
+            apiGroup = "rbac.authorization.k8s.io";
+            kind = "ClusterRole";
+            name = "system:kubelet-api-admin";
+          };
+          subjects = [
+            {
+              kind = "User";
+              name = "system:kube-apiserver";
+            }
+          ];
+        };
+
+      };
+
+      services.kubernetes.pki.certs = with top.lib; {
+        apiServer = mkCert {
+          name = "kube-apiserver";
+          CN = "kubernetes";
+          hosts = [
+            "kubernetes.default.svc"
+            "kubernetes.default.svc.${top.addons.dns.clusterDomain}"
+            cfg.advertiseAddress
+            top.masterAddress
+            apiserverServiceIP
+            "127.0.0.1"
+          ]
+          ++ cfg.extraSANs;
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverProxyClient = mkCert {
+          name = "kube-apiserver-proxy-client";
+          CN = "front-proxy-client";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverKubeletClient = mkCert {
+          name = "kube-apiserver-kubelet-client";
+          CN = "system:kube-apiserver";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        apiserverEtcdClient = mkCert {
+          name = "kube-apiserver-etcd-client";
+          CN = "etcd-client";
+          action = "systemctl restart kube-apiserver.service";
+        };
+        clusterAdmin = mkCert {
+          name = "cluster-admin";
+          CN = "cluster-admin";
+          fields = {
+            O = "system:masters";
+          };
+          privateKeyOwner = "root";
+        };
+        etcd = mkCert {
+          name = "etcd";
+          CN = top.masterAddress;
+          hosts = [
+            "etcd.local"
+            "etcd.${top.addons.dns.clusterDomain}"
+            top.masterAddress
+            cfg.advertiseAddress
+          ];
+          privateKeyOwner = "etcd";
+          action = "systemctl restart etcd.service";
+        };
+      };
+
+    })
+
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/controller-manager.nix

@@ -0,0 +1,182 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.controllerManager;
+in
+{
+  imports = [
+    (lib.mkRenamedOptionModule
+      [ "services" "kubernetes" "controllerManager" "address" ]
+      [ "services" "kubernetes" "controllerManager" "bindAddress" ]
+    )
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "controllerManager" "insecurePort" ] "")
+  ];
+
+  ###### interface
+  options.services.kubernetes.controllerManager = with lib.types; {
+
+    allocateNodeCIDRs = lib.mkOption {
+      description = "Whether to automatically allocate CIDR ranges for cluster nodes.";
+      default = true;
+      type = bool;
+    };
+
+    bindAddress = lib.mkOption {
+      description = "Kubernetes controller manager listening address.";
+      default = "127.0.0.1";
+      type = str;
+    };
+
+    clusterCidr = lib.mkOption {
+      description = "Kubernetes CIDR Range for Pods in cluster.";
+      default = top.clusterCidr;
+      defaultText = lib.literalExpression "config.${otop.clusterCidr}";
+      type = str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes controller manager";
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes controller manager extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes controller manager";
+
+    leaderElect = lib.mkOption {
+      description = "Whether to start leader election before executing main loop.";
+      type = bool;
+      default = true;
+    };
+
+    rootCaFile = lib.mkOption {
+      description = ''
+        Kubernetes controller manager certificate authority file included in
+        service account's token secret.
+      '';
+      default = top.caFile;
+      defaultText = lib.literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    securePort = lib.mkOption {
+      description = "Kubernetes controller manager secure listening port.";
+      default = 10252;
+      type = int;
+    };
+
+    serviceAccountKeyFile = lib.mkOption {
+      description = ''
+        Kubernetes controller manager PEM-encoded private RSA key file used to
+        sign service account tokens
+      '';
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsCertFile = lib.mkOption {
+      description = "Kubernetes controller-manager certificate file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = lib.mkOption {
+      description = "Kubernetes controller-manager private key file.";
+      default = null;
+      type = nullOr path;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    systemd.services.kube-controller-manager = {
+      description = "Kubernetes Controller Manager Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      serviceConfig = {
+        RestartSec = "30s";
+        Restart = "on-failure";
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-controller-manager \
+                    --allocate-node-cidrs=${lib.boolToString cfg.allocateNodeCIDRs} \
+                    --bind-address=${cfg.bindAddress} \
+                    ${lib.optionalString (cfg.clusterCidr != null) "--cluster-cidr=${cfg.clusterCidr}"} \
+                    ${
+                      lib.optionalString (cfg.featureGates != { })
+                        "--feature-gates=${
+                          lib.concatStringsSep "," (
+                            builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                          )
+                        }"
+                    } \
+                    --kubeconfig=${top.lib.mkKubeConfig "kube-controller-manager" cfg.kubeconfig} \
+                    --leader-elect=${lib.boolToString cfg.leaderElect} \
+                    ${lib.optionalString (cfg.rootCaFile != null) "--root-ca-file=${cfg.rootCaFile}"} \
+                    --secure-port=${toString cfg.securePort} \
+                    ${
+                      lib.optionalString (
+                        cfg.serviceAccountKeyFile != null
+                      ) "--service-account-private-key-file=${cfg.serviceAccountKeyFile}"
+                    } \
+                    ${lib.optionalString (cfg.tlsCertFile != null) "--tls-cert-file=${cfg.tlsCertFile}"} \
+                    ${
+                      lib.optionalString (cfg.tlsKeyFile != null) "--tls-private-key-file=${cfg.tlsKeyFile}"
+                    } \
+                    ${lib.optionalString (lib.elem "RBAC" top.apiserver.authorizationMode) "--use-service-account-credentials"} \
+                    ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                    ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+      path = top.path;
+    };
+
+    services.kubernetes.pki.certs = with top.lib; {
+      controllerManager = mkCert {
+        name = "kube-controller-manager";
+        CN = "kube-controller-manager";
+        action = "systemctl restart kube-controller-manager.service";
+      };
+      controllerManagerClient = mkCert {
+        name = "kube-controller-manager-client";
+        CN = "system:kube-controller-manager";
+        action = "systemctl restart kube-controller-manager.service";
+      };
+    };
+
+    services.kubernetes.controllerManager.kubeconfig.server = lib.mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/default.nix

@@ -0,0 +1,356 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.kubernetes;
+  opt = options.services.kubernetes;
+
+  defaultContainerdSettings = {
+    version = 2;
+    root = "/var/lib/containerd";
+    state = "/run/containerd";
+    oom_score = 0;
+
+    grpc = {
+      address = "/run/containerd/containerd.sock";
+    };
+
+    plugins."io.containerd.grpc.v1.cri" = {
+      sandbox_image = "pause:latest";
+
+      cni = {
+        bin_dir = "/opt/cni/bin";
+        max_conf_num = 0;
+      };
+
+      containerd.runtimes.runc = {
+        runtime_type = "io.containerd.runc.v2";
+        options.SystemdCgroup = true;
+      };
+    };
+  };
+
+  mkKubeConfig =
+    name: conf:
+    pkgs.writeText "${name}-kubeconfig" (
+      builtins.toJSON {
+        apiVersion = "v1";
+        kind = "Config";
+        clusters = [
+          {
+            name = "local";
+            cluster.certificate-authority = conf.caFile or cfg.caFile;
+            cluster.server = conf.server;
+          }
+        ];
+        users = [
+          {
+            inherit name;
+            user = {
+              client-certificate = conf.certFile;
+              client-key = conf.keyFile;
+            };
+          }
+        ];
+        contexts = [
+          {
+            context = {
+              cluster = "local";
+              user = name;
+            };
+            name = "local";
+          }
+        ];
+        current-context = "local";
+      }
+    );
+
+  caCert = secret "ca";
+
+  etcdEndpoints = [ "https://${cfg.masterAddress}:2379" ];
+
+  mkCert =
+    {
+      name,
+      CN,
+      hosts ? [ ],
+      fields ? { },
+      action ? "",
+      privateKeyOwner ? "kubernetes",
+      privateKeyGroup ? "kubernetes",
+    }:
+    rec {
+      inherit
+        name
+        caCert
+        CN
+        hosts
+        fields
+        action
+        ;
+      cert = secret name;
+      key = secret "${name}-key";
+      privateKeyOptions = {
+        owner = privateKeyOwner;
+        group = privateKeyGroup;
+        mode = "0600";
+        path = key;
+      };
+    };
+
+  secret = name: "${cfg.secretsPath}/${name}.pem";
+
+  mkKubeConfigOptions = prefix: {
+    server = lib.mkOption {
+      description = "${prefix} kube-apiserver server address.";
+      type = lib.types.str;
+    };
+
+    caFile = lib.mkOption {
+      description = "${prefix} certificate authority file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = cfg.caFile;
+      defaultText = lib.literalExpression "config.${opt.caFile}";
+    };
+
+    certFile = lib.mkOption {
+      description = "${prefix} client certificate file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    keyFile = lib.mkOption {
+      description = "${prefix} client key file used to connect to kube-apiserver.";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+  };
+in
+{
+
+  imports = [
+    (lib.mkRemovedOptionModule [
+      "services"
+      "kubernetes"
+      "addons"
+      "dashboard"
+    ] "Removed due to it being an outdated version")
+    (lib.mkRemovedOptionModule [ "services" "kubernetes" "verbose" ] "")
+  ];
+
+  ###### interface
+
+  options.services.kubernetes = {
+    roles = lib.mkOption {
+      description = ''
+        Kubernetes role that this machine should take.
+
+        Master role will enable etcd, apiserver, scheduler, controller manager
+        addon manager, flannel and proxy services.
+        Node role will enable flannel, docker, kubelet and proxy services.
+      '';
+      default = [ ];
+      type = lib.types.listOf (
+        lib.types.enum [
+          "master"
+          "node"
+        ]
+      );
+    };
+
+    package = lib.mkPackageOption pkgs "kubernetes" { };
+
+    kubeconfig = mkKubeConfigOptions "Default kubeconfig";
+
+    apiserverAddress = lib.mkOption {
+      description = ''
+        Clusterwide accessible address for the kubernetes apiserver,
+        including protocol and optional port.
+      '';
+      example = "https://kubernetes-apiserver.example.com:6443";
+      type = lib.types.str;
+    };
+
+    caFile = lib.mkOption {
+      description = "Default kubernetes certificate authority";
+      type = lib.types.nullOr lib.types.path;
+      default = null;
+    };
+
+    dataDir = lib.mkOption {
+      description = "Kubernetes root directory for managing kubelet files.";
+      default = "/var/lib/kubernetes";
+      type = lib.types.path;
+    };
+
+    easyCerts = lib.mkOption {
+      description = "Automatically setup x509 certificates and keys for the entire cluster.";
+      default = false;
+      type = lib.types.bool;
+    };
+
+    featureGates = lib.mkOption {
+      description = "List set of feature gates.";
+      default = { };
+      type = lib.types.attrsOf lib.types.bool;
+    };
+
+    masterAddress = lib.mkOption {
+      description = "Clusterwide available network address or hostname for the kubernetes master server.";
+      example = "master.example.com";
+      type = lib.types.str;
+    };
+
+    path = lib.mkOption {
+      description = "Packages added to the services' PATH environment variable. Both the bin and sbin subdirectories of each package are added.";
+      type = lib.types.listOf lib.types.package;
+      default = [ ];
+    };
+
+    clusterCidr = lib.mkOption {
+      description = "Kubernetes controller manager and proxy CIDR Range for Pods in cluster.";
+      default = "10.1.0.0/16";
+      type = lib.types.nullOr lib.types.str;
+    };
+
+    lib = lib.mkOption {
+      description = "Common functions for the kubernetes modules.";
+      default = {
+        inherit mkCert;
+        inherit mkKubeConfig;
+        inherit mkKubeConfigOptions;
+      };
+      type = lib.types.attrs;
+    };
+
+    secretsPath = lib.mkOption {
+      description = "Default location for kubernetes secrets. Not a store location.";
+      type = lib.types.path;
+      default = cfg.dataDir + "/secrets";
+      defaultText = lib.literalExpression ''
+        config.${opt.dataDir} + "/secrets"
+      '';
+    };
+  };
+
+  ###### implementation
+
+  config = lib.mkMerge [
+
+    (lib.mkIf cfg.easyCerts {
+      services.kubernetes.pki.enable = lib.mkDefault true;
+      services.kubernetes.caFile = caCert;
+    })
+
+    (lib.mkIf (lib.elem "master" cfg.roles) {
+      services.kubernetes.apiserver.enable = lib.mkDefault true;
+      services.kubernetes.scheduler.enable = lib.mkDefault true;
+      services.kubernetes.controllerManager.enable = lib.mkDefault true;
+      services.kubernetes.addonManager.enable = lib.mkDefault true;
+      services.kubernetes.proxy.enable = lib.mkDefault true;
+      services.etcd.enable = true; # Cannot mkDefault because of flannel default options
+      services.kubernetes.kubelet = {
+        enable = lib.mkDefault true;
+        taints = lib.mkIf (!(lib.elem "node" cfg.roles)) {
+          master = {
+            key = "node-role.kubernetes.io/master";
+            value = "true";
+            effect = "NoSchedule";
+          };
+        };
+      };
+    })
+
+    (lib.mkIf (lib.all (el: el == "master") cfg.roles) {
+      # if this node is only a master make it unschedulable by default
+      services.kubernetes.kubelet.unschedulable = lib.mkDefault true;
+    })
+
+    (lib.mkIf (lib.elem "node" cfg.roles) {
+      services.kubernetes.kubelet.enable = lib.mkDefault true;
+      services.kubernetes.proxy.enable = lib.mkDefault true;
+    })
+
+    # Using "services.kubernetes.roles" will automatically enable easyCerts and flannel
+    (lib.mkIf (cfg.roles != [ ]) {
+      services.kubernetes.flannel.enable = lib.mkDefault true;
+      services.flannel.etcd.endpoints = lib.mkDefault etcdEndpoints;
+      services.kubernetes.easyCerts = lib.mkDefault true;
+    })
+
+    (lib.mkIf cfg.apiserver.enable {
+      services.kubernetes.pki.etcClusterAdminKubeconfig = lib.mkDefault "kubernetes/cluster-admin.kubeconfig";
+      services.kubernetes.apiserver.etcd.servers = lib.mkDefault etcdEndpoints;
+    })
+
+    (lib.mkIf cfg.kubelet.enable {
+      virtualisation.containerd = {
+        enable = lib.mkDefault true;
+        settings = lib.mapAttrsRecursive (name: lib.mkDefault) defaultContainerdSettings;
+      };
+    })
+
+    (lib.mkIf (cfg.apiserver.enable || cfg.controllerManager.enable) {
+      services.kubernetes.pki.certs = {
+        serviceAccount = mkCert {
+          name = "service-account";
+          CN = "system:service-account-signer";
+          action = ''
+            systemctl restart \
+              kube-apiserver.service \
+              kube-controller-manager.service
+          '';
+        };
+      };
+    })
+
+    (lib.mkIf
+      (
+        cfg.apiserver.enable
+        || cfg.scheduler.enable
+        || cfg.controllerManager.enable
+        || cfg.kubelet.enable
+        || cfg.proxy.enable
+        || cfg.addonManager.enable
+      )
+      {
+        systemd.targets.kubernetes = {
+          description = "Kubernetes";
+          wantedBy = [ "multi-user.target" ];
+        };
+
+        systemd.tmpfiles.rules = [
+          "d /opt/cni/bin 0755 root root -"
+          "d /run/kubernetes 0755 kubernetes kubernetes -"
+          "d ${cfg.dataDir} 0755 kubernetes kubernetes -"
+        ];
+
+        users.users.kubernetes = {
+          uid = config.ids.uids.kubernetes;
+          description = "Kubernetes user";
+          group = "kubernetes";
+          home = cfg.dataDir;
+          createHome = true;
+          homeMode = "755";
+        };
+        users.groups.kubernetes.gid = config.ids.gids.kubernetes;
+
+        # dns addon is enabled by default
+        services.kubernetes.addons.dns.enable = lib.mkDefault true;
+
+        services.kubernetes.apiserverAddress = lib.mkDefault "https://${
+          if cfg.apiserver.advertiseAddress != null then
+            cfg.apiserver.advertiseAddress
+          else
+            "${cfg.masterAddress}:${toString cfg.apiserver.securePort}"
+        }";
+      }
+    )
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/flannel.nix

@@ -0,0 +1,126 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  cfg = top.flannel;
+
+  # we want flannel to use kubernetes itself as configuration backend, not direct etcd
+  storageBackend = "kubernetes";
+in
+{
+  ###### interface
+  options.services.kubernetes.flannel = {
+    enable = lib.mkEnableOption "flannel networking";
+
+    openFirewallPorts = lib.mkOption {
+      description = ''Whether to open the Flannel UDP ports in the firewall on all interfaces.'';
+      type = lib.types.bool;
+      default = true;
+    };
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    services.flannel = {
+
+      enable = lib.mkDefault true;
+      network = lib.mkDefault top.clusterCidr;
+      inherit storageBackend;
+      nodeName = config.services.kubernetes.kubelet.hostname;
+    };
+
+    services.kubernetes.kubelet = {
+      cni.config = lib.mkDefault [
+        {
+          name = "mynet";
+          type = "flannel";
+          cniVersion = "0.3.1";
+          delegate = {
+            isDefaultGateway = true;
+            hairpinMode = true;
+            bridge = "mynet";
+          };
+        }
+      ];
+    };
+
+    networking = {
+      firewall.allowedUDPPorts = lib.mkIf cfg.openFirewallPorts [
+        8285 # flannel udp
+        8472 # flannel vxlan
+      ];
+      dhcpcd.denyInterfaces = [
+        "mynet*"
+        "flannel*"
+      ];
+    };
+
+    services.kubernetes.pki.certs = {
+      flannelClient = top.lib.mkCert {
+        name = "flannel-client";
+        CN = "flannel-client";
+        action = "systemctl restart flannel.service";
+      };
+    };
+
+    # give flannel some kubernetes rbac permissions if applicable
+    services.kubernetes.addonManager.bootstrapAddons =
+      lib.mkIf ((storageBackend == "kubernetes") && (lib.elem "RBAC" top.apiserver.authorizationMode))
+        {
+
+          flannel-cr = {
+            apiVersion = "rbac.authorization.k8s.io/v1";
+            kind = "ClusterRole";
+            metadata = {
+              name = "flannel";
+            };
+            rules = [
+              {
+                apiGroups = [ "" ];
+                resources = [ "pods" ];
+                verbs = [ "get" ];
+              }
+              {
+                apiGroups = [ "" ];
+                resources = [ "nodes" ];
+                verbs = [
+                  "list"
+                  "watch"
+                ];
+              }
+              {
+                apiGroups = [ "" ];
+                resources = [ "nodes/status" ];
+                verbs = [ "patch" ];
+              }
+            ];
+          };
+
+          flannel-crb = {
+            apiVersion = "rbac.authorization.k8s.io/v1";
+            kind = "ClusterRoleBinding";
+            metadata = {
+              name = "flannel";
+            };
+            roleRef = {
+              apiGroup = "rbac.authorization.k8s.io";
+              kind = "ClusterRole";
+              name = "flannel";
+            };
+            subjects = [
+              {
+                kind = "User";
+                name = "flannel-client";
+              }
+            ];
+          };
+
+        };
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/kubelet.nix

@@ -0,0 +1,444 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.kubelet;
+
+  cniConfig =
+    if cfg.cni.config != [ ] && cfg.cni.configDir != null then
+      throw "Verbatim CNI-config and CNI configDir cannot both be set."
+    else if cfg.cni.configDir != null then
+      cfg.cni.configDir
+    else
+      (pkgs.buildEnv {
+        name = "kubernetes-cni-config";
+        paths = imap (
+          i: entry: pkgs.writeTextDir "${toString (10 + i)}-${entry.type}.conf" (builtins.toJSON entry)
+        ) cfg.cni.config;
+      });
+
+  infraContainer = pkgs.dockerTools.buildImage {
+    name = "pause";
+    tag = "latest";
+    copyToRoot = pkgs.buildEnv {
+      name = "image-root";
+      pathsToLink = [ "/bin" ];
+      paths = [ top.package.pause ];
+    };
+    config.Cmd = [ "/bin/pause" ];
+  };
+
+  kubeconfig = top.lib.mkKubeConfig "kubelet" cfg.kubeconfig;
+
+  # Flag based settings are deprecated, use the `--config` flag with a
+  # `KubeletConfiguration` struct.
+  # https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/
+  #
+  # NOTE: registerWithTaints requires a []core/v1.Taint, therefore requires
+  # additional work to be put in config format.
+  #
+  kubeletConfig = pkgs.writeText "kubelet-config" (
+    builtins.toJSON (
+      {
+        apiVersion = "kubelet.config.k8s.io/v1beta1";
+        kind = "KubeletConfiguration";
+        address = cfg.address;
+        port = cfg.port;
+        authentication = {
+          x509 = lib.optionalAttrs (cfg.clientCaFile != null) { clientCAFile = cfg.clientCaFile; };
+          webhook = {
+            enabled = true;
+            cacheTTL = "10s";
+          };
+        };
+        authorization = {
+          mode = "Webhook";
+        };
+        cgroupDriver = "systemd";
+        hairpinMode = "hairpin-veth";
+        registerNode = cfg.registerNode;
+        containerRuntimeEndpoint = cfg.containerRuntimeEndpoint;
+        healthzPort = cfg.healthz.port;
+        healthzBindAddress = cfg.healthz.bind;
+      }
+      // lib.optionalAttrs (cfg.tlsCertFile != null) { tlsCertFile = cfg.tlsCertFile; }
+      // lib.optionalAttrs (cfg.tlsKeyFile != null) { tlsPrivateKeyFile = cfg.tlsKeyFile; }
+      // lib.optionalAttrs (cfg.clusterDomain != "") { clusterDomain = cfg.clusterDomain; }
+      // lib.optionalAttrs (cfg.clusterDns != [ ]) { clusterDNS = cfg.clusterDns; }
+      // lib.optionalAttrs (cfg.featureGates != { }) { featureGates = cfg.featureGates; }
+      // lib.optionalAttrs (cfg.extraConfig != { }) cfg.extraConfig
+    )
+  );
+
+  manifestPath = "kubernetes/manifests";
+
+  taintOptions =
+    with lib.types;
+    { name, ... }:
+    {
+      options = {
+        key = mkOption {
+          description = "Key of taint.";
+          default = name;
+          defaultText = literalMD "Name of this submodule.";
+          type = str;
+        };
+        value = mkOption {
+          description = "Value of taint.";
+          type = str;
+        };
+        effect = mkOption {
+          description = "Effect of taint.";
+          example = "NoSchedule";
+          type = enum [
+            "NoSchedule"
+            "PreferNoSchedule"
+            "NoExecute"
+          ];
+        };
+      };
+    };
+
+  taints = concatMapStringsSep "," (v: "${v.key}=${v.value}:${v.effect}") (
+    mapAttrsToList (n: v: v) cfg.taints
+  );
+in
+{
+  imports = [
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "applyManifests" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "cadvisorPort" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "allowPrivileged" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "networkPlugin" ] "")
+    (mkRemovedOptionModule [ "services" "kubernetes" "kubelet" "containerRuntime" ] "")
+  ];
+
+  ###### interface
+  options.services.kubernetes.kubelet = with lib.types; {
+
+    address = mkOption {
+      description = "Kubernetes kubelet info server listening address.";
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    clusterDns = mkOption {
+      description = "Use alternative DNS.";
+      default = [ "10.1.0.1" ];
+      type = listOf str;
+    };
+
+    clusterDomain = mkOption {
+      description = "Use alternative domain.";
+      default = config.services.kubernetes.addons.dns.clusterDomain;
+      defaultText = literalExpression "config.${options.services.kubernetes.addons.dns.clusterDomain}";
+      type = str;
+    };
+
+    clientCaFile = mkOption {
+      description = "Kubernetes apiserver CA file for client authentication.";
+      default = top.caFile;
+      defaultText = literalExpression "config.${otop.caFile}";
+      type = nullOr path;
+    };
+
+    cni = {
+      packages = mkOption {
+        description = "List of network plugin packages to install.";
+        type = listOf package;
+        default = [ ];
+      };
+
+      config = mkOption {
+        description = "Kubernetes CNI configuration.";
+        type = listOf attrs;
+        default = [ ];
+        example = literalExpression ''
+          [{
+            "cniVersion": "0.3.1",
+            "name": "mynet",
+            "type": "bridge",
+            "bridge": "cni0",
+            "isGateway": true,
+            "ipMasq": true,
+            "ipam": {
+                "type": "host-local",
+                "subnet": "10.22.0.0/16",
+                "routes": [
+                    { "dst": "0.0.0.0/0" }
+                ]
+            }
+          } {
+            "cniVersion": "0.3.1",
+            "type": "loopback"
+          }]
+        '';
+      };
+
+      configDir = mkOption {
+        description = "Path to Kubernetes CNI configuration directory.";
+        type = nullOr path;
+        default = null;
+      };
+    };
+
+    containerRuntimeEndpoint = mkOption {
+      description = "Endpoint at which to find the container runtime api interface/socket";
+      type = str;
+      default = "unix:///run/containerd/containerd.sock";
+    };
+
+    enable = mkEnableOption "Kubernetes kubelet";
+
+    extraOpts = mkOption {
+      description = "Kubernetes kubelet extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    extraConfig = mkOption {
+      description = ''
+        Kubernetes kubelet extra configuration file entries.
+
+        See also [Set Kubelet Parameters Via A Configuration File](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)
+        and [Kubelet Configuration](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/).
+      '';
+      default = { };
+      type = attrsOf ((pkgs.formats.json { }).type);
+    };
+
+    featureGates = mkOption {
+      description = "Attribute set of feature gate";
+      default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    healthz = {
+      bind = mkOption {
+        description = "Kubernetes kubelet healthz listening address.";
+        default = "127.0.0.1";
+        type = str;
+      };
+
+      port = mkOption {
+        description = "Kubernetes kubelet healthz port.";
+        default = 10248;
+        type = port;
+      };
+    };
+
+    hostname = mkOption {
+      description = "Kubernetes kubelet hostname override.";
+      defaultText = literalExpression "config.networking.fqdnOrHostName";
+      type = str;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubelet";
+
+    manifests = mkOption {
+      description = "List of manifests to bootstrap with kubelet (only pods can be created as manifest entry)";
+      type = attrsOf attrs;
+      default = { };
+    };
+
+    nodeIp = mkOption {
+      description = "IP address of the node. If set, kubelet will use this IP address for the node.";
+      default = null;
+      type = nullOr str;
+    };
+
+    registerNode = mkOption {
+      description = "Whether to auto register kubelet with API server.";
+      default = true;
+      type = bool;
+    };
+
+    port = mkOption {
+      description = "Kubernetes kubelet info server listening port.";
+      default = 10250;
+      type = port;
+    };
+
+    seedDockerImages = mkOption {
+      description = "List of docker images to preload on system";
+      default = [ ];
+      type = listOf package;
+    };
+
+    taints = mkOption {
+      description = "Node taints (https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).";
+      default = { };
+      type = attrsOf (submodule [ taintOptions ]);
+    };
+
+    tlsCertFile = mkOption {
+      description = "File containing x509 Certificate for HTTPS.";
+      default = null;
+      type = nullOr path;
+    };
+
+    tlsKeyFile = mkOption {
+      description = "File containing x509 private key matching tlsCertFile.";
+      default = null;
+      type = nullOr path;
+    };
+
+    unschedulable = mkOption {
+      description = "Whether to set node taint to unschedulable=true as it is the case of node that has only master role.";
+      default = false;
+      type = bool;
+    };
+
+    verbosity = mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = mkMerge [
+    (mkIf cfg.enable {
+
+      environment.etc."cni/net.d".source = cniConfig;
+
+      services.kubernetes.kubelet.seedDockerImages = [ infraContainer ];
+
+      boot.kernel.sysctl = {
+        "net.bridge.bridge-nf-call-iptables" = 1;
+        "net.ipv4.ip_forward" = 1;
+        "net.bridge.bridge-nf-call-ip6tables" = 1;
+      };
+
+      systemd.services.kubelet = {
+        description = "Kubernetes Kubelet Service";
+        wantedBy = [ "kubernetes.target" ];
+        after = [
+          "containerd.service"
+          "network.target"
+          "kube-apiserver.service"
+        ];
+        path =
+          with pkgs;
+          [
+            gitMinimal
+            openssh
+            util-linuxMinimal
+            iproute2
+            ethtool
+            thin-provisioning-tools
+            iptables
+            socat
+          ]
+          ++ lib.optional config.boot.zfs.enabled config.boot.zfs.package
+          ++ top.path;
+        preStart = ''
+          ${concatMapStrings (img: ''
+            echo "Seeding container image: ${img}"
+            ${
+              if (lib.hasSuffix "gz" img) then
+                ''${pkgs.gzip}/bin/zcat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+              else
+                ''${pkgs.coreutils}/bin/cat "${img}" | ${pkgs.containerd}/bin/ctr -n k8s.io image import -''
+            }
+          '') cfg.seedDockerImages}
+
+          rm /opt/cni/bin/* || true
+          ${concatMapStrings (package: ''
+            echo "Linking cni package: ${package}"
+            ln -fs ${package}/bin/* /opt/cni/bin
+          '') cfg.cni.packages}
+        '';
+        serviceConfig = {
+          Slice = "kubernetes.slice";
+          CPUAccounting = true;
+          MemoryAccounting = true;
+          Restart = "on-failure";
+          RestartSec = "1000ms";
+          ExecStart = ''
+            ${top.package}/bin/kubelet \
+                        --config=${kubeletConfig} \
+                        --hostname-override=${cfg.hostname} \
+                        --kubeconfig=${kubeconfig} \
+                        ${optionalString (cfg.nodeIp != null) "--node-ip=${cfg.nodeIp}"} \
+                        --pod-infra-container-image=pause \
+                        ${optionalString (cfg.manifests != { }) "--pod-manifest-path=/etc/${manifestPath}"} \
+                        ${optionalString (taints != "") "--register-with-taints=${taints}"} \
+                        --root-dir=${top.dataDir} \
+                        ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                        ${cfg.extraOpts}
+          '';
+          WorkingDirectory = top.dataDir;
+        };
+        unitConfig = {
+          StartLimitIntervalSec = 0;
+        };
+      };
+
+      # Always include cni plugins
+      services.kubernetes.kubelet.cni.packages = [
+        pkgs.cni-plugins
+        pkgs.cni-plugin-flannel
+      ];
+
+      boot.kernelModules = [
+        "br_netfilter"
+        "overlay"
+      ];
+
+      services.kubernetes.kubelet.hostname = mkDefault (lib.toLower config.networking.fqdnOrHostName);
+
+      services.kubernetes.pki.certs = with top.lib; {
+        kubelet = mkCert {
+          name = "kubelet";
+          CN = top.kubelet.hostname;
+          action = "systemctl restart kubelet.service";
+
+        };
+        kubeletClient = mkCert {
+          name = "kubelet-client";
+          CN = "system:node:${top.kubelet.hostname}";
+          fields = {
+            O = "system:nodes";
+          };
+          action = "systemctl restart kubelet.service";
+        };
+      };
+
+      services.kubernetes.kubelet.kubeconfig.server = mkDefault top.apiserverAddress;
+    })
+
+    (mkIf (cfg.enable && cfg.manifests != { }) {
+      environment.etc = mapAttrs' (
+        name: manifest:
+        nameValuePair "${manifestPath}/${name}.json" {
+          text = builtins.toJSON manifest;
+          mode = "0755";
+        }
+      ) cfg.manifests;
+    })
+
+    (mkIf (cfg.unschedulable && cfg.enable) {
+      services.kubernetes.kubelet.taints.unschedulable = {
+        value = "true";
+        effect = "NoSchedule";
+      };
+    })
+
+  ];
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/pki.nix

@@ -0,0 +1,437 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  cfg = top.pki;
+
+  csrCA = pkgs.writeText "kube-pki-cacert-csr.json" (
+    builtins.toJSON {
+      key = {
+        algo = "rsa";
+        size = 2048;
+      };
+      names = singleton cfg.caSpec;
+    }
+  );
+
+  csrCfssl = pkgs.writeText "kube-pki-cfssl-csr.json" (
+    builtins.toJSON {
+      key = {
+        algo = "rsa";
+        size = 2048;
+      };
+      CN = top.masterAddress;
+      hosts = [ top.masterAddress ] ++ cfg.cfsslAPIExtraSANs;
+    }
+  );
+
+  cfsslAPITokenBaseName = "apitoken.secret";
+  cfsslAPITokenPath = "${config.services.cfssl.dataDir}/${cfsslAPITokenBaseName}";
+  certmgrAPITokenPath = "${top.secretsPath}/${cfsslAPITokenBaseName}";
+  cfsslAPITokenLength = 32;
+
+  clusterAdminKubeconfig =
+    with cfg.certs.clusterAdmin;
+    top.lib.mkKubeConfig "cluster-admin" {
+      server = top.apiserverAddress;
+      certFile = cert;
+      keyFile = key;
+    };
+
+  remote = with config.services; "https://${kubernetes.masterAddress}:${toString cfssl.port}";
+in
+{
+  ###### interface
+  options.services.kubernetes.pki = with lib.types; {
+
+    enable = mkEnableOption "easyCert issuer service";
+
+    certs = mkOption {
+      description = "List of certificate specs to feed to cert generator.";
+      default = { };
+      type = attrs;
+    };
+
+    genCfsslCACert = mkOption {
+      description = ''
+        Whether to automatically generate cfssl CA certificate and key,
+        if they don't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    genCfsslAPICerts = mkOption {
+      description = ''
+        Whether to automatically generate cfssl API webserver TLS cert and key,
+        if they don't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    cfsslAPIExtraSANs = mkOption {
+      description = ''
+        Extra x509 Subject Alternative Names to be added to the cfssl API webserver TLS cert.
+      '';
+      default = [ ];
+      example = [ "subdomain.example.com" ];
+      type = listOf str;
+    };
+
+    genCfsslAPIToken = mkOption {
+      description = ''
+        Whether to automatically generate cfssl API-token secret,
+        if they doesn't exist.
+      '';
+      default = true;
+      type = bool;
+    };
+
+    pkiTrustOnBootstrap = mkOption {
+      description = "Whether to always trust remote cfssl server upon initial PKI bootstrap.";
+      default = true;
+      type = bool;
+    };
+
+    caCertPathPrefix = mkOption {
+      description = ''
+        Path-prefrix for the CA-certificate to be used for cfssl signing.
+        Suffixes ".pem" and "-key.pem" will be automatically appended for
+        the public and private keys respectively.
+      '';
+      default = "${config.services.cfssl.dataDir}/ca";
+      defaultText = literalExpression ''"''${config.services.cfssl.dataDir}/ca"'';
+      type = str;
+    };
+
+    caSpec = mkOption {
+      description = "Certificate specification for the auto-generated CAcert.";
+      default = {
+        CN = "kubernetes-cluster-ca";
+        O = "NixOS";
+        OU = "services.kubernetes.pki.caSpec";
+        L = "auto-generated";
+      };
+      type = attrs;
+    };
+
+    etcClusterAdminKubeconfig = mkOption {
+      description = ''
+        Symlink a kubeconfig with cluster-admin privileges to environment path
+        (/etc/\<path\>).
+      '';
+      default = null;
+      type = nullOr str;
+    };
+
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable (
+    let
+      cfsslCertPathPrefix = "${config.services.cfssl.dataDir}/cfssl";
+      cfsslCert = "${cfsslCertPathPrefix}.pem";
+      cfsslKey = "${cfsslCertPathPrefix}-key.pem";
+    in
+    {
+
+      services.cfssl = mkIf (top.apiserver.enable) {
+        enable = true;
+        address = "0.0.0.0";
+        tlsCert = cfsslCert;
+        tlsKey = cfsslKey;
+        configFile = toString (
+          pkgs.writeText "cfssl-config.json" (
+            builtins.toJSON {
+              signing = {
+                profiles = {
+                  default = {
+                    usages = [ "digital signature" ];
+                    auth_key = "default";
+                    expiry = "720h";
+                  };
+                };
+              };
+              auth_keys = {
+                default = {
+                  type = "standard";
+                  key = "file:${cfsslAPITokenPath}";
+                };
+              };
+            }
+          )
+        );
+      };
+
+      systemd.services.cfssl.preStart =
+        with pkgs;
+        with config.services.cfssl;
+        mkIf (top.apiserver.enable) (
+          concatStringsSep "\n" [
+            "set -e"
+            (optionalString cfg.genCfsslCACert ''
+              if [ ! -f "${cfg.caCertPathPrefix}.pem" ]; then
+                ${cfssl}/bin/cfssl genkey -initca ${csrCA} | \
+                  ${cfssl}/bin/cfssljson -bare ${cfg.caCertPathPrefix}
+              fi
+            '')
+            (optionalString cfg.genCfsslAPICerts ''
+              if [ ! -f "${dataDir}/cfssl.pem" ]; then
+                ${cfssl}/bin/cfssl gencert -ca "${cfg.caCertPathPrefix}.pem" -ca-key "${cfg.caCertPathPrefix}-key.pem" ${csrCfssl} | \
+                  ${cfssl}/bin/cfssljson -bare ${cfsslCertPathPrefix}
+              fi
+            '')
+            (optionalString cfg.genCfsslAPIToken ''
+              if [ ! -f "${cfsslAPITokenPath}" ]; then
+                install -o cfssl -m 400 <(head -c ${
+                  toString (cfsslAPITokenLength / 2)
+                } /dev/urandom | od -An -t x | tr -d ' ') "${cfsslAPITokenPath}"
+              fi
+            '')
+          ]
+        );
+
+      systemd.services.kube-certmgr-bootstrap = {
+        description = "Kubernetes certmgr bootstrapper";
+        wantedBy = [ "certmgr.service" ];
+        after = [ "cfssl.target" ];
+        script = concatStringsSep "\n" [
+          ''
+            set -e
+
+            # If there's a cfssl (cert issuer) running locally, then don't rely on user to
+            # manually paste it in place. Just symlink.
+            # otherwise, create the target file, ready for users to insert the token
+
+            mkdir -p "$(dirname "${certmgrAPITokenPath}")"
+            if [ -f "${cfsslAPITokenPath}" ]; then
+              ln -fs "${cfsslAPITokenPath}" "${certmgrAPITokenPath}"
+            elif [ ! -f "${certmgrAPITokenPath}" ]; then
+              # Don't remove the token if it already exists
+              install -m 600 /dev/null "${certmgrAPITokenPath}"
+            fi
+          ''
+          (optionalString (cfg.pkiTrustOnBootstrap) ''
+            if [ ! -f "${top.caFile}" ] || [ $(cat "${top.caFile}" | wc -c) -lt 1 ]; then
+              ${pkgs.curl}/bin/curl --fail-early -f -kd '{}' ${remote}/api/v1/cfssl/info | \
+                ${pkgs.cfssl}/bin/cfssljson -stdout >${top.caFile}
+            fi
+          '')
+        ];
+        serviceConfig = {
+          RestartSec = "10s";
+          Restart = "on-failure";
+        };
+      };
+
+      services.certmgr = {
+        enable = true;
+        package = pkgs.certmgr;
+        svcManager = "command";
+        specs =
+          let
+            mkSpec = _: cert: {
+              inherit (cert) action;
+              authority = {
+                inherit remote;
+                root_ca = cert.caCert;
+                profile = "default";
+                auth_key_file = certmgrAPITokenPath;
+              };
+              certificate = {
+                path = cert.cert;
+              };
+              private_key = cert.privateKeyOptions;
+              request = {
+                hosts = [ cert.CN ] ++ cert.hosts;
+                inherit (cert) CN;
+                key = {
+                  algo = "rsa";
+                  size = 2048;
+                };
+                names = [ cert.fields ];
+              };
+            };
+          in
+          mapAttrs mkSpec cfg.certs;
+      };
+
+      #TODO: Get rid of kube-addon-manager in the future for the following reasons
+      # - it is basically just a shell script wrapped around kubectl
+      # - it assumes that it is clusterAdmin or can gain clusterAdmin rights through serviceAccount
+      # - it is designed to be used with k8s system components only
+      # - it would be better with a more Nix-oriented way of managing addons
+      systemd.services.kube-addon-manager = mkIf top.addonManager.enable (mkMerge [
+        {
+          environment.KUBECONFIG =
+            with cfg.certs.addonManager;
+            top.lib.mkKubeConfig "addon-manager" {
+              server = top.apiserverAddress;
+              certFile = cert;
+              keyFile = key;
+            };
+        }
+
+        (optionalAttrs (top.addonManager.bootstrapAddons != { }) {
+          serviceConfig.PermissionsStartOnly = true;
+          preStart =
+            with pkgs;
+            let
+              files = mapAttrsToList (
+                n: v: writeText "${n}.json" (builtins.toJSON v)
+              ) top.addonManager.bootstrapAddons;
+            in
+            ''
+              export KUBECONFIG=${clusterAdminKubeconfig}
+              ${top.package}/bin/kubectl apply -f ${concatStringsSep " \\\n -f " files}
+            '';
+        })
+      ]);
+
+      environment.etc.${cfg.etcClusterAdminKubeconfig}.source = mkIf (
+        cfg.etcClusterAdminKubeconfig != null
+      ) clusterAdminKubeconfig;
+
+      environment.systemPackages = mkIf (top.kubelet.enable || top.proxy.enable) [
+        (pkgs.writeScriptBin "nixos-kubernetes-node-join" ''
+          set -e
+          exec 1>&2
+
+          if [ $# -gt 0 ]; then
+            echo "Usage: $(basename $0)"
+            echo ""
+            echo "No args. Apitoken must be provided on stdin."
+            echo "To get the apitoken, execute: 'sudo cat ${certmgrAPITokenPath}' on the master node."
+            exit 1
+          fi
+
+          if [ $(id -u) != 0 ]; then
+            echo "Run as root please."
+            exit 1
+          fi
+
+          read -r token
+          if [ ''${#token} != ${toString cfsslAPITokenLength} ]; then
+            echo "Token must be of length ${toString cfsslAPITokenLength}."
+            exit 1
+          fi
+
+          install -m 0600 <(echo $token) ${certmgrAPITokenPath}
+
+          echo "Restarting certmgr..." >&1
+          systemctl restart certmgr
+
+          echo "Waiting for certs to appear..." >&1
+
+          ${optionalString top.kubelet.enable ''
+            while [ ! -f ${cfg.certs.kubelet.cert} ]; do sleep 1; done
+            echo "Restarting kubelet..." >&1
+            systemctl restart kubelet
+          ''}
+
+          ${optionalString top.proxy.enable ''
+            while [ ! -f ${cfg.certs.kubeProxyClient.cert} ]; do sleep 1; done
+            echo "Restarting kube-proxy..." >&1
+            systemctl restart kube-proxy
+          ''}
+
+          ${optionalString top.flannel.enable ''
+            while [ ! -f ${cfg.certs.flannelClient.cert} ]; do sleep 1; done
+            echo "Restarting flannel..." >&1
+            systemctl restart flannel
+          ''}
+
+          echo "Node joined successfully"
+        '')
+      ];
+
+      # isolate etcd on loopback at the master node
+      # easyCerts doesn't support multimaster clusters anyway atm.
+      services.etcd = with cfg.certs.etcd; {
+        listenClientUrls = [ "https://127.0.0.1:2379" ];
+        listenPeerUrls = [ "https://127.0.0.1:2380" ];
+        advertiseClientUrls = [ "https://etcd.local:2379" ];
+        initialCluster = [ "${top.masterAddress}=https://etcd.local:2380" ];
+        initialAdvertisePeerUrls = [ "https://etcd.local:2380" ];
+        certFile = mkDefault cert;
+        keyFile = mkDefault key;
+        trustedCaFile = mkDefault caCert;
+      };
+      networking.extraHosts = mkIf (config.services.etcd.enable) ''
+        127.0.0.1 etcd.${top.addons.dns.clusterDomain} etcd.local
+      '';
+
+      services.flannel = with cfg.certs.flannelClient; {
+        kubeconfig = top.lib.mkKubeConfig "flannel" {
+          server = top.apiserverAddress;
+          certFile = cert;
+          keyFile = key;
+        };
+      };
+
+      services.kubernetes = {
+
+        apiserver = mkIf top.apiserver.enable (
+          with cfg.certs.apiServer;
+          {
+            etcd = with cfg.certs.apiserverEtcdClient; {
+              servers = [ "https://etcd.local:2379" ];
+              certFile = mkDefault cert;
+              keyFile = mkDefault key;
+              caFile = mkDefault caCert;
+            };
+            clientCaFile = mkDefault caCert;
+            tlsCertFile = mkDefault cert;
+            tlsKeyFile = mkDefault key;
+            serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.cert;
+            serviceAccountSigningKeyFile = mkDefault cfg.certs.serviceAccount.key;
+            kubeletClientCaFile = mkDefault caCert;
+            kubeletClientCertFile = mkDefault cfg.certs.apiserverKubeletClient.cert;
+            kubeletClientKeyFile = mkDefault cfg.certs.apiserverKubeletClient.key;
+            proxyClientCertFile = mkDefault cfg.certs.apiserverProxyClient.cert;
+            proxyClientKeyFile = mkDefault cfg.certs.apiserverProxyClient.key;
+          }
+        );
+        controllerManager = mkIf top.controllerManager.enable {
+          serviceAccountKeyFile = mkDefault cfg.certs.serviceAccount.key;
+          rootCaFile = cfg.certs.controllerManagerClient.caCert;
+          kubeconfig = with cfg.certs.controllerManagerClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        scheduler = mkIf top.scheduler.enable {
+          kubeconfig = with cfg.certs.schedulerClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        kubelet = mkIf top.kubelet.enable {
+          clientCaFile = mkDefault cfg.certs.kubelet.caCert;
+          tlsCertFile = mkDefault cfg.certs.kubelet.cert;
+          tlsKeyFile = mkDefault cfg.certs.kubelet.key;
+          kubeconfig = with cfg.certs.kubeletClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+        proxy = mkIf top.proxy.enable {
+          kubeconfig = with cfg.certs.kubeProxyClient; {
+            certFile = mkDefault cert;
+            keyFile = mkDefault key;
+          };
+        };
+      };
+    }
+  );
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/proxy.nix

@@ -0,0 +1,120 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+
+with lib;
+
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.proxy;
+in
+{
+  imports = [
+    (mkRenamedOptionModule
+      [ "services" "kubernetes" "proxy" "address" ]
+      [ "services" "kubernetes" "proxy" "bindAddress" ]
+    )
+  ];
+
+  ###### interface
+  options.services.kubernetes.proxy = with lib.types; {
+
+    bindAddress = mkOption {
+      description = "Kubernetes proxy listening address.";
+      default = "0.0.0.0";
+      type = str;
+    };
+
+    enable = mkEnableOption "Kubernetes proxy";
+
+    extraOpts = mkOption {
+      description = "Kubernetes proxy extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    hostname = mkOption {
+      description = "Kubernetes proxy hostname override.";
+      default = config.networking.hostName;
+      defaultText = literalExpression "config.networking.hostName";
+      type = str;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes proxy";
+
+    verbosity = mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = mkIf cfg.enable {
+    systemd.services.kube-proxy = {
+      description = "Kubernetes Proxy Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      path = with pkgs; [
+        iptables
+        conntrack-tools
+      ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-proxy \
+          --bind-address=${cfg.bindAddress} \
+          ${optionalString (top.clusterCidr != null) "--cluster-cidr=${top.clusterCidr}"} \
+          ${
+            optionalString (cfg.featureGates != { })
+              "--feature-gates=${
+                concatStringsSep "," (
+                  builtins.attrValues (mapAttrs (n: v: "${n}=${trivial.boolToString v}") cfg.featureGates)
+                )
+              }"
+          } \
+          --hostname-override=${cfg.hostname} \
+          --kubeconfig=${top.lib.mkKubeConfig "kube-proxy" cfg.kubeconfig} \
+          ${optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+          ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.proxy.hostname = with config.networking; mkDefault hostName;
+
+    services.kubernetes.pki.certs = {
+      kubeProxyClient = top.lib.mkCert {
+        name = "kube-proxy-client";
+        CN = "system:kube-proxy";
+        action = "systemctl restart kube-proxy.service";
+      };
+    };
+
+    services.kubernetes.proxy.kubeconfig.server = mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/kubernetes/scheduler.nix

@@ -0,0 +1,111 @@
+{
+  config,
+  lib,
+  options,
+  pkgs,
+  ...
+}:
+let
+  top = config.services.kubernetes;
+  otop = options.services.kubernetes;
+  cfg = top.scheduler;
+in
+{
+  ###### interface
+  options.services.kubernetes.scheduler = with lib.types; {
+
+    address = lib.mkOption {
+      description = "Kubernetes scheduler listening address.";
+      default = "127.0.0.1";
+      type = str;
+    };
+
+    enable = lib.mkEnableOption "Kubernetes scheduler";
+
+    extraOpts = lib.mkOption {
+      description = "Kubernetes scheduler extra command line options.";
+      default = "";
+      type = separatedString " ";
+    };
+
+    featureGates = lib.mkOption {
+      description = "Attribute set of feature gates.";
+      default = top.featureGates;
+      defaultText = lib.literalExpression "config.${otop.featureGates}";
+      type = attrsOf bool;
+    };
+
+    kubeconfig = top.lib.mkKubeConfigOptions "Kubernetes scheduler";
+
+    leaderElect = lib.mkOption {
+      description = "Whether to start leader election before executing main loop.";
+      type = bool;
+      default = true;
+    };
+
+    port = lib.mkOption {
+      description = "Kubernetes scheduler listening port.";
+      default = 10251;
+      type = port;
+    };
+
+    verbosity = lib.mkOption {
+      description = ''
+        Optional glog verbosity level for logging statements. See
+        <https://github.com/kubernetes/community/blob/master/contributors/devel/logging.md>
+      '';
+      default = null;
+      type = nullOr int;
+    };
+
+  };
+
+  ###### implementation
+  config = lib.mkIf cfg.enable {
+    systemd.services.kube-scheduler = {
+      description = "Kubernetes Scheduler Service";
+      wantedBy = [ "kubernetes.target" ];
+      after = [ "kube-apiserver.service" ];
+      serviceConfig = {
+        Slice = "kubernetes.slice";
+        ExecStart = ''
+          ${top.package}/bin/kube-scheduler \
+                    --bind-address=${cfg.address} \
+                    ${
+                      lib.optionalString (cfg.featureGates != { })
+                        "--feature-gates=${
+                          lib.concatStringsSep "," (
+                            builtins.attrValues (lib.mapAttrs (n: v: "${n}=${lib.trivial.boolToString v}") cfg.featureGates)
+                          )
+                        }"
+                    } \
+                    --kubeconfig=${top.lib.mkKubeConfig "kube-scheduler" cfg.kubeconfig} \
+                    --leader-elect=${lib.boolToString cfg.leaderElect} \
+                    --secure-port=${toString cfg.port} \
+                    ${lib.optionalString (cfg.verbosity != null) "--v=${toString cfg.verbosity}"} \
+                    ${cfg.extraOpts}
+        '';
+        WorkingDirectory = top.dataDir;
+        User = "kubernetes";
+        Group = "kubernetes";
+        Restart = "on-failure";
+        RestartSec = 5;
+      };
+      unitConfig = {
+        StartLimitIntervalSec = 0;
+      };
+    };
+
+    services.kubernetes.pki.certs = {
+      schedulerClient = top.lib.mkCert {
+        name = "kube-scheduler-client";
+        CN = "system:kube-scheduler";
+        action = "systemctl restart kube-scheduler.service";
+      };
+    };
+
+    services.kubernetes.scheduler.kubeconfig.server = lib.mkDefault top.apiserverAddress;
+  };
+
+  meta.buildDocsInSandbox = false;
+}

nixos/modules/services/cluster/pacemaker/default.nix

@@ -0,0 +1,52 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.pacemaker;
+in
+{
+  # interface
+  options.services.pacemaker = {
+    enable = lib.mkEnableOption "pacemaker";
+
+    package = lib.mkPackageOption pkgs "pacemaker" { };
+  };
+
+  # implementation
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = config.services.corosync.enable;
+        message = ''
+          Enabling services.pacemaker requires a services.corosync configuration.
+        '';
+      }
+    ];
+
+    environment.systemPackages = [ cfg.package ];
+
+    # required by pacemaker
+    users.users.hacluster = {
+      isSystemUser = true;
+      group = "pacemaker";
+      home = "/var/lib/pacemaker";
+    };
+    users.groups.pacemaker = { };
+
+    systemd.tmpfiles.rules = [
+      "d /var/log/pacemaker 0700 hacluster pacemaker -"
+    ];
+
+    systemd.packages = [ cfg.package ];
+    systemd.services.pacemaker = {
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        StateDirectory = "pacemaker";
+        StateDirectoryMode = "0700";
+      };
+    };
+  };
+}

nixos/modules/services/cluster/patroni/default.nix

@@ -0,0 +1,303 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.patroni;
+  defaultUser = "patroni";
+  defaultGroup = "patroni";
+  format = pkgs.formats.yaml { };
+
+  configFileName = "patroni-${cfg.scope}-${cfg.name}.yaml";
+  configFile = format.generate configFileName cfg.settings;
+in
+{
+  imports = [
+    (lib.mkRemovedOptionModule [ "services" "patroni" "raft" ] ''
+      Raft has been deprecated by upstream.
+    '')
+    (lib.mkRemovedOptionModule [ "services" "patroni" "raftPort" ] ''
+      Raft has been deprecated by upstream.
+    '')
+  ];
+
+  options.services.patroni = {
+
+    enable = lib.mkEnableOption "Patroni";
+
+    postgresqlPackage = lib.mkOption {
+      type = lib.types.package;
+      example = lib.literalExpression "pkgs.postgresql_14";
+      description = ''
+        PostgreSQL package to use.
+        Plugins can be enabled like this `pkgs.postgresql_14.withPackages (p: [ p.pg_safeupdate p.postgis ])`.
+      '';
+    };
+
+    postgresqlDataDir = lib.mkOption {
+      type = lib.types.path;
+      defaultText = lib.literalExpression ''"/var/lib/postgresql/''${config.services.patroni.postgresqlPackage.psqlSchema}"'';
+      example = "/var/lib/postgresql/14";
+      default = "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}";
+      description = ''
+        The data directory for PostgreSQL. If left as the default value
+        this directory will automatically be created before the PostgreSQL server starts, otherwise
+        the sysadmin is responsible for ensuring the directory exists with appropriate ownership
+        and permissions.
+      '';
+    };
+
+    postgresqlPort = lib.mkOption {
+      type = lib.types.port;
+      default = 5432;
+      description = ''
+        The port on which PostgreSQL listens.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = defaultUser;
+      example = "postgres";
+      description = ''
+        The user for the service. If left as the default value this user will automatically be created,
+        otherwise the sysadmin is responsible for ensuring the user exists.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = defaultGroup;
+      example = "postgres";
+      description = ''
+        The group for the service. If left as the default value this group will automatically be created,
+        otherwise the sysadmin is responsible for ensuring the group exists.
+      '';
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/patroni";
+      description = ''
+        Folder where Patroni data will be written, this is where the pgpass password file will be written.
+      '';
+    };
+
+    scope = lib.mkOption {
+      type = lib.types.str;
+      example = "cluster1";
+      description = ''
+        Cluster name.
+      '';
+    };
+
+    name = lib.mkOption {
+      type = lib.types.str;
+      example = "node1";
+      description = ''
+        The name of the host. Must be unique for the cluster.
+      '';
+    };
+
+    namespace = lib.mkOption {
+      type = lib.types.str;
+      default = "/service";
+      description = ''
+        Path within the configuration store where Patroni will keep information about the cluster.
+      '';
+    };
+
+    nodeIp = lib.mkOption {
+      type = lib.types.str;
+      example = "192.168.1.1";
+      description = ''
+        IP address of this node.
+      '';
+    };
+
+    otherNodesIps = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      example = [
+        "192.168.1.2"
+        "192.168.1.3"
+      ];
+      description = ''
+        IP addresses of the other nodes.
+      '';
+    };
+
+    restApiPort = lib.mkOption {
+      type = lib.types.port;
+      default = 8008;
+      description = ''
+        The port on Patroni's REST api listens.
+      '';
+    };
+
+    softwareWatchdog = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        This will configure Patroni to use the software watchdog built into the Linux kernel
+        as described in the [documentation](https://patroni.readthedocs.io/en/latest/watchdog.html#setting-up-software-watchdog-on-linux).
+      '';
+    };
+
+    settings = lib.mkOption {
+      type = format.type;
+      default = { };
+      example = {
+        bootstrap = {
+          initdb = [
+            "encoding=UTF-8"
+            "data-checksums"
+          ];
+        };
+        postgresql = {
+          parameters = {
+            unix_socket_directories = "/tmp";
+          };
+        };
+      };
+      description = ''
+        The primary patroni configuration. See the [documentation](https://patroni.readthedocs.io/en/latest/yaml_configuration.html)
+        for possible values.
+        Secrets should be passed in by using the `environmentFiles` option.
+      '';
+    };
+
+    environmentFiles = lib.mkOption {
+      type =
+        with lib.types;
+        attrsOf (
+          nullOr (oneOf [
+            str
+            path
+            package
+          ])
+        );
+      default = { };
+      example = {
+        PATRONI_REPLICATION_PASSWORD = "/secret/file";
+        PATRONI_SUPERUSER_PASSWORD = "/secret/file";
+      };
+      description = "Environment variables made available to Patroni as files content, useful for providing secrets from files.";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion =
+          !(
+            cfg.enable
+            && config.services.postgresql.enable
+            && cfg.postgresqlDataDir == config.services.postgresql.dataDir
+          );
+        message = ''
+          Both services.patroni and services.postgresql are enabled and
+          services.patroni.postgresqlDataDir == services.postgresql.dataDir
+          Disable one or the other, or configure them to use different directories.
+        '';
+      }
+    ];
+
+    services.patroni.settings = {
+      scope = cfg.scope;
+      name = cfg.name;
+      namespace = cfg.namespace;
+
+      restapi = {
+        listen = "${cfg.nodeIp}:${toString cfg.restApiPort}";
+        connect_address = "${cfg.nodeIp}:${toString cfg.restApiPort}";
+      };
+
+      postgresql = {
+        listen = "${cfg.nodeIp}:${toString cfg.postgresqlPort}";
+        connect_address = "${cfg.nodeIp}:${toString cfg.postgresqlPort}";
+        data_dir = cfg.postgresqlDataDir;
+        bin_dir = "${cfg.postgresqlPackage}/bin";
+        pgpass = "${cfg.dataDir}/pgpass";
+      };
+
+      watchdog = lib.mkIf cfg.softwareWatchdog {
+        mode = "required";
+        device = "/dev/watchdog";
+        safety_margin = 5;
+      };
+    };
+
+    users = {
+      users = lib.mkIf (cfg.user == defaultUser) {
+        patroni = {
+          group = cfg.group;
+          isSystemUser = true;
+        };
+      };
+      groups = lib.mkIf (cfg.group == defaultGroup) {
+        patroni = { };
+      };
+    };
+
+    systemd.services = {
+      patroni = {
+        description = "Runners to orchestrate a high-availability PostgreSQL";
+
+        wantedBy = [ "multi-user.target" ];
+        after = [ "network.target" ];
+
+        script = ''
+          ${lib.concatStringsSep "\n" (
+            lib.attrValues (
+              lib.mapAttrs (name: path: ''export ${name}="$(< ${lib.escapeShellArg path})"'') cfg.environmentFiles
+            )
+          )}
+          exec ${pkgs.patroni}/bin/patroni ${configFile}
+        '';
+
+        serviceConfig = lib.mkMerge [
+          {
+            User = cfg.user;
+            Group = cfg.group;
+            Type = "simple";
+            Restart = "on-failure";
+            TimeoutSec = 30;
+            ExecReload = "${pkgs.coreutils}/bin/kill -s HUP $MAINPID";
+            KillMode = "process";
+          }
+          (lib.mkIf
+            (
+              cfg.postgresqlDataDir == "/var/lib/postgresql/${cfg.postgresqlPackage.psqlSchema}"
+              && cfg.dataDir == "/var/lib/patroni"
+            )
+            {
+              StateDirectory = "patroni postgresql postgresql/${cfg.postgresqlPackage.psqlSchema}";
+              StateDirectoryMode = "0750";
+            }
+          )
+        ];
+      };
+    };
+
+    boot.kernelModules = lib.mkIf cfg.softwareWatchdog [ "softdog" ];
+
+    services.udev.extraRules = lib.mkIf cfg.softwareWatchdog ''
+      KERNEL=="watchdog", OWNER="${cfg.user}", GROUP="${cfg.group}", MODE="0600"
+    '';
+
+    environment.systemPackages = [
+      pkgs.patroni
+      cfg.postgresqlPackage
+    ];
+
+    environment.etc."${configFileName}".source = configFile;
+
+    environment.sessionVariables = {
+      PATRONICTL_CONFIG_FILE = "/etc/${configFileName}";
+    };
+  };
+
+  meta.maintainers = [ lib.maintainers.phfroidmont ];
+}

nixos/modules/services/cluster/rke2/default.nix

@@ -0,0 +1,338 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.rke2;
+in
+{
+  imports = [ ];
+
+  options.services.rke2 = {
+    enable = lib.mkEnableOption "rke2";
+
+    package = lib.mkPackageOption pkgs "rke2" { };
+
+    role = lib.mkOption {
+      type = lib.types.enum [
+        "server"
+        "agent"
+      ];
+      description = ''
+        Whether rke2 should run as a server or agent.
+
+        If it's a server:
+
+        - By default it also runs workloads as an agent.
+        - any optionals is allowed.
+
+        If it's an agent:
+
+        - `serverAddr` is required.
+        - `token` or `tokenFile` is required.
+        - `agentToken` or `agentTokenFile` or `disable` or `cni` are not allowed.
+      '';
+      default = "server";
+    };
+
+    configPath = lib.mkOption {
+      type = lib.types.path;
+      description = "Load configuration from FILE.";
+      default = "/etc/rancher/rke2/config.yaml";
+    };
+
+    debug = lib.mkOption {
+      type = lib.types.bool;
+      description = "Turn on debug logs.";
+      default = false;
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      description = "The folder to hold state in.";
+      default = "/var/lib/rancher/rke2";
+    };
+
+    token = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        Shared secret used to join a server or agent to a cluster.
+
+        > WARNING: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `tokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    tokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing rke2 token to use when connecting to the server.";
+      default = null;
+    };
+
+    disable = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Do not deploy packaged components and delete any deployed components.";
+      default = [ ];
+    };
+
+    nodeName = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = "Node name.";
+      default = null;
+    };
+
+    nodeLabel = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Registering and starting kubelet with set of labels.";
+      default = [ ];
+    };
+
+    nodeTaint = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = "Registering kubelet with set of taints.";
+      default = [ ];
+    };
+
+    nodeIP = lib.mkOption {
+      type = lib.types.nullOr lib.types.str;
+      description = "IPv4/IPv6 addresses to advertise for node.";
+      default = null;
+    };
+
+    agentToken = lib.mkOption {
+      type = lib.types.str;
+      description = ''
+        Shared secret used to join agents to the cluster, but not servers.
+
+        > **WARNING**: This option will expose store your token unencrypted world-readable in the nix store.
+        If this is undesired use the `agentTokenFile` option instead.
+      '';
+      default = "";
+    };
+
+    agentTokenFile = lib.mkOption {
+      type = lib.types.nullOr lib.types.path;
+      description = "File path containing rke2 agent token to use when connecting to the server.";
+      default = null;
+    };
+
+    serverAddr = lib.mkOption {
+      type = lib.types.str;
+      description = "The rke2 server to connect to, used to join a cluster.";
+      example = "https://10.0.0.10:6443";
+      default = "";
+    };
+
+    selinux = lib.mkOption {
+      type = lib.types.bool;
+      description = "Enable SELinux in containerd.";
+      default = false;
+    };
+
+    cni = lib.mkOption {
+      type = lib.types.enum [
+        "none"
+        "canal"
+        "cilium"
+        "calico"
+        "flannel"
+      ];
+      description = ''
+        CNI Plugins to deploy, one of `none`, `calico`, `canal`, `cilium` or `flannel`.
+
+        All CNI plugins get installed via a helm chart after the main components are up and running
+        and can be [customized by modifying the helm chart options](https://docs.rke2.io/helm).
+
+        [Learn more about RKE2 and CNI plugins](https://docs.rke2.io/networking/basic_network_options)
+
+        > **WARNING**: Flannel support in RKE2 is currently experimental.
+      '';
+      default = "canal";
+    };
+
+    cisHardening = lib.mkOption {
+      type = lib.types.bool;
+      description = ''
+        Enable CIS Hardening for RKE2.
+
+        It will set the configurations and controls required to address Kubernetes benchmark controls
+        from the Center for Internet Security (CIS).
+
+        Learn more about [CIS Hardening for RKE2](https://docs.rke2.io/security/hardening_guide).
+
+        > **NOTICE**:
+        >
+        > You may need restart the `systemd-sysctl` muaually by:
+        >
+        > ```shell
+        > sudo systemctl restart systemd-sysctl
+        > ```
+      '';
+      default = false;
+    };
+
+    extraFlags = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      description = ''
+        Extra flags to pass to the rke2 service/agent.
+
+        Here you can find all the available flags:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+      '';
+      example = [
+        "--disable-kube-proxy"
+        "--cluster-cidr=10.24.0.0/16"
+      ];
+      default = [ ];
+    };
+
+    environmentVars = lib.mkOption {
+      type = lib.types.attrsOf lib.types.str;
+      description = ''
+        Environment variables for configuring the rke2 service/agent.
+
+        Here you can find all the available environment variables:
+
+        - [Server Configuration Reference](https://docs.rke2.io/reference/server_config)
+        - [Agent Configuration Reference](https://docs.rke2.io/reference/linux_agent_config)
+
+        Besides the options above, you can also active environment variables by edit/create those files:
+
+        - `/etc/default/rke2`
+        - `/etc/sysconfig/rke2`
+        - `/usr/local/lib/systemd/system/rke2.env`
+      '';
+      # See: https://github.com/rancher/rke2/blob/master/bundle/lib/systemd/system/rke2-server.env#L1
+      default = {
+        HOME = "/root";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.role == "agent" -> (builtins.pathExists cfg.configPath || cfg.serverAddr != "");
+        message = "serverAddr or configPath (with 'server' key) should be set if role is 'agent'";
+      }
+      {
+        assertion =
+          cfg.role == "agent"
+          -> (builtins.pathExists cfg.configPath || cfg.tokenFile != null || cfg.token != "");
+        message = "token or tokenFile or configPath (with 'token' or 'token-file' keys) should be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.agentTokenFile != null || cfg.agentToken != "");
+        message = "agentToken or agentTokenFile should NOT be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.disable != [ ]);
+        message = "disable should not be set if role is 'agent'";
+      }
+      {
+        assertion = cfg.role == "agent" -> !(cfg.cni != "canal");
+        message = "cni should not be set if role is 'agent'";
+      }
+    ];
+
+    environment.systemPackages = [ config.services.rke2.package ];
+    # To configure NetworkManager to ignore calico/flannel related network interfaces.
+    # See: https://docs.rke2.io/known_issues#networkmanager
+    environment.etc."NetworkManager/conf.d/rke2-canal.conf" = {
+      enable = config.networking.networkmanager.enable;
+      text = ''
+        [keyfile]
+        unmanaged-devices=interface-name:cali*;interface-name:flannel*
+      '';
+    };
+    # See: https://docs.rke2.io/security/hardening_guide#set-kernel-parameters
+    boot.kernel.sysctl = lib.mkIf cfg.cisHardening {
+      "vm.panic_on_oom" = 0;
+      "vm.overcommit_memory" = 1;
+      "kernel.panic" = 10;
+      "kernel.panic_on_oops" = 1;
+    };
+
+    systemd.services."rke2-${cfg.role}" = {
+      description = "Rancher Kubernetes Engine v2";
+      documentation = [ "https://github.com/rancher/rke2#readme" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        Type = if cfg.role == "agent" then "exec" else "notify";
+        EnvironmentFile = [
+          "-/etc/default/%N"
+          "-/etc/sysconfig/%N"
+          "-/usr/local/lib/systemd/system/%N.env"
+        ];
+        Environment = lib.mapAttrsToList (k: v: "${k}=${v}") cfg.environmentVars;
+        KillMode = "process";
+        Delegate = "yes";
+        LimitNOFILE = 1048576;
+        LimitNPROC = "infinity";
+        LimitCORE = "infinity";
+        TasksMax = "infinity";
+        TimeoutStartSec = 0;
+        Restart = "always";
+        RestartSec = "5s";
+        ExecStartPre = [
+          # There is a conflict between RKE2 and `nm-cloud-setup.service`. This service add a routing table that
+          # interfere with the CNI plugin's configuration. This script checks if the service is enabled and if so,
+          # failed the RKE2 start.
+          # See: https://github.com/rancher/rke2/issues/1053
+          (pkgs.writeScript "check-nm-cloud-setup.sh" ''
+            #! ${pkgs.runtimeShell}
+            set -x
+            ! /run/current-system/systemd/bin/systemctl is-enabled --quiet nm-cloud-setup.service
+          '')
+          "-${pkgs.kmod}/bin/modprobe br_netfilter"
+          "-${pkgs.kmod}/bin/modprobe overlay"
+        ];
+        ExecStart = "${cfg.package}/bin/rke2 '${cfg.role}' ${
+          lib.escapeShellArgs (
+            (lib.optional (cfg.configPath != "/etc/rancher/rke2/config.yaml") "--config=${cfg.configPath}")
+            ++ (lib.optional cfg.debug "--debug")
+            ++ (lib.optional (cfg.dataDir != "/var/lib/rancher/rke2") "--data-dir=${cfg.dataDir}")
+            ++ (lib.optional (cfg.token != "") "--token=${cfg.token}")
+            ++ (lib.optional (cfg.tokenFile != null) "--token-file=${cfg.tokenFile}")
+            ++ (lib.optionals (cfg.role == "server" && cfg.disable != [ ]) (
+              map (d: "--disable=${d}") cfg.disable
+            ))
+            ++ (lib.optional (cfg.nodeName != null) "--node-name=${cfg.nodeName}")
+            ++ (lib.optionals (cfg.nodeLabel != [ ]) (map (l: "--node-label=${l}") cfg.nodeLabel))
+            ++ (lib.optionals (cfg.nodeTaint != [ ]) (map (t: "--node-taint=${t}") cfg.nodeTaint))
+            ++ (lib.optional (cfg.nodeIP != null) "--node-ip=${cfg.nodeIP}")
+            ++ (lib.optional (cfg.role == "server" && cfg.agentToken != "") "--agent-token=${cfg.agentToken}")
+            ++ (lib.optional (
+              cfg.role == "server" && cfg.agentTokenFile != null
+            ) "--agent-token-file=${cfg.agentTokenFile}")
+            ++ (lib.optional (cfg.serverAddr != "") "--server=${cfg.serverAddr}")
+            ++ (lib.optional cfg.selinux "--selinux")
+            ++ (lib.optional (cfg.role == "server" && cfg.cni != "canal") "--cni=${cfg.cni}")
+            ++ (lib.optional cfg.cisHardening "--profile=${
+              if cfg.package.version >= "1.25" then "cis-1.23" else "cis-1.6"
+            }")
+            ++ cfg.extraFlags
+          )
+        }";
+        ExecStopPost =
+          let
+            killProcess = pkgs.writeScript "kill-process.sh" ''
+              #! ${pkgs.runtimeShell}
+              /run/current-system/systemd/bin/systemd-cgls /system.slice/$1 | \
+              ${pkgs.gnugrep}/bin/grep -Eo '[0-9]+ (containerd|kubelet)' | \
+              ${pkgs.gawk}/bin/awk '{print $1}' | \
+              ${pkgs.findutils}/bin/xargs -r ${pkgs.util-linux}/bin/kill
+            '';
+          in
+          "-${killProcess} %n";
+      };
+    };
+  };
+}

nixos/modules/services/cluster/spark/default.nix

@@ -0,0 +1,177 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+let
+  cfg = config.services.spark;
+in
+{
+  options = {
+    services.spark = {
+      master = {
+        enable = lib.mkEnableOption "Spark master service";
+        bind = lib.mkOption {
+          type = lib.types.str;
+          description = "Address the spark master binds to.";
+          default = "127.0.0.1";
+          example = "0.0.0.0";
+        };
+        restartIfChanged = lib.mkOption {
+          type = lib.types.bool;
+          description = ''
+            Automatically restart master service on config change.
+            This can be set to false to defer restarts on clusters running critical applications.
+            Please consider the security implications of inadvertently running an older version,
+            and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+          '';
+          default = true;
+        };
+        extraEnvironment = lib.mkOption {
+          type = lib.types.attrsOf lib.types.str;
+          description = "Extra environment variables to pass to spark master. See spark-standalone documentation.";
+          default = { };
+          example = {
+            SPARK_MASTER_WEBUI_PORT = 8181;
+            SPARK_MASTER_OPTS = "-Dspark.deploy.defaultCores=5";
+          };
+        };
+      };
+      worker = {
+        enable = lib.mkEnableOption "Spark worker service";
+        workDir = lib.mkOption {
+          type = lib.types.path;
+          description = "Spark worker work dir.";
+          default = "/var/lib/spark";
+        };
+        master = lib.mkOption {
+          type = lib.types.str;
+          description = "Address of the spark master.";
+          default = "127.0.0.1:7077";
+        };
+        restartIfChanged = lib.mkOption {
+          type = lib.types.bool;
+          description = ''
+            Automatically restart worker service on config change.
+            This can be set to false to defer restarts on clusters running critical applications.
+            Please consider the security implications of inadvertently running an older version,
+            and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+          '';
+          default = true;
+        };
+        extraEnvironment = lib.mkOption {
+          type = lib.types.attrsOf lib.types.str;
+          description = "Extra environment variables to pass to spark worker.";
+          default = { };
+          example = {
+            SPARK_WORKER_CORES = 5;
+            SPARK_WORKER_MEMORY = "2g";
+          };
+        };
+      };
+      confDir = lib.mkOption {
+        type = lib.types.path;
+        description = "Spark configuration directory. Spark will use the configuration files (spark-defaults.conf, spark-env.sh, log4j.properties, etc) from this directory.";
+        default = "${cfg.package}/conf";
+        defaultText = lib.literalExpression ''"''${package}/conf"'';
+      };
+      logDir = lib.mkOption {
+        type = lib.types.path;
+        description = "Spark log directory.";
+        default = "/var/log/spark";
+      };
+      package = lib.mkPackageOption pkgs "spark" {
+        example = ''
+          spark.overrideAttrs (super: rec {
+            pname = "spark";
+            version = "2.4.4";
+
+            src = pkgs.fetchzip {
+              url    = "mirror://apache/spark/"''${pname}-''${version}/''${pname}-''${version}-bin-without-hadoop.tgz";
+              sha256 = "1a9w5k0207fysgpxx6db3a00fs5hdc2ncx99x4ccy2s0v5ndc66g";
+            };
+          })
+        '';
+      };
+    };
+  };
+  config = lib.mkIf (cfg.worker.enable || cfg.master.enable) {
+    environment.systemPackages = [ cfg.package ];
+    systemd = {
+      services = {
+        spark-master = lib.mkIf cfg.master.enable {
+          path = with pkgs; [
+            procps
+            openssh
+            net-tools
+          ];
+          description = "spark master service.";
+          after = [ "network.target" ];
+          wantedBy = [ "multi-user.target" ];
+          restartIfChanged = cfg.master.restartIfChanged;
+          environment = cfg.master.extraEnvironment // {
+            SPARK_MASTER_HOST = cfg.master.bind;
+            SPARK_CONF_DIR = cfg.confDir;
+            SPARK_LOG_DIR = cfg.logDir;
+          };
+          serviceConfig = {
+            Type = "forking";
+            User = "spark";
+            Group = "spark";
+            WorkingDirectory = "${cfg.package}/";
+            ExecStart = "${cfg.package}/sbin/start-master.sh";
+            ExecStop = "${cfg.package}/sbin/stop-master.sh";
+            TimeoutSec = 300;
+            Restart = "always";
+          };
+          unitConfig = {
+            StartLimitBurst = 10;
+          };
+        };
+        spark-worker = lib.mkIf cfg.worker.enable {
+          path = with pkgs; [
+            procps
+            openssh
+            net-tools
+            rsync
+          ];
+          description = "spark master service.";
+          after = [ "network.target" ];
+          wantedBy = [ "multi-user.target" ];
+          restartIfChanged = cfg.worker.restartIfChanged;
+          environment = cfg.worker.extraEnvironment // {
+            SPARK_MASTER = cfg.worker.master;
+            SPARK_CONF_DIR = cfg.confDir;
+            SPARK_LOG_DIR = cfg.logDir;
+            SPARK_WORKER_DIR = cfg.worker.workDir;
+          };
+          serviceConfig = {
+            Type = "forking";
+            User = "spark";
+            WorkingDirectory = "${cfg.package}/";
+            ExecStart = "${cfg.package}/sbin/start-worker.sh spark://${cfg.worker.master}";
+            ExecStop = "${cfg.package}/sbin/stop-worker.sh";
+            TimeoutSec = 300;
+            Restart = "always";
+          };
+          unitConfig = {
+            StartLimitBurst = 10;
+          };
+        };
+      };
+      tmpfiles.rules = [
+        "d '${cfg.worker.workDir}' - spark spark - -"
+        "d '${cfg.logDir}' - spark spark - -"
+      ];
+    };
+    users = {
+      users.spark = {
+        description = "spark user.";
+        group = "spark";
+        isSystemUser = true;
+      };
+      groups.spark = { };
+    };
+  };
+}

nixos/modules/services/cluster/temporal/default.nix

@@ -0,0 +1,146 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  cfg = config.services.temporal;
+
+  settingsFormat = pkgs.formats.yaml { };
+
+  usingDefaultDataDir = cfg.dataDir == "/var/lib/temporal";
+  usingDefaultUserAndGroup = cfg.user == "temporal" && cfg.group == "temporal";
+in
+{
+  meta.maintainers = [ lib.maintainers.jpds ];
+
+  options.services.temporal = {
+    enable = lib.mkEnableOption "Temporal";
+
+    package = lib.mkPackageOption pkgs "Temporal" {
+      default = [ "temporal" ];
+    };
+
+    settings = lib.mkOption {
+      type = lib.types.submodule {
+        freeformType = settingsFormat.type;
+      };
+
+      description = ''
+        Temporal configuration.
+
+        See <https://docs.temporal.io/references/configuration> for more
+        information about Temporal configuration options
+      '';
+    };
+
+    dataDir = lib.mkOption {
+      type = lib.types.path;
+      default = "/var/lib/temporal";
+      apply = lib.converge (lib.removeSuffix "/");
+      description = ''
+        Data directory for Temporal. If you change this, you need to
+        manually create the directory. You also need to create the
+        `temporal` user and group, or change
+        [](#opt-services.temporal.user) and
+        [](#opt-services.temporal.group) to existing ones with
+        access to the directory.
+      '';
+    };
+
+    user = lib.mkOption {
+      type = lib.types.str;
+      default = "temporal";
+      description = ''
+        The user Temporal runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    group = lib.mkOption {
+      type = lib.types.str;
+      default = "temporal";
+      description = ''
+        The group temporal runs as. Should be left at default unless
+        you have very specific needs.
+      '';
+    };
+
+    restartIfChanged = lib.mkOption {
+      type = lib.types.bool;
+      description = ''
+        Automatically restart the service on config change.
+        This can be set to false to defer restarts on a server or cluster.
+        Please consider the security implications of inadvertently running an older version,
+        and the possibility of unexpected behavior caused by inconsistent versions across a cluster when disabling this option.
+      '';
+      default = true;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    environment.etc."temporal/temporal-server.yaml".source =
+      settingsFormat.generate "temporal-server.yaml" cfg.settings;
+
+    systemd.services.temporal = {
+      description = "Temporal server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      inherit (cfg) restartIfChanged;
+      restartTriggers = [ config.environment.etc."temporal/temporal-server.yaml".source ];
+      environment = {
+        HOME = cfg.dataDir;
+      };
+      serviceConfig = {
+        ExecStart = ''
+          ${cfg.package}/bin/temporal-server --root / --config /etc/temporal/ -e temporal-server start
+        '';
+        User = cfg.user;
+        Group = cfg.group;
+        Restart = "on-failure";
+        DynamicUser = usingDefaultUserAndGroup && usingDefaultDataDir;
+        CapabilityBoundingSet = [ "" ];
+        DevicePolicy = "closed";
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectControlGroups = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        ReadWritePaths = [
+          cfg.dataDir
+        ];
+        RestrictAddressFamilies = [
+          "AF_NETLINK"
+          "AF_INET"
+          "AF_INET6"
+        ];
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          # 1. allow a reasonable set of syscalls
+          "@system-service @resources"
+          # 2. and deny unreasonable ones
+          "~@privileged"
+          # 3. then allow the required subset within denied groups
+          "@chown"
+        ];
+      }
+      // (lib.optionalAttrs usingDefaultDataDir {
+        StateDirectory = "temporal";
+        StateDirectoryMode = "0700";
+      });
+    };
+  };
+}