push sheeet

nixos/tests/postgresql/anonymizer.nix

@@ -0,0 +1,113 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    makeTest {
+      name = "postgresql_anonymizer-${package.name}";
+      meta.maintainers = lib.teams.flyingcircus.members;
+
+      nodes.machine =
+        { pkgs, ... }:
+        {
+          environment.systemPackages = [ (pkgs.pg-dump-anon.override { postgresql = package; }) ];
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            extensions = ps: [ ps.anonymizer ];
+            settings.shared_preload_libraries = [ "anon" ];
+          };
+        };
+
+      testScript = ''
+        start_all()
+        machine.wait_for_unit("multi-user.target")
+        machine.wait_for_unit("postgresql.target")
+
+        with subtest("Setup"):
+            machine.succeed("sudo -u postgres psql --command 'create database demo'")
+            machine.succeed(
+                "sudo -u postgres psql -d demo -f ${pkgs.writeText "init.sql" ''
+                  create extension anon cascade;
+                  select anon.init();
+                  create table player(id serial, name text, points int);
+                  insert into player(id,name,points) values (1,'Foo', 23);
+                  insert into player(id,name,points) values (2,'Bar',42);
+                  security label for anon on column player.name is 'MASKED WITH FUNCTION anon.fake_last_name()';
+                  security label for anon on column player.points is 'MASKED WITH VALUE NULL';
+                ''}"
+            )
+
+        def get_player_table_contents():
+            return [
+                x.split(',') for x in machine.succeed("sudo -u postgres psql -d demo --csv --command 'select * from player'").splitlines()[1:]
+            ]
+
+        def check_anonymized_row(row, id, original_name):
+            assert row[0] == id, f"Expected first row to have ID {id}, but got {row[0]}"
+            assert row[1] != original_name, f"Expected first row to have a name other than {original_name}"
+            assert not bool(row[2]), "Expected points to be NULL in first row"
+
+        def find_xsv_in_dump(dump, sep=','):
+            """
+            Expecting to find a CSV (for pg_dump_anon) or TSV (for pg_dump) structure, looking like
+
+                COPY public.player ...
+                1,Shields,
+                2,Salazar,
+                \\.
+
+            in the given dump (the commas are tabs in case of pg_dump).
+                  Extract the CSV lines and split by `sep`.
+            """
+
+            try:
+                from itertools import dropwhile, takewhile
+                return [x.split(sep) for x in list(takewhile(
+                    lambda x: x != "\\.",
+                    dropwhile(
+                        lambda x: not x.startswith("COPY public.player"),
+                        dump.splitlines()
+                    )
+                ))[1:]]
+            except:
+                print(f"Dump to process: {dump}")
+                raise
+
+        def check_original_data(output):
+            assert output[0] == ['1','Foo','23'], f"Expected first row from player table to be 1,Foo,23; got {output[0]}"
+            assert output[1] == ['2','Bar','42'], f"Expected first row from player table to be 2,Bar,42; got {output[1]}"
+
+        def check_anonymized_rows(output):
+            check_anonymized_row(output[0], '1', 'Foo')
+            check_anonymized_row(output[1], '2', 'Bar')
+
+        with subtest("Check initial state"):
+            check_original_data(get_player_table_contents())
+
+        with subtest("Anonymous dumps"):
+            check_original_data(find_xsv_in_dump(
+                machine.succeed("sudo -u postgres pg_dump demo"),
+                sep='\t'
+            ))
+            check_anonymized_rows(find_xsv_in_dump(
+                machine.succeed("sudo -u postgres pg_dump_anon -U postgres -h /run/postgresql -d demo"),
+                sep=','
+            ))
+
+        with subtest("Anonymize"):
+            machine.succeed("sudo -u postgres psql -d demo --command 'select anon.anonymize_database();'")
+            check_anonymized_rows(get_player_table_contents())
+      '';
+    };
+in
+genTests {
+  inherit makeTestFor;
+  filter = _: p: !p.pkgs.anonymizer.meta.broken;
+}

nixos/tests/postgresql/default.nix

@@ -0,0 +1,41 @@
+{
+  system ? builtins.currentSystem,
+  config ? { },
+  pkgs ? import ../../.. { inherit system config; },
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+
+let
+  inherit (pkgs.lib)
+    recurseIntoAttrs
+    filterAttrs
+    mapAttrs
+    const
+    ;
+  genTests =
+    {
+      makeTestFor,
+      filter ? (_: _: true),
+    }:
+    recurseIntoAttrs (
+      mapAttrs (const makeTestFor) (filterAttrs filter pkgs.postgresqlVersions)
+      // {
+        passthru.override = makeTestFor;
+      }
+    );
+
+  importWithArgs = path: import path { inherit pkgs makeTest genTests; };
+in
+{
+  # postgresql
+  postgresql = importWithArgs ./postgresql.nix;
+  postgresql-jit = importWithArgs ./postgresql-jit.nix;
+  postgresql-wal-receiver = importWithArgs ./postgresql-wal-receiver.nix;
+  postgresql-tls-client-cert = importWithArgs ./postgresql-tls-client-cert.nix;
+
+  # extensions
+  anonymizer = importWithArgs ./anonymizer.nix;
+  pgjwt = importWithArgs ./pgjwt.nix;
+  wal2json = importWithArgs ./wal2json.nix;
+}

nixos/tests/postgresql/pgjwt.nix

@@ -0,0 +1,53 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    makeTest {
+      name = "pgjwt-${package.name}";
+      meta = with lib.maintainers; {
+        maintainers = [
+          spinus
+        ];
+      };
+
+      nodes.master =
+        { ... }:
+        {
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            extensions =
+              ps: with ps; [
+                pgjwt
+                pgtap
+              ];
+          };
+        };
+
+      testScript =
+        { nodes, ... }:
+        let
+          sqlSU = "${nodes.master.services.postgresql.superUser}";
+          pgProve = "${pkgs.perlPackages.TAPParserSourceHandlerpgTAP}";
+          inherit (nodes.master.services.postgresql.package.pkgs) pgjwt;
+        in
+        ''
+          start_all()
+          master.wait_for_unit("postgresql.target")
+          master.succeed(
+              "${pkgs.sudo}/bin/sudo -u ${sqlSU} ${pgProve}/bin/pg_prove -d postgres -v -f ${pgjwt.src}/test.sql"
+          )
+        '';
+    };
+in
+genTests {
+  inherit makeTestFor;
+  filter = _: p: !p.pkgs.pgjwt.meta.broken;
+}

nixos/tests/postgresql/postgresql-jit.nix

@@ -0,0 +1,54 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    makeTest {
+      name = "postgresql-jit-${package.name}";
+      meta.maintainers = with lib.maintainers; [ ma27 ];
+
+      nodes.machine =
+        { pkgs, ... }:
+        {
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            enableJIT = true;
+            initialScript = pkgs.writeText "init.sql" ''
+              create table demo (id int);
+              insert into demo (id) select generate_series(1, 5);
+            '';
+          };
+        };
+
+      testScript = ''
+        machine.start()
+        machine.wait_for_unit("postgresql.target")
+
+        with subtest("JIT is enabled"):
+            machine.succeed("sudo -u postgres psql <<<'show jit;' | grep 'on'")
+
+        with subtest("Test JIT works fine"):
+            output = machine.succeed(
+                "cat ${pkgs.writeText "test.sql" ''
+                  set jit_above_cost = 1;
+                  EXPLAIN ANALYZE SELECT CONCAT('jit result = ', SUM(id)) FROM demo;
+                  SELECT CONCAT('jit result = ', SUM(id)) from demo;
+                ''} | sudo -u postgres psql"
+            )
+            assert "JIT:" in output
+            assert "jit result = 15" in output
+
+        machine.shutdown()
+      '';
+    };
+in
+genTests {
+  inherit makeTestFor;
+}

nixos/tests/postgresql/postgresql-tls-client-cert.nix

@@ -0,0 +1,131 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  runWithOpenSSL =
+    file: cmd:
+    pkgs.runCommand file {
+      buildInputs = [ pkgs.openssl ];
+    } cmd;
+  caKey = runWithOpenSSL "ca.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out";
+  caCert = runWithOpenSSL "ca.crt" ''
+    openssl req -new -x509 -sha256 -key ${caKey} -out $out -subj "/CN=test.example" -days 36500
+  '';
+  serverKey = runWithOpenSSL "server.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out";
+  serverKeyPath = "/var/lib/postgresql";
+  serverCert = runWithOpenSSL "server.crt" ''
+    openssl req -new -sha256 -key ${serverKey} -out server.csr -subj "/CN=db.test.example"
+    openssl x509 -req -in server.csr -CA ${caCert} -CAkey ${caKey} \
+      -CAcreateserial -out $out -days 36500 -sha256
+  '';
+  clientKey = runWithOpenSSL "client.key" "openssl ecparam -name prime256v1 -genkey -noout -out $out";
+  clientCert = runWithOpenSSL "client.crt" ''
+    openssl req -new -sha256 -key ${clientKey} -out client.csr -subj "/CN=test"
+    openssl x509 -req -in client.csr -CA ${caCert} -CAkey ${caKey} \
+      -CAcreateserial -out $out -days 36500 -sha256
+  '';
+  clientKeyPath = "/root";
+
+  makeTestFor =
+    package:
+    makeTest {
+      name = "postgresql-tls-client-cert-${package.name}";
+      meta.maintainers = with lib.maintainers; [ erictapen ];
+
+      nodes.server =
+        { ... }:
+        {
+          system.activationScripts = {
+            keyPlacement.text = ''
+              mkdir -p '${serverKeyPath}'
+              cp '${serverKey}' '${serverKeyPath}/server.key'
+              chown postgres:postgres '${serverKeyPath}/server.key'
+              chmod 600 '${serverKeyPath}/server.key'
+            '';
+          };
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            enableTCPIP = true;
+            ensureUsers = [
+              {
+                name = "test";
+                ensureDBOwnership = true;
+              }
+            ];
+            ensureDatabases = [ "test" ];
+            settings = {
+              ssl = "on";
+              ssl_ca_file = toString caCert;
+              ssl_cert_file = toString serverCert;
+              ssl_key_file = "${serverKeyPath}/server.key";
+            };
+            authentication = ''
+              hostssl test test ::/0 cert clientcert=verify-full
+            '';
+          };
+          networking = {
+            interfaces.eth1 = {
+              ipv6.addresses = [
+                {
+                  address = "fc00::1";
+                  prefixLength = 120;
+                }
+              ];
+            };
+            firewall.allowedTCPPorts = [ 5432 ];
+          };
+        };
+
+      nodes.client =
+        { ... }:
+        {
+          system.activationScripts = {
+            keyPlacement.text = ''
+              mkdir -p '${clientKeyPath}'
+              cp '${clientKey}' '${clientKeyPath}/client.key'
+              chown root:root '${clientKeyPath}/client.key'
+              chmod 600 '${clientKeyPath}/client.key'
+            '';
+          };
+          environment = {
+            variables = {
+              PGHOST = "db.test.example";
+              PGPORT = "5432";
+              PGDATABASE = "test";
+              PGUSER = "test";
+              PGSSLMODE = "verify-full";
+              PGSSLCERT = clientCert;
+              PGSSLKEY = "${clientKeyPath}/client.key";
+              PGSSLROOTCERT = caCert;
+            };
+            systemPackages = [ package ];
+          };
+          networking = {
+            interfaces.eth1 = {
+              ipv6.addresses = [
+                {
+                  address = "fc00::2";
+                  prefixLength = 120;
+                }
+              ];
+            };
+            hosts = {
+              "fc00::1" = [ "db.test.example" ];
+            };
+          };
+        };
+
+      testScript = ''
+        server.wait_for_unit("multi-user.target")
+        client.wait_for_unit("multi-user.target")
+        client.succeed("psql -c \"SELECT 1;\"")
+      '';
+    };
+in
+genTests { inherit makeTestFor; }

nixos/tests/postgresql/postgresql-wal-receiver.nix

@@ -0,0 +1,111 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    let
+      postgresqlDataDir = "/var/lib/postgresql/${package.psqlSchema}";
+      replicationUser = "wal_receiver_user";
+      replicationSlot = "wal_receiver_slot";
+      replicationConn = "postgresql://${replicationUser}@localhost";
+      baseBackupDir = "/var/cache/wals/pg_basebackup";
+      walBackupDir = "/var/cache/wals/pg_wal";
+      recoveryFile = pkgs.writeTextDir "recovery.signal" "";
+    in
+    makeTest {
+      name = "postgresql-wal-receiver-${package.name}";
+      meta.maintainers = with lib.maintainers; [ euxane ];
+
+      nodes.machine =
+        { ... }:
+        {
+          systemd.tmpfiles.rules = [
+            "d /var/cache/wals 0750 postgres postgres - -"
+          ];
+
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            settings = {
+              max_replication_slots = 10;
+              max_wal_senders = 10;
+              recovery_end_command = "touch recovery.done";
+              restore_command = "cp ${walBackupDir}/%f %p";
+              wal_level = "archive"; # alias for replica on pg >= 9.6
+            };
+            authentication = ''
+              host replication ${replicationUser} all trust
+            '';
+            initialScript = pkgs.writeText "init.sql" ''
+              create user ${replicationUser} replication;
+              select * from pg_create_physical_replication_slot('${replicationSlot}');
+            '';
+          };
+
+          services.postgresqlWalReceiver.receivers.main = {
+            postgresqlPackage = package;
+            connection = replicationConn;
+            slot = replicationSlot;
+            directory = walBackupDir;
+          };
+          # This is only to speedup test, it isn't time racing. Service is set to autorestart always,
+          # default 60sec is fine for real system, but is too much for a test
+          systemd.services.postgresql-wal-receiver-main.serviceConfig.RestartSec = lib.mkForce 5;
+          systemd.services.postgresql.serviceConfig.ReadWritePaths = [ "/var/cache/wals" ];
+        };
+
+      testScript = ''
+        # make an initial base backup
+        machine.wait_for_unit("postgresql.target")
+        machine.wait_for_unit("postgresql-wal-receiver-main")
+        # WAL receiver healthchecks PG every 5 seconds, so let's be sure they have connected each other
+        # required only for 9.4
+        machine.sleep(5)
+        machine.succeed(
+            "${package}/bin/pg_basebackup --dbname=${replicationConn} --pgdata=${baseBackupDir}"
+        )
+
+        # create a dummy table with 100 records
+        machine.succeed(
+            "sudo -u postgres psql --command='create table dummy as select * from generate_series(1, 100) as val;'"
+        )
+
+        # stop postgres and destroy data
+        machine.systemctl("stop postgresql")
+        machine.systemctl("stop postgresql-wal-receiver-main")
+        machine.succeed("rm -r ${postgresqlDataDir}/{base,global,pg_*}")
+
+        # restore the base backup
+        machine.succeed(
+            "cp -r ${baseBackupDir}/* ${postgresqlDataDir} && chown postgres:postgres -R ${postgresqlDataDir}"
+        )
+
+        # prepare WAL and recovery
+        machine.succeed("chmod a+rX -R ${walBackupDir}")
+        machine.execute(
+            "for part in ${walBackupDir}/*.partial; do mv $part ''${part%%.*}; done"
+        )  # make use of partial segments too
+        machine.succeed(
+            "cp ${recoveryFile}/* ${postgresqlDataDir}/ && chmod 666 ${postgresqlDataDir}/recovery*"
+        )
+
+        # replay WAL
+        machine.systemctl("start postgresql")
+        machine.wait_for_file("${postgresqlDataDir}/recovery.done")
+        machine.systemctl("restart postgresql")
+        machine.wait_for_unit("postgresql.target")
+
+        # check that our records have been restored
+        machine.succeed(
+            "test $(sudo -u postgres psql --pset='pager=off' --tuples-only --command='select count(distinct val) from dummy;') -eq 100"
+        )
+      '';
+    };
+in
+genTests { inherit makeTestFor; }

nixos/tests/postgresql/postgresql.nix

@@ -0,0 +1,265 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    lib.recurseIntoAttrs {
+      postgresql = makeTestForWithBackupAll package false;
+      postgresql-backup-all = makeTestForWithBackupAll package true;
+      postgresql-clauses = makeEnsureTestFor package;
+    };
+
+  test-sql = pkgs.writeText "postgresql-test" ''
+    CREATE EXTENSION pgcrypto; -- just to check if lib loading works
+    CREATE TABLE sth (
+      id int
+    );
+    INSERT INTO sth (id) VALUES (1);
+    INSERT INTO sth (id) VALUES (1);
+    INSERT INTO sth (id) VALUES (1);
+    INSERT INTO sth (id) VALUES (1);
+    INSERT INTO sth (id) VALUES (1);
+    CREATE TABLE xmltest ( doc xml );
+    INSERT INTO xmltest (doc) VALUES ('<test>ok</test>'); -- check if libxml2 enabled
+
+    -- check if hardening gets relaxed
+    CREATE EXTENSION plv8;
+    -- try to trigger the V8 JIT, which requires MemoryDenyWriteExecute
+    DO $$
+      let xs = [];
+      for (let i = 0, n = 400000; i < n; i++) {
+          xs.push(Math.round(Math.random() * n))
+      }
+      console.log(xs.reduce((acc, x) => acc + x, 0));
+    $$ LANGUAGE plv8;
+  '';
+
+  makeTestForWithBackupAll =
+    package: backupAll:
+    makeTest {
+      name = "postgresql${lib.optionalString backupAll "-backup-all"}-${package.name}";
+      meta = with lib.maintainers; {
+        maintainers = [ zagy ];
+      };
+
+      nodes.machine =
+        { config, ... }:
+        {
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            identMap = ''
+              postgres root postgres
+            '';
+            # TODO(@Ma27) split this off into its own VM test and move a few other
+            # extension tests to use postgresqlTestExtension.
+            extensions = ps: with ps; [ plv8 ];
+          };
+
+          services.postgresqlBackup = {
+            enable = true;
+            databases = lib.optional (!backupAll) "postgres";
+            pgdumpOptions = "--restrict-key=ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+            pgdumpAllOptions = "--restrict-key=ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+          };
+        };
+
+      testScript =
+        let
+          backupName = if backupAll then "all" else "postgres";
+          backupService = if backupAll then "postgresqlBackup" else "postgresqlBackup-postgres";
+          backupFileBase = "/var/backup/postgresql/${backupName}";
+        in
+        ''
+          def check_count(statement, lines):
+              return 'test $(psql -U postgres postgres -tAc "{}"|wc -l) -eq {}'.format(
+                  statement, lines
+              )
+
+
+          machine.start()
+          machine.wait_for_unit("postgresql.target")
+
+          with subtest("Postgresql is available just after unit start"):
+              machine.succeed(
+                  "cat ${test-sql} | sudo -u postgres psql"
+              )
+
+          with subtest("Postgresql survives restart (bug #1735)"):
+              machine.shutdown()
+              import time
+              time.sleep(2)
+              machine.start()
+              machine.wait_for_unit("postgresql.target")
+
+          machine.fail(check_count("SELECT * FROM sth;", 3))
+          machine.succeed(check_count("SELECT * FROM sth;", 5))
+          machine.fail(check_count("SELECT * FROM sth;", 4))
+          machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1))
+
+          with subtest("killing postgres process should trigger an automatic restart"):
+              machine.succeed("systemctl kill -s KILL postgresql")
+
+              machine.wait_until_succeeds("systemctl is-active postgresql.service")
+              machine.wait_until_succeeds("systemctl is-active postgresql.target")
+
+          with subtest("Backup service works"):
+              machine.succeed(
+                  "systemctl start ${backupService}.service",
+                  "zcat ${backupFileBase}.sql.gz | grep '<test>ok</test>'",
+                  "ls -hal /var/backup/postgresql/ >/dev/console",
+                  "stat -c '%a' ${backupFileBase}.sql.gz | grep 600",
+              )
+          with subtest("Backup service removes prev files"):
+              machine.succeed(
+                  # Create dummy prev files.
+                  "touch ${backupFileBase}.prev.sql{,.gz,.zstd}",
+                  "chown postgres:postgres ${backupFileBase}.prev.sql{,.gz,.zstd}",
+
+                  # Run backup.
+                  "systemctl start ${backupService}.service",
+                  "ls -hal /var/backup/postgresql/ >/dev/console",
+
+                  # Since nothing has changed in the database, the cur and prev files
+                  # should match.
+                  "zcat ${backupFileBase}.sql.gz | grep '<test>ok</test>'",
+                  "cmp ${backupFileBase}.sql.gz ${backupFileBase}.prev.sql.gz",
+
+                  # The prev files with unused suffix should be removed.
+                  "[ ! -f '${backupFileBase}.prev.sql' ]",
+                  "[ ! -f '${backupFileBase}.prev.sql.zstd' ]",
+
+                  # Both cur and prev file should only be accessible by the postgres user.
+                  "stat -c '%a' ${backupFileBase}.sql.gz | grep 600",
+                  "stat -c '%a' '${backupFileBase}.prev.sql.gz' | grep 600",
+              )
+          with subtest("Backup service fails gracefully"):
+              # Sabotage the backup process
+              machine.succeed("rm /run/postgresql/.s.PGSQL.5432")
+              machine.fail(
+                  "systemctl start ${backupService}.service",
+              )
+              machine.succeed(
+                  "ls -hal /var/backup/postgresql/ >/dev/console",
+                  "zcat ${backupFileBase}.prev.sql.gz | grep '<test>ok</test>'",
+                  "stat ${backupFileBase}.in-progress.sql.gz",
+              )
+              # In a previous version, the second run would overwrite prev.sql.gz,
+              # so we test a second run as well.
+              machine.fail(
+                  "systemctl start ${backupService}.service",
+              )
+              machine.succeed(
+                  "stat ${backupFileBase}.in-progress.sql.gz",
+                  "zcat ${backupFileBase}.prev.sql.gz | grep '<test>ok</test>'",
+              )
+
+
+          with subtest("Initdb works"):
+              machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2")
+
+          machine.log(machine.execute("systemd-analyze security postgresql.service | grep -v ✓")[1])
+
+          machine.shutdown()
+        '';
+    };
+
+  makeEnsureTestFor =
+    package:
+    makeTest {
+      name = "postgresql-clauses-${package.name}";
+      meta = with lib.maintainers; {
+        maintainers = [ zagy ];
+      };
+
+      nodes.machine =
+        { ... }:
+        {
+          services.postgresql = {
+            inherit package;
+            enable = true;
+            ensureUsers = [
+              {
+                name = "all-clauses";
+                ensureClauses = {
+                  superuser = true;
+                  createdb = true;
+                  createrole = true;
+                  "inherit" = true;
+                  login = true;
+                  replication = true;
+                  bypassrls = true;
+                };
+              }
+              {
+                name = "default-clauses";
+              }
+            ];
+          };
+        };
+
+      testScript =
+        let
+          getClausesQuery =
+            user:
+            lib.concatStringsSep " " [
+              "SELECT row_to_json(row)"
+              "FROM ("
+              "SELECT"
+              "rolsuper,"
+              "rolinherit,"
+              "rolcreaterole,"
+              "rolcreatedb,"
+              "rolcanlogin,"
+              "rolreplication,"
+              "rolbypassrls"
+              "FROM pg_roles"
+              "WHERE rolname = '${user}'"
+              ") row;"
+            ];
+        in
+        ''
+          import json
+          machine.start()
+          machine.wait_for_unit("postgresql.target")
+
+          with subtest("All user permissions are set according to the ensureClauses attr"):
+              clauses = json.loads(
+                machine.succeed(
+                    "sudo -u postgres psql -tc \"${getClausesQuery "all-clauses"}\""
+                )
+              )
+              print(clauses)
+              assert clauses['rolsuper'], 'expected user with clauses to have superuser clause'
+              assert clauses['rolinherit'], 'expected user with clauses to have inherit clause'
+              assert clauses['rolcreaterole'], 'expected user with clauses to have create role clause'
+              assert clauses['rolcreatedb'], 'expected user with clauses to have create db clause'
+              assert clauses['rolcanlogin'], 'expected user with clauses to have login clause'
+              assert clauses['rolreplication'], 'expected user with clauses to have replication clause'
+              assert clauses['rolbypassrls'], 'expected user with clauses to have bypassrls clause'
+
+          with subtest("All user permissions default when ensureClauses is not provided"):
+              clauses = json.loads(
+                machine.succeed(
+                    "sudo -u postgres psql -tc \"${getClausesQuery "default-clauses"}\""
+                )
+              )
+              assert not clauses['rolsuper'], 'expected user with no clauses set to have default superuser clause'
+              assert clauses['rolinherit'], 'expected user with no clauses set to have default inherit clause'
+              assert not clauses['rolcreaterole'], 'expected user with no clauses set to have default create role clause'
+              assert not clauses['rolcreatedb'], 'expected user with no clauses set to have default create db clause'
+              assert clauses['rolcanlogin'], 'expected user with no clauses set to have default login clause'
+              assert not clauses['rolreplication'], 'expected user with no clauses set to have default replication clause'
+              assert not clauses['rolbypassrls'], 'expected user with no clauses set to have default bypassrls clause'
+
+          machine.shutdown()
+        '';
+    };
+in
+genTests { inherit makeTestFor; }

nixos/tests/postgresql/wal2json.nix

@@ -0,0 +1,49 @@
+{
+  pkgs,
+  makeTest,
+  genTests,
+}:
+
+let
+  inherit (pkgs) lib;
+
+  makeTestFor =
+    package:
+    makeTest {
+      name = "wal2json-${package.name}";
+      meta.maintainers = with pkgs.lib.maintainers; [ euank ];
+
+      nodes.machine = {
+        services.postgresql = {
+          inherit package;
+          enable = true;
+          extensions = with package.pkgs; [ wal2json ];
+          settings = {
+            wal_level = "logical";
+            max_replication_slots = "10";
+            max_wal_senders = "10";
+          };
+        };
+      };
+
+      testScript = ''
+        machine.wait_for_unit("postgresql.target")
+        machine.succeed(
+            "sudo -u postgres psql -qAt -f ${./wal2json/example2.sql} postgres > /tmp/example2.out"
+        )
+        machine.succeed(
+            "diff ${./wal2json/example2.out} /tmp/example2.out"
+        )
+        machine.succeed(
+            "sudo -u postgres psql -qAt -f ${./wal2json/example3.sql} postgres > /tmp/example3.out"
+        )
+        machine.succeed(
+            "diff ${./wal2json/example3.out} /tmp/example3.out"
+        )
+      '';
+    };
+in
+genTests {
+  inherit makeTestFor;
+  filter = _: p: !p.pkgs.wal2json.meta.broken;
+}

nixos/tests/postgresql/wal2json/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2013-2024, Euler Taveira de Oliveira
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the Euler Taveira de Oliveira nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

nixos/tests/postgresql/wal2json/README.md

@@ -0,0 +1,11 @@
+Data in this folder taken from the wal2json README's examples [here](https://github.com/eulerto/wal2json/tree/75629c2e1e81a12350cc9d63782fc53252185d8d#sql-functions)
+
+They are used under the terms of the BSD-3 License, a copy of which is included
+in this directory.
+
+These files have been lightly modified in order to make their output more reproducible.
+
+Changes:
+- `\o /dev/null` has been added before commands that print LSNs since LSNs aren't reproducible
+- `now()` has been replaced with a hardcoded timestamp string for reproducibility
+- The test is run with `--quiet`, and the expected output has been trimmed accordingly

nixos/tests/postgresql/wal2json/example2.out

@@ -0,0 +1,74 @@
+init
+{
+	"change": [
+		{
+			"kind": "message",
+			"transactional": false,
+			"prefix": "wal2json",
+			"content": "this non-transactional message will be delivered even if you rollback the transaction"
+		}
+	]
+}
+{
+	"change": [
+		{
+			"kind": "insert",
+			"schema": "public",
+			"table": "table2_with_pk",
+			"columnnames": ["a", "b", "c"],
+			"columntypes": ["integer", "character varying(30)", "timestamp without time zone"],
+			"columnvalues": [1, "Backup and Restore", "2018-03-27 12:05:29.914496"]
+		}
+		,{
+			"kind": "insert",
+			"schema": "public",
+			"table": "table2_with_pk",
+			"columnnames": ["a", "b", "c"],
+			"columntypes": ["integer", "character varying(30)", "timestamp without time zone"],
+			"columnvalues": [2, "Tuning", "2018-03-27 12:05:29.914496"]
+		}
+		,{
+			"kind": "insert",
+			"schema": "public",
+			"table": "table2_with_pk",
+			"columnnames": ["a", "b", "c"],
+			"columntypes": ["integer", "character varying(30)", "timestamp without time zone"],
+			"columnvalues": [3, "Replication", "2018-03-27 12:05:29.914496"]
+		}
+		,{
+			"kind": "message",
+			"transactional": true,
+			"prefix": "wal2json",
+			"content": "this message will be delivered"
+		}
+		,{
+			"kind": "delete",
+			"schema": "public",
+			"table": "table2_with_pk",
+			"oldkeys": {
+				"keynames": ["a", "c"],
+				"keytypes": ["integer", "timestamp without time zone"],
+				"keyvalues": [1, "2018-03-27 12:05:29.914496"]
+			}
+		}
+		,{
+			"kind": "delete",
+			"schema": "public",
+			"table": "table2_with_pk",
+			"oldkeys": {
+				"keynames": ["a", "c"],
+				"keytypes": ["integer", "timestamp without time zone"],
+				"keyvalues": [2, "2018-03-27 12:05:29.914496"]
+			}
+		}
+		,{
+			"kind": "insert",
+			"schema": "public",
+			"table": "table2_without_pk",
+			"columnnames": ["a", "b", "c"],
+			"columntypes": ["integer", "numeric(5,2)", "text"],
+			"columnvalues": [1, 2.34, "Tapir"]
+		}
+	]
+}
+stop

nixos/tests/postgresql/wal2json/example2.sql

@@ -0,0 +1,31 @@
+CREATE TABLE table2_with_pk (a SERIAL, b VARCHAR(30), c TIMESTAMP NOT NULL, PRIMARY KEY(a, c));
+CREATE TABLE table2_without_pk (a SERIAL, b NUMERIC(5,2), c TEXT);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'wal2json');
+
+BEGIN;
+INSERT INTO table2_with_pk (b, c) VALUES('Backup and Restore', '2018-03-27 12:05:29.914496');
+INSERT INTO table2_with_pk (b, c) VALUES('Tuning', '2018-03-27 12:05:29.914496');
+INSERT INTO table2_with_pk (b, c) VALUES('Replication', '2018-03-27 12:05:29.914496');
+
+-- Avoid printing wal LSNs since they're not reproducible, so harder to assert on
+\o /dev/null
+SELECT pg_logical_emit_message(true, 'wal2json', 'this message will be delivered');
+SELECT pg_logical_emit_message(true, 'pgoutput', 'this message will be filtered');
+\o
+
+DELETE FROM table2_with_pk WHERE a < 3;
+\o /dev/null
+SELECT pg_logical_emit_message(false, 'wal2json', 'this non-transactional message will be delivered even if you rollback the transaction');
+\o
+
+INSERT INTO table2_without_pk (b, c) VALUES(2.34, 'Tapir');
+-- it is not added to stream because there isn't a pk or a replica identity
+UPDATE table2_without_pk SET c = 'Anta' WHERE c = 'Tapir';
+COMMIT;
+
+SELECT data FROM pg_logical_slot_get_changes('test_slot', NULL, NULL, 'pretty-print', '1', 'add-msg-prefixes', 'wal2json');
+SELECT 'stop' FROM pg_drop_replication_slot('test_slot');
+
+DROP TABLE table2_with_pk;
+DROP TABLE table2_without_pk;

nixos/tests/postgresql/wal2json/example3.out

@@ -0,0 +1,12 @@
+init
+{"action":"M","transactional":false,"prefix":"wal2json","content":"this non-transactional message will be delivered even if you rollback the transaction"}
+{"action":"B"}
+{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":1},{"name":"b","type":"character varying(30)","value":"Backup and Restore"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]}
+{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":2},{"name":"b","type":"character varying(30)","value":"Tuning"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]}
+{"action":"I","schema":"public","table":"table3_with_pk","columns":[{"name":"a","type":"integer","value":3},{"name":"b","type":"character varying(30)","value":"Replication"},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]}
+{"action":"M","transactional":true,"prefix":"wal2json","content":"this message will be delivered"}
+{"action":"D","schema":"public","table":"table3_with_pk","identity":[{"name":"a","type":"integer","value":1},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]}
+{"action":"D","schema":"public","table":"table3_with_pk","identity":[{"name":"a","type":"integer","value":2},{"name":"c","type":"timestamp without time zone","value":"2019-12-29 04:58:34.806671"}]}
+{"action":"I","schema":"public","table":"table3_without_pk","columns":[{"name":"a","type":"integer","value":1},{"name":"b","type":"numeric(5,2)","value":2.34},{"name":"c","type":"text","value":"Tapir"}]}
+{"action":"C"}
+stop

nixos/tests/postgresql/wal2json/example3.sql

@@ -0,0 +1,26 @@
+CREATE TABLE table3_with_pk (a SERIAL, b VARCHAR(30), c TIMESTAMP NOT NULL, PRIMARY KEY(a, c));
+CREATE TABLE table3_without_pk (a SERIAL, b NUMERIC(5,2), c TEXT);
+
+SELECT 'init' FROM pg_create_logical_replication_slot('test_slot', 'wal2json');
+
+BEGIN;
+INSERT INTO table3_with_pk (b, c) VALUES('Backup and Restore', '2019-12-29 04:58:34.806671');
+INSERT INTO table3_with_pk (b, c) VALUES('Tuning', '2019-12-29 04:58:34.806671');
+INSERT INTO table3_with_pk (b, c) VALUES('Replication', '2019-12-29 04:58:34.806671');
+\o /dev/null
+SELECT pg_logical_emit_message(true, 'wal2json', 'this message will be delivered');
+SELECT pg_logical_emit_message(true, 'pgoutput', 'this message will be filtered');
+DELETE FROM table3_with_pk WHERE a < 3;
+SELECT pg_logical_emit_message(false, 'wal2json', 'this non-transactional message will be delivered even if you rollback the transaction');
+\o
+
+INSERT INTO table3_without_pk (b, c) VALUES(2.34, 'Tapir');
+-- it is not added to stream because there isn't a pk or a replica identity
+UPDATE table3_without_pk SET c = 'Anta' WHERE c = 'Tapir';
+COMMIT;
+
+SELECT data FROM pg_logical_slot_get_changes('test_slot', NULL, NULL, 'format-version', '2', 'add-msg-prefixes', 'wal2json');
+SELECT 'stop' FROM pg_drop_replication_slot('test_slot');
+
+DROP TABLE table3_with_pk;
+DROP TABLE table3_without_pk;