push sheeet

nixos/tests/nextcloud/basic.nix

@@ -0,0 +1,121 @@
+{
+  name,
+  pkgs,
+  testBase,
+  system,
+  ...
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+runTest (
+  { config, lib, ... }:
+  {
+    inherit name;
+    meta = {
+      maintainers = lib.teams.nextcloud.members;
+    };
+
+    imports = [ testBase ];
+
+    nodes = {
+      # The only thing the client needs to do is download a file.
+      client =
+        { ... }:
+        {
+          services.davfs2.enable = true;
+          systemd.tmpfiles.settings.nextcloud = {
+            "/tmp/davfs2-secrets"."f+" = {
+              mode = "0600";
+              argument = "http://nextcloud/remote.php/dav/files/${config.adminuser} ${config.adminuser} ${config.adminpass}";
+            };
+          };
+          virtualisation.fileSystems = {
+            "/mnt/dav" = {
+              device = "http://nextcloud/remote.php/dav/files/${config.adminuser}";
+              fsType = "davfs";
+              options =
+                let
+                  davfs2Conf = (pkgs.writeText "davfs2.conf" "secrets /tmp/davfs2-secrets");
+                in
+                [
+                  "conf=${davfs2Conf}"
+                  "x-systemd.automount"
+                  "noauto"
+                ];
+            };
+          };
+        };
+
+      nextcloud =
+        { config, pkgs, ... }:
+        {
+          systemd.tmpfiles.rules = [
+            "d /var/lib/nextcloud-data 0750 nextcloud nginx - -"
+          ];
+
+          services.nextcloud = {
+            enable = true;
+            config.dbtype = "sqlite";
+            datadir = "/var/lib/nextcloud-data";
+            autoUpdateApps = {
+              enable = true;
+              startAt = "20:00";
+            };
+            phpExtraExtensions = all: [ all.bz2 ];
+            nginx.enableFastcgiRequestBuffering = true;
+          };
+
+          specialisation.withoutMagick.configuration = {
+            services.nextcloud.enableImagemagick = false;
+          };
+        };
+    };
+
+    test-helpers.extraTests =
+      { nodes, ... }:
+      let
+        findInClosure =
+          what: drv:
+          pkgs.runCommand "find-in-closure"
+            {
+              exportReferencesGraph = [
+                "graph"
+                drv
+              ];
+              inherit what;
+            }
+            ''
+              test -e graph
+              grep "$what" graph >$out || true
+            '';
+        nexcloudWithImagick = findInClosure "imagick" nodes.nextcloud.system.build.vm;
+        nextcloudWithoutImagick = findInClosure "imagick" nodes.nextcloud.specialisation.withoutMagick.configuration.system.build.vm;
+      in
+      # python
+      ''
+        with subtest("File is in proper nextcloud home"):
+            nextcloud.succeed("test -f ${nodes.nextcloud.services.nextcloud.datadir}/data/root/files/test-shared-file")
+
+        with subtest("Closure checks"):
+            assert open("${nexcloudWithImagick}").read() != ""
+            assert open("${nextcloudWithoutImagick}").read() == ""
+
+        with subtest("Davfs2"):
+            assert "hi" in client.succeed("cat /mnt/dav/test-shared-file")
+
+        with subtest("Ensure SSE is disabled by default"):
+            nextcloud.succeed("grep -vE '^HBEGIN:oc_encryption_module' /var/lib/nextcloud-data/data/root/files/test-shared-file")
+
+        with subtest("Create non-empty files with Transfer-Encoding: chunked"):
+            client.succeed(
+              'dd if=/dev/urandom of=testfile.bin bs=1M count=10',
+              'curl --fail -v -X PUT --header "Transfer-Encoding: chunked" --data-binary @testfile.bin "http://nextcloud/remote.php/webdav/testfile.bin" -u ${config.adminuser}:${config.adminpass}',
+            )
+
+            # Verify the local and remote copies of the file are identical.
+            client_hash = client.succeed("nix-hash testfile.bin").strip()
+            nextcloud_hash = nextcloud.succeed("nix-hash /var/lib/nextcloud-data/data/root/files/testfile.bin").strip()
+            t.assertEqual(client_hash, nextcloud_hash)
+      '';
+  }
+)

nixos/tests/nextcloud/default.nix

@@ -0,0 +1,144 @@
+{
+  system ? builtins.currentSystem,
+  config ? { },
+  pkgs ? import ../../.. { inherit system config; },
+}:
+
+with pkgs.lib;
+
+let
+  baseModule =
+    { config, ... }:
+    {
+      imports = [
+        {
+          options.test-helpers = {
+            rclone = mkOption { type = types.str; };
+            upload-sample = mkOption { type = types.str; };
+            check-sample = mkOption { type = types.str; };
+            init = mkOption {
+              type = types.str;
+              default = "";
+            };
+            extraTests = mkOption {
+              type = types.either types.str (types.functionTo types.str);
+              default = "";
+            };
+          };
+          options.adminuser = mkOption { type = types.str; };
+          options.adminpass = mkOption { type = types.str; };
+        }
+      ];
+
+      adminuser = "root";
+      adminpass = "hunter2";
+
+      test-helpers.rclone = "${pkgs.writeShellScript "rclone" ''
+        set -euo pipefail
+        export PATH="${pkgs.rclone}/bin:$PATH"
+        export RCLONE_CONFIG_NEXTCLOUD_TYPE=webdav
+        export RCLONE_CONFIG_NEXTCLOUD_URL="http://nextcloud/remote.php/dav/files/${config.adminuser}"
+        export RCLONE_CONFIG_NEXTCLOUD_VENDOR="nextcloud"
+        export RCLONE_CONFIG_NEXTCLOUD_USER="${config.adminuser}"
+        export RCLONE_CONFIG_NEXTCLOUD_PASS="$(rclone obscure ${config.adminpass})"
+        exec "$@"
+      ''}";
+      test-helpers.upload-sample = "${pkgs.writeShellScript "rclone-upload" ''
+        <<<'hi' rclone rcat nextcloud:test-shared-file
+      ''}";
+      test-helpers.check-sample = "${pkgs.writeShellScript "check-sample" ''
+        set -e
+        diff <(echo 'hi') <(rclone cat nextcloud:test-shared-file)
+      ''}";
+
+      nodes = {
+        client = { ... }: { };
+        nextcloud =
+          { lib, ... }:
+          {
+            networking.firewall.allowedTCPPorts = [ 80 ];
+            services.nextcloud = {
+              enable = true;
+              hostName = "nextcloud";
+              https = false;
+              database.createLocally = lib.mkDefault true;
+              config = {
+                adminpassFile = "${pkgs.writeText "adminpass" config.adminpass}"; # Don't try this at home!
+              };
+            };
+          };
+      };
+
+      testScript =
+        args@{ nodes, ... }:
+        let
+          inherit (config) test-helpers;
+        in
+        mkBefore ''
+          nextcloud.start()
+          client.start()
+          nextcloud.wait_for_unit("multi-user.target")
+
+          ${test-helpers.init}
+
+          with subtest("Ensure nextcloud-occ is working"):
+              nextcloud.succeed("nextcloud-occ status")
+              nextcloud.succeed("curl -sSf http://nextcloud/login")
+
+          with subtest("Upload/Download test"):
+              nextcloud.succeed(
+                  "${test-helpers.rclone} ${test-helpers.upload-sample}"
+              )
+              client.wait_for_unit("multi-user.target")
+              client.succeed(
+                  "${test-helpers.rclone} ${test-helpers.check-sample}"
+              )
+
+          ${
+            if pkgs.lib.isFunction test-helpers.extraTests then
+              test-helpers.extraTests args
+            else
+              test-helpers.extraTests
+          }
+        '';
+    };
+
+  genTests =
+    version:
+    let
+      testBase.imports = [
+        baseModule
+        {
+          nodes.nextcloud =
+            { pkgs, ... }:
+            {
+              services.nextcloud.package = pkgs.${"nextcloud${toString version}"};
+            };
+        }
+      ];
+
+      callNextcloudTest =
+        path:
+        let
+          name = "${removeSuffix ".nix" (baseNameOf path)}${toString version}";
+        in
+        nameValuePair name (
+          import path {
+            inherit system pkgs testBase;
+            name = "nextcloud-${name}";
+          }
+        );
+    in
+    map callNextcloudTest [
+      ./basic.nix
+      ./with-declarative-redis-and-secrets.nix
+      ./with-mysql-and-memcached.nix
+      ./with-postgresql-and-redis.nix
+      ./with-objectstore.nix
+    ];
+in
+listToAttrs (
+  concatMap genTests [
+    31
+  ]
+)

nixos/tests/nextcloud/with-declarative-redis-and-secrets.nix

@@ -0,0 +1,108 @@
+{
+  name,
+  pkgs,
+  testBase,
+  system,
+  ...
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+runTest (
+  { config, lib, ... }:
+  let
+    inherit (config) adminuser;
+  in
+  {
+    inherit name;
+    meta.maintainers = lib.teams.nextcloud.members;
+
+    imports = [ testBase ];
+
+    nodes = {
+      nextcloud =
+        { config, pkgs, ... }:
+        {
+          environment.systemPackages = [ pkgs.jq ];
+          services.nextcloud = {
+            caching = {
+              apcu = false;
+              redis = true;
+              memcached = false;
+            };
+            notify_push = {
+              enable = true;
+              bendDomainToLocalhost = true;
+              logLevel = "debug";
+            };
+            extraAppsEnable = true;
+            extraApps.notify_push = config.services.nextcloud.package.packages.apps.notify_push;
+            # This test also validates that we can use an "external" database
+            database.createLocally = false;
+            config = {
+              dbtype = "pgsql";
+              dbname = "nextcloud";
+              dbuser = adminuser;
+              dbpassFile = config.services.nextcloud.config.adminpassFile;
+            };
+
+            secretFile = "/etc/nextcloud-secrets.json";
+
+            settings = {
+              allow_local_remote_servers = true;
+              redis = {
+                dbindex = 0;
+                timeout = 1.5;
+                # password handled via secretfile below
+              };
+            };
+            configureRedis = true;
+          };
+
+          services.redis.servers."nextcloud" = {
+            enable = true;
+            port = 6379;
+            requirePass = "secret";
+          };
+
+          systemd.services.nextcloud-setup = {
+            requires = [ "postgresql.target" ];
+            after = [ "postgresql.target" ];
+          };
+
+          services.postgresql = {
+            enable = true;
+          };
+          systemd.services.postgresql-setup.postStart = ''
+            password=$(cat ${config.services.nextcloud.config.dbpassFile})
+            ${config.services.postgresql.package}/bin/psql <<EOF
+              CREATE ROLE ${adminuser} WITH LOGIN PASSWORD '$password' CREATEDB;
+              CREATE DATABASE nextcloud;
+              ALTER DATABASE nextcloud OWNER to ${adminuser};
+            EOF
+          '';
+
+          # This file is meant to contain secret options which should
+          # not go into the nix store. Here it is just used to set the
+          # redis password.
+          environment.etc."nextcloud-secrets.json" = {
+            mode = "0600";
+            text = builtins.toJSON {
+              redis.password = "secret";
+            };
+          };
+        };
+    };
+
+    test-helpers.extraTests = ''
+      with subtest("non-empty redis cache"):
+          # redis cache should not be empty
+          assert nextcloud.succeed('redis-cli --pass secret --json KEYS "*" | jq length').strip() != "0", """
+            redis-cli for keys * returned 0 entries
+          """
+
+      with subtest("notify-push"):
+          client.execute("${lib.getExe pkgs.nextcloud-notify_push.passthru.test_client} http://nextcloud ${config.adminuser} ${config.adminpass} >&2 &")
+          nextcloud.wait_until_succeeds("journalctl -u nextcloud-notify_push | grep -q \"Sending ping to ${config.adminuser}\"")
+    '';
+  }
+)

nixos/tests/nextcloud/with-mysql-and-memcached.nix

@@ -0,0 +1,49 @@
+{
+  name,
+  pkgs,
+  testBase,
+  system,
+  ...
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+runTest (
+  { config, lib, ... }:
+  {
+    inherit name;
+
+    meta.maintainers = lib.teams.nextcloud.members;
+
+    imports = [ testBase ];
+
+    nodes = {
+      nextcloud =
+        { config, pkgs, ... }:
+        {
+          services.nextcloud = {
+            caching = {
+              apcu = true;
+              memcached = true;
+            };
+            config.dbtype = "mysql";
+            configureRedis = false;
+          };
+
+          services.memcached.enable = true;
+        };
+    };
+
+    test-helpers.init =
+      let
+        configureMemcached = pkgs.writeScript "configure-memcached" ''
+          nextcloud-occ config:system:set memcached_servers 0 0 --value 127.0.0.1 --type string
+          nextcloud-occ config:system:set memcached_servers 0 1 --value 11211 --type integer
+          nextcloud-occ config:system:set memcache.local --value '\OC\Memcache\APCu' --type string
+          nextcloud-occ config:system:set memcache.distributed --value '\OC\Memcache\Memcached' --type string
+        '';
+      in
+      ''
+        nextcloud.succeed("${configureMemcached}")
+      '';
+  }
+)

nixos/tests/nextcloud/with-objectstore.nix

@@ -0,0 +1,169 @@
+{
+  name,
+  pkgs,
+  testBase,
+  system,
+  ...
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+runTest (
+  { config, lib, ... }:
+  let
+    accessKey = "BKIKJAA5BMMU2RHO6IBB";
+    secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
+
+    rootCredentialsFile = pkgs.writeText "minio-credentials-full" ''
+      MINIO_ROOT_USER=${accessKey}
+      MINIO_ROOT_PASSWORD=${secretKey}
+    '';
+  in
+  {
+    inherit name;
+    meta.maintainers = lib.teams.nextcloud.members;
+
+    imports = [ testBase ];
+
+    nodes = {
+      nextcloud =
+        {
+          config,
+          pkgs,
+          nodes,
+          ...
+        }:
+        {
+          services.nextcloud.config.dbtype = "sqlite";
+
+          services.nextcloud.config.objectstore.s3 = {
+            enable = true;
+            bucket = "nextcloud";
+            autocreate = true;
+            key = accessKey;
+            secretFile = "${pkgs.writeText "secretKey" secretKey}";
+            hostname = "acme.test";
+            useSsl = true;
+            port = 443;
+            usePathStyle = true;
+            region = "us-east-1";
+          };
+
+          security.pki.certificates = [
+            (builtins.readFile ../common/acme/server/ca.cert.pem)
+          ];
+
+          environment.systemPackages = [ pkgs.minio-client ];
+
+          # The dummy certs are for acme.test, so we pretend that's the FQDN
+          # of the minio VM.
+          networking.extraHosts = ''
+            ${nodes.minio.networking.primaryIPAddress} acme.test
+          '';
+        };
+
+      client =
+        { nodes, ... }:
+        {
+          security.pki.certificates = [
+            (builtins.readFile ../common/acme/server/ca.cert.pem)
+          ];
+          networking.extraHosts = ''
+            ${nodes.minio.networking.primaryIPAddress} acme.test
+          '';
+        };
+
+      minio =
+        { ... }:
+        {
+          security.pki.certificates = [
+            (builtins.readFile ../common/acme/server/ca.cert.pem)
+          ];
+
+          services.nginx = {
+            enable = true;
+            recommendedProxySettings = true;
+
+            virtualHosts."acme.test" = {
+              onlySSL = true;
+              sslCertificate = ../common/acme/server/acme.test.cert.pem;
+              sslCertificateKey = ../common/acme/server/acme.test.key.pem;
+              locations."/".proxyPass = "http://127.0.0.1:9000";
+            };
+          };
+
+          networking.extraHosts = ''
+            127.0.0.1 acme.test
+          '';
+
+          networking.firewall.allowedTCPPorts = [
+            9000
+            80
+            443
+          ];
+
+          services.minio = {
+            enable = true;
+            listenAddress = "0.0.0.0:9000";
+            consoleAddress = "0.0.0.0:9001";
+            inherit rootCredentialsFile;
+          };
+        };
+    };
+
+    test-helpers.init = ''
+      minio.start()
+      minio.wait_for_open_port(9000)
+      minio.wait_for_unit("nginx.service")
+      minio.wait_for_open_port(443)
+    '';
+
+    test-helpers.extraTests =
+      { nodes, ... }:
+      ''
+
+        with subtest("File is not on the filesystem"):
+            nextcloud.succeed("test ! -e ${nodes.nextcloud.services.nextcloud.home}/data/root/files/test-shared-file")
+
+        with subtest("Check if file is in S3"):
+            nextcloud.succeed(
+                "mc alias set minio https://acme.test ${accessKey} ${secretKey} --api s3v4"
+            )
+            files = nextcloud.succeed('mc ls minio/nextcloud|sort').strip().split('\n')
+
+            # Cannot assert an exact number here, nc27 writes more stuff initially into S3.
+            # For now let's assume it's always the most recently added file.
+            assert len(files) > 0, f"""
+              Expected to have at least one object in minio/nextcloud. But `mc ls` gave output:
+
+              '{files}'
+            """
+
+            import re
+            ptrn = re.compile("^\[[A-Z0-9 :-]+\] +(?P<details>[A-Za-z0-9 :]+)$")
+            match = ptrn.match(files[-1].strip())
+            assert match, "Cannot match mc client output!"
+            size, type_, file = tuple(match.group('details').split(' '))
+
+            assert size == "3B", f"""
+              Expected size of uploaded file to be 3 bytes, got {size}
+            """
+
+            assert type_ == 'STANDARD', f"""
+              Expected type of bucket entry to be a file, i.e. 'STANDARD'. Got {type_}
+            """
+
+            assert file.startswith('urn:oid'), """
+              Expected filename to start with 'urn:oid', instead got '{file}.
+            """
+
+        with subtest("Test download from S3"):
+            client.succeed(
+                "env AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey} "
+                + f"${lib.getExe pkgs.awscli2} s3 cp s3://nextcloud/{file} test --endpoint-url https://acme.test "
+                + "--region us-east-1 --ca-bundle /etc/ssl/certs/ca-bundle.crt"
+            )
+
+            client.succeed("test hi = $(cat test)")
+      '';
+  }
+)

nixos/tests/nextcloud/with-postgresql-and-redis.nix

@@ -0,0 +1,80 @@
+{
+  name,
+  pkgs,
+  testBase,
+  system,
+  ...
+}:
+
+with import ../../lib/testing-python.nix { inherit system pkgs; };
+runTest (
+  { config, lib, ... }:
+  {
+    inherit name;
+    meta.maintainers = lib.teams.nextcloud.members;
+
+    imports = [ testBase ];
+
+    nodes = {
+      nextcloud =
+        {
+          config,
+          pkgs,
+          lib,
+          ...
+        }:
+        {
+          environment.systemPackages = [ pkgs.jq ];
+          services.nextcloud = {
+            caching = {
+              apcu = false;
+              redis = true;
+              memcached = false;
+            };
+            config.dbtype = "pgsql";
+            notify_push = {
+              enable = true;
+              bendDomainToLocalhost = true;
+              logLevel = "debug";
+            };
+            extraAppsEnable = true;
+            extraApps = with config.services.nextcloud.package.packages.apps; {
+              inherit notify_push notes;
+            };
+            settings.trusted_proxies = [ "::1" ];
+          };
+
+          services.redis.servers."nextcloud".enable = true;
+          services.redis.servers."nextcloud".port = 6379;
+        };
+    };
+
+    test-helpers.init =
+      let
+        configureRedis = pkgs.writeScript "configure-redis" ''
+          nextcloud-occ config:system:set redis 'host' --value 'localhost' --type string
+          nextcloud-occ config:system:set redis 'port' --value 6379 --type integer
+          nextcloud-occ config:system:set memcache.local --value '\OC\Memcache\Redis' --type string
+          nextcloud-occ config:system:set memcache.locking --value '\OC\Memcache\Redis' --type string
+        '';
+      in
+      ''
+        nextcloud.succeed("${configureRedis}")
+      '';
+
+    test-helpers.extraTests = ''
+      with subtest("notify-push"):
+          client.execute("${lib.getExe pkgs.nextcloud-notify_push.passthru.test_client} http://nextcloud ${config.adminuser} ${config.adminpass} >&2 &")
+          nextcloud.wait_until_succeeds("journalctl -u nextcloud-notify_push | grep -q \"Sending ping to ${config.adminuser}\"")
+
+      with subtest("Redis is used for caching"):
+          # redis cache should not be empty
+          assert nextcloud.succeed('redis-cli --json KEYS "*" | jq length').strip() != "0", """
+            redis-cli for keys * returned 0 entries
+          """
+
+      with subtest("No code is returned when requesting PHP files (regression test)"):
+          nextcloud.fail("curl -f http://nextcloud/nix-apps/notes/lib/AppInfo/Application.php")
+    '';
+  }
+)