nixpkgs/nixos/tests/garage/with-3node-replication.nix

{
  lib,
  mkNode,
  package,
  testScriptSetup,
  ...
}:
let
  extraSettings =
    if (lib.versionAtLeast package.version "2") then
      {
        replication_factor = 3;
        consistency_mode = "consistent";
      }
    else
      {
        replication_mode = "3";
      };
in
{
  name = "garage-3node-replication";

  nodes = {
    node1 = mkNode {
      inherit extraSettings;
      publicV6Address = "fc00:1::1";
    };
    node2 = mkNode {
      inherit extraSettings;
      publicV6Address = "fc00:1::2";
    };
    node3 = mkNode {
      inherit extraSettings;
      publicV6Address = "fc00:1::3";
    };
    node4 = mkNode {
      inherit extraSettings;
      publicV6Address = "fc00:1::4";
    };
  };

  testScript = # python
    ''
      ${testScriptSetup}

      with subtest("Garage works as a multi-node S3 storage"):
        nodes = ('node1', 'node2', 'node3', 'node4')
        rev_machines = {m.name: m for m in machines}
        def get_machine(key): return rev_machines[key]
        for key in nodes:
          node = get_machine(key)
          node.wait_for_unit("garage.service")
          node.wait_for_open_port(3900)

        # Garage is initialized on all nodes.
        node_ids = {key: get_node_fqn(get_machine(key)) for key in nodes}

        for key in nodes:
          for other_key in nodes:
            if other_key != key:
              other_id = node_ids[other_key]
              get_machine(key).succeed(f"garage node connect {other_id.node_id}@{other_id.host}")

        # Provide multiple zones for the nodes.
        zones = ["nixcon", "nixcon", "paris_meetup", "fosdem"]
        apply_garage_layout(node1,
        [
          f'{ndata.node_id} -z {zones[index]} -c 1G'
          for index, ndata in enumerate(node_ids.values())
        ])
        # Now Garage is operational.
        test_bucket_writes(node1)
        for node in nodes:
           test_bucket_over_http(get_machine(node))
    '';
}