push sheeet

pkgs/tools/networking/maubot/default.nix

@@ -0,0 +1,150 @@
+{
+  lib,
+  fetchPypi,
+  fetchpatch,
+  callPackage,
+  runCommand,
+  python,
+  encryptionSupport ? true,
+  sqliteSupport ? true,
+}:
+
+let
+  # save for overriding it
+  python' = python;
+in
+let
+  python = python'.override {
+    self = python;
+    packageOverrides = final: prev: {
+      # SQLAlchemy>=1,<1.4
+      # SQLAlchemy 2.0's derivation is very different, so don't override, just write it from scratch
+      # (see https://github.com/NixOS/nixpkgs/blob/65dbed73949e4c0207e75dcc7271b29f9e457670/pkgs/development/python-modules/sqlalchemy/default.nix)
+      sqlalchemy = final.buildPythonPackage rec {
+        pname = "SQLAlchemy";
+        version = "1.3.24";
+        format = "setuptools";
+
+        src = fetchPypi {
+          inherit pname version;
+          sha256 = "sha256-67t3fL+TEjWbiXv4G6ANrg9ctp+6KhgmXcwYpvXvdRk=";
+        };
+
+        postInstall = ''
+          sed -e 's:--max-worker-restart=5::g' -i setup.cfg
+        '';
+
+        # tests are pretty annoying to set up for this version, and these dependency overrides are already long enough
+        doCheck = false;
+      };
+    };
+  };
+
+  maubot = python.pkgs.buildPythonPackage rec {
+    pname = "maubot";
+    version = "0.5.1";
+    format = "setuptools";
+    disabled = python.pythonOlder "3.10";
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-0UtelZ3w0QUw825AGhSc8wfhYaL9FSYJXCvYZEefWPQ=";
+    };
+
+    patches = [
+      # add entry point - https://github.com/maubot/maubot/pull/146
+      (fetchpatch {
+        url = "https://github.com/maubot/maubot/commit/ef6e23eccb530187dd3447b6aac2047d4a32fb83.patch";
+        hash = "sha256-d5fu47F93aXZmk6MiSsxTE8pHjMKNL0FUdU+ynUqY2o=";
+      })
+    ];
+
+    propagatedBuildInputs =
+      with python.pkgs;
+      [
+        # requirements.txt
+        (mautrix.override { withOlm = encryptionSupport; })
+        aiohttp
+        yarl
+        asyncpg
+        aiosqlite
+        commonmark
+        ruamel-yaml
+        attrs
+        bcrypt
+        packaging
+        click
+        colorama
+        questionary
+        jinja2
+        setuptools
+      ]
+      # optional-requirements.txt
+      ++ lib.optionals encryptionSupport [
+        python-olm
+        pycryptodome
+        unpaddedbase64
+      ]
+      ++ lib.optionals sqliteSupport [
+        sqlalchemy
+      ];
+
+    # used for plugin tests
+    propagatedNativeBuildInputs = with python.pkgs; [
+      pytest
+      pytest-asyncio
+    ];
+
+    postInstall = ''
+      rm $out/example-config.yaml
+    '';
+
+    pythonImportsCheck = [
+      "maubot"
+    ];
+
+    passthru =
+      let
+        wrapper = callPackage ./wrapper.nix {
+          unwrapped = maubot;
+          python3 = python;
+        };
+      in
+      {
+        tests = {
+          simple = runCommand "${pname}-tests" { } ''
+            ${maubot}/bin/mbc --help > $out
+          '';
+        };
+
+        inherit python;
+
+        plugins = callPackage ./plugins {
+          maubot = maubot;
+          python3 = python;
+        };
+
+        withPythonPackages = pythonPackages: wrapper { inherit pythonPackages; };
+
+        # This adds the plugins to lib/maubot-plugins
+        withPlugins = plugins: wrapper { inherit plugins; };
+
+        # This changes example-config.yaml in module directory
+        withBaseConfig = baseConfig: wrapper { inherit baseConfig; };
+      };
+
+    meta = with lib; {
+      description = "Plugin-based Matrix bot system written in Python";
+      homepage = "https://maubot.xyz/";
+      changelog = "https://github.com/maubot/maubot/blob/v${version}/CHANGELOG.md";
+      license = licenses.agpl3Plus;
+      # Presumably, people running "nix run nixpkgs#maubot" will want to run the tool
+      # for interacting with Maubot rather than Maubot itself, which should be used as
+      # a NixOS module.
+      mainProgram = "mbc";
+      maintainers = with maintainers; [ chayleaf ];
+    };
+  };
+
+in
+maubot

pkgs/tools/networking/maubot/plugins/default.nix

@@ -0,0 +1,90 @@
+{
+  lib,
+  fetchgit,
+  fetchFromGitHub,
+  fetchFromGitLab,
+  fetchFromGitea,
+  stdenvNoCC,
+  callPackage,
+  ensureNewerSourcesForZipFilesHook,
+  maubot,
+  python3,
+  poetry,
+  formats,
+}:
+
+let
+  # pname: plugin id (example: xyz.maubot.echo)
+  # version: plugin version
+  # other attributes are passed directly to stdenv.mkDerivation (you at least need src)
+  buildMaubotPlugin =
+    attrs@{
+      version,
+      pname,
+      base_config ? null,
+      ...
+    }:
+    stdenvNoCC.mkDerivation (
+      removeAttrs attrs [ "base_config" ]
+      // {
+        pluginName = "${pname}-v${version}.mbp";
+        nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [
+          ensureNewerSourcesForZipFilesHook
+          maubot
+        ];
+        buildPhase = ''
+          runHook preBuild
+
+          mbc build
+
+          runHook postBuild
+        '';
+
+        postPatch =
+          lib.optionalString (base_config != null) ''
+            [ -e base-config.yaml ] || (echo "base-config.yaml doesn't exist, can't override it" && exit 1)
+            cp "${
+              if builtins.isPath base_config || lib.isDerivation base_config then
+                base_config
+              else if builtins.isString base_config then
+                builtins.toFile "base-config.yaml" base_config
+              else
+                (formats.yaml { }).generate "base-config.yaml" base_config
+            }" base-config.yaml
+          ''
+          + attrs.postPatch or "";
+
+        installPhase = ''
+          runHook preInstall
+
+          mkdir -p $out/lib/maubot-plugins
+          install -m 444 $pluginName $out/lib/maubot-plugins
+
+          runHook postInstall
+        '';
+      }
+    );
+
+  generated = import ./generated.nix {
+    inherit
+      lib
+      fetchgit
+      fetchFromGitHub
+      fetchFromGitLab
+      fetchFromGitea
+      python3
+      poetry
+      buildMaubotPlugin
+      ;
+  };
+in
+generated
+// {
+  inherit buildMaubotPlugin;
+
+  allOfficialPlugins = builtins.filter (x: x.isOfficial && !x.meta.broken) (
+    builtins.attrValues generated
+  );
+
+  allPlugins = builtins.filter (x: !x.meta.broken) (builtins.attrValues generated);
+}

pkgs/tools/networking/maubot/plugins/generated.nix

@@ -0,0 +1,94 @@
+{
+  lib,
+  fetchgit,
+  fetchFromGitHub,
+  fetchFromGitLab,
+  fetchFromGitea,
+  python3,
+  poetry,
+  buildMaubotPlugin,
+}:
+
+let
+  json = builtins.fromJSON (builtins.readFile ./generated.json);
+in
+
+lib.flip builtins.mapAttrs json (
+  name: entry:
+  let
+    inherit (entry) manifest;
+
+    resolveDeps =
+      deps:
+      map (
+        name:
+        let
+          packageName = builtins.head (builtins.match "([^~=<>@]*).*" name);
+          lower = lib.toLower packageName;
+          dash = builtins.replaceStrings [ "_" ] [ "-" ] packageName;
+          lowerDash = builtins.replaceStrings [ "_" ] [ "-" ] lower;
+        in
+        python3.pkgs.${packageName} or python3.pkgs.${lower} or python3.pkgs.${dash}
+          or python3.pkgs.${lowerDash} or null
+      ) (builtins.filter (x: x != "maubot" && x != null) deps);
+
+    reqDeps = resolveDeps (lib.toList (manifest.dependencies or null));
+    optDeps = resolveDeps (lib.toList (manifest.soft_dependencies or null));
+  in
+
+  lib.makeOverridable buildMaubotPlugin (
+    entry.attrs
+    // {
+      pname = manifest.id;
+      inherit (manifest) version;
+
+      src =
+        if entry ? github then
+          fetchFromGitHub entry.github
+        else if entry ? git then
+          fetchgit entry.git
+        else if entry ? gitlab then
+          fetchFromGitLab entry.gitlab
+        else if entry ? gitea then
+          fetchFromGitea entry.gitea
+        else
+          throw "Invalid generated entry for ${manifest.id}: missing source";
+
+      propagatedBuildInputs = builtins.filter (x: x != null) (reqDeps ++ optDeps);
+
+      passthru.isOfficial = entry.isOfficial or false;
+
+      meta = entry.attrs.meta // {
+        license =
+          let
+            spdx = entry.attrs.meta.license or manifest.license or "unfree";
+            spdxLicenses = builtins.listToAttrs (
+              map (x: lib.nameValuePair x.spdxId x) (
+                builtins.filter (x: x ? spdxId) (builtins.attrValues lib.licenses)
+              )
+            );
+          in
+          spdxLicenses.${spdx};
+        broken = builtins.any (x: x == null) reqDeps;
+      };
+    }
+    // lib.optionalAttrs (entry.isPoetry or false) {
+      nativeBuildInputs = [
+        poetry
+        (python3.withPackages (
+          p: with p; [
+            toml
+            ruamel-yaml
+            isort
+          ]
+        ))
+      ];
+
+      preBuild = lib.optionalString (entry ? attrs.preBuild) (entry.attrs.preBuild + "\n") + ''
+        export HOME=$(mktemp -d)
+        [[ ! -d scripts ]] || patchShebangs --build scripts
+        make maubot.yaml
+      '';
+    }
+  )
+)

pkgs/tools/networking/maubot/plugins/update.py

@@ -0,0 +1,228 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"
+
+import git
+import json
+import os
+import subprocess
+import ruamel.yaml
+import sys
+import toml
+import zipfile
+
+HOSTNAMES = {
+    "git.skeg1.se": "gitlab",
+    "edugit.org": "gitlab",
+    "codeberg.org": "gitea",
+}
+PLUGINS: dict[str, dict] = {}
+# https://github.com/maubot/plugins.maubot.xyz/pull/45
+SKIP = {"characterai"}
+DIRS = {"matrix-to-discourse": "plugin"}
+
+yaml = ruamel.yaml.YAML(typ="safe")
+
+TMP = os.environ.get("TEMPDIR", "/tmp")
+
+
+def process_repo(path: str, official: bool):
+    global PLUGINS
+    with open(path, "rt") as f:
+        data = yaml.load(f)
+    name, repourl, license, desc = (
+        data["name"],
+        data["repo"],
+        data["license"],
+        data["description"],
+    )
+    if name in SKIP:
+        return
+    origurl = repourl
+    if "/" in name or " " in name:
+        name = os.path.split(path)[-1].removesuffix(".yaml")
+    name = name.replace("_", "-").lower()
+    if name in PLUGINS.keys():
+        raise ValueError(f"Duplicate plugin {name}, refusing to continue")
+    repodir = os.path.join(TMP, "maubot-plugins", name)
+    plugindir = repodir
+    if "/tree/" in repourl:
+        repourl, rev_path = repourl.split("/tree/")
+        rev, subdir = rev_path.strip("/").split("/")
+        plugindir = os.path.join(plugindir, subdir)
+    elif name in DIRS.keys():
+        subdir = DIRS[name]
+        plugindir = os.path.join(plugindir, subdir)
+    else:
+        rev = None
+        subdir = None
+
+    if repourl.startswith("http:"):
+        repourl = "https" + repourl[4:]
+    repourl = repourl.rstrip("/")
+    if not os.path.exists(repodir):
+        print("Fetching", name)
+        repo = git.Repo.clone_from(repourl + ".git", repodir)
+    else:
+        repo = git.Repo(repodir)
+    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
+    tags = list(filter(lambda x: "rc" not in str(x), tags))
+    if tags:
+        repo.git.checkout(tags[-1])
+        rev = str(tags[-1])
+    else:
+        rev = str(repo.commit("HEAD"))
+    ret: dict = {"attrs": {}}
+    if subdir:
+        ret["attrs"]["postPatch"] = f"cd {subdir}"
+    domain, query = repourl.removeprefix("https://").split("/", 1)
+    hash = subprocess.run(
+        ["nurl", "--hash", f"file://{repodir}", rev], capture_output=True, check=True
+    ).stdout.decode("utf-8")
+    ret["attrs"]["meta"] = {
+        "description": desc,
+        "homepage": origurl,
+    }
+    if domain == "github.com":
+        owner, repo = query.split("/")
+        ret["github"] = {
+            "owner": owner,
+            "repo": repo,
+            "rev": rev,
+            "hash": hash,
+        }
+        ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
+        ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
+        repobase = f"{repourl}/blob/{rev}"
+    elif (
+        HOSTNAMES.get(
+            domain, "gitea" if "gitea." in domain or "forgejo." in domain else None
+        )
+        == "gitea"
+    ):
+        owner, repo = query.split("/")
+        ret["gitea"] = {
+            "domain": domain,
+            "owner": owner,
+            "repo": repo,
+            "rev": rev,
+            "hash": hash,
+        }
+        repobase = f"{repourl}/src/commit/{rev}"
+        ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
+        ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
+    elif HOSTNAMES.get(domain, "gitlab" if "gitlab." in domain else None) == "gitlab":
+        owner, repo = query.split("/")
+        ret["gitlab"] = {
+            "owner": owner,
+            "repo": repo,
+            "rev": rev,
+            "hash": hash,
+        }
+        if domain != "gitlab.com":
+            ret["gitlab"]["domain"] = domain
+        repobase = f"{repourl}/-/blob/{rev}"
+    else:
+        raise ValueError(
+            f"Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!"
+        )
+    if os.path.exists(os.path.join(plugindir, "CHANGELOG.md")):
+        ret["attrs"]["meta"]["changelog"] = f"{repobase}/CHANGELOG.md"
+    if os.path.exists(os.path.join(plugindir, "maubot.yaml")):
+        with open(os.path.join(plugindir, "maubot.yaml"), "rt") as f:
+            ret["manifest"] = yaml.load(f)
+    elif os.path.exists(os.path.join(plugindir, "pyproject.toml")):
+        ret["isPoetry"] = True
+        with open(os.path.join(plugindir, "pyproject.toml"), "rt") as f:
+            data = toml.load(f)
+        deps = []
+        for key, val in data["tool"]["poetry"].get("dependencies", {}).items():
+            if key in ["maubot", "mautrix", "python"]:
+                continue
+            reqs = []
+            for req in val.split(","):
+                reqs.extend(poetry_to_pep(req))
+            deps.append(key + ", ".join(reqs))
+        ret["manifest"] = data["tool"]["maubot"]
+        ret["manifest"]["id"] = data["tool"]["poetry"]["name"]
+        ret["manifest"]["version"] = data["tool"]["poetry"]["version"]
+        ret["manifest"]["license"] = data["tool"]["poetry"]["license"]
+        if deps:
+            ret["manifest"]["dependencies"] = deps
+    else:
+        raise ValueError(f"No maubot.yaml or pyproject.toml found in {repodir}")
+    # normalize non-spdx-conformant licenses this way
+    # (and fill out missing license info)
+    if "license" not in ret["manifest"] or ret["manifest"]["license"] in [
+        "GPLv3",
+        "AGPL 3.0",
+    ]:
+        ret["attrs"]["meta"]["license"] = license
+    elif ret["manifest"]["license"] != license:
+        print(
+            f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}"
+        )
+    if official:
+        ret["isOfficial"] = official
+    PLUGINS[name] = ret
+
+
+def next_incomp(ver_s: str) -> str:
+    ver = ver_s.split(".")
+    zero = False
+    for i in range(len(ver)):
+        try:
+            seg = int(ver[i])
+        except ValueError:
+            if zero:
+                ver = ver[:i]
+                break
+            continue
+        if zero:
+            ver[i] = "0"
+        elif seg:
+            ver[i] = str(seg + 1)
+            zero = True
+    return ".".join(ver)
+
+
+def poetry_to_pep(ver_req: str) -> list[str]:
+    if "*" in ver_req:
+        raise NotImplementedError("Wildcard poetry versions not implemented!")
+    if ver_req.startswith("^"):
+        return [">=" + ver_req[1:], "<" + next_incomp(ver_req[1:])]
+    if ver_req.startswith("~"):
+        return ["~=" + ver_req[1:]]
+    return [ver_req]
+
+
+def main():
+    cache_path = os.path.join(TMP, "maubot-plugins")
+    if not os.path.exists(cache_path):
+        os.makedirs(cache_path)
+        git.Repo.clone_from(
+            "https://github.com/maubot/plugins.maubot.xyz",
+            os.path.join(cache_path, "_repo"),
+        )
+    else:
+        pass
+
+    repodir = os.path.join(cache_path, "_repo")
+
+    for suffix, official in (("official", True), ("thirdparty", False)):
+        directory = os.path.join(repodir, "data", "plugins", suffix)
+        for plugin_name in os.listdir(directory):
+            process_repo(os.path.join(directory, plugin_name), official)
+
+    if os.path.isdir("pkgs/tools/networking/maubot/plugins"):
+        generated = "pkgs/tools/networking/maubot/plugins/generated.json"
+    else:
+        script_dir = os.path.dirname(os.path.realpath(__file__))
+        generated = os.path.join(script_dir, "generated.json")
+
+    with open(generated, "wt") as file:
+        json.dump(PLUGINS, file, indent="  ", separators=(",", ": "), sort_keys=True)
+        file.write("\n")
+
+
+if __name__ == "__main__":
+    main()

pkgs/tools/networking/maubot/wrapper.nix

@@ -0,0 +1,89 @@
+{
+  lib,
+  symlinkJoin,
+  runCommand,
+  unwrapped,
+  python3,
+  formats,
+}:
+
+let
+  wrapper =
+    {
+      pythonPackages ? (_: [ ]),
+      plugins ? (_: [ ]),
+      baseConfig ? null,
+    }:
+    let
+      plugins' = plugins unwrapped.plugins;
+      extraPythonPackages = builtins.concatLists (map (p: p.propagatedBuildInputs or [ ]) plugins');
+    in
+    symlinkJoin {
+      name = "${unwrapped.pname}-with-plugins-${unwrapped.version}";
+
+      inherit unwrapped;
+      paths = lib.optional (baseConfig != null) unwrapped ++ plugins';
+      pythonPath =
+        lib.optional (baseConfig == null) unwrapped ++ pythonPackages python3.pkgs ++ extraPythonPackages;
+
+      nativeBuildInputs = [ python3.pkgs.wrapPython ];
+
+      postBuild = ''
+        rm -f $out/nix-support/propagated-build-inputs
+        rmdir $out/nix-support || true
+        ${lib.optionalString (baseConfig != null) ''
+          rm $out/${python3.sitePackages}/maubot/example-config.yaml
+          substituteAll ${
+            (formats.yaml { }).generate "example-config.yaml" (
+              lib.recursiveUpdate baseConfig {
+                plugin_directories = lib.optionalAttrs (plugins' != [ ]) {
+                  load = [ "@out@/lib/maubot-plugins" ] ++ (baseConfig.plugin_directories.load or [ ]);
+                };
+                # Normally it should be set to false by default to take it from package
+                # root, but aiohttp doesn't follow symlinks when serving static files
+                # unless follow_symlinks=True is passed. Instead of patching maubot, use
+                # this non-invasive approach
+                # XXX: would patching maubot be better? See:
+                # https://github.com/maubot/maubot/blob/75879cfb9370aade6fa0e84e1dde47222625139a/maubot/server.py#L106
+                server.override_resource_path =
+                  if isNull (baseConfig.server.override_resource_path or null) then
+                    "${unwrapped}/${python3.sitePackages}/maubot/management/frontend/build"
+                  else
+                    baseConfig.server.override_resource_path;
+              }
+            )
+          } $out/${python3.sitePackages}/maubot/example-config.yaml
+          rm -rf $out/bin
+        ''}
+        mkdir -p $out/bin
+        cp $unwrapped/bin/.mbc-wrapped $out/bin/mbc
+        cp $unwrapped/bin/.maubot-wrapped $out/bin/maubot
+        wrapPythonProgramsIn "$out/bin" "${lib.optionalString (baseConfig != null) "$out "}$pythonPath"
+      '';
+
+      passthru = {
+        inherit unwrapped;
+        python = python3;
+        withPythonPackages =
+          filter:
+          wrapper {
+            pythonPackages = pkgs: pythonPackages pkgs ++ filter pkgs;
+            inherit plugins baseConfig;
+          };
+        withPlugins =
+          filter:
+          wrapper {
+            plugins = pkgs: plugins pkgs ++ filter pkgs;
+            inherit pythonPackages baseConfig;
+          };
+        withBaseConfig =
+          baseConfig:
+          wrapper {
+            inherit baseConfig pythonPackages plugins;
+          };
+      };
+
+      meta.priority = (unwrapped.meta.priority or lib.meta.defaultPriority) - 1;
+    };
+in
+wrapper