push sheeet

maintainers/scripts/kde/collect-licenses.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p gnutar jq reuse
+set -eu
+cd "$(dirname "$(readlink -f "$0")")"/../../..
+
+TMPDIR=$(mktemp -d)
+trap 'rm -rf $TMPDIR' EXIT
+
+echo "# Prebuilding sources..."
+nix-build -A kdePackages.sources --no-link || true
+
+echo "# Evaluating sources..."
+declare -A sources
+eval "$(nix-instantiate --eval -A kdePackages.sources --json --strict | jq 'to_entries[] | "sources[" + .key + "]=" + .value' -r)"
+
+echo "# Collecting licenses..."
+for k in "${!sources[@]}"; do
+    echo "- Processing $k..."
+
+    if [ ! -f "${sources[$k]}" ]; then
+        echo "Not found!"
+        continue
+    fi
+
+    mkdir "$TMPDIR/$k"
+    tar -C "$TMPDIR/$k" -xf "${sources[$k]}"
+
+    (cd "$TMPDIR/$k"; reuse lint --json) | jq --arg name "$k" '{$name: .summary.used_licenses | sort}' -c > "$TMPDIR/$k.json"
+done
+
+jq -s 'add' -S "$TMPDIR"/*.json > pkgs/kde/generated/licenses.json

maintainers/scripts/kde/collect-logs.nu

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i nu -p nushell
+cd $"($env.FILE_PWD)/../../.."
+
+mkdir logs
+nix-env -qaP -f . -A kdePackages --json --out-path | from json | values | par-each { |it|
+    echo $"Processing ($it.pname)..."
+    if "outputs" in $it {
+        nix-store --read-log $it.outputs.out | save -f $"logs/($it.pname).log"
+    }
+}

maintainers/scripts/kde/collect-metadata.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.click ps.pyyaml ])"
+import pathlib
+
+import click
+
+import utils
+
+@click.command
+@click.argument(
+    "repo-metadata",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        path_type=pathlib.Path,
+    ),
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+@click.option(
+    "--unstable",
+    default=False,
+    is_flag=True
+)
+def main(repo_metadata: pathlib.Path, nixpkgs: pathlib.Path, unstable: bool):
+    metadata = utils.KDERepoMetadata.from_repo_metadata_checkout(repo_metadata, unstable)
+    out_dir = nixpkgs / "pkgs/kde/generated"
+    metadata.write_json(out_dir)
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/collect-missing-deps.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p python3
+import pathlib
+
+OK_MISSING = {
+    # we don't use precompiled QML
+    'Qt6QuickCompiler',
+    'Qt6QmlCompilerPlusPrivate',
+    # usually used for version numbers
+    'Git',
+    # useless by itself, will warn if something else is not found
+    'PkgConfig',
+    # license verification
+    'ReuseTool',
+    # dev only
+    'ClangFormat',
+    # doesn't exist
+    'Qt6X11Extras',
+}
+
+OK_MISSING_BY_PACKAGE = {
+    "angelfish": {
+        "Qt6Feedback",  # we don't have it
+    },
+    "attica": {
+        "Python3",  # only used for license checks
+    },
+    "discover": {
+        "ApkQt",  # we don't have APK (duh)
+        "rpm-ostree-1",  # we don't have rpm-ostree (duh)
+        "Snapd",  # we don't have snaps and probably never will
+        "packagekitqt6",  # intentionally disabled
+    },
+    "elisa": {
+        "UPNPQT",  # upstream says it's broken
+    },
+    "extra-cmake-modules": {
+        "Sphinx",  # only used for docs, bloats closure size
+        "QCollectionGenerator"
+    },
+    "gwenview": {
+        "Tiff",  # duplicate?
+    },
+    "kio-extras-kf5": {
+        "KDSoapWSDiscoveryClient",  # actually vendored on KF5 version
+    },
+    "kitinerary": {
+        "OsmTools",  # used for map data updates, we use prebuilt
+    },
+    "kosmindoormap": {
+        "OsmTools",  # same
+        "Protobuf",
+    },
+    "kpty": {
+        "UTEMPTER",  # we don't have it and it probably wouldn't work anyway
+    },
+    "kpublictransport": {
+        "OsmTools",  # same
+        "PolyClipping",
+        "Protobuf",
+    },
+    "krfb": {
+        "Qt6XkbCommonSupport",  # not real
+    },
+    "ksystemstats": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "kuserfeedback": {
+        "Qt6Svg",  # all used for backend console stuff we don't ship
+        "QmlLint",
+        "Qt6Charts",
+        "FLEX",
+        "BISON",
+        "Php",
+        "PhpUnit",
+    },
+    "kwin": {
+        "display-info",  # newer versions identify as libdisplay-info
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "kwin-x11": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "libksysguard": {
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "mlt": {
+        "Qt5",  # intentionally disabled
+        "SWIG",
+    },
+    "plasma-desktop": {
+        "scim",  # upstream is dead, not packaged in Nixpkgs
+        "KAccounts6",  # dead upstream
+        "AccountsQt6",  # dead upstream
+        "signon-oauth2plugin",  # dead upstream
+    },
+    "plasma-dialer": {
+        "KTactileFeedback",  # dead?
+    },
+    "poppler-qt6": {
+        "gobject-introspection-1.0",  # we don't actually want to build the GTK variant
+        "gdk-pixbuf-2.0",
+        "gtk+-3.0",
+    },
+    "powerdevil": {
+        "DDCUtil",  # cursed, intentionally disabled
+        "Libcap",  # used to call setcap at build time and nothing else
+    },
+    "print-manager": {
+        "PackageKitQt6",  # used for auto-installing drivers which does not work for obvious reasons
+    },
+    "pulseaudio-qt": {
+        "Qt6Qml",  # tests only
+        "Qt6Quick",
+    },
+    "skladnik": {
+        "POVRay",  # too expensive to rerender all the assets
+    },
+    "syntax-highlighting": {
+        "XercesC",  # only used for extra validation at build time
+    }
+}
+
+def main():
+    here = pathlib.Path(__file__).parent.parent.parent.parent
+    logs = (here / "logs").glob("*.log")
+
+    for log in sorted(logs):
+        pname = log.stem
+
+        missing = []
+        is_in_block = False
+        with log.open(errors="replace") as fd:
+            for line in fd:
+                line = line.strip()
+                if line.startswith("--   No package '"):
+                    package = line.removeprefix("--   No package '").removesuffix("' found")
+                    missing.append(package)
+                if line == "-- The following OPTIONAL packages have not been found:" or line == "-- The following RECOMMENDED packages have not been found:":
+                    is_in_block = True
+                elif line.startswith("--") and is_in_block:
+                    is_in_block = False
+                elif line.startswith("*") and is_in_block:
+                    package = line.removeprefix("* ")
+                    missing.append(package)
+
+        missing = {
+            package
+            for package in missing
+            if not any(package.startswith(i) for i in OK_MISSING | OK_MISSING_BY_PACKAGE.get(pname, set()))
+        }
+
+        if missing:
+            print(pname + ":")
+            for line in missing:
+                print("  -", line)
+            print()
+
+if __name__ == '__main__':
+    main()

maintainers/scripts/kde/generate-sources.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i python3 -p "python3.withPackages(ps: [ ps.beautifulsoup4 ps.click ps.httpx ps.jinja2 ps.packaging ps.pyyaml ])" nix-update
+import base64
+import binascii
+import hashlib
+import json
+import pathlib
+import subprocess
+from urllib.parse import urljoin, urlparse
+
+import bs4
+import click
+import httpx
+import jinja2
+import packaging.version as v
+
+import utils
+
+
+LEAF_TEMPLATE = jinja2.Template('''
+{ mkKdeDerivation }:
+mkKdeDerivation {
+  pname = "{{ pname }}";
+}
+'''.strip())
+
+ROOT_TEMPLATE = jinja2.Template('''
+{ callPackage }:
+{
+  {%- for p in packages %}
+  {{ p }} = callPackage ./{{ p }} { };
+  {%- endfor %}
+}
+'''.strip())
+
+PROJECTS_WITH_RUST = {
+    "akonadi-search",
+    "angelfish",
+    "kdepim-addons",
+}
+
+def to_sri(hash):
+    raw = binascii.unhexlify(hash)
+    b64 = base64.b64encode(raw).decode()
+    return f"sha256-{b64}"
+
+
+@click.command
+@click.argument(
+    "pkgset",
+    type=click.Choice(["frameworks", "gear", "plasma"]),
+    required=True
+)
+@click.argument(
+    "version",
+    type=str,
+    required=True
+)
+@click.option(
+    "--nixpkgs",
+    type=click.Path(
+        exists=True,
+        file_okay=False,
+        resolve_path=True,
+        writable=True,
+        path_type=pathlib.Path,
+    ),
+    default=pathlib.Path(__file__).parent.parent.parent.parent
+)
+@click.option(
+    "--sources-url",
+    type=str,
+    default=None,
+)
+def main(pkgset: str, version: str, nixpkgs: pathlib.Path, sources_url: str | None):
+    root_dir = nixpkgs / "pkgs/kde"
+    set_dir = root_dir / pkgset
+    generated_dir = root_dir / "generated"
+    metadata = utils.KDERepoMetadata.from_json(generated_dir)
+
+    if sources_url is None:
+        set_url = {
+            "frameworks": f"frameworks/{version}/",
+            "gear": f"release-service/{version}/src/",
+            "plasma": f"plasma/{version}/",
+        }[pkgset]
+        sources_url = f"https://download.kde.org/stable/{set_url}"
+
+    client = httpx.Client()
+    sources = client.get(sources_url)
+    sources.raise_for_status()
+    bs = bs4.BeautifulSoup(sources.text, features="html.parser")
+
+    results = {}
+    projects_to_update_rust = set()
+    for item in bs.select("tr")[3:]:
+        link = item.select_one("td:nth-child(2) a")
+        if not link:
+            continue
+
+        project_name, version_and_ext = link.text.rsplit("-", maxsplit=1)
+
+        if project_name not in metadata.projects_by_name:
+            print(f"Warning: unknown tarball: {project_name}")
+
+        if project_name in PROJECTS_WITH_RUST:
+            projects_to_update_rust.add(project_name)
+
+        if version_and_ext.endswith(".sig"):
+            continue
+
+        version = version_and_ext.removesuffix(".tar.xz")
+
+        url = urljoin(sources_url, link.attrs["href"])
+
+        hash = client.get(url + ".sha256").text.strip()
+
+        if hash == "Hash type not supported":
+            print(f"{url} missing hash on CDN, downloading...")
+            hasher = hashlib.sha256()
+            with client.stream("GET", url, follow_redirects=True) as r:
+                for data in r.iter_bytes():
+                    hasher.update(data)
+            hash = hasher.hexdigest()
+        else:
+            hash = hash.split(" ", maxsplit=1)[0]
+
+        if existing := results.get(project_name):
+            old_version = existing["version"]
+            if v.parse(old_version) > v.parse(version):
+                print(f"{project_name} {old_version} is newer than {version}, skipping...")
+                continue
+
+        results[project_name] = {
+            "version": version,
+            "url": "mirror://kde" + urlparse(url).path,
+            "hash": to_sri(hash)
+        }
+
+        pkg_dir = set_dir / project_name
+        pkg_file = pkg_dir / "default.nix"
+
+        if not pkg_file.exists():
+            print(f"Generated new package: {pkgset}/{project_name}")
+            pkg_dir.mkdir(parents=True, exist_ok=True)
+            with pkg_file.open("w") as fd:
+                fd.write(LEAF_TEMPLATE.render(pname=project_name) + "\n")
+
+    set_dir.mkdir(parents=True, exist_ok=True)
+    with (set_dir / "default.nix").open("w") as fd:
+        fd.write(ROOT_TEMPLATE.render(packages=sorted(results.keys())) + "\n")
+
+    sources_dir = generated_dir / "sources"
+    sources_dir.mkdir(parents=True, exist_ok=True)
+    with (sources_dir / f"{pkgset}.json").open("w") as fd:
+        json.dump(results, fd, indent=2)
+
+    for project_name in projects_to_update_rust:
+            print(f"Updating cargoDeps hash for {pkgset}/{project_name}...")
+            subprocess.run([
+                "nix-update",
+                f"kdePackages.{project_name}",
+                "--version",
+                "skip",
+                "--override-filename",
+                pkg_file
+            ])
+
+
+if __name__ == "__main__":
+    main()  # type: ignore

maintainers/scripts/kde/utils.py

@@ -0,0 +1,185 @@
+import collections
+import dataclasses
+import functools
+import json
+import pathlib
+import subprocess
+
+import yaml
+
+class DataclassEncoder(json.JSONEncoder):
+    def default(self, it):
+        if dataclasses.is_dataclass(it):
+            return dataclasses.asdict(it)
+        return super().default(it)
+
+
+@dataclasses.dataclass
+class Project:
+    name: str
+    description: str | None
+    project_path: str
+    repo_path: str | None
+
+    def __hash__(self) -> int:
+        return hash(self.name)
+
+    @classmethod
+    def from_yaml(cls, path: pathlib.Path):
+        data = yaml.safe_load(path.open())
+        return cls(
+            name=data["identifier"],
+            description=data["description"],
+            project_path=data["projectpath"],
+            repo_path=data["repopath"]
+        )
+
+
+def get_git_commit(path: pathlib.Path):
+    return subprocess.check_output(["git", "-C", path, "rev-parse", "--short", "HEAD"]).decode().strip()
+
+
+def validate_unique(projects: list[Project], attr: str):
+    seen = set()
+    for item in projects:
+        attr_value = getattr(item, attr)
+        if attr_value in seen:
+            raise Exception(f"Duplicate {attr}: {attr_value}")
+        seen.add(attr_value)
+
+
+THIRD_PARTY = {
+    "third-party/appstream": "appstream-qt",
+    "third-party/cmark": "cmark",
+    "third-party/gpgme": "gpgme",
+    "third-party/kdsoap": "kdsoap",
+    "third-party/libaccounts-qt": "accounts-qt",
+    "third-party/libgpg-error": "libgpg-error",
+    "third-party/libquotient": "libquotient",
+    "third-party/packagekit-qt": "packagekit-qt",
+    "third-party/poppler": "poppler",
+    "third-party/qcoro": "qcoro",
+    "third-party/qmltermwidget": "qmltermwidget",
+    "third-party/qtkeychain": "qtkeychain",
+    "third-party/signond": "signond",
+    "third-party/taglib": "taglib",
+    "third-party/wayland-protocols": "wayland-protocols",
+    "third-party/wayland": "wayland",
+    "third-party/zxing-cpp": "zxing-cpp",
+}
+
+IGNORE = {
+    "kdesupport/phonon-directshow",
+    "kdesupport/phonon-mmf",
+    "kdesupport/phonon-mplayer",
+    "kdesupport/phonon-quicktime",
+    "kdesupport/phonon-waveout",
+    "kdesupport/phonon-xine"
+}
+
+WARNED = set()
+
+
+@dataclasses.dataclass
+class KDERepoMetadata:
+    version: str
+    projects: list[Project]
+    dep_graph: dict[Project, set[Project]]
+
+    @functools.cached_property
+    def projects_by_name(self):
+        return {p.name: p for p in self.projects}
+
+    @functools.cached_property
+    def projects_by_path(self):
+        return {p.project_path: p for p in self.projects}
+
+    def try_lookup_package(self, path):
+        if path in IGNORE:
+            return None
+        project = self.projects_by_path.get(path)
+        if project is None and path not in WARNED:
+            WARNED.add(path)
+            print(f"Warning: unknown project {path}")
+        return project
+
+    @classmethod
+    def from_repo_metadata_checkout(cls, repo_metadata: pathlib.Path, unstable=False):
+        projects = [
+            Project.from_yaml(metadata_file)
+            for metadata_file in repo_metadata.glob("projects-invent/**/metadata.yaml")
+        ] + [
+            Project(id, None, project_path, None)
+            for project_path, id in THIRD_PARTY.items()
+        ]
+
+        validate_unique(projects, "name")
+        validate_unique(projects, "project_path")
+
+        self = cls(
+            version=get_git_commit(repo_metadata),
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_graph = collections.defaultdict(set)
+
+        if unstable:
+            spec_name = "dependency-data-kf6-qt6"
+        else:
+            spec_name = "dependency-data-stable-kf6-qt6"
+
+        spec_path = repo_metadata / "dependencies" / spec_name
+        for line in spec_path.open():
+            line = line.strip()
+            if line.startswith("#"):
+                continue
+            if not line:
+                continue
+
+            dependent, dependency = line.split(": ")
+
+            dependent = self.try_lookup_package(dependent)
+            if dependent is None:
+                continue
+
+            dependency = self.try_lookup_package(dependency)
+            if dependency is None:
+                continue
+
+            dep_graph[dependent].add(dependency)
+
+        self.dep_graph = dep_graph
+
+        return self
+
+    def write_json(self, root: pathlib.Path):
+        root.mkdir(parents=True, exist_ok=True)
+
+        with (root / "projects.json").open("w") as fd:
+            json.dump(self.projects_by_name, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+        with (root / "dependencies.json").open("w") as fd:
+            deps = {k.name: sorted(dep.name for dep in v) for k, v in self.dep_graph.items()}
+            json.dump({"version": self.version, "dependencies": deps}, fd, cls=DataclassEncoder, sort_keys=True, indent=2)
+
+    @classmethod
+    def from_json(cls, root: pathlib.Path):
+        projects = [
+            Project(**v) for v in json.load((root / "projects.json").open()).values()
+        ]
+
+        deps = json.load((root / "dependencies.json").open())
+        self = cls(
+            version=deps["version"],
+            projects=projects,
+            dep_graph={},
+        )
+
+        dep_graph = collections.defaultdict(set)
+        for dependent, dependencies in deps["dependencies"].items():
+            for dependency in dependencies:
+                dep_graph[self.projects_by_name[dependent]].add(self.projects_by_name[dependency])
+
+        self.dep_graph = dep_graph
+        return self