push sheeet

pkgs/development/tools/continuous-integration/buildbot/default.nix

@@ -0,0 +1,32 @@
+{
+  lib,
+  newScope,
+  python3,
+  recurseIntoAttrs,
+}:
+# Take packages from self first, then python.pkgs (and secondarily pkgs)
+lib.makeScope (self: newScope (self.python.pkgs // self)) (self: {
+  python = python3;
+
+  buildbot-pkg = self.callPackage ./pkg.nix { };
+
+  buildbot-worker = self.callPackage ./worker.nix { };
+
+  buildbot = self.callPackage ./master.nix { };
+
+  buildbot-plugins = recurseIntoAttrs (self.callPackage ./plugins.nix { });
+
+  buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]);
+
+  buildbot-full = self.buildbot.withPlugins (
+    with self.buildbot-plugins;
+    [
+      www
+      console-view
+      waterfall-view
+      grid-view
+      wsgi-dashboards
+      badges
+    ]
+  );
+})

pkgs/development/tools/continuous-integration/buildbot/master.nix

@@ -0,0 +1,175 @@
+{
+  lib,
+  stdenv,
+  buildPythonApplication,
+  fetchFromGitHub,
+  makeWrapper,
+  # Tie withPlugins through the fixed point here, so it will receive an
+  # overridden version properly
+  buildbot,
+  pythonOlder,
+  python,
+  twisted,
+  jinja2,
+  msgpack,
+  zope-interface,
+  sqlalchemy,
+  alembic,
+  python-dateutil,
+  txaio,
+  autobahn,
+  pyjwt,
+  pyyaml,
+  treq,
+  txrequests,
+  pypugjs,
+  boto3,
+  moto,
+  markdown,
+  lz4,
+  brotli,
+  zstandard,
+  setuptools-trial,
+  buildbot-worker,
+  buildbot-plugins,
+  buildbot-pkg,
+  parameterized,
+  git,
+  openssh,
+  setuptools,
+  croniter,
+  importlib-resources,
+  packaging,
+  unidiff,
+  nixosTests,
+}:
+
+let
+  withPlugins =
+    plugins:
+    buildPythonApplication {
+      pname = "${buildbot.pname}-with-plugins";
+      inherit (buildbot) version;
+      format = "other";
+
+      dontUnpack = true;
+      dontBuild = true;
+      doCheck = false;
+
+      nativeBuildInputs = [
+        makeWrapper
+      ];
+
+      propagatedBuildInputs = plugins ++ buildbot.propagatedBuildInputs;
+
+      installPhase = ''
+        makeWrapper ${buildbot}/bin/buildbot $out/bin/buildbot \
+          --prefix PYTHONPATH : "${buildbot}/${python.sitePackages}:$PYTHONPATH"
+        ln -sfv ${buildbot}/lib $out/lib
+      '';
+
+      passthru = buildbot.passthru // {
+        withPlugins = morePlugins: withPlugins (morePlugins ++ plugins);
+      };
+    };
+in
+buildPythonApplication rec {
+  pname = "buildbot";
+  version = "4.3.0";
+  format = "pyproject";
+
+  disabled = pythonOlder "3.8";
+
+  src = fetchFromGitHub {
+    owner = "buildbot";
+    repo = "buildbot";
+    rev = "v${version}";
+    hash = "sha256-yUtOJRI04/clCMImh5sokpj6MeBIXjEAdf9xnToqJZs=";
+  };
+
+  build-system = [
+  ];
+
+  pythonRelaxDeps = [
+    "twisted"
+  ];
+
+  propagatedBuildInputs = [
+    # core
+    twisted
+    jinja2
+    msgpack
+    zope-interface
+    sqlalchemy
+    alembic
+    python-dateutil
+    txaio
+    autobahn
+    pyjwt
+    pyyaml
+    setuptools
+    croniter
+    importlib-resources
+    packaging
+    unidiff
+    treq
+    brotli
+    zstandard
+  ]
+  # tls
+  ++ twisted.optional-dependencies.tls;
+
+  nativeCheckInputs = [
+    treq
+    txrequests
+    pypugjs
+    boto3
+    moto
+    markdown
+    lz4
+    setuptools-trial
+    buildbot-worker
+    buildbot-pkg
+    buildbot-plugins.www
+    parameterized
+    git
+    openssh
+  ];
+
+  patches = [
+    # This patch disables the test that tries to read /etc/os-release which
+    # is not accessible in sandboxed builds.
+    ./skip_test_linux_distro.patch
+  ];
+
+  postPatch = ''
+    cd master
+    touch buildbot/py.typed
+    substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)"
+  '';
+
+  # TimeoutErrors on slow machines -> aarch64
+  doCheck = !stdenv.hostPlatform.isAarch64;
+
+  preCheck = ''
+    export PATH="$out/bin:$PATH"
+  '';
+
+  passthru = {
+    inherit withPlugins python;
+    updateScript = ./update.sh;
+  }
+  // lib.optionalAttrs stdenv.hostPlatform.isLinux {
+    tests = {
+      inherit (nixosTests) buildbot;
+    };
+  };
+
+  meta = with lib; {
+    description = "Open-source continuous integration framework for automating software build, test, and release processes";
+    homepage = "https://buildbot.net/";
+    changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}";
+    teams = [ teams.buildbot ];
+    license = licenses.gpl2Only;
+  };
+}

pkgs/development/tools/continuous-integration/buildbot/pkg.nix

@@ -0,0 +1,33 @@
+{
+  lib,
+  buildPythonPackage,
+  isPy3k,
+  buildbot,
+}:
+
+buildPythonPackage {
+  format = "setuptools";
+  pname = "buildbot_pkg";
+  inherit (buildbot) src version;
+
+  postPatch = ''
+    cd pkg
+    # Their listdir function filters out `node_modules` folders.
+    # Do we have to care about that with Nix...?
+    substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" ""
+  '';
+
+  # No tests
+  doCheck = false;
+
+  pythonImportsCheck = [ "buildbot_pkg" ];
+
+  disabled = !isPy3k;
+
+  meta = with lib; {
+    homepage = "https://buildbot.net/";
+    description = "Buildbot Packaging Helper";
+    teams = [ teams.buildbot ];
+    license = licenses.gpl2;
+  };
+}

pkgs/development/tools/continuous-integration/buildbot/plugins.nix

@@ -0,0 +1,167 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchurl,
+  callPackage,
+  mock,
+  cairosvg,
+  klein,
+  jinja2,
+  buildbot-pkg,
+}:
+{
+  # this is exposed for potential plugins to use and for nix-update
+  inherit buildbot-pkg;
+  www = buildPythonPackage rec {
+    format = "setuptools";
+    pname = "buildbot_www";
+    inherit (buildbot-pkg) version;
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-mn55+Fb2cU2rNB5Nwt41nWXjcZfgd07ijYAAnZnnnwI=";
+    };
+
+    # Remove unnecessary circular dependency on buildbot
+    postPatch = ''
+      sed -i "s/'buildbot'//" setup.py
+    '';
+
+    buildInputs = [
+      buildbot-pkg
+      mock
+    ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot UI";
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+  console-view = buildPythonPackage rec {
+    pname = "buildbot_console_view";
+    inherit (buildbot-pkg) version;
+    format = "setuptools";
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-VA6xqJBjD4XmQabTN8M+PLvfrG7Hq2ooxChtz2jAT8A=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Console View Plugin";
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+  waterfall-view = buildPythonPackage rec {
+    pname = "buildbot_waterfall_view";
+    inherit (buildbot-pkg) version;
+    format = "setuptools";
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-c/Nmr0Uscalnndq72Y6jPM1JDs5OyOCERtuX/GXkxp8=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Waterfall View Plugin";
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+  grid-view = buildPythonPackage rec {
+    pname = "buildbot_grid_view";
+    inherit (buildbot-pkg) version;
+    format = "setuptools";
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-AmY8RkFX0POmVpW71nNz4+dFbr0FHGhNR3RJymDNoaw=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Grid View Plugin";
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+  wsgi-dashboards = buildPythonPackage rec {
+    pname = "buildbot_wsgi_dashboards";
+    inherit (buildbot-pkg) version;
+    format = "setuptools";
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-vofKxpIfbAs7HR43Y7ojHLQEn6/WIdjZPgZieBMsz74=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot WSGI dashboards Plugin";
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+  badges = buildPythonPackage rec {
+    pname = "buildbot_badges";
+    inherit (buildbot-pkg) version;
+    format = "setuptools";
+
+    src = fetchurl {
+      url = "https://github.com/buildbot/buildbot/releases/download/v${version}/${pname}-${version}.tar.gz";
+      hash = "sha256-u7HF6X+ClT4rT3LJcTHXWi5oSxCKPXoUDH+QFRI2S0w=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+    propagatedBuildInputs = [
+      cairosvg
+      klein
+      jinja2
+    ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Badges Plugin";
+      maintainers = [ maintainers.julienmalka ];
+      teams = [ teams.buildbot ];
+      license = licenses.gpl2;
+    };
+  };
+
+}

pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch

@@ -0,0 +1,11 @@
+diff -Nur buildbot-0.9.6/master/buildbot/test/unit/test_buildbot_net_usage_data.py buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py
+--- buildbot-0.9.6/master/buildbot/test/unit/test_buildbot_net_usage_data.py	2017-04-19 16:57:02.000000000 +0200
++++ buildbot-0.9.6.patched/master/buildbot/test/unit/test_buildbot_net_usage_data.py	2017-05-04 12:22:54.575762551 +0200
+@@ -147,6 +147,7 @@
+         _sendBuildbotNetUsageData({'foo': 'bar'})
+ 
+     def test_linux_distro(self):
++        raise SkipTest("NixOS sandboxed builds hides /etc/os-release")
+         system = platform.system()
+         if system != "Linux":
+             raise SkipTest("test is only for linux")

pkgs/development/tools/continuous-integration/buildbot/update.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p nix-update
+set -eu -o pipefail
+
+nix-update buildbot
+nix-update --version=skip buildbot-plugins.www
+nix-update --version=skip buildbot-plugins.console-view
+nix-update --version=skip buildbot-plugins.waterfall-view
+nix-update --version=skip buildbot-plugins.grid-view
+nix-update --version=skip buildbot-plugins.wsgi-dashboards
+nix-update --version=skip buildbot-plugins.badges

pkgs/development/tools/continuous-integration/buildbot/worker.nix

@@ -0,0 +1,61 @@
+{
+  lib,
+  buildPythonPackage,
+  buildbot,
+  stdenv,
+
+  # patch
+  coreutils,
+
+  # propagates
+  autobahn,
+  msgpack,
+  twisted,
+
+  # tests
+  parameterized,
+  psutil,
+  setuptools-trial,
+
+  # passthru
+  nixosTests,
+}:
+
+buildPythonPackage {
+  pname = "buildbot_worker";
+  inherit (buildbot) src version;
+  format = "setuptools";
+
+  postPatch = ''
+    cd worker
+    touch buildbot_worker/py.typed
+    substituteInPlace buildbot_worker/scripts/logwatcher.py \
+      --replace /usr/bin/tail "${coreutils}/bin/tail"
+  '';
+
+  nativeBuildInputs = [
+    setuptools-trial
+  ];
+
+  propagatedBuildInputs = [
+    autobahn
+    msgpack
+    twisted
+  ];
+
+  nativeCheckInputs = [
+    parameterized
+    psutil
+  ];
+
+  passthru.tests = {
+    smoke-test = nixosTests.buildbot;
+  };
+
+  meta = with lib; {
+    homepage = "https://buildbot.net/";
+    description = "Buildbot Worker Daemon";
+    teams = [ teams.buildbot ];
+    license = licenses.gpl2;
+  };
+}