push sheeet
Some checks failed
Periodic Merges (6h) / master → staging-nixos (push) Failing after 12m50s
Periodic Merges (6h) / master → staging-next (push) Failing after 12m54s
Periodic Merges (24h) / merge-base(master,staging) → haskell-updates (push) Failing after 11m54s
Periodic Merges (6h) / staging-next → staging (push) Failing after 12m13s
Periodic Merges (24h) / staging-next-25.05 → staging-25.05 (push) Failing after 13m24s
Periodic Merges (24h) / release-25.05 → staging-next-25.05 (push) Failing after 14m28s

This commit is contained in:
Dark Steveneq
2025-10-09 14:15:47 +02:00
commit 646b892680
49168 changed files with 5897842 additions and 0 deletions

View File

@@ -0,0 +1,340 @@
# This is is arrow-cpp < 20 used as a workaround for
# Ceph not supporting >= yet, taken from nixpkgs commit
# 97ae53798f6a7c7c3c259ad8c2cbcede6ca34b2a~
# This should be entirely removed when upstream bug
# https://tracker.ceph.com/issues/71269
# is fixed.
{
stdenv,
lib,
fetchurl,
fetchpatch2,
fetchFromGitHub,
fixDarwinDylibNames,
autoconf,
aws-sdk-cpp,
aws-sdk-cpp-arrow ? aws-sdk-cpp.override {
apis = [
"cognito-identity"
"config"
"identity-management"
"s3"
"sts"
"transfer"
];
},
boost,
brotli,
bzip2,
cmake,
crc32c,
curl,
flatbuffers,
gflags,
glog,
google-cloud-cpp,
grpc,
gtest,
libbacktrace,
lz4,
minio,
ninja,
nlohmann_json,
openssl,
perl,
pkg-config,
protobuf,
python3,
rapidjson,
re2,
snappy,
sqlite,
thrift,
tzdata,
utf8proc,
which,
zlib,
zstd,
testers,
enableShared ? !stdenv.hostPlatform.isStatic,
enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform,
# Disable also on RiscV
# configure: error: cannot determine number of significant virtual address bits
enableJemalloc ?
!stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64 && !stdenv.hostPlatform.isRiscV64,
enableS3 ? true,
# google-cloud-cpp fails to build on RiscV
enableGcs ? !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isRiscV64,
}:
let
arrow-testing = fetchFromGitHub {
name = "arrow-testing";
owner = "apache";
repo = "arrow-testing";
rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c";
hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o=";
};
parquet-testing = fetchFromGitHub {
name = "parquet-testing";
owner = "apache";
repo = "parquet-testing";
rev = "c7cf1374cf284c0c73024cd1437becea75558bf8";
hash = "sha256-DThjyZ34LajHwXZy1IhYKUGUG/ejQ9WvBNuI8eUKmSs=";
};
version = "19.0.1";
in
stdenv.mkDerivation (finalAttrs: {
pname = "arrow-cpp";
inherit version;
src = fetchFromGitHub {
owner = "apache";
repo = "arrow";
rev = "apache-arrow-${version}";
hash = "sha256-toHwUIOZRpgR0K7pQtT5nqWpO9G7AuHYTcvA6UVg9lA=";
};
sourceRoot = "${finalAttrs.src.name}/cpp";
patches = [
(fetchpatch2 {
name = "protobuf-30-compat.patch";
url = "https://github.com/apache/arrow/pull/46136.patch";
hash = "sha256-WTpe/eT3himlCHN/R78w1sF0HG859mE2ZN70U+9N8Ag=";
stripLen = 1;
})
];
# versions are all taken from
# https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt
# jemalloc: arrow uses a custom prefix to prevent default allocator symbol
# collisions as well as custom build flags
${if enableJemalloc then "ARROW_JEMALLOC_URL" else null} = fetchurl {
url = "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2";
hash = "sha256-LbgtHnEZ3z5xt2QCGbbf6EeJvAU3mDw7esT3GJrs/qo=";
};
# mimalloc: arrow uses custom build flags for mimalloc
ARROW_MIMALLOC_URL = fetchFromGitHub {
owner = "microsoft";
repo = "mimalloc";
rev = "v2.0.6";
hash = "sha256-u2ITXABBN/dwU+mCIbL3tN1f4c17aBuSdNTV+Adtohc=";
};
ARROW_XSIMD_URL = fetchFromGitHub {
owner = "xtensor-stack";
repo = "xsimd";
rev = "13.0.0";
hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE=";
};
ARROW_SUBSTRAIT_URL = fetchFromGitHub {
owner = "substrait-io";
repo = "substrait";
rev = "v0.44.0";
hash = "sha256-V739IFTGPtbGPlxcOi8sAaYSDhNUEpITvN9IqdPReug=";
};
nativeBuildInputs = [
cmake
pkg-config
ninja
autoconf # for vendored jemalloc
flatbuffers
]
++ lib.optional stdenv.hostPlatform.isDarwin fixDarwinDylibNames;
buildInputs = [
boost
brotli
bzip2
flatbuffers
gflags
glog
gtest
libbacktrace
lz4
nlohmann_json # alternative JSON parser to rapidjson
protobuf # substrait requires protobuf
rapidjson
re2
snappy
thrift
utf8proc
zlib
zstd
]
++ lib.optionals enableFlight [
grpc
openssl
protobuf
sqlite
]
++ lib.optionals enableS3 [
aws-sdk-cpp-arrow
openssl
]
++ lib.optionals enableGcs [
crc32c
curl
google-cloud-cpp
grpc
nlohmann_json
];
preConfigure = ''
patchShebangs build-support/
substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \
--replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
'';
cmakeFlags = [
"-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
"-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
"-DARROW_BUILD_TESTS=${if enableShared then "ON" else "OFF"}"
"-DARROW_BUILD_INTEGRATION=ON"
"-DARROW_BUILD_UTILITIES=ON"
"-DARROW_EXTRA_ERROR_CONTEXT=ON"
"-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
"-DARROW_DEPENDENCY_SOURCE=SYSTEM"
"-Dxsimd_SOURCE=AUTO"
"-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
"-DARROW_COMPUTE=ON"
"-DARROW_CSV=ON"
"-DARROW_DATASET=ON"
"-DARROW_FILESYSTEM=ON"
"-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}"
"-DARROW_HDFS=ON"
"-DARROW_IPC=ON"
"-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}"
"-DARROW_JSON=ON"
"-DARROW_USE_GLOG=ON"
"-DARROW_WITH_BACKTRACE=ON"
"-DARROW_WITH_BROTLI=ON"
"-DARROW_WITH_BZ2=ON"
"-DARROW_WITH_LZ4=ON"
"-DARROW_WITH_NLOHMANN_JSON=ON"
"-DARROW_WITH_SNAPPY=ON"
"-DARROW_WITH_UTF8PROC=ON"
"-DARROW_WITH_ZLIB=ON"
"-DARROW_WITH_ZSTD=ON"
"-DARROW_MIMALLOC=ON"
"-DARROW_SUBSTRAIT=ON"
"-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}"
"-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}"
"-DARROW_S3=${if enableS3 then "ON" else "OFF"}"
"-DARROW_GCS=${if enableGcs then "ON" else "OFF"}"
# Parquet options:
"-DARROW_PARQUET=ON"
"-DPARQUET_BUILD_EXECUTABLES=ON"
"-DPARQUET_REQUIRE_ENCRYPTION=ON"
]
++ lib.optionals (!enableShared) [ "-DARROW_TEST_LINKAGE=static" ]
++ lib.optionals stdenv.hostPlatform.isDarwin [
"-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables
]
++ lib.optionals (!stdenv.hostPlatform.isx86_64) [ "-DARROW_USE_SIMD=OFF" ]
++ lib.optionals enableS3 [
"-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h"
];
doInstallCheck = true;
ARROW_TEST_DATA = lib.optionalString finalAttrs.doInstallCheck "${arrow-testing}/data";
PARQUET_TEST_DATA = lib.optionalString finalAttrs.doInstallCheck "${parquet-testing}/data";
GTEST_FILTER =
let
# Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398
filteredTests =
lib.optionals stdenv.hostPlatform.isAarch64 [
"TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
"TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
"TestCompareKernel.PrimitiveRandomTests"
]
++ lib.optionals enableS3 [
"S3OptionsTest.FromUri"
"S3RegionResolutionTest.NonExistentBucket"
"S3RegionResolutionTest.PublicBucket"
"S3RegionResolutionTest.RestrictedBucket"
"TestMinioServer.Connect"
"TestS3FS.*"
"TestS3FSGeneric.*"
]
++ lib.optionals stdenv.hostPlatform.isDarwin [
# TODO: revisit at 12.0.0 or when
# https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989
# is available, see
# https://github.com/apache/arrow/pull/15288#discussion_r1071244661
"ExecPlanExecution.StressSourceSinkStopped"
];
in
lib.optionalString finalAttrs.doInstallCheck "-${lib.concatStringsSep ":" filteredTests}";
__darwinAllowLocalNetworking = true;
nativeInstallCheckInputs = [
perl
which
sqlite
]
++ lib.optionals enableS3 [ minio ]
++ lib.optionals enableFlight [ python3 ];
installCheckPhase =
let
disabledTests = [
# flaky
"arrow-flight-test"
# requires networking
"arrow-gcsfs-test"
"arrow-flight-integration-test"
];
in
''
runHook preInstallCheck
ctest -L unittest --exclude-regex '^(${lib.concatStringsSep "|" disabledTests})$'
runHook postInstallCheck
'';
meta = with lib; {
description = "Cross-language development platform for in-memory data";
homepage = "https://arrow.apache.org/docs/cpp/";
license = licenses.asl20;
platforms = platforms.unix;
maintainers = with maintainers; [
tobim
veprbl
cpcloud
];
pkgConfigModules = [
"arrow"
"arrow-acero"
"arrow-compute"
"arrow-csv"
"arrow-dataset"
"arrow-filesystem"
"arrow-flight"
"arrow-flight-sql"
"arrow-flight-testing"
"arrow-json"
"arrow-substrait"
"arrow-testing"
"parquet"
];
};
passthru = {
inherit
enableFlight
enableJemalloc
enableS3
enableGcs
;
tests.pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage;
};
})

View File

@@ -0,0 +1,69 @@
Backported from <https://github.com/ceph/ceph/commit/857eedbe6c9ed80ed0625bd0aa27b1a1e85f8d59>.
Original author: Adam Emerson <aemerson@redhat.com>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bbd63a6a006..bbd7c737feb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -666,7 +666,7 @@ option(WITH_SYSTEM_BOOST "require and build with system Boost" OFF)
# Boost::thread depends on Boost::atomic, so list it explicitly.
set(BOOST_COMPONENTS
atomic chrono thread system regex random program_options date_time
- iostreams context coroutine)
+ iostreams context coroutine url)
set(BOOST_HEADER_COMPONENTS container)
if(WITH_MGR)
diff --git a/src/mds/BoostUrlImpl.cc b/src/mds/BoostUrlImpl.cc
deleted file mode 100644
index 479f4c6d75d..00000000000
--- a/src/mds/BoostUrlImpl.cc
+++ /dev/null
@@ -1,8 +0,0 @@
-/*
- * https://www.boost.org/doc/libs/1_82_0/libs/url/doc/html/url/overview.html#url.overview.requirements
- *
- * To use the library as header-only; that is, to eliminate the requirement
- * to link a program to a static or dynamic Boost.URL library,
- * simply place the following line in exactly one source file in your project.
- */
-#include <boost/url/src.hpp>
diff --git a/src/mds/CMakeLists.txt b/src/mds/CMakeLists.txt
index 0c6c31a3c51..5c98db76e4d 100644
--- a/src/mds/CMakeLists.txt
+++ b/src/mds/CMakeLists.txt
@@ -45,12 +45,12 @@ set(mds_srcs
QuiesceDbManager.cc
QuiesceAgent.cc
MDSRankQuiesce.cc
- BoostUrlImpl.cc
${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
${CMAKE_SOURCE_DIR}/src/common/MemoryModel.cc
${CMAKE_SOURCE_DIR}/src/osdc/Journaler.cc
${CMAKE_SOURCE_DIR}/src/mgr/MDSPerfMetricTypes.cc)
add_library(mds STATIC ${mds_srcs})
target_link_libraries(mds PRIVATE
+ Boost::url
heap_profiler cpu_profiler osdc ${LUA_LIBRARIES})
target_include_directories(mds PRIVATE "${LUA_INCLUDE_DIR}")
diff --git a/src/test/mds/CMakeLists.txt b/src/test/mds/CMakeLists.txt
index f80abe75083..18ebb648e68 100644
--- a/src/test/mds/CMakeLists.txt
+++ b/src/test/mds/CMakeLists.txt
@@ -18,11 +18,10 @@ target_link_libraries(unittest_mds_sessionfilter mds osdc ceph-common global ${B
add_executable(unittest_mds_quiesce_db
TestQuiesceDb.cc
../../../src/mds/QuiesceDbManager.cc
- ../../../src/mds/BoostUrlImpl.cc
$<TARGET_OBJECTS:unit-main>
)
add_ceph_unittest(unittest_mds_quiesce_db)
-target_link_libraries(unittest_mds_quiesce_db ceph-common global)
+target_link_libraries(unittest_mds_quiesce_db ceph-common global Boost::url)
# unittest_mds_quiesce_agent
add_executable(unittest_mds_quiesce_agent
--
2.47.0

View File

@@ -0,0 +1,20 @@
Excerpted from <https://aur.archlinux.org/cgit/aur.git/commit/?h=ceph&id=8c5cc7d8deec002f7596b6d0860859a0a718f12b>.
Original author: Bazaah <github@luxolus.com>
diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc
index 084cf3ffc1e..010a1177a88 100644
--- a/src/mgr/PyModule.cc
+++ b/src/mgr/PyModule.cc
@@ -36,6 +36,11 @@ std::string PyModule::mgr_store_prefix = "mgr/";
// Courtesy of http://stackoverflow.com/questions/1418015/how-to-get-python-exception-text
#define BOOST_BIND_GLOBAL_PLACEHOLDERS
+// Fix instances of "'BOOST_PP_ITERATION_02' was not declared in this scope; did you mean 'BOOST_PP_ITERATION_05'"
+// and related macro error bullshit that spans 300 lines of errors
+//
+// Apparently you can't include boost/python stuff _and_ have this header defined
+#undef BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
// Boost apparently can't be bothered to fix its own usage of its own
// deprecated features.
#include <boost/python/extract.hpp>

View File

@@ -0,0 +1,621 @@
{
lib,
stdenv,
runCommand,
fetchurl,
fetchFromGitHub,
fetchPypi,
fetchpatch2,
# Build time
autoconf,
automake,
cmake,
ensureNewerSourcesHook,
fmt,
git,
libtool,
makeWrapper,
nasm,
pkg-config,
which,
openssl,
# Tests
nixosTests,
# Runtime dependencies
arrow-cpp,
babeltrace,
# Note when trying to upgrade boost:
# * When upgrading Ceph, it's recommended to check which boost version Ceph uses on Fedora,
# and default to that.
# * The version that Ceph downloads if `-DWITH_SYSTEM_BOOST:BOOL=ON` is not given
# is declared in `cmake/modules/BuildBoost.cmake` line `set(boost_version ...)`.
#
# If you want to upgrade to boost >= 1.86, you need a Ceph version that
# has this PR in:
# https://github.com/ceph/ceph/pull/61312
boost183,
bzip2,
cryptsetup,
cunit,
e2fsprogs,
doxygen,
gperf,
graphviz,
gnugrep,
gtest,
icu,
kmod,
libcap,
libcap_ng,
libnbd,
libnl,
libxml2,
lmdb,
lttng-ust,
lua,
lvm2,
lz4,
oath-toolkit,
openldap,
parted,
python311, # to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable)
rdkafka,
rocksdb,
snappy,
openssh,
sqlite,
utf8proc,
xfsprogs,
zlib,
zstd,
# Dependencies of overridden Python dependencies, hopefully we can remove these soon.
rustPlatform,
# Optional Dependencies
curl ? null,
expat ? null,
fuse ? null,
libatomic_ops ? null,
libedit ? null,
libs3 ? null,
yasm ? null,
# Mallocs
gperftools ? null,
jemalloc ? null,
# Crypto Dependencies
cryptopp ? null,
nspr ? null,
nss ? null,
# Linux Only Dependencies
linuxHeaders,
util-linux,
libuuid,
udev,
keyutils,
rdma-core,
rabbitmq-c,
libaio ? null,
libxfs ? null,
liburing ? null,
zfs ? null,
...
}:
# We must have one crypto library
assert cryptopp != null || (nss != null && nspr != null);
let
shouldUsePkg =
pkg: if pkg != null && lib.meta.availableOn stdenv.hostPlatform pkg then pkg else null;
optYasm = shouldUsePkg yasm;
optExpat = shouldUsePkg expat;
optCurl = shouldUsePkg curl;
optFuse = shouldUsePkg fuse;
optLibedit = shouldUsePkg libedit;
optLibatomic_ops = shouldUsePkg libatomic_ops;
optLibs3 = shouldUsePkg libs3;
optJemalloc = shouldUsePkg jemalloc;
optGperftools = shouldUsePkg gperftools;
optCryptopp = shouldUsePkg cryptopp;
optNss = shouldUsePkg nss;
optNspr = shouldUsePkg nspr;
optLibaio = shouldUsePkg libaio;
optLibxfs = shouldUsePkg libxfs;
optZfs = shouldUsePkg zfs;
# Downgrade rocksdb, 7.10 breaks ceph
rocksdb' = rocksdb.overrideAttrs {
version = "7.9.2";
src = fetchFromGitHub {
owner = "facebook";
repo = "rocksdb";
rev = "refs/tags/v7.9.2";
hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
};
};
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
# Malloc implementation (can be jemalloc, tcmalloc or null)
malloc = if optJemalloc != null then optJemalloc else optGperftools;
# We prefer nss over cryptopp
cryptoStr =
if optNss != null && optNspr != null then
"nss"
else if optCryptopp != null then
"cryptopp"
else
"none";
cryptoLibsMap = {
nss = [
optNss
optNspr
];
cryptopp = [ optCryptopp ];
none = [ ];
};
getMeta = description: {
homepage = "https://ceph.io/en/";
inherit description;
license = with lib.licenses; [
lgpl21
gpl2Only
bsd3
mit
publicDomain
];
maintainers = with lib.maintainers; [
adev
ak
johanot
krav
nh2
benaryorg
];
platforms = [
"x86_64-linux"
"aarch64-linux"
];
};
ceph-common =
with python.pkgs;
buildPythonPackage {
pname = "ceph-common";
format = "setuptools";
inherit src version;
sourceRoot = "ceph-${version}/src/python-common";
propagatedBuildInputs = [
pyyaml
];
nativeCheckInputs = [
pytestCheckHook
];
disabledTests = [
# requires network access
"test_valid_addr"
];
meta = getMeta "Ceph common module for code shared by manager modules";
};
# Watch out for python <> boost compatibility
python = python311.override {
self = python;
packageOverrides =
self: super:
let
bcryptOverrideVersion = "4.0.1";
in
{
# Ceph does not support the following yet:
# * `bcrypt` > 4.0
# * `cryptography` > 40
# See:
# * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
# * Upstream issue: https://tracker.ceph.com/issues/63529
# > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
# * Moved to issue: https://tracker.ceph.com/issues/64213
# > MGR modules incompatible with later PyO3 versions - PyO3 modules may only be initialized once per interpreter process
bcrypt = super.bcrypt.overridePythonAttrs (old: rec {
pname = "bcrypt";
version = bcryptOverrideVersion;
src = fetchPypi {
inherit pname version;
hash = "sha256-J9N1kDrIJhz+QEf2cJ0W99GNObHskqr3KvmJVSplDr0=";
};
cargoRoot = "src/_bcrypt";
cargoDeps = rustPlatform.fetchCargoVendor {
inherit
pname
version
src
cargoRoot
;
hash = "sha256-8PyCgh/rUO8uynzGdgylAsb5k55dP9fCnf40UOTCR/M=";
};
});
# We pin the older `cryptography` 40 here;
# this also forces us to pin other packages, see below
cryptography = self.callPackage ./old-python-packages/cryptography.nix { };
# This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
# See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
# and https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec {
version = "23.1.1";
src = fetchPypi {
pname = "pyOpenSSL";
inherit version;
hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
};
disabledTests = old.disabledTests or [ ] ++ [
"test_export_md5_digest"
];
disabledTestPaths = old.disabledTestPaths or [ ] ++ [
"tests/test_ssl.py"
];
propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [
self.flaky
];
# hack: avoid building docs due to incompatibility with current sphinx
nativeBuildInputs = [ openssl ]; # old.nativeBuildInputs but without sphinx*
outputs = lib.filter (o: o != "doc") old.outputs;
});
# This is the most recent version of `trustme` that's still compatible with `cryptography` 40.
# See https://github.com/NixOS/nixpkgs/issues/359723
# and https://github.com/python-trio/trustme/commit/586f7759d5c27beb44da60615a71848eb2a5a490
trustme = self.callPackage ./old-python-packages/trustme.nix { };
fastapi = super.fastapi.overridePythonAttrs (old: {
# Flaky test:
# ResourceWarning: Unclosed <MemoryObjectSendStream>
# Unclear whether it's flaky in general or only in this overridden package set.
doCheck = false;
});
# Ceph does not support `kubernetes` >= 19, see:
# https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
version = "18.20.0";
src = fetchFromGitHub {
owner = "kubernetes-client";
repo = "python";
rev = "v${version}";
sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
fetchSubmodules = true;
};
});
};
};
boost' = boost183.override {
enablePython = true;
inherit python;
};
# TODO: split this off in build and runtime environment
ceph-python-env = python.withPackages (
ps: with ps; [
ceph-common
# build time
cython_0
# debian/control
bcrypt
cherrypy
influxdb
jinja2
kubernetes
natsort
numpy
pecan
prettytable
pyjwt
pyopenssl
python-dateutil
pyyaml
requests
routes
scikit-learn
scipy
setuptools
sphinx
virtualenv
werkzeug
# src/cephadm/zipapp-reqs.txt
markupsafe
# src/pybind/mgr/requirements-required.txt
cryptography
jsonpatch
# src/tools/cephfs/shell/setup.py
cmd2
colorama
]
);
inherit (ceph-python-env.python) sitePackages;
version = "19.2.3";
src = fetchurl {
url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
hash = "sha256-zlgp28C81SZbaFJ4yvQk4ZgYz4K/aZqtcISTO8LscSU=";
};
in
rec {
ceph = stdenv.mkDerivation {
pname = "ceph";
inherit src version;
patches = [
./boost-1.85.patch
(fetchpatch2 {
name = "ceph-boost-1.86-uuid.patch";
url = "https://github.com/ceph/ceph/commit/01306208eac492ee0e67bff143fc32d0551a2a6f.patch?full_index=1";
hash = "sha256-OnDrr72inzGXXYxPFQevsRZImSvI0uuqFHqtFU2dPQE=";
})
# See:
# * <https://github.com/boostorg/python/issues/394>
# * <https://aur.archlinux.org/cgit/aur.git/commit/?h=ceph&id=8c5cc7d8deec002f7596b6d0860859a0a718f12b>
# * <https://github.com/ceph/ceph/pull/60999>
./boost-1.86-PyModule.patch
];
nativeBuildInputs = [
autoconf # `autoreconf` is called, e.g. for `qatlib_ext`
automake # `aclocal` is called, e.g. for `qatlib_ext`
cmake
fmt
git
makeWrapper
libtool # used e.g. for `qatlib_ext`
nasm
pkg-config
python
python.pkgs.python # for the toPythonPath function
python.pkgs.wrapPython
which
(ensureNewerSourcesHook { year = "1980"; })
# for building docs/man-pages presumably
doxygen
graphviz
];
buildInputs =
cryptoLibsMap.${cryptoStr}
++ [
arrow-cpp
babeltrace
boost'
bzip2
# Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build.
# This is important, otherwise the build system may not find the Python deps and then
# silently skip installing ceph-volume and other Ceph python tools.
ceph-python-env
cryptsetup
cunit
e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
gperf
gtest
icu
libcap
libnbd
libnl
libxml2
lmdb
lttng-ust
lua
lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py`
lz4
malloc
oath-toolkit
openldap
optLibatomic_ops
optLibs3
optYasm
parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py`
rdkafka
rocksdb'
snappy
openssh # according to `debian/control` file, `ssh` command used by `cephadm`
sqlite
utf8proc
xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
zlib
zstd
]
++ lib.optionals stdenv.hostPlatform.isLinux [
keyutils
libcap_ng
liburing
libuuid
linuxHeaders
optLibaio
optLibxfs
optZfs
rabbitmq-c
rdma-core
udev
util-linux
]
++ lib.optionals hasRadosgw [
optCurl
optExpat
optFuse
optLibedit
];
# Picked up, amongst others, by `wrapPythonPrograms`.
pythonPath = [
ceph-python-env
"${placeholder "out"}/${ceph-python-env.sitePackages}"
];
# * `unset AS` because otherwise the Ceph CMake build errors with
# configure: error: No modern nasm or yasm found as required. Nasm should be v2.11.01 or later (v2.13 for AVX512) and yasm should be 1.2.0 or later.
# because the code at
# https://github.com/intel/isa-l/blob/633add1b569fe927bace3960d7c84ed9c1b38bb9/configure.ac#L99-L191
# doesn't even consider using `nasm` or `yasm` but instead uses `$AS`
# from `gcc-wrapper`.
# (Ceph's error message is extra confusing, because it says
# `No modern nasm or yasm found` when in fact it found e.g. `nasm`
# but then uses `$AS` instead.
# * replace /sbin and /bin based paths with direct nix store paths
# * increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters
preConfigure = ''
unset AS
substituteInPlace src/common/module.c \
--replace "char command[128];" "char command[256];" \
--replace "/sbin/modinfo" "${kmod}/bin/modinfo" \
--replace "/sbin/modprobe" "${kmod}/bin/modprobe" \
--replace "/bin/grep" "${gnugrep}/bin/grep"
# Patch remount to use full path to mount(8), otherwise ceph-fuse fails when run
# from a systemd unit for example.
substituteInPlace src/client/fuse_ll.cc \
--replace-fail "mount -i -o remount" "${util-linux}/bin/mount -i -o remount"
# The install target needs to be in PYTHONPATH for "*.pth support" check to succeed
export PYTHONPATH=$PYTHONPATH:$lib/${sitePackages}:$out/${sitePackages}
patchShebangs src/
'';
cmakeFlags = [
"-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
"-DWITH_CEPHFS_SHELL:BOOL=ON"
"-DWITH_SYSTEMD:BOOL=OFF"
# `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
# CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
# Command failed: 2
#
# 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
#
# See also
#
# /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
# and that file contains:
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
# 71 | ::THRIFT_CLOSESOCKET(socket_);
# | ^~~~~~~~~~~~~~~~~~
# Looks like `close()` is somehow not included.
# But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
# So it's probably not worth trying to fix that for this Ceph version,
# and instead just disable Ceph's Jaeger support.
"-DWITH_JAEGER:BOOL=OFF"
"-DWITH_TESTS:BOOL=OFF"
# Use our own libraries, where possible
"-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
"-DWITH_SYSTEM_BOOST:BOOL=ON"
"-DWITH_SYSTEM_GTEST:BOOL=ON"
"-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
"-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
"-DWITH_SYSTEM_ZSTD:BOOL=ON"
# Use our own python libraries too, see:
# https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329
"-DCEPHADM_BUNDLED_DEPENDENCIES=none"
# TODO breaks with sandbox, tries to download stuff with npm
"-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
# WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
]
++ lib.optional stdenv.hostPlatform.isLinux "-DWITH_SYSTEM_LIBURING=ON";
preBuild =
# The legacy-option-headers target is not correctly empbedded in the build graph.
# It also contains some internal race conditions that we work around by building with `-j 1`.
# Upstream discussion for additional context at https://tracker.ceph.com/issues/63402.
''
cmake --build . --target legacy-option-headers -j 1
'';
postFixup = ''
wrapPythonPrograms
wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
# Test that ceph-volume exists since the build system has a tendency to
# silently drop it with misconfigurations.
test -f $out/bin/ceph-volume
'';
outputs = [
"out"
"lib"
"dev"
"doc"
"man"
];
doCheck = false; # uses pip to install things from the internet
# Takes 7+h to build with 2 cores.
requiredSystemFeatures = [ "big-parallel" ];
meta = getMeta "Distributed storage system";
passthru = {
inherit version;
inherit python; # to be able to test our overridden packages above individually with `nix-build -A`
tests = {
inherit (nixosTests)
ceph-multi-node
ceph-single-node
ceph-single-node-bluestore
ceph-single-node-bluestore-dmcrypt
;
};
};
};
ceph-client =
runCommand "ceph-client-${version}"
{
meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
}
''
mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
cp -r ${ceph}/bin/rbd-replay* $out/bin
cp -r ${ceph}/sbin/mount.ceph $out/bin
cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
ln -s bin $out/sbin
cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
# wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
substituteInPlace $out/bin/ceph --replace ${ceph} $out
substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
'';
}

View File

@@ -0,0 +1,36 @@
# This older version only exists because `ceph` needs it, see `cryptography.nix`.
{
buildPythonPackage,
fetchPypi,
lib,
cryptography,
}:
buildPythonPackage rec {
pname = "cryptography-vectors";
# The test vectors must have the same version as the cryptography package
inherit (cryptography) version;
format = "setuptools";
src = fetchPypi {
pname = "cryptography_vectors";
inherit version;
hash = "sha256-hGBwa1tdDOSoVXHKM4nPiPcAu2oMYTPcn+D1ovW9oEE=";
};
# No tests included
doCheck = false;
pythonImportsCheck = [ "cryptography_vectors" ];
meta = with lib; {
description = "Test vectors for the cryptography package";
homepage = "https://cryptography.io/en/latest/development/test-vectors/";
# Source: https://github.com/pyca/cryptography/tree/master/vectors;
license = with licenses; [
asl20
bsd3
];
maintainers = with maintainers; [ nh2 ];
};
}

View File

@@ -0,0 +1,138 @@
# This older version only exists because `ceph` needs it, see its package.
{
lib,
stdenv,
callPackage,
buildPythonPackage,
fetchPypi,
fetchpatch,
rustPlatform,
cargo,
rustc,
setuptoolsRustBuildHook,
openssl,
Security ? null,
isPyPy,
cffi,
pkg-config,
pytestCheckHook,
pytest-subtests,
pythonOlder,
pretend,
libiconv,
libxcrypt,
iso8601,
py,
pytz,
hypothesis,
}:
let
cryptography-vectors = callPackage ./cryptography-vectors.nix { };
in
buildPythonPackage rec {
pname = "cryptography";
version = "40.0.1"; # Also update the hash in vectors.nix
format = "setuptools";
disabled = pythonOlder "3.6";
src = fetchPypi {
inherit pname version;
hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
};
cargoDeps = rustPlatform.fetchCargoVendor {
inherit
pname
version
src
cargoRoot
;
hash = "sha256-pZHu3Oo9DWRAtldU0UvrH1FIg0bEvyfizPUhj9IBL58=";
};
# Since Cryptography v40 is quite outdated, we need to backport
# security fixes that are only available in newer versions.
patches = [
# Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
# See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
# Fix https://nvd.nist.gov/vuln/detail/CVE-2024-26130
# See https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55
(fetchpatch {
name = "python-cryptography-CVE-2024-26130-dont-crash-when-a-PKCS-12-key-and-cert-dont-match-mmap-mode.patch";
url = "https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55.patch";
hash = "sha256-l45NOzOWhHW4nY4OIRpdjYQRvUW8BROGWdpkAtvVn0Y=";
})
];
postPatch = ''
substituteInPlace pyproject.toml \
--replace "--benchmark-disable" ""
'';
cargoRoot = "src/rust";
nativeBuildInputs = [
rustPlatform.cargoSetupHook
setuptoolsRustBuildHook
cargo
rustc
pkg-config
]
++ lib.optionals (!isPyPy) [ cffi ];
buildInputs = [
openssl
]
++ lib.optionals stdenv.hostPlatform.isDarwin [
libiconv
]
++ lib.optionals (pythonOlder "3.9") [ libxcrypt ];
propagatedBuildInputs = lib.optionals (!isPyPy) [ cffi ];
nativeCheckInputs = [
cryptography-vectors
hypothesis
iso8601
pretend
py
pytestCheckHook
pytest-subtests
pytz
];
pytestFlags = [ "--disable-pytest-warnings" ];
disabledTestPaths = [
# save compute time by not running benchmarks
"tests/bench"
]
++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [
# aarch64-darwin forbids W+X memory, but this tests depends on it:
# * https://cffi.readthedocs.io/en/latest/using.html#callbacks
"tests/hazmat/backends/test_openssl_memleak.py"
];
meta = with lib; {
description = "Package which provides cryptographic recipes and primitives";
longDescription = ''
Cryptography includes both high level recipes and low level interfaces to
common cryptographic algorithms such as symmetric ciphers, message
digests, and key derivation functions.
Our goal is for it to be your "cryptographic standard library". It
supports Python 2.7, Python 3.5+, and PyPy 5.4+.
'';
homepage = "https://github.com/pyca/cryptography";
changelog =
"https://cryptography.io/en/latest/changelog/#v" + replaceStrings [ "." ] [ "-" ] version;
license = with licenses; [
asl20
bsd3
psfl
];
maintainers = with maintainers; [ nh2 ];
};
}

View File

@@ -0,0 +1,44 @@
commit 08dbad8552ecca85d3e38072f94eb864b143d218
Author: Niklas Hambüchen <mail@nh2.me>
Date: Mon Jan 22 12:52:06 2024 +0000
Cherry-pick fix for CVE-2023-49083
Cherry-Picked-From: f09c261ca10a31fe41b1262306db7f8f1da0e48a
Cherry-Picked-By: Niklas Hambüchen <mail@nh2.me>
diff --git a/src/cryptography/hazmat/backends/openssl/backend.py b/src/cryptography/hazmat/backends/openssl/backend.py
index a3fe1bce4..58e7207c4 100644
--- a/src/cryptography/hazmat/backends/openssl/backend.py
+++ b/src/cryptography/hazmat/backends/openssl/backend.py
@@ -2383,9 +2383,12 @@ class Backend:
_Reasons.UNSUPPORTED_SERIALIZATION,
)
+ certs: list[x509.Certificate] = []
+ if p7.d.sign == self._ffi.NULL:
+ return certs
+
sk_x509 = p7.d.sign.cert
num = self._lib.sk_X509_num(sk_x509)
- certs = []
for i in range(num):
x509 = self._lib.sk_X509_value(sk_x509, i)
self.openssl_assert(x509 != self._ffi.NULL)
diff --git a/tests/hazmat/primitives/test_pkcs7.py b/tests/hazmat/primitives/test_pkcs7.py
index 4e61c5ef5..d8170bfb9 100644
--- a/tests/hazmat/primitives/test_pkcs7.py
+++ b/tests/hazmat/primitives/test_pkcs7.py
@@ -89,6 +89,12 @@ class TestPKCS7Loading:
mode="rb",
)
+ def test_load_pkcs7_empty_certificates(self, backend):
+ der = b"\x30\x0B\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x02"
+
+ certificates = pkcs7.load_der_pkcs7_certificates(der)
+ assert certificates == []
+
# We have no public verification API and won't be adding one until we get
# some requirements from users so this function exists to give us basic

View File

@@ -0,0 +1,52 @@
# This older version only exists because `ceph` needs it, see its package.
{
lib,
buildPythonPackage,
cryptography,
fetchPypi,
idna,
pyopenssl,
pytestCheckHook,
pythonOlder,
service-identity,
}:
buildPythonPackage rec {
pname = "trustme";
version = "1.1.0";
format = "setuptools";
disabled = pythonOlder "3.8";
src = fetchPypi {
inherit pname version;
hash = "sha256-U3Wtf7QnB0vslWWS4NTuKkz02miTThukvPQhcSa8ReY=";
};
propagatedBuildInputs = [
cryptography
idna
];
nativeCheckInputs = [
pyopenssl
pytestCheckHook
service-identity
];
# Some of the tests use localhost networking.
__darwinAllowLocalNetworking = true;
pythonImportsCheck = [ "trustme" ];
meta = with lib; {
description = "High quality TLS certs while you wait, for the discerning tester";
homepage = "https://github.com/python-trio/trustme";
changelog = "https://trustme.readthedocs.io/en/latest/#change-history";
license = with licenses; [
mit
asl20
];
maintainers = with maintainers; [ jfly ];
};
}