571 lines
16 KiB
Nix
571 lines
16 KiB
Nix
{
|
|
lib,
|
|
stdenv,
|
|
runCommand,
|
|
fetchurl,
|
|
fetchFromGitHub,
|
|
fetchPypi,
|
|
fetchpatch2,
|
|
|
|
# Build time
|
|
autoconf,
|
|
automake,
|
|
cmake,
|
|
ensureNewerSourcesHook,
|
|
fmt,
|
|
git,
|
|
libtool,
|
|
makeWrapper,
|
|
nasm,
|
|
pkg-config,
|
|
which,
|
|
|
|
# Tests
|
|
nixosTests,
|
|
|
|
# Runtime dependencies
|
|
arrow-cpp,
|
|
babeltrace,
|
|
boost182, # using the version installed by ceph's `install-deps.sh`
|
|
bzip2,
|
|
cryptsetup,
|
|
cunit,
|
|
e2fsprogs,
|
|
doxygen,
|
|
gperf,
|
|
graphviz,
|
|
gnugrep,
|
|
gtest,
|
|
icu,
|
|
kmod,
|
|
libcap,
|
|
libcap_ng,
|
|
libnl,
|
|
libxml2,
|
|
lmdb,
|
|
lttng-ust,
|
|
lua,
|
|
lvm2,
|
|
lz4,
|
|
oath-toolkit,
|
|
openldap,
|
|
parted,
|
|
python311, # to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable)
|
|
rdkafka,
|
|
rocksdb,
|
|
snappy,
|
|
openssh,
|
|
sqlite,
|
|
utf8proc,
|
|
xfsprogs,
|
|
zlib,
|
|
zstd,
|
|
|
|
# Dependencies of overridden Python dependencies, hopefully we can remove these soon.
|
|
rustPlatform,
|
|
|
|
# Optional Dependencies
|
|
curl ? null,
|
|
expat ? null,
|
|
fuse ? null,
|
|
libatomic_ops ? null,
|
|
libedit ? null,
|
|
libs3 ? null,
|
|
yasm ? null,
|
|
|
|
# Mallocs
|
|
gperftools ? null,
|
|
jemalloc ? null,
|
|
|
|
# Crypto Dependencies
|
|
cryptopp ? null,
|
|
nspr ? null,
|
|
nss ? null,
|
|
|
|
# Linux Only Dependencies
|
|
linuxHeaders,
|
|
util-linux,
|
|
libuuid,
|
|
udev,
|
|
keyutils,
|
|
rdma-core,
|
|
rabbitmq-c,
|
|
libaio ? null,
|
|
libxfs ? null,
|
|
liburing ? null,
|
|
zfs ? null,
|
|
...
|
|
}:
|
|
|
|
# We must have one crypto library
|
|
assert cryptopp != null || (nss != null && nspr != null);
|
|
|
|
let
|
|
shouldUsePkg =
|
|
pkg: if pkg != null && lib.meta.availableOn stdenv.hostPlatform pkg then pkg else null;
|
|
|
|
optYasm = shouldUsePkg yasm;
|
|
optExpat = shouldUsePkg expat;
|
|
optCurl = shouldUsePkg curl;
|
|
optFuse = shouldUsePkg fuse;
|
|
optLibedit = shouldUsePkg libedit;
|
|
optLibatomic_ops = shouldUsePkg libatomic_ops;
|
|
optLibs3 = shouldUsePkg libs3;
|
|
|
|
optJemalloc = shouldUsePkg jemalloc;
|
|
optGperftools = shouldUsePkg gperftools;
|
|
|
|
optCryptopp = shouldUsePkg cryptopp;
|
|
optNss = shouldUsePkg nss;
|
|
optNspr = shouldUsePkg nspr;
|
|
|
|
optLibaio = shouldUsePkg libaio;
|
|
optLibxfs = shouldUsePkg libxfs;
|
|
optZfs = shouldUsePkg zfs;
|
|
|
|
# Downgrade rocksdb, 7.10 breaks ceph
|
|
rocksdb' = rocksdb.overrideAttrs {
|
|
version = "7.9.2";
|
|
src = fetchFromGitHub {
|
|
owner = "facebook";
|
|
repo = "rocksdb";
|
|
rev = "refs/tags/v7.9.2";
|
|
hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
|
|
};
|
|
};
|
|
|
|
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
|
|
|
|
# Malloc implementation (can be jemalloc, tcmalloc or null)
|
|
malloc = if optJemalloc != null then optJemalloc else optGperftools;
|
|
|
|
# We prefer nss over cryptopp
|
|
cryptoStr =
|
|
if optNss != null && optNspr != null then
|
|
"nss"
|
|
else if optCryptopp != null then
|
|
"cryptopp"
|
|
else
|
|
"none";
|
|
|
|
cryptoLibsMap = {
|
|
nss = [
|
|
optNss
|
|
optNspr
|
|
];
|
|
cryptopp = [ optCryptopp ];
|
|
none = [ ];
|
|
};
|
|
|
|
getMeta = description: {
|
|
homepage = "https://ceph.io/en/";
|
|
inherit description;
|
|
license = with lib.licenses; [
|
|
lgpl21
|
|
gpl2Only
|
|
bsd3
|
|
mit
|
|
publicDomain
|
|
];
|
|
maintainers = with lib.maintainers; [
|
|
adev
|
|
ak
|
|
johanot
|
|
krav
|
|
nh2
|
|
];
|
|
platforms = [
|
|
"x86_64-linux"
|
|
"aarch64-linux"
|
|
];
|
|
};
|
|
|
|
ceph-common =
|
|
with python.pkgs;
|
|
buildPythonPackage {
|
|
pname = "ceph-common";
|
|
inherit src version;
|
|
|
|
sourceRoot = "ceph-${version}/src/python-common";
|
|
|
|
propagatedBuildInputs = [
|
|
pyyaml
|
|
];
|
|
|
|
nativeCheckInputs = [
|
|
pytestCheckHook
|
|
];
|
|
|
|
disabledTests = [
|
|
# requires network access
|
|
"test_valid_addr"
|
|
];
|
|
|
|
meta = getMeta "Ceph common module for code shared by manager modules";
|
|
};
|
|
|
|
# Watch out for python <> boost compatibility
|
|
python = python311.override {
|
|
self = python;
|
|
packageOverrides =
|
|
self: super:
|
|
let
|
|
bcryptOverrideVersion = "4.0.1";
|
|
in
|
|
{
|
|
# Ceph does not support the following yet:
|
|
# * `bcrypt` > 4.0
|
|
# * `cryptography` > 40
|
|
# See:
|
|
# * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
|
|
# * Upstream issue: https://tracker.ceph.com/issues/63529
|
|
# > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
|
|
# * Moved to issue: https://tracker.ceph.com/issues/64213
|
|
# > MGR modules incompatible with later PyO3 versions - PyO3 modules may only be initialized once per interpreter process
|
|
|
|
bcrypt = super.bcrypt.overridePythonAttrs (old: rec {
|
|
pname = "bcrypt";
|
|
version = bcryptOverrideVersion;
|
|
src = fetchPypi {
|
|
inherit pname version;
|
|
hash = "sha256-J9N1kDrIJhz+QEf2cJ0W99GNObHskqr3KvmJVSplDr0=";
|
|
};
|
|
cargoRoot = "src/_bcrypt";
|
|
cargoDeps = rustPlatform.fetchCargoTarball {
|
|
inherit src;
|
|
sourceRoot = "${pname}-${version}/${cargoRoot}";
|
|
name = "${pname}-${version}";
|
|
hash = "sha256-lDWX69YENZFMu7pyBmavUZaalGvFqbHSHfkwkzmDQaY=";
|
|
};
|
|
});
|
|
|
|
# We pin the older `cryptography` 40 here;
|
|
# this also forces us to pin an older `pyopenssl` because the current one
|
|
# is not compatible with older `cryptography`, see:
|
|
# https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
|
|
cryptography = self.callPackage ./old-python-packages/cryptography.nix { };
|
|
|
|
# This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
|
|
# See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
|
|
pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec {
|
|
version = "23.1.1";
|
|
src = fetchPypi {
|
|
pname = "pyOpenSSL";
|
|
inherit version;
|
|
hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
|
|
};
|
|
disabledTests = old.disabledTests or [ ] ++ [
|
|
"test_export_md5_digest"
|
|
];
|
|
propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [
|
|
self.flaky
|
|
];
|
|
});
|
|
|
|
fastapi = super.fastapi.overridePythonAttrs (old: rec {
|
|
# Flaky test:
|
|
# ResourceWarning: Unclosed <MemoryObjectSendStream>
|
|
# Unclear whether it's flaky in general or only in this overridden package set.
|
|
doCheck = false;
|
|
});
|
|
|
|
# Ceph does not support `kubernetes` >= 19, see:
|
|
# https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
|
|
kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
|
|
version = "18.20.0";
|
|
src = fetchFromGitHub {
|
|
owner = "kubernetes-client";
|
|
repo = "python";
|
|
rev = "v${version}";
|
|
sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
|
|
fetchSubmodules = true;
|
|
};
|
|
});
|
|
|
|
};
|
|
};
|
|
|
|
boost = boost182.override {
|
|
enablePython = true;
|
|
inherit python;
|
|
};
|
|
|
|
# TODO: split this off in build and runtime environment
|
|
ceph-python-env = python.withPackages (
|
|
ps: with ps; [
|
|
ceph-common
|
|
|
|
# build time
|
|
cython_0
|
|
|
|
# debian/control
|
|
bcrypt
|
|
cherrypy
|
|
influxdb
|
|
jinja2
|
|
kubernetes
|
|
natsort
|
|
numpy
|
|
pecan
|
|
prettytable
|
|
pyjwt
|
|
pyopenssl
|
|
python-dateutil
|
|
pyyaml
|
|
requests
|
|
routes
|
|
scikit-learn
|
|
scipy
|
|
setuptools
|
|
sphinx
|
|
virtualenv
|
|
werkzeug
|
|
|
|
# src/cephadm/zipapp-reqs.txt
|
|
markupsafe
|
|
|
|
# src/pybind/mgr/requirements-required.txt
|
|
cryptography
|
|
jsonpatch
|
|
|
|
# src/tools/cephfs/shell/setup.py
|
|
cmd2
|
|
colorama
|
|
]
|
|
);
|
|
inherit (ceph-python-env.python) sitePackages;
|
|
|
|
version = "19.2.0";
|
|
src = fetchurl {
|
|
url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
|
|
hash = "sha256-30vkW1j49hFIxyxzkssSKVSq0VqiwLfDtOb62xfxadM=";
|
|
};
|
|
in
|
|
rec {
|
|
ceph = stdenv.mkDerivation {
|
|
pname = "ceph";
|
|
inherit src version;
|
|
|
|
patches = [
|
|
(fetchpatch2 {
|
|
name = "ceph-s3select-arrow-18-compat.patch";
|
|
url = "https://github.com/ceph/s3select/commit/f333ec82e6e8a3f7eb9ba1041d1442b2c7cd0f05.patch";
|
|
hash = "sha256-21fi5tMIs/JmuhwPYMWtampv/aqAe+EoPAXZLJlOvgo=";
|
|
stripLen = 1;
|
|
extraPrefix = "src/s3select/";
|
|
})
|
|
];
|
|
|
|
nativeBuildInputs = [
|
|
autoconf # `autoreconf` is called, e.g. for `qatlib_ext`
|
|
automake # `aclocal` is called, e.g. for `qatlib_ext`
|
|
cmake
|
|
fmt
|
|
git
|
|
makeWrapper
|
|
libtool # used e.g. for `qatlib_ext`
|
|
nasm
|
|
pkg-config
|
|
python
|
|
python.pkgs.python # for the toPythonPath function
|
|
python.pkgs.wrapPython
|
|
which
|
|
(ensureNewerSourcesHook { year = "1980"; })
|
|
# for building docs/man-pages presumably
|
|
doxygen
|
|
graphviz
|
|
];
|
|
|
|
buildInputs =
|
|
cryptoLibsMap.${cryptoStr}
|
|
++ [
|
|
arrow-cpp
|
|
babeltrace
|
|
boost
|
|
bzip2
|
|
# Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build.
|
|
# This is important, otherwise the build system may not find the Python deps and then
|
|
# silently skip installing ceph-volume and other Ceph python tools.
|
|
ceph-python-env
|
|
cryptsetup
|
|
cunit
|
|
e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
|
gperf
|
|
gtest
|
|
icu
|
|
libcap
|
|
libnl
|
|
libxml2
|
|
lmdb
|
|
lttng-ust
|
|
lua
|
|
lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py`
|
|
lz4
|
|
malloc
|
|
oath-toolkit
|
|
openldap
|
|
optLibatomic_ops
|
|
optLibs3
|
|
optYasm
|
|
parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py`
|
|
rdkafka
|
|
rocksdb'
|
|
snappy
|
|
openssh # according to `debian/control` file, `ssh` command used by `cephadm`
|
|
sqlite
|
|
utf8proc
|
|
xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
|
zlib
|
|
zstd
|
|
]
|
|
++ lib.optionals stdenv.hostPlatform.isLinux [
|
|
keyutils
|
|
libcap_ng
|
|
liburing
|
|
libuuid
|
|
linuxHeaders
|
|
optLibaio
|
|
optLibxfs
|
|
optZfs
|
|
rabbitmq-c
|
|
rdma-core
|
|
udev
|
|
util-linux
|
|
]
|
|
++ lib.optionals hasRadosgw [
|
|
optCurl
|
|
optExpat
|
|
optFuse
|
|
optLibedit
|
|
];
|
|
|
|
# Picked up, amongst others, by `wrapPythonPrograms`.
|
|
pythonPath = [
|
|
ceph-python-env
|
|
"${placeholder "out"}/${ceph-python-env.sitePackages}"
|
|
];
|
|
|
|
# replace /sbin and /bin based paths with direct nix store paths
|
|
# increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters
|
|
preConfigure = ''
|
|
substituteInPlace src/common/module.c \
|
|
--replace "char command[128];" "char command[256];" \
|
|
--replace "/sbin/modinfo" "${kmod}/bin/modinfo" \
|
|
--replace "/sbin/modprobe" "${kmod}/bin/modprobe" \
|
|
--replace "/bin/grep" "${gnugrep}/bin/grep"
|
|
|
|
# The install target needs to be in PYTHONPATH for "*.pth support" check to succeed
|
|
export PYTHONPATH=$PYTHONPATH:$lib/${sitePackages}:$out/${sitePackages}
|
|
patchShebangs src/
|
|
'';
|
|
|
|
cmakeFlags = [
|
|
"-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
|
|
|
|
"-DWITH_CEPHFS_SHELL:BOOL=ON"
|
|
"-DWITH_SYSTEMD:BOOL=OFF"
|
|
# `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
|
|
# CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
|
|
# Command failed: 2
|
|
#
|
|
# 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
|
|
#
|
|
# See also
|
|
#
|
|
# /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
|
|
# and that file contains:
|
|
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
|
|
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
|
|
# 71 | ::THRIFT_CLOSESOCKET(socket_);
|
|
# | ^~~~~~~~~~~~~~~~~~
|
|
# Looks like `close()` is somehow not included.
|
|
# But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
|
|
# So it's proably not worth trying to fix that for this Ceph version,
|
|
# and instead just disable Ceph's Jaeger support.
|
|
"-DWITH_JAEGER:BOOL=OFF"
|
|
"-DWITH_TESTS:BOOL=OFF"
|
|
|
|
# Use our own libraries, where possible
|
|
"-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
|
|
"-DWITH_SYSTEM_BOOST:BOOL=ON"
|
|
"-DWITH_SYSTEM_GTEST:BOOL=ON"
|
|
"-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
|
|
"-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
|
|
"-DWITH_SYSTEM_ZSTD:BOOL=ON"
|
|
|
|
# Use our own python libraries too, see:
|
|
# https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329
|
|
"-DCEPHADM_BUNDLED_DEPENDENCIES=none"
|
|
|
|
# TODO breaks with sandbox, tries to download stuff with npm
|
|
"-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
|
|
# WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
|
|
''-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}''
|
|
] ++ lib.optional stdenv.hostPlatform.isLinux "-DWITH_SYSTEM_LIBURING=ON";
|
|
|
|
preBuild =
|
|
# The legacy-option-headers target is not correctly empbedded in the build graph.
|
|
# It also contains some internal race conditions that we work around by building with `-j 1`.
|
|
# Upstream discussion for additional context at https://tracker.ceph.com/issues/63402.
|
|
''
|
|
cmake --build . --target legacy-option-headers -j 1
|
|
'';
|
|
|
|
postFixup = ''
|
|
wrapPythonPrograms
|
|
wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
|
|
|
|
# Test that ceph-volume exists since the build system has a tendency to
|
|
# silently drop it with misconfigurations.
|
|
test -f $out/bin/ceph-volume
|
|
'';
|
|
|
|
outputs = [
|
|
"out"
|
|
"lib"
|
|
"dev"
|
|
"doc"
|
|
"man"
|
|
];
|
|
|
|
doCheck = false; # uses pip to install things from the internet
|
|
|
|
# Takes 7+h to build with 2 cores.
|
|
requiredSystemFeatures = [ "big-parallel" ];
|
|
|
|
meta = getMeta "Distributed storage system";
|
|
|
|
passthru = {
|
|
inherit version;
|
|
inherit python; # to be able to test our overridden packages above individually with `nix-build -A`
|
|
tests = {
|
|
inherit (nixosTests)
|
|
ceph-multi-node
|
|
ceph-single-node
|
|
ceph-single-node-bluestore
|
|
ceph-single-node-bluestore-dmcrypt
|
|
;
|
|
};
|
|
};
|
|
};
|
|
|
|
ceph-client =
|
|
runCommand "ceph-client-${version}"
|
|
{
|
|
meta = getMeta "Tools needed to mount Ceph's RADOS Block Devices/Cephfs";
|
|
}
|
|
''
|
|
mkdir -p $out/{bin,etc,${sitePackages},share/bash-completion/completions}
|
|
cp -r ${ceph}/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $out/bin
|
|
cp -r ${ceph}/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $out/bin
|
|
cp -r ${ceph}/bin/rbd-replay* $out/bin
|
|
cp -r ${ceph}/sbin/mount.ceph $out/bin
|
|
cp -r ${ceph}/sbin/mount.fuse.ceph $out/bin
|
|
ln -s bin $out/sbin
|
|
cp -r ${ceph}/${sitePackages}/* $out/${sitePackages}
|
|
cp -r ${ceph}/etc/bash_completion.d $out/share/bash-completion/completions
|
|
# wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
|
|
substituteInPlace $out/bin/ceph --replace ${ceph} $out
|
|
substituteInPlace $out/bin/.ceph-wrapped --replace ${ceph} $out
|
|
'';
|
|
}
|