diff --git a/.github/workflows/ceph-custom-test.yaml b/.github/workflows/ceph-custom-test.yaml new file mode 100644 index 00000000..31de8ec7 --- /dev/null +++ b/.github/workflows/ceph-custom-test.yaml @@ -0,0 +1,22 @@ +name: "ceph-custom-cmd-test" +on: + pull_request: + paths: + - "ceph/**" + push: + paths: + - "ceph/**" +jobs: + test: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: "go.mod" + - name: Build the custom ceph command + run: ./build.sh + working-directory: ceph + - name: Run test for the custom ceph command + run: ./test.sh + working-directory: ceph diff --git a/.github/workflows/e2e-multiple-k8s-clusters.yaml b/.github/workflows/e2e-multiple-k8s-clusters.yaml index 7ee443f4..1fd1cf42 100644 --- a/.github/workflows/e2e-multiple-k8s-clusters.yaml +++ b/.github/workflows/e2e-multiple-k8s-clusters.yaml @@ -3,10 +3,12 @@ on: pull_request: paths-ignore: - "**/*.md" + - "ceph/**" - "CODEOWNERS" push: paths-ignore: - "**/*.md" + - "ceph/**" - "CODEOWNERS" branches: - "main" diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index a3d0cf9a..298296d6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3,10 +3,12 @@ on: pull_request: paths-ignore: - "**/*.md" + - "ceph/**" - "CODEOWNERS" push: paths-ignore: - "**/*.md" + - "ceph/**" - "CODEOWNERS" branches: - "main" diff --git a/ceph/TAG b/ceph/TAG new file mode 100644 index 00000000..b5345383 --- /dev/null +++ b/ceph/TAG @@ -0,0 +1,2 @@ +source: v18.2.4 +release: v18.2.4.1 diff --git a/ceph/build.sh b/ceph/build.sh new file mode 100644 index 00000000..0e02d269 --- /dev/null +++ b/ceph/build.sh @@ -0,0 +1,62 @@ +#!/bin/sh +set -eu + +CEPH_DIR=$(readlink -f $(dirname $0)) + +if [ $# -ne 1 ]; then + echo "Usage: $0 VERSION" + exit 1 +fi + +VERSION="$1" + +# Checkout Ceph source +mkdir -p src/workspace/rocksdb/ +cd src +git clone -b v${VERSION} --depth=1 --recurse-submodules --shallow-submodules https://github.com/ceph/ceph.git +cd ceph + +# Install dependencies +apt-get update + +# Workaround for github actions runner. +# Ceph depends on this library, but it is not automatically installed +# because libraries that conflict with this library are installed. +# Therefore, it should be installed explicitly. +# See. https://github.com/actions/runner-images/issues/6399#issuecomment-1286050292 +apt-get install -y libunwind-dev + +apt-get install -y curl + +# Ceph v18.2.4 has a bug that causes the ceph-volume command to fail on the OSD prepare pod. +# This bug was caused by the wrong way of installing the Python package. +# As a workaround, the following patch is applied ahead of upstream. +# https://github.com/ceph/ceph/commit/729fd8e25ff2bfbcf99790d6cd08489d1c4e2ede +# Apply the packing-1.patch(Commit 80edcd4) simply because the changes depend on it. +patch -p1 < ${CEPH_DIR}/packing-1.patch +patch -p1 < ${CEPH_DIR}/packing-2.patch +./install-deps.sh +apt-get install -y python3-routes + +# Prebuild ceph source to generate files in `src/pybind/mgr/dashboard/frontend/dist` needed by CMake +./make-dist + +# Build Ceph packages +sed -i -e 's/WITH_CEPHFS_JAVA=ON/WITH_CEPHFS_JAVA=OFF/' debian/rules +sed -i -e 's@usr/share/java/libcephfs-test.jar@@' debian/ceph-test.install +# CMake in the self-build environment did not allow space-separated URLs. +# As a workaround, the following patch is applied ahead of upstream. +# https://github.com/ceph/ceph/commit/35435420781f84e9b71f72b10e6842a89c06de7f +patch -p1 < ${CEPH_DIR}/boost-url.patch +rm debian/libcephfs-java.jlibs debian/libcephfs-jni.install debian/ceph-mgr-dashboard* +# To avoid OOM killer, use 10 parallelism instead of 20 (max vCPU). +dpkg-buildpackage --build=binary -uc -us -j10 +rm ../*-dbg_*.deb ../ceph-test_*.deb +mv ../*.deb ../workspace/ +mv COPYING* ../workspace + +# Intall libgflags to build rocksdb tools +apt-get install --no-install-recommends -y libgflags-dev +# Build rocksdb tools +make -C src/rocksdb release -j10 +find src/rocksdb -maxdepth 1 -type f -executable -exec mv {} ../workspace/rocksdb \; diff --git a/ceph/custom-export.patch b/ceph/custom-export.patch new file mode 100644 index 00000000..5176baad --- /dev/null +++ b/ceph/custom-export.patch @@ -0,0 +1,196 @@ +--- a/src/tools/rbd/ArgumentTypes.h ++++ b/src/tools/rbd/ArgumentTypes.h +@@ -55,6 +55,9 @@ static const std::string DEST_SNAPSHOT_NAME("dest-snap"); + static const std::string PATH("path"); + static const std::string FROM_SNAPSHOT_NAME("from-snap"); + static const std::string WHOLE_OBJECT("whole-object"); ++static const std::string READ_OFFSET("read-offset"); ++static const std::string READ_LENGTH("read-length"); ++static const std::string MID_SNAP_PREFIX("mid-snap-prefix"); + + // encryption arguments + static const std::string ENCRYPTION_FORMAT("encryption-format"); +diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc +index ddcf0f2c30cef..0569823e67a8f 100644 +--- a/src/tools/rbd/action/Export.cc ++++ b/src/tools/rbd/action/Export.cc +@@ -123,8 +123,8 @@ class C_ExportDiff : public Context { + + + int do_export_diff_fd(librbd::Image& image, const char *fromsnapname, +- const char *endsnapname, bool whole_object, +- int fd, bool no_progress, int export_format) ++ const char *endsnapname, uint64_t offset, uint64_t length, ++ bool whole_object, int fd, bool no_progress, int export_format) + { + int r; + librbd::image_info_t info; +@@ -133,6 +133,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname, + if (r < 0) + return r; + ++ if (offset == 0 && length == 0) { ++ length = info.size; ++ } ++ + { + // header + bufferlist bl; +@@ -192,10 +196,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname, + return r; + } + } +- ExportDiffContext edc(&image, fd, info.size, ++ ExportDiffContext edc(&image, fd, length + offset, + g_conf().get_val("rbd_concurrent_management_ops"), + no_progress, export_format); +- r = image.diff_iterate2(fromsnapname, 0, info.size, true, whole_object, ++ r = image.diff_iterate2(fromsnapname, offset, length, true, whole_object, + &C_ExportDiff::export_diff_cb, (void *)&edc); + if (r < 0) { + goto out; +@@ -223,8 +227,8 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname, + } + + int do_export_diff(librbd::Image& image, const char *fromsnapname, +- const char *endsnapname, bool whole_object, +- const char *path, bool no_progress) ++ const char *endsnapname, uint64_t offset, uint64_t length, ++ bool whole_object, const char *path, bool no_progress) + { + int r; + int fd; +@@ -236,7 +240,9 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname, + if (fd < 0) + return -errno; + +- r = do_export_diff_fd(image, fromsnapname, endsnapname, whole_object, fd, no_progress, 1); ++ r = do_export_diff_fd(image, fromsnapname, endsnapname, ++ offset, length, ++ whole_object, fd, no_progress, 1); + + if (fd != 1) + close(fd); +@@ -260,7 +266,11 @@ void get_arguments_diff(po::options_description *positional, + options->add_options() + (at::FROM_SNAPSHOT_NAME.c_str(), po::value(), + "snapshot starting point") +- (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object"); ++ (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object") ++ (at::READ_OFFSET.c_str(), po::value(), "offset in bytes") ++ (at::READ_LENGTH.c_str(), po::value(), "length in bytes") ++ (at::MID_SNAP_PREFIX.c_str(), po::value(), ++ "the prefix of snapshot name in output diff when specifying offset and length"); + at::add_no_progress_option(options); + } + +@@ -290,6 +300,26 @@ int execute_diff(const po::variables_map &vm, + from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as(); + } + ++ if (vm.count(at::READ_OFFSET) != vm.count(at::READ_LENGTH)) { ++ std::cerr << "rbd: must specify both --read-offset and --read-length" << std::endl; ++ return -EINVAL; ++ } ++ ++ uint64_t offset = 0; ++ if (vm.count(at::READ_OFFSET)) { ++ offset = vm[at::READ_OFFSET].as(); ++ } ++ ++ uint64_t length = 0; ++ if (vm.count(at::READ_LENGTH)) { ++ length = vm[at::READ_LENGTH].as(); ++ } ++ ++ std::string mid_snap_prefix("mid-snap"); ++ if (vm.count(at::MID_SNAP_PREFIX)) { ++ mid_snap_prefix = vm[at::MID_SNAP_PREFIX].as(); ++ } ++ + librados::Rados rados; + librados::IoCtx io_ctx; + librbd::Image image; +@@ -299,9 +329,19 @@ int execute_diff(const po::variables_map &vm, + return r; + } + ++ if (offset != 0 || length != 0) { ++ r = get_snapshot_name_for_offset_length(image, mid_snap_prefix, ++ &from_snap_name, &snap_name, ++ &offset, &length); ++ if (r < 0) { ++ return r; ++ } ++ } ++ + r = do_export_diff(image, + from_snap_name.empty() ? nullptr : from_snap_name.c_str(), + snap_name.empty() ? nullptr : snap_name.c_str(), ++ *offset, *length, + vm[at::WHOLE_OBJECT].as(), path.c_str(), + vm[at::NO_PROGRESS].as()); + if (r < 0) { +@@ -311,6 +351,44 @@ int execute_diff(const po::variables_map &vm, + return 0; + } + ++int get_snapshot_name_for_offset_length(librbd::Image& image, ++ const std::string& mid_snap_prefix, ++ std::string* from_snap_name, ++ std::string* snap_name, ++ uint64_t* offset, uint64_t* length) ++{ ++ int r; ++ librbd::image_info_t info; ++ ++ r = image.stat(info, sizeof(info)); ++ if (r < 0) ++ return r; ++ ++ // ⚠️ should be test when offset == info.size ++ if (*offset > info.size) { ++ std::cerr << "rbd: offset " << *offset << " exceeds image size " ++ << info.size << std::endl; ++ return -EINVAL; ++ } ++ ++ if (*offset > 0) { ++ *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset); ++ } ++ ++ if (*length == 0) { ++ *length = info.size - *offset; ++ return 0; ++ } ++ ++ if (*offset + *length < info.size) { ++ *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length); ++ } else { ++ *length = info.size - *offset; ++ } ++ ++ return 0; ++} ++ + Shell::Action action_diff( + {"export-diff"}, {}, "Export incremental diff to file.", "", + &get_arguments_diff, &execute_diff); +@@ -501,7 +579,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd + const char *last_snap = NULL; + for (size_t i = 0; i < snaps.size(); ++i) { + utils::snap_set(image, snaps[i].name.c_str()); +- r = do_export_diff_fd(image, last_snap, snaps[i].name.c_str(), false, fd, true, 2); ++ r = do_export_diff_fd(image, last_snap, snaps[i].name.c_str(), 0, 0, false, fd, true, 2); + if (r < 0) { + return r; + } +@@ -509,7 +587,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd + last_snap = snaps[i].name.c_str(); + } + utils::snap_set(image, std::string("")); +- r = do_export_diff_fd(image, last_snap, nullptr, false, fd, true, 2); ++ r = do_export_diff_fd(image, last_snap, nullptr, 0, 0, false, fd, true, 2); + if (r < 0) { + return r; + } diff --git a/ceph/release.sh b/ceph/release.sh new file mode 100644 index 00000000..92b70ea0 --- /dev/null +++ b/ceph/release.sh @@ -0,0 +1,2 @@ +#!/bin/sh +set -eu diff --git a/ceph/test.sh b/ceph/test.sh new file mode 100644 index 00000000..92b70ea0 --- /dev/null +++ b/ceph/test.sh @@ -0,0 +1,2 @@ +#!/bin/sh +set -eu