Added the custom ceph command build workflow

Signed-off-by: Yuji Ito <[email protected]>

.github/workflows/ceph-custom-test.yaml

@@ -0,0 +1,22 @@
+name: "ceph-custom-cmd-test"
+on:
+  pull_request:
+    paths:
+      - "ceph/**"
+  push:
+    paths:
+      - "ceph/**"
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+      - name: Build the custom ceph command
+        run: ./build.sh
+        working-directory: ceph
+      - name: Run test for the custom ceph command
+        run: ./test.sh
+        working-directory: ceph

.github/workflows/e2e-multiple-k8s-clusters.yaml

@@ -3,10 +3,12 @@ on:
   pull_request:
     paths-ignore:
       - "**/*.md"
+      - "ceph/**"
       - "CODEOWNERS"
   push:
     paths-ignore:
       - "**/*.md"
+      - "ceph/**"
       - "CODEOWNERS"
     branches:
       - "main"

.github/workflows/e2e.yaml

@@ -3,10 +3,12 @@ on:
   pull_request:
     paths-ignore:
       - "**/*.md"
+      - "ceph/**"
       - "CODEOWNERS"
   push:
     paths-ignore:
       - "**/*.md"
+      - "ceph/**"
       - "CODEOWNERS"
     branches:
       - "main"

ceph/TAG

@@ -0,0 +1,2 @@
+source: v18.2.4
+release: v18.2.4.1

ceph/build.sh

@@ -0,0 +1,62 @@
+#!/bin/sh
+set -eu
+
+CEPH_DIR=$(readlink -f $(dirname $0))
+
+if [ $# -ne 1 ]; then
+    echo "Usage: $0 VERSION"
+    exit 1
+fi
+
+VERSION="$1"
+
+# Checkout Ceph source
+mkdir -p src/workspace/rocksdb/
+cd src
+git clone -b v${VERSION} --depth=1 --recurse-submodules --shallow-submodules https://github.com/ceph/ceph.git
+cd ceph
+
+# Install dependencies
+apt-get update
+
+# Workaround for github actions runner.
+# Ceph depends on this library, but it is not automatically installed
+# because libraries that conflict with this library are installed.
+# Therefore, it should be installed explicitly.
+# See. https://github.com/actions/runner-images/issues/6399#issuecomment-1286050292
+apt-get install -y libunwind-dev
+
+apt-get install -y curl
+
+# Ceph v18.2.4 has a bug that causes the ceph-volume command to fail on the OSD prepare pod.
+# This bug was caused by the wrong way of installing the Python package.
+# As a workaround, the following patch is applied ahead of upstream.
+# https://github.com/ceph/ceph/commit/729fd8e25ff2bfbcf99790d6cd08489d1c4e2ede
+# Apply the packing-1.patch(Commit 80edcd4) simply because the changes depend on it.
+patch -p1 < ${CEPH_DIR}/packing-1.patch
+patch -p1 < ${CEPH_DIR}/packing-2.patch
+./install-deps.sh
+apt-get install -y python3-routes
+
+# Prebuild ceph source to generate files in `src/pybind/mgr/dashboard/frontend/dist` needed by CMake
+./make-dist
+
+# Build Ceph packages
+sed -i -e 's/WITH_CEPHFS_JAVA=ON/WITH_CEPHFS_JAVA=OFF/' debian/rules
+sed -i -e 's@usr/share/java/libcephfs-test.jar@@' debian/ceph-test.install
+# CMake in the self-build environment did not allow space-separated URLs.
+# As a workaround, the following patch is applied ahead of upstream.
+# https://github.com/ceph/ceph/commit/35435420781f84e9b71f72b10e6842a89c06de7f
+patch -p1 < ${CEPH_DIR}/boost-url.patch
+rm debian/libcephfs-java.jlibs debian/libcephfs-jni.install debian/ceph-mgr-dashboard*
+# To avoid OOM killer, use 10 parallelism instead of 20 (max vCPU).
+dpkg-buildpackage --build=binary -uc -us -j10
+rm ../*-dbg_*.deb ../ceph-test_*.deb
+mv ../*.deb ../workspace/
+mv COPYING* ../workspace
+
+# Intall libgflags to build rocksdb tools
+apt-get install --no-install-recommends -y libgflags-dev
+# Build rocksdb tools
+make -C src/rocksdb release -j10
+find src/rocksdb -maxdepth 1 -type f -executable -exec mv {} ../workspace/rocksdb \;

ceph/custom-export.patch

@@ -0,0 +1,196 @@
+--- a/src/tools/rbd/ArgumentTypes.h
++++ b/src/tools/rbd/ArgumentTypes.h
+@@ -55,6 +55,9 @@ static const std::string DEST_SNAPSHOT_NAME("dest-snap");
+ static const std::string PATH("path");
+ static const std::string FROM_SNAPSHOT_NAME("from-snap");
+ static const std::string WHOLE_OBJECT("whole-object");
++static const std::string READ_OFFSET("read-offset");
++static const std::string READ_LENGTH("read-length");
++static const std::string MID_SNAP_PREFIX("mid-snap-prefix");
+
+ // encryption arguments
+ static const std::string ENCRYPTION_FORMAT("encryption-format");
+diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
+index ddcf0f2c30cef..0569823e67a8f 100644
+--- a/src/tools/rbd/action/Export.cc
++++ b/src/tools/rbd/action/Export.cc
+@@ -123,8 +123,8 @@ class C_ExportDiff : public Context {
+
+
+ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
+-		   const char *endsnapname, bool whole_object,
+-		   int fd, bool no_progress, int export_format)
++		   const char *endsnapname, uint64_t offset, uint64_t length,
++       bool whole_object, int fd, bool no_progress, int export_format)
+ {
+   int r;
+   librbd::image_info_t info;
+@@ -133,6 +133,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
+   if (r < 0)
+     return r;
+
++  if (offset == 0 && length == 0) {
++    length = info.size;
++  }
++
+   {
+     // header
+     bufferlist bl;
+@@ -192,10 +196,10 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
+       return r;
+     }
+   }
+-  ExportDiffContext edc(&image, fd, info.size,
++  ExportDiffContext edc(&image, fd, length + offset,
+                         g_conf().get_val<uint64_t>("rbd_concurrent_management_ops"),
+                         no_progress, export_format);
+-  r = image.diff_iterate2(fromsnapname, 0, info.size, true, whole_object,
++  r = image.diff_iterate2(fromsnapname, offset, length, true, whole_object,
+                           &C_ExportDiff::export_diff_cb, (void *)&edc);
+   if (r < 0) {
+     goto out;
+@@ -223,8 +227,8 @@ int do_export_diff_fd(librbd::Image& image, const char *fromsnapname,
+ }
+
+ int do_export_diff(librbd::Image& image, const char *fromsnapname,
+-                const char *endsnapname, bool whole_object,
+-                const char *path, bool no_progress)
++                const char *endsnapname, uint64_t offset, uint64_t length,
++                bool whole_object, const char *path, bool no_progress)
+ {
+   int r;
+   int fd;
+@@ -236,7 +240,9 @@ int do_export_diff(librbd::Image& image, const char *fromsnapname,
+   if (fd < 0)
+     return -errno;
+
+-  r = do_export_diff_fd(image, fromsnapname, endsnapname, whole_object, fd, no_progress, 1);
++  r = do_export_diff_fd(image, fromsnapname, endsnapname,
++                        offset, length,
++                        whole_object, fd, no_progress, 1);
+
+   if (fd != 1)
+     close(fd);
+@@ -260,7 +266,11 @@ void get_arguments_diff(po::options_description *positional,
+   options->add_options()
+     (at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
+      "snapshot starting point")
+-    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object");
++    (at::WHOLE_OBJECT.c_str(), po::bool_switch(), "compare whole object")
++    (at::READ_OFFSET.c_str(), po::value<uint64_t>(), "offset in bytes")
++    (at::READ_LENGTH.c_str(), po::value<uint64_t>(), "length in bytes")
++    (at::MID_SNAP_PREFIX.c_str(), po::value<std::string>(),
++     "the prefix of snapshot name in output diff when specifying offset and length");
+   at::add_no_progress_option(options);
+ }
+
+@@ -290,6 +300,26 @@ int execute_diff(const po::variables_map &vm,
+     from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
+   }
+
++  if (vm.count(at::READ_OFFSET) != vm.count(at::READ_LENGTH)) {
++    std::cerr << "rbd: must specify both --read-offset and --read-length" << std::endl;
++    return -EINVAL;
++  }
++
++  uint64_t offset = 0;
++  if (vm.count(at::READ_OFFSET)) {
++    offset = vm[at::READ_OFFSET].as<uint64_t>();
++  }
++
++  uint64_t length = 0;
++  if (vm.count(at::READ_LENGTH)) {
++    length = vm[at::READ_LENGTH].as<uint64_t>();
++  }
++
++  std::string mid_snap_prefix("mid-snap");
++  if (vm.count(at::MID_SNAP_PREFIX)) {
++    mid_snap_prefix = vm[at::MID_SNAP_PREFIX].as<std::string>();
++  }
++
+   librados::Rados rados;
+   librados::IoCtx io_ctx;
+   librbd::Image image;
+@@ -299,9 +329,19 @@ int execute_diff(const po::variables_map &vm,
+     return r;
+   }
+
++  if (offset != 0 || length != 0) {
++    r = get_snapshot_name_for_offset_length(image, mid_snap_prefix,
++                                            &from_snap_name, &snap_name,
++                                            &offset, &length);
++    if (r < 0) {
++      return r;
++    }
++  }
++
+   r = do_export_diff(image,
+                      from_snap_name.empty() ? nullptr : from_snap_name.c_str(),
+                      snap_name.empty() ? nullptr : snap_name.c_str(),
++                     *offset, *length,
+                      vm[at::WHOLE_OBJECT].as<bool>(), path.c_str(),
+                      vm[at::NO_PROGRESS].as<bool>());
+   if (r < 0) {
+@@ -311,6 +351,44 @@ int execute_diff(const po::variables_map &vm,
+   return 0;
+ }
+
++int get_snapshot_name_for_offset_length(librbd::Image& image,
++                                        const std::string& mid_snap_prefix,
++                                        std::string* from_snap_name,
++                                        std::string* snap_name,
++                                        uint64_t* offset, uint64_t* length)
++{
++  int r;
++  librbd::image_info_t info;
++
++  r = image.stat(info, sizeof(info));
++  if (r < 0)
++    return r;
++  
++  // ⚠️ should be test when offset == info.size
++  if (*offset > info.size) {
++    std::cerr << "rbd: offset " << *offset << " exceeds image size "
++              << info.size << std::endl;
++    return -EINVAL;
++  }
++  
++  if (*offset > 0) {
++    *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset);
++  }
++
++  if (*length == 0) {
++    *length = info.size - *offset;
++    return 0;
++  }
++
++  if (*offset + *length < info.size) {
++    *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length);
++  } else {
++    *length = info.size - *offset;
++  }
++
++  return 0;
++}
++
+ Shell::Action action_diff(
+   {"export-diff"}, {}, "Export incremental diff to file.", "",
+   &get_arguments_diff, &execute_diff);
+@@ -501,7 +579,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
+   const char *last_snap = NULL;
+   for (size_t i = 0; i < snaps.size(); ++i) {
+     utils::snap_set(image, snaps[i].name.c_str());
+-    r = do_export_diff_fd(image, last_snap, snaps[i].name.c_str(), false, fd, true, 2);
++    r = do_export_diff_fd(image, last_snap, snaps[i].name.c_str(), 0, 0, false, fd, true, 2);
+     if (r < 0) {
+       return r;
+     }
+@@ -509,7 +587,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
+     last_snap = snaps[i].name.c_str();
+   }
+   utils::snap_set(image, std::string(""));
+-  r = do_export_diff_fd(image, last_snap, nullptr, false, fd, true, 2);
++  r = do_export_diff_fd(image, last_snap, nullptr, 0, 0, false, fd, true, 2);
+   if (r < 0) {
+     return r;
+   }

ceph/release.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+set -eu

ceph/test.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+set -eu