wip

Signed-off-by: Yuji Ito <[email protected]>

.gitignore

@@ -25,7 +25,9 @@ Dockerfile.cross
 *.swo
 *~
 
-ceph/src/
+ceph/go.mod
+ceph/go.sum
+ceph/packages/
 test/e2e/testdata/persistentvolumes.yaml
 test/e2e/testdata/values-mantle-primary.yaml
 test/e2e/testdata/values-mantle-secondary.yaml

ceph/Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/cybozu/ubuntu:22.04 AS builder
+FROM ghcr.io/cybozu/ubuntu:22.04 AS build-ceph
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 ENV DEBIAN_FRONTEND=noninteractive
@@ -49,3 +49,53 @@ RUN patch -p1 < ${WORKSPACE}/boost-url.patch
 RUN rm debian/libcephfs-java.jlibs debian/libcephfs-jni.install debian/ceph-mgr-dashboard*
 # To avoid OOM killer, use 10 parallelism instead of 20 (max vCPU).
 RUN dpkg-buildpackage --build=binary -uc -us -j10
+RUN rm ${WORKSPACE}/*-dbg_*.deb ${WORKSPACE}/ceph-test_*.deb
+
+FROM ghcr.io/cybozu/golang:1.22-jammy AS build-test
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV WORKSPACE=/workspace
+
+COPY go.mod go.sum ${WORKSPACE}/
+COPY test/ ${WORKSPACE}/test/
+
+WORKDIR ${WORKSPACE}
+RUN go test -o test.bin ./test/
+
+
+FROM ghcr.io/cybozu/ubuntu:22.04
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+ENV DEBIAN_FRONTEND=noninteractive
+ENV WORKSPACE=/workspace
+ENV PACKAGES=/packages
+
+# Since the package file is extracted outside the docker image at the end, keep the files.
+COPY --from=build-ceph ${WORKSPACE}/*.deb ${PACKAGES}/
+COPY --from=build-test ${WORKSPACE}/test.bin /test
+
+WORKDIR ${WORKSPACE}
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/usr/local/bin/kubectl"
+RUN mkdir -p /var/run/ceph && \
+    apt-get update && apt-get install --no-install-recommends -y \
+        libstdc++-11-dev jq kmod lvm2 gdisk ca-certificates e2fsprogs attr udev libgflags2.2 ${PACKAGES}/*.deb && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    sed -i -e 's/udev_rules = 1/udev_rules = 0/' -e 's/udev_sync = 1/udev_sync = 0/' -e 's/obtain_device_list_from_udev = 1/obtain_device_list_from_udev = 0/' /etc/lvm/lvm.conf && \
+    # validate the sed command worked as expected
+    grep -sqo "udev_sync = 0" /etc/lvm/lvm.conf && \
+    grep -sqo "udev_rules = 0" /etc/lvm/lvm.conf && \
+    grep -sqo "obtain_device_list_from_udev = 0" /etc/lvm/lvm.conf && \
+    # Clean common files like /tmp, /var/lib, etc.
+    rm -rf \
+        /etc/{selinux,systemd,udev} \
+        /lib/{lsb,udev} \
+        /tmp/* \
+        /usr/lib{,64}/{locale,systemd,udev,dracut} \
+        /usr/share/{doc,info,locale,man} \
+        /usr/share/{bash-completion,pkgconfig/bash-completion.pc} \
+        /var/log/* \
+        /var/tmp/* && \
+    find / -xdev \( -name "*.pyc" -o -name "*.pyo" \) -exec rm -f {} \; && \
+    mkdir -p /usr/local/share/doc/ceph && \
+    chown ceph:ceph -R /run/ceph /var/lib/ceph

ceph/build.sh

@@ -1,4 +1,13 @@
 #!/bin/sh
 set -eu
 
-docker build .
+# Build the custom Ceph packages
+cp ../go.mod ../go.sum .
+docker build -f Dockerfile -t ceph-custom .
+rm go.mod go.sum
+
+# Extract the custom Ceph packages
+mkdir -p packages
+docker create --name ceph-custom ceph-custom
+docker cp ceph-custom:/packages/* packages/
+docker rm ceph-custom

ceph/custom-export.patch

@@ -11,7 +11,7 @@
  // encryption arguments
  static const std::string ENCRYPTION_FORMAT("encryption-format");
 diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc
-index ddcf0f2c30cef..491d38d0d5c5e 100644
+index ddcf0f2c30cef..fbe2a57488516 100644
 --- a/src/tools/rbd/action/Export.cc
 +++ b/src/tools/rbd/action/Export.cc
 @@ -123,8 +123,8 @@ class C_ExportDiff : public Context {
@@ -71,7 +71,7 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644
 
    if (fd != 1)
      close(fd);
-@@ -260,7 +266,11 @@ void get_arguments_diff(po::options_description *positional,
+@@ -260,10 +266,52 @@ void get_arguments_diff(po::options_description *positional,
    options->add_options()
      (at::FROM_SNAPSHOT_NAME.c_str(), po::value<std::string>(),
       "snapshot starting point")
@@ -84,7 +84,48 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644
    at::add_no_progress_option(options);
  }
 
-@@ -290,6 +300,26 @@ int execute_diff(const po::variables_map &vm,
++int get_snapshot_name_for_offset_length(librbd::Image& image,
++                                        const std::string& mid_snap_prefix,
++                                        std::string* from_snap_name,
++                                        std::string* snap_name,
++                                        uint64_t* offset, uint64_t* length)
++{
++  int r;
++  librbd::image_info_t info;
++
++  r = image.stat(info, sizeof(info));
++  if (r < 0)
++    return r;
++  
++  // ⚠️ should be test when offset == info.size
++  if (*offset > info.size) {
++    std::cerr << "rbd: offset " << *offset << " exceeds image size "
++              << info.size << std::endl;
++    return -EINVAL;
++  }
++  
++  if (*offset > 0) {
++    *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset);
++  }
++
++  if (*length == 0) {
++    *length = info.size - *offset;
++    return 0;
++  }
++
++  if (*offset + *length < info.size) {
++    *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length);
++  } else {
++    *length = info.size - *offset;
++  }
++
++  return 0;
++}
++
+ int execute_diff(const po::variables_map &vm,
+                  const std::vector<std::string> &ceph_global_init_args) {
+   size_t arg_index = 0;
+@@ -290,6 +338,26 @@ int execute_diff(const po::variables_map &vm,
      from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as<std::string>();
    }
 
@@ -111,7 +152,7 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644
    librados::Rados rados;
    librados::IoCtx io_ctx;
    librbd::Image image;
-@@ -299,9 +329,19 @@ int execute_diff(const po::variables_map &vm,
+@@ -299,9 +367,19 @@ int execute_diff(const po::variables_map &vm,
      return r;
    }
 
@@ -131,51 +172,6 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644
                       vm[at::WHOLE_OBJECT].as<bool>(), path.c_str(),
                       vm[at::NO_PROGRESS].as<bool>());
    if (r < 0) {
-@@ -311,6 +351,44 @@ int execute_diff(const po::variables_map &vm,
-   return 0;
- }
-
-+int get_snapshot_name_for_offset_length(librbd::Image& image,
-+                                        const std::string& mid_snap_prefix,
-+                                        std::string* from_snap_name,
-+                                        std::string* snap_name,
-+                                        uint64_t* offset, uint64_t* length)
-+{
-+  int r;
-+  librbd::image_info_t info;
-+
-+  r = image.stat(info, sizeof(info));
-+  if (r < 0)
-+    return r;
-+  
-+  // ⚠️ should be test when offset == info.size
-+  if (*offset > info.size) {
-+    std::cerr << "rbd: offset " << *offset << " exceeds image size "
-+              << info.size << std::endl;
-+    return -EINVAL;
-+  }
-+  
-+  if (*offset > 0) {
-+    *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset);
-+  }
-+
-+  if (*length == 0) {
-+    *length = info.size - *offset;
-+    return 0;
-+  }
-+
-+  if (*offset + *length < info.size) {
-+    *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length);
-+  } else {
-+    *length = info.size - *offset;
-+  }
-+
-+  return 0;
-+}
-+
- Shell::Action action_diff(
-   {"export-diff"}, {}, "Export incremental diff to file.", "",
-   &get_arguments_diff, &execute_diff);
 @@ -501,7 +579,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd
    const char *last_snap = NULL;
    for (size_t i = 0; i < snaps.size(); ++i) {

ceph/test/suite_test.go

@@ -0,0 +1,7 @@
+package ceph
+
+import "testing"
+
+func TestCustomCeph(t *testing.T) {
+	t.Log("implement me!")
+}