diff --git a/.gitignore b/.gitignore index 08a2eb00..0d07a8d2 100644 --- a/.gitignore +++ b/.gitignore @@ -25,7 +25,9 @@ Dockerfile.cross *.swo *~ -ceph/src/ +ceph/go.mod +ceph/go.sum +ceph/packages/ test/e2e/testdata/persistentvolumes.yaml test/e2e/testdata/values-mantle-primary.yaml test/e2e/testdata/values-mantle-secondary.yaml diff --git a/ceph/Dockerfile b/ceph/Dockerfile index c371b6e2..cc543440 100644 --- a/ceph/Dockerfile +++ b/ceph/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/cybozu/ubuntu:22.04 AS builder +FROM ghcr.io/cybozu/ubuntu:22.04 AS build-ceph SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND=noninteractive @@ -49,3 +49,53 @@ RUN patch -p1 < ${WORKSPACE}/boost-url.patch RUN rm debian/libcephfs-java.jlibs debian/libcephfs-jni.install debian/ceph-mgr-dashboard* # To avoid OOM killer, use 10 parallelism instead of 20 (max vCPU). RUN dpkg-buildpackage --build=binary -uc -us -j10 +RUN rm ${WORKSPACE}/*-dbg_*.deb ${WORKSPACE}/ceph-test_*.deb + +FROM ghcr.io/cybozu/golang:1.22-jammy AS build-test + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV WORKSPACE=/workspace + +COPY go.mod go.sum ${WORKSPACE}/ +COPY test/ ${WORKSPACE}/test/ + +WORKDIR ${WORKSPACE} +RUN go test -o test.bin ./test/ + + +FROM ghcr.io/cybozu/ubuntu:22.04 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +ENV DEBIAN_FRONTEND=noninteractive +ENV WORKSPACE=/workspace +ENV PACKAGES=/packages + +# Since the package file is extracted outside the docker image at the end, keep the files. +COPY --from=build-ceph ${WORKSPACE}/*.deb ${PACKAGES}/ +COPY --from=build-test ${WORKSPACE}/test.bin /test + +WORKDIR ${WORKSPACE} +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/usr/local/bin/kubectl" +RUN mkdir -p /var/run/ceph && \ + apt-get update && apt-get install --no-install-recommends -y \ + libstdc++-11-dev jq kmod lvm2 gdisk ca-certificates e2fsprogs attr udev libgflags2.2 ${PACKAGES}/*.deb && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* && \ + sed -i -e 's/udev_rules = 1/udev_rules = 0/' -e 's/udev_sync = 1/udev_sync = 0/' -e 's/obtain_device_list_from_udev = 1/obtain_device_list_from_udev = 0/' /etc/lvm/lvm.conf && \ + # validate the sed command worked as expected + grep -sqo "udev_sync = 0" /etc/lvm/lvm.conf && \ + grep -sqo "udev_rules = 0" /etc/lvm/lvm.conf && \ + grep -sqo "obtain_device_list_from_udev = 0" /etc/lvm/lvm.conf && \ + # Clean common files like /tmp, /var/lib, etc. + rm -rf \ + /etc/{selinux,systemd,udev} \ + /lib/{lsb,udev} \ + /tmp/* \ + /usr/lib{,64}/{locale,systemd,udev,dracut} \ + /usr/share/{doc,info,locale,man} \ + /usr/share/{bash-completion,pkgconfig/bash-completion.pc} \ + /var/log/* \ + /var/tmp/* && \ + find / -xdev \( -name "*.pyc" -o -name "*.pyo" \) -exec rm -f {} \; && \ + mkdir -p /usr/local/share/doc/ceph && \ + chown ceph:ceph -R /run/ceph /var/lib/ceph diff --git a/ceph/build.sh b/ceph/build.sh index e01eef52..9124b1c0 100755 --- a/ceph/build.sh +++ b/ceph/build.sh @@ -1,4 +1,13 @@ #!/bin/sh set -eu -docker build . +# Build the custom Ceph packages +cp ../go.mod ../go.sum . +docker build -f Dockerfile -t ceph-custom . +rm go.mod go.sum + +# Extract the custom Ceph packages +mkdir -p packages +docker create --name ceph-custom ceph-custom +docker cp ceph-custom:/packages/* packages/ +docker rm ceph-custom diff --git a/ceph/custom-export.patch b/ceph/custom-export.patch index 998fa156..cd37e5f3 100644 --- a/ceph/custom-export.patch +++ b/ceph/custom-export.patch @@ -11,7 +11,7 @@ // encryption arguments static const std::string ENCRYPTION_FORMAT("encryption-format"); diff --git a/src/tools/rbd/action/Export.cc b/src/tools/rbd/action/Export.cc -index ddcf0f2c30cef..491d38d0d5c5e 100644 +index ddcf0f2c30cef..fbe2a57488516 100644 --- a/src/tools/rbd/action/Export.cc +++ b/src/tools/rbd/action/Export.cc @@ -123,8 +123,8 @@ class C_ExportDiff : public Context { @@ -71,7 +71,7 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644 if (fd != 1) close(fd); -@@ -260,7 +266,11 @@ void get_arguments_diff(po::options_description *positional, +@@ -260,10 +266,52 @@ void get_arguments_diff(po::options_description *positional, options->add_options() (at::FROM_SNAPSHOT_NAME.c_str(), po::value(), "snapshot starting point") @@ -84,7 +84,48 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644 at::add_no_progress_option(options); } -@@ -290,6 +300,26 @@ int execute_diff(const po::variables_map &vm, ++int get_snapshot_name_for_offset_length(librbd::Image& image, ++ const std::string& mid_snap_prefix, ++ std::string* from_snap_name, ++ std::string* snap_name, ++ uint64_t* offset, uint64_t* length) ++{ ++ int r; ++ librbd::image_info_t info; ++ ++ r = image.stat(info, sizeof(info)); ++ if (r < 0) ++ return r; ++ ++ // ⚠️ should be test when offset == info.size ++ if (*offset > info.size) { ++ std::cerr << "rbd: offset " << *offset << " exceeds image size " ++ << info.size << std::endl; ++ return -EINVAL; ++ } ++ ++ if (*offset > 0) { ++ *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset); ++ } ++ ++ if (*length == 0) { ++ *length = info.size - *offset; ++ return 0; ++ } ++ ++ if (*offset + *length < info.size) { ++ *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length); ++ } else { ++ *length = info.size - *offset; ++ } ++ ++ return 0; ++} ++ + int execute_diff(const po::variables_map &vm, + const std::vector &ceph_global_init_args) { + size_t arg_index = 0; +@@ -290,6 +338,26 @@ int execute_diff(const po::variables_map &vm, from_snap_name = vm[at::FROM_SNAPSHOT_NAME].as(); } @@ -111,7 +152,7 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644 librados::Rados rados; librados::IoCtx io_ctx; librbd::Image image; -@@ -299,9 +329,19 @@ int execute_diff(const po::variables_map &vm, +@@ -299,9 +367,19 @@ int execute_diff(const po::variables_map &vm, return r; } @@ -131,51 +172,6 @@ index ddcf0f2c30cef..491d38d0d5c5e 100644 vm[at::WHOLE_OBJECT].as(), path.c_str(), vm[at::NO_PROGRESS].as()); if (r < 0) { -@@ -311,6 +351,44 @@ int execute_diff(const po::variables_map &vm, - return 0; - } - -+int get_snapshot_name_for_offset_length(librbd::Image& image, -+ const std::string& mid_snap_prefix, -+ std::string* from_snap_name, -+ std::string* snap_name, -+ uint64_t* offset, uint64_t* length) -+{ -+ int r; -+ librbd::image_info_t info; -+ -+ r = image.stat(info, sizeof(info)); -+ if (r < 0) -+ return r; -+ -+ // ⚠️ should be test when offset == info.size -+ if (*offset > info.size) { -+ std::cerr << "rbd: offset " << *offset << " exceeds image size " -+ << info.size << std::endl; -+ return -EINVAL; -+ } -+ -+ if (*offset > 0) { -+ *from_snap_name = mid_snap_prefix + "-" + std::to_string(*offset); -+ } -+ -+ if (*length == 0) { -+ *length = info.size - *offset; -+ return 0; -+ } -+ -+ if (*offset + *length < info.size) { -+ *snap_name = mid_snap_prefix + "-" + std::to_string(*offset + *length); -+ } else { -+ *length = info.size - *offset; -+ } -+ -+ return 0; -+} -+ - Shell::Action action_diff( - {"export-diff"}, {}, "Export incremental diff to file.", "", - &get_arguments_diff, &execute_diff); @@ -501,7 +579,7 @@ static int do_export_v2(librbd::Image& image, librbd::image_info_t &info, int fd const char *last_snap = NULL; for (size_t i = 0; i < snaps.size(); ++i) { diff --git a/ceph/test/suite_test.go b/ceph/test/suite_test.go new file mode 100644 index 00000000..19e93ee6 --- /dev/null +++ b/ceph/test/suite_test.go @@ -0,0 +1,7 @@ +package ceph + +import "testing" + +func TestCustomCeph(t *testing.T) { + t.Log("implement me!") +}