Merge pull request rook#14488 from sp98/set-min-compat-client-to-reef

core: set min-compat-client to reef for upmap-read

deploy/charts/rook-ceph/templates/resources.yaml

@@ -1601,6 +1601,7 @@ spec:
                                   - ""
                                   - crush-compat
                                   - upmap
+                                  - read
                                   - upmap-read
                                 type: string
                             type: object

deploy/examples/crds.yaml

@@ -1599,6 +1599,7 @@ spec:
                                   - ""
                                   - crush-compat
                                   - upmap
+                                  - read
                                   - upmap-read
                                 type: string
                             type: object

pkg/apis/ceph.rook.io/v1/types.go

@@ -679,7 +679,7 @@ type Module struct {
 
 type ModuleSettings struct {
 	// BalancerMode sets the `balancer` module with different modes like `upmap`, `crush-compact` etc
-	// +kubebuilder:validation:Enum="";crush-compat;upmap;upmap-read
+	// +kubebuilder:validation:Enum="";crush-compat;upmap;read;upmap-read
 	BalancerMode string `json:"balancerMode,omitempty"`
 }
 

pkg/daemon/ceph/client/mgr.go

@@ -22,12 +22,18 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 )
 
 var (
 	moduleEnableWaitTime = 5 * time.Second
 )
 
+const (
+	readBalancerMode      = "read"
+	upmapReadBalancerMode = "upmap-read"
+)
+
 func CephMgrMap(context *clusterd.Context, clusterInfo *ClusterInfo) (*MgrMap, error) {
 	args := []string{"mgr", "dump"}
 	buf, err := NewCephCommand(context, clusterInfo, args).Run()
@@ -132,12 +138,12 @@ func setBalancerMode(context *clusterd.Context, clusterInfo *ClusterInfo, mode s
 	return nil
 }
 
-// setMinCompatClientLuminous set the minimum compatibility for clients to Luminous
-func setMinCompatClientLuminous(context *clusterd.Context, clusterInfo *ClusterInfo) error {
-	args := []string{"osd", "set-require-min-compat-client", "luminous", "--yes-i-really-mean-it"}
+// setMinCompatClient set the minimum compatibility for clients
+func setMinCompatClient(context *clusterd.Context, clusterInfo *ClusterInfo, version string) error {
+	args := []string{"osd", "set-require-min-compat-client", version, "--yes-i-really-mean-it"}
 	_, err := NewCephCommand(context, clusterInfo, args).Run()
 	if err != nil {
-		return errors.Wrap(err, "failed to set set-require-min-compat-client to luminous")
+		return errors.Wrapf(err, "failed to set set-require-min-compat-client to %q", version)
 	}
 
 	return nil
@@ -165,8 +171,12 @@ func mgrSetBalancerMode(context *clusterd.Context, clusterInfo *ClusterInfo, bal
 
 // ConfigureBalancerModule configures the balancer module
 func ConfigureBalancerModule(context *clusterd.Context, clusterInfo *ClusterInfo, balancerModuleMode string) error {
-	// Set min compat client to luminous before enabling the balancer mode "upmap"
-	err := setMinCompatClientLuminous(context, clusterInfo)
+	minCompatClientVersion, err := desiredMinCompatClientVersion(clusterInfo, balancerModuleMode)
+	if err != nil {
+		return errors.Wrap(err, "failed to get minimum compatibility client version")
+	}
+
+	err = setMinCompatClient(context, clusterInfo, minCompatClientVersion)
 	if err != nil {
 		return errors.Wrap(err, "failed to set minimum compatibility client")
 	}
@@ -179,3 +189,17 @@ func ConfigureBalancerModule(context *clusterd.Context, clusterInfo *ClusterInfo
 
 	return nil
 }
+
+func desiredMinCompatClientVersion(clusterInfo *ClusterInfo, balancerModuleMode string) (string, error) {
+	// Set min compat client to luminous before enabling the balancer mode "upmap"
+	minCompatClientVersion := "luminous"
+	if balancerModuleMode == readBalancerMode || balancerModuleMode == upmapReadBalancerMode {
+		if !clusterInfo.CephVersion.IsAtLeast(cephver.CephVersion{Major: 19}) {
+			return "", errors.New("minimum ceph v19 (Squid) is required for upmap-read or read balancer modes")
+		}
+		// Set min compat client to reef before enabling the balancer mode "upmap-read" or "read"
+		minCompatClientVersion = "reef"
+	}
+
+	return minCompatClientVersion, nil
+}

pkg/daemon/ceph/client/mgr_test.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rook/rook/pkg/clusterd"
+	cephver "github.com/rook/rook/pkg/operator/ceph/version"
 	exectest "github.com/rook/rook/pkg/util/exec/test"
 	"github.com/stretchr/testify/assert"
 )
@@ -135,3 +136,37 @@ func TestSetBalancerMode(t *testing.T) {
 	err := setBalancerMode(&clusterd.Context{Executor: executor}, AdminTestClusterInfo("mycluster"), "upmap")
 	assert.NoError(t, err)
 }
+
+func TestGetMinCompatClientVersion(t *testing.T) {
+	clusterInfo := AdminTestClusterInfo("mycluster")
+	t.Run("upmap-read balancer mode with ceph v19", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, upmapReadBalancerMode)
+		assert.NoError(t, err)
+		assert.Equal(t, "reef", result)
+	})
+
+	t.Run("read balancer mode with ceph v19", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, readBalancerMode)
+		assert.NoError(t, err)
+		assert.Equal(t, "reef", result)
+	})
+	t.Run("upmap-read balancer mode with ceph below v19 should fail", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 18}
+		_, err := desiredMinCompatClientVersion(clusterInfo, upmapReadBalancerMode)
+		assert.Error(t, err)
+	})
+	t.Run("read balancer mode with ceph below v19 should fail", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 18}
+		_, err := desiredMinCompatClientVersion(clusterInfo, readBalancerMode)
+		assert.Error(t, err)
+	})
+
+	t.Run("upmap balancer set min compat client to luminous", func(t *testing.T) {
+		clusterInfo.CephVersion = cephver.CephVersion{Major: 19}
+		result, err := desiredMinCompatClientVersion(clusterInfo, "upmap")
+		assert.NoError(t, err)
+		assert.Equal(t, "luminous", result)
+	})
+}