pdms: randomly run pdms in e2e test

examples/basic/pd-micro-service-cluster.yaml

@@ -17,8 +17,8 @@ spec:
     image: alpine:3.16.0
   pd:
     # TODO: use a stable version
-    baseImage: pingcap/pd
-    version: nightly
+    baseImage: hub-new.pingcap.net/orchestration/pd
+    version: v8.0.0
     maxFailoverCount: 0
     replicas: 1
     # if storageClassName is not set, the default Storage Class of the Kubernetes cluster will be used
@@ -29,16 +29,18 @@ spec:
     mode: "ms"
   pdms:
     - name: "tso"
-      baseImage: pingcap/pd
-      version: nightly
+      # TODO: use a stable version
+      baseImage: hub-new.pingcap.net/orchestration/pd
+      version: v8.0.0
       replicas: 2
     - name: "scheduling"
-      baseImage: pingcap/pd
-      version: nightly
+      # TODO: use a stable version
+      baseImage: hub-new.pingcap.net/orchestration/pd
+      version: v8.0.0
       replicas: 1
   tikv:
     baseImage: pingcap/tikv
-    version: v7.4.0
+    version: v7.5.0
     maxFailoverCount: 0
     # If only 1 TiKV is deployed, the TiKV region leader
     # cannot be transferred during upgrade, so we have
@@ -61,7 +63,7 @@ spec:
         max-open-files: 256
   tidb:
     baseImage: pingcap/tidb
-    version: v7.4.0
+    version: v7.5.0
     maxFailoverCount: 0
     replicas: 1
     service:

pkg/controller/tidbcluster/tidb_cluster_control.go

@@ -14,21 +14,20 @@
 package tidbcluster
 
 import (
-	"github.com/pingcap/tidb-operator/pkg/features"
-	v1 "k8s.io/api/core/v1"
-	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	errorutils "k8s.io/apimachinery/pkg/util/errors"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/klog/v2"
-
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1/defaulting"
 	v1alpha1validation "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1/validation"
 	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/manager"
 	"github.com/pingcap/tidb-operator/pkg/manager/member"
 	"github.com/pingcap/tidb-operator/pkg/manager/volumes"
 	"github.com/pingcap/tidb-operator/pkg/metrics"
+	v1 "k8s.io/api/core/v1"
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	errorutils "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/klog/v2"
 )
 
 // ControlInterface implements the control logic for updating TidbClusters and their children StatefulSets.

pkg/manager/member/pd_member_manager.go

@@ -80,8 +80,9 @@ func (m *pdMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 		return nil
 	}
 
-	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil {
-		return fmt.Errorf("tidbcluster: [%s/%s]'s enable micro service but pdMS spec is nil, please check `PDMS`", tc.GetNamespace(), tc.GetName())
+	if (tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil) ||
+		(tc.Spec.PDMS != nil && tc.Spec.PD.Mode != "ms") {
+		klog.Infof("tidbcluster: [%s/%s]'s enable micro service failed, please check `PD.Mode` and `PDMS`", tc.GetNamespace(), tc.GetName())
 	}
 
 	// skip sync if pd is suspended

pkg/manager/member/pd_ms_member_manager.go

@@ -59,37 +59,40 @@ func NewPDMSMemberManager(dependencies *controller.Dependencies, pdMSScaler Scal
 
 // Sync for all PD Micro Service components.
 func (m *pdMSMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
-	if tc.Spec.PDMS == nil {
-		return nil
-	}
-
 	// Need to start PD API
-	if tc.Spec.PD == nil {
+	if tc.Spec.PDMS != nil && tc.Spec.PD == nil {
 		klog.Infof("PD Micro Service is enabled, but PD is not enabled, skip syncing PD Micro Service")
 		return nil
 	}
-	if tc.Spec.PD.Mode != "ms" {
-		// remove all micro service components
-		for _, comp := range tc.Spec.PDMS {
+	// remove all micro service components if PDMS is not enabled
+	// PDMS need to be enabled when PD.Mode is ms && PDMS is not nil
+	if tc.Spec.PDMS == nil || (tc.Spec.PD != nil && tc.Spec.PD.Mode != "ms") {
+		for _, comp := range tc.Status.PDMS {
 			ns := tc.GetNamespace()
 			tcName := tc.GetName()
 			curService := comp.Name
 
 			oldPDMSSetTmp, err := m.deps.StatefulSetLister.StatefulSets(ns).Get(controller.PDMSMemberName(tcName, curService))
-			if err != nil && !errors.IsNotFound(err) {
+			if err != nil {
+				if errors.IsNotFound(err) {
+					continue
+				}
 				return fmt.Errorf("syncPDMSStatefulSet: fail to get sts %s PDMS component %s for cluster [%s/%s], error: %s",
 					controller.PDMSMemberName(tcName, curService), curService, ns, tcName, err)
 			}
 
 			oldPDMSSet := oldPDMSSetTmp.DeepCopy()
 			newPDMSSet := oldPDMSSetTmp.DeepCopy()
+			if oldPDMSSet.Status.Replicas == 0 {
+				continue
+			}
+			tc.Status.PDMS[curService].Synced = true
 			*newPDMSSet.Spec.Replicas = 0
 			if err := m.scaler.Scale(tc, oldPDMSSet, newPDMSSet); err != nil {
 				return err
 			}
 			mngerutils.UpdateStatefulSetWithPrecheck(m.deps, tc, "FailedUpdatePDMSSTS", newPDMSSet, oldPDMSSet)
 		}
-		klog.Infof("PD Micro Service is enabled, but PD is not in `ms` mode, skip syncing PD Micro Service")
 		return nil
 	}
 

pkg/manager/member/startscript/v1/render_script.go

@@ -74,7 +74,7 @@ func RenderPDStartScript(tc *v1alpha1.TidbCluster) (string, error) {
 
 	pdStartSubScript := ``
 	mode := ""
-	if tc.Spec.PD.Mode == "ms" {
+	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS != nil {
 		mode = "api"
 	}
 	pdStartScriptTpl := template.Must(

pkg/manager/member/startscript/v2/pd_start_script.go

@@ -94,7 +94,7 @@ func RenderPDStartScript(tc *v1alpha1.TidbCluster) (string, error) {
 		tc.Spec.StartScriptV2FeatureFlags, v1alpha1.StartScriptV2FeatureFlagWaitForDnsNameIpMatch)
 
 	mode := ""
-	if tc.Spec.PD.Mode == "ms" {
+	if tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS != nil {
 		mode = "api"
 	}
 	pdStartScriptTpl := template.Must(

pkg/manager/member/tikv_member_manager.go

@@ -111,17 +111,14 @@ func (m *tikvMemberManager) Sync(tc *v1alpha1.TidbCluster) error {
 	if tc.Spec.PD != nil && !tc.PDIsAvailable() {
 		return controller.RequeueErrorf("TidbCluster: [%s/%s], waiting for PD cluster running", ns, tcName)
 	}
-	// Need to wait for PDMS
-	if tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms" && tc.Spec.PDMS == nil {
-		return controller.RequeueErrorf("TidbCluster: [%s/%s], please make sure pdms is not nil, "+
-			"then now waiting for PD's micro service running", ns, tcName)
-	}
 	// Check if all PD Micro Services are available
-	for _, pdms := range tc.Spec.PDMS {
-		_, err = controller.GetPDMSClient(m.deps.PDControl, tc, pdms.Name)
-		if err != nil {
-			return controller.RequeueErrorf("PDMS component %s for TidbCluster: [%s/%s], "+
-				"waiting for PD micro service cluster running, error: %v", pdms.Name, ns, tcName, err)
+	if tc.Spec.PDMS != nil && (tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms") {
+		for _, pdms := range tc.Spec.PDMS {
+			_, err = controller.GetPDMSClient(m.deps.PDControl, tc, pdms.Name)
+			if err != nil {
+				return controller.RequeueErrorf("PDMS component %s for TidbCluster: [%s/%s], "+
+					"waiting for PD micro service cluster running, error: %v", pdms.Name, ns, tcName, err)
+			}
 		}
 	}
 

tests/actions.go

@@ -683,6 +683,20 @@ func (oa *OperatorActions) memberCheckContextForTC(tc *v1alpha1.TidbCluster, com
 		expectedImage = tc.PDImage()
 		services = []string{controller.PDMemberName(name), controller.PDPeerMemberName(name)}
 		checkComponent = oa.isPDMembersReady
+	case v1alpha1.PDMSTSOMemberType, v1alpha1.PDMSSchedulingMemberType:
+		skip = true
+		if tc.Spec.PD != nil && tc.Spec.PD.Mode == "ms" {
+			curService := component.String()
+			for _, service := range tc.Spec.PDMS {
+				if curService == service.Name {
+					skip = false
+					break
+				}
+			}
+			expectedImage = tc.PDImage()
+			services = []string{controller.PDMSMemberName(name, curService), controller.PDMSPeerMemberName(name, curService)}
+			checkComponent = oa.isPDMSMembersReady
+		}
 	case v1alpha1.TiDBMemberType:
 		skip = tc.Spec.TiDB == nil
 		expectedImage = tc.TiDBImage()
@@ -878,6 +892,36 @@ func (oa *OperatorActions) isPDMembersReady(tc *v1alpha1.TidbCluster, sts *v1.St
 	return nil
 }
 
+func (oa *OperatorActions) isPDMSMembersReady(tc *v1alpha1.TidbCluster, sts *v1.StatefulSet) error {
+	curService := controller.PDMSTrimName(sts.Name)
+	if tc.Status.PDMS[curService] == nil || tc.Status.PDMS[curService].StatefulSet == nil {
+		return fmt.Errorf("sts in tc status is nil, pdms curService is %s", curService)
+	}
+
+	var replicas int32
+	for _, component := range tc.Spec.PDMS {
+		if strings.Contains(component.Name, curService) {
+			replicas = component.Replicas
+			break
+		}
+	}
+
+	if *sts.Spec.Replicas != replicas {
+		return fmt.Errorf("sts.spec.Replicas(%d) != %d, pdms curService is %s\", curService)",
+			*sts.Spec.Replicas, replicas, curService)
+	}
+	if sts.Status.ReadyReplicas != replicas {
+		return fmt.Errorf("sts.status.ReadyReplicas(%d) != %d, pdms curService is %s\", curService)",
+			sts.Status.ReadyReplicas, tc.Spec.PD.Replicas, curService)
+	}
+	if sts.Status.ReadyReplicas != sts.Status.Replicas {
+		return fmt.Errorf("sts.status.ReadyReplicas(%d) != sts.status.Replicas(%d), pdms curService is %s\", curService)",
+			sts.Status.ReadyReplicas, sts.Status.Replicas, curService)
+	}
+
+	return nil
+}
+
 func (oa *OperatorActions) isTiDBMembersReady(tc *v1alpha1.TidbCluster, sts *v1.StatefulSet) error {
 	if tc.Status.TiDB.StatefulSet == nil {
 		return fmt.Errorf("sts in tc status is nil")
@@ -1320,6 +1364,8 @@ func (oa *OperatorActions) WaitForTidbClusterReady(tc *v1alpha1.TidbCluster, tim
 		}
 
 		components := []v1alpha1.MemberType{
+			v1alpha1.PDMSTSOMemberType,
+			v1alpha1.PDMSSchedulingMemberType,
 			v1alpha1.PDMemberType,
 			v1alpha1.TiKVMemberType,
 			v1alpha1.TiDBMemberType,