Merge branch 'release-6.1' into cherry-pick-7415-to-release-6.1

OWNERS

@@ -0,0 +1,26 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+approvers:
+  - AndreMouche
+  - binshi-bing
+  - bufferflies
+  - CabinfeverB
+  - Connor1996
+  - disksing
+  - huachaohuang
+  - HunDunDM
+  - HuSharp
+  - JmPotato
+  - lhy1024
+  - nolouch
+  - overvenus
+  - qiuyesuifeng
+  - rleungx
+  - siddontang
+  - Yisaer
+  - zhouqiang-cl
+reviewers:
+  - BusyJay
+  - howardlau1999
+  - Luffbee
+  - shafreeck
+  - xhebox

OWNERS_ALIASES

@@ -0,0 +1,6 @@
+# Sort the member alphabetically.
+aliases:
+  sig-critical-approvers-config:
+    - easonn7
+    - kevin-xianliu
+    - niubell

client/resource_group/controller/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

client/tlsutil/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|tlsconfig\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

conf/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.toml)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/encryption/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/mcs/resourcemanager/server/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/mcs/scheduling/server/config/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/mcs/tso/server/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/schedule/config/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|(config|store_config)\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/schedule/schedulers/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|hot_region_config\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

pkg/typeutil/size_test.go

@@ -17,6 +17,7 @@ package typeutil
 import (
 	"encoding/json"
 
+	"github.com/docker/go-units"
 	. "github.com/pingcap/check"
 )
 
@@ -41,23 +42,31 @@ func (s *testSizeSuite) TestJSON(c *C) {
 }
 
 func (s *testSizeSuite) TestParseMbFromText(c *C) {
+	const defaultValue = 2
+
 	testdata := []struct {
 		body []string
 		size uint64
 	}{{
 		body: []string{"10Mib", "10MiB", "10M", "10MB"},
-		size: uint64(10),
+		size: 10,
 	}, {
 		body: []string{"10GiB", "10Gib", "10G", "10GB"},
-		size: uint64(10 * 1024),
+		size: 10 * units.GiB / units.MiB,
+	}, {
+		body: []string{"1024KiB", "1048576"},
+		size: 1,
+	}, {
+		body: []string{"100KiB", "1023KiB", "1048575", "0"},
+		size: 0,
 	}, {
 		body: []string{"10yiB", "10aib"},
-		size: uint64(1),
+		size: defaultValue,
 	}}
 
 	for _, t := range testdata {
 		for _, b := range t.body {
-			c.Assert(int(ParseMBFromText(b, 1)), Equals, int(t.size))
+			c.Assert(ParseMBFromText(b, defaultValue), Equals, t.size)
 		}
 	}
 }

server/api/label_test.go

@@ -17,13 +17,15 @@ package api
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	. "github.com/pingcap/check"
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/kvproto/pkg/pdpb"
 	tu "github.com/tikv/pd/pkg/testutil"
 	"github.com/tikv/pd/server"
 	"github.com/tikv/pd/server/config"
+	"github.com/tikv/pd/server/core"
 )
 
 var _ = Suite(&testLabelsStoreSuite{})
@@ -264,19 +266,47 @@ func (s *testStrictlyLabelsStoreSuite) TestStoreMatch(c *C) {
 			valid:       false,
 			expectError: "key matching the label was not found",
 		},
+		{
+			store: &metapb.Store{
+				Id:      3,
+				Address: "tiflash1",
+				State:   metapb.StoreState_Up,
+				Labels: []*metapb.StoreLabel{
+					{
+						Key:   "zone",
+						Value: "us-west-1",
+					},
+					{
+						Key:   "disk",
+						Value: "ssd",
+					},
+					{
+						Key:   core.EngineKey,
+						Value: core.EngineTiFlash,
+					},
+				},
+				Version: "3.0.0",
+			},
+			valid:       true,
+			expectError: "placement rules is disabled",
+		},
 	}
 
 	for _, t := range cases {
 		resp, err := s.grpcSvr.PutStore(context.Background(), &pdpb.PutStoreRequest{
 			Header: &pdpb.RequestHeader{ClusterId: s.svr.ClusterID()},
 			Store: &metapb.Store{
 				Id:      t.store.Id,
-				Address: fmt.Sprintf("tikv%d", t.store.Id),
+				Address: t.store.Address,
 				State:   t.store.State,
 				Labels:  t.store.Labels,
 				Version: t.store.Version,
 			},
 		})
+		if t.store.Address == "tiflash1" {
+			c.Assert(strings.Contains(resp.GetHeader().GetError().String(), t.expectError), IsTrue)
+			continue
+		}
 		if t.valid {
 			c.Assert(err, IsNil)
 		} else {
@@ -291,12 +321,13 @@ func (s *testStrictlyLabelsStoreSuite) TestStoreMatch(c *C) {
 			Header: &pdpb.RequestHeader{ClusterId: s.svr.ClusterID()},
 			Store: &metapb.Store{
 				Id:      t.store.Id,
-				Address: fmt.Sprintf("tikv%d", t.store.Id),
+				Address: t.store.Address,
 				State:   t.store.State,
 				Labels:  t.store.Labels,
 				Version: t.store.Version,
 			},
 		})
+
 		if t.valid {
 			c.Assert(err, IsNil)
 		} else {

server/cluster/cluster.go

@@ -63,6 +63,8 @@ import (
 var (
 	// DefaultMinResolvedTSPersistenceInterval is the default value of min resolved ts persistence interval.
 	DefaultMinResolvedTSPersistenceInterval = 10 * time.Second
+
+	denySchedulersByLabelerCounter = schedule.LabelerEventCounter.WithLabelValues("schedulers", "deny")
 )
 
 // regionLabelGCInterval is the interval to run region-label's GC work.
@@ -1128,6 +1130,9 @@ func (c *RaftCluster) checkStoreLabels(s *core.StoreInfo) error {
 	}
 	for _, label := range s.GetLabels() {
 		key := label.GetKey()
+		if key == core.EngineKey {
+			continue
+		}
 		if _, ok := keysSet[key]; !ok {
 			log.Warn("not found the key match with the store label",
 				zap.Stringer("store", s.GetMeta()),
@@ -1451,8 +1456,8 @@ func (c *RaftCluster) checkStores() {
 				}
 			} else if c.IsPrepared() {
 				threshold := c.getThreshold(stores, store)
-				log.Debug("store serving threshold", zap.Uint64("store-id", storeID), zap.Float64("threshold", threshold))
 				regionSize := float64(store.GetRegionSize())
+				log.Debug("store serving threshold", zap.Uint64("store-id", storeID), zap.Float64("threshold", threshold), zap.Float64("region-size", regionSize))
 				if regionSize >= threshold {
 					if err := c.ReadyToServe(storeID); err != nil {
 						log.Error("change store to serving failed",

server/cluster/coordinator.go

@@ -893,8 +893,23 @@ func (s *scheduleController) Schedule() []*operator.Operator {
 		}
 		cacheCluster := newCacheCluster(s.cluster)
 		// If we have schedule, reset interval to the minimal interval.
-		if ops := s.Scheduler.Schedule(cacheCluster); len(ops) > 0 {
+		ops := s.Scheduler.Schedule(cacheCluster)
+		foundDisabled := false
+		for _, op := range ops {
+			if labelMgr := s.cluster.GetRegionLabeler(); labelMgr != nil {
+				if labelMgr.ScheduleDisabled(s.cluster.GetRegion(op.RegionID())) {
+					denySchedulersByLabelerCounter.Inc()
+					foundDisabled = true
+					break
+				}
+			}
+		}
+		if len(ops) > 0 {
 			s.nextInterval = s.Scheduler.GetMinInterval()
+			// try regenerating operators
+			if foundDisabled {
+				continue
+			}
 			return ops
 		}
 	}

server/cluster/coordinator_test.go

@@ -373,11 +373,22 @@ func (s *testCoordinatorSuite) TestCheckRegionWithScheduleDeny(c *C) {
 		Data:     []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}},
 	})
 
+	// should allow to do rule checker
 	c.Assert(labelerManager.ScheduleDisabled(region), IsTrue)
-	s.checkRegion(c, tc, co, 1, 0)
+	s.checkRegion(c, tc, co, 1, 1)
+	// should not allow to merge
+	tc.opt.SetSplitMergeInterval(time.Duration(0))
+
+	c.Assert(tc.addLeaderRegion(2, 2, 3, 4), IsNil)
+	c.Assert(tc.addLeaderRegion(3, 2, 3, 4), IsNil)
+	region = tc.GetRegion(2)
+	c.Assert(labelerManager.ScheduleDisabled(region), IsTrue)
+	s.checkRegion(c, tc, co, 2, 0)
+
+	// delete label rule, should allow to do merge
 	labelerManager.DeleteLabelRule("schedulelabel")
 	c.Assert(labelerManager.ScheduleDisabled(region), IsFalse)
-	s.checkRegion(c, tc, co, 1, 1)
+	s.checkRegion(c, tc, co, 2, 2)
 }
 
 func (s *testCoordinatorSuite) TestCheckerIsBusy(c *C) {
@@ -864,6 +875,43 @@ func (s *testCoordinatorSuite) TestRemoveScheduler(c *C) {
 	co.wg.Wait()
 }
 
+func (s *testCoordinatorSuite) TestDenyScheduler(c *C) {
+	tc, co, cleanup := prepare(nil, nil, func(co *coordinator) {
+		labelerManager := co.cluster.GetRegionLabeler()
+		labelerManager.SetLabelRule(&labeler.LabelRule{
+			ID:       "schedulelabel",
+			Labels:   []labeler.RegionLabel{{Key: "schedule", Value: "deny"}},
+			RuleType: labeler.KeyRange,
+			Data:     []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}},
+		})
+		co.run()
+	}, c)
+	defer cleanup()
+
+	c.Assert(len(co.schedulers), Equals, len(config.DefaultSchedulers))
+
+	// Transfer peer from store 4 to store 1 if not set deny.
+	c.Assert(tc.addRegionStore(4, 40), IsNil)
+	c.Assert(tc.addRegionStore(3, 30), IsNil)
+	c.Assert(tc.addRegionStore(2, 20), IsNil)
+	c.Assert(tc.addRegionStore(1, 10), IsNil)
+	c.Assert(tc.addLeaderRegion(1, 2, 3, 4), IsNil)
+
+	// Transfer leader from store 4 to store 2 if not set deny.
+	c.Assert(tc.updateLeaderCount(4, 1000), IsNil)
+	c.Assert(tc.updateLeaderCount(3, 50), IsNil)
+	c.Assert(tc.updateLeaderCount(2, 20), IsNil)
+	c.Assert(tc.updateLeaderCount(1, 10), IsNil)
+	c.Assert(tc.addLeaderRegion(2, 4, 3, 2), IsNil)
+
+	// there should no balance leader/region operator
+	for i := 0; i < 10; i++ {
+		c.Assert(co.opController.GetOperator(1), IsNil)
+		c.Assert(co.opController.GetOperator(2), IsNil)
+		time.Sleep(10 * time.Millisecond)
+	}
+}
+
 func (s *testCoordinatorSuite) TestRestart(c *C) {
 	tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) {
 		// Turn off balance, we test add replica only.

server/config/OWNERS

@@ -0,0 +1,7 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+options:
+  no_parent_owners: true
+filters:
+  "(OWNERS|(config|service_middleware_config)\\.go)$":
+    approvers:
+      - sig-critical-approvers-config

server/config/store_config.go

@@ -129,9 +129,14 @@ func (c *StoreConfig) CheckRegionSize(size, mergeSize uint64) error {
 	if size < c.GetRegionMaxSize() {
 		return nil
 	}
-
+	// This could happen when the region split size is set to a value less than 1MiB,
+	// which is a very extreme case, we just pass the check here to prevent panic.
+	regionSplitSize := c.GetRegionSplitSize()
+	if regionSplitSize == 0 {
+		return nil
+	}
 	// the smallest of the split regions can not be merge again, so it's size should less merge size.
-	if smallSize := size % c.GetRegionSplitSize(); smallSize <= mergeSize && smallSize != 0 {
+	if smallSize := size % regionSplitSize; smallSize <= mergeSize && smallSize != 0 {
 		log.Debug("region size is too small", zap.Uint64("size", size), zap.Uint64("merge-size", mergeSize), zap.Uint64("small-size", smallSize))
 		return errs.ErrCheckerMergeAgain.FastGenByArgs("the smallest region of the split regions is less than max-merge-region-size, " +
 			"it will be merged again")

server/config/store_config_test.go

@@ -147,4 +147,8 @@ func (t *testTiKVConfigSuite) TestMergeCheck(c *C) {
 			c.Assert(config.CheckRegionKeys(v.keys, v.mergeKeys), NotNil)
 		}
 	}
+	// Test CheckRegionSize when the region split size is 0.
+	config.RegionSplitSize = "100KiB"
+	c.Assert(config.GetRegionSplitSize(), Equals, uint64(0))
+	c.Assert(config.CheckRegionSize(defaultRegionMaxSize, 50), IsNil)
 }