diff --git a/pkg/mock/mockcluster/mockcluster.go b/pkg/mock/mockcluster/mockcluster.go index b1b18cad74e..bf32d9476b2 100644 --- a/pkg/mock/mockcluster/mockcluster.go +++ b/pkg/mock/mockcluster/mockcluster.go @@ -24,6 +24,7 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" + "github.com/pkg/errors" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" @@ -63,7 +64,7 @@ type Cluster struct { // NewCluster creates a new Cluster func NewCluster(ctx context.Context, opts *config.PersistOptions) *Cluster { - clus := &Cluster{ + c := &Cluster{ ctx: ctx, BasicCluster: core.NewBasicCluster(), IDAllocator: mockid.NewIDAllocator(), @@ -74,13 +75,13 @@ func NewCluster(ctx context.Context, opts *config.PersistOptions) *Cluster { StoreConfigManager: config.NewTestStoreConfigManager(nil), Storage: storage.NewStorageWithMemoryBackend(), } - if clus.PersistOptions.GetReplicationConfig().EnablePlacementRules { - clus.initRuleManager() + if c.PersistOptions.GetReplicationConfig().EnablePlacementRules { + c.initRuleManager() } // It should be updated to the latest feature version. - clus.PersistOptions.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.HotScheduleWithQuery)) - clus.RegionLabeler, _ = labeler.NewRegionLabeler(ctx, storage.NewStorageWithMemoryBackend(), time.Second*5) - return clus + c.PersistOptions.SetClusterVersion(versioninfo.MinSupportedVersion(versioninfo.HotScheduleWithQuery)) + c.RegionLabeler, _ = labeler.NewRegionLabeler(ctx, c.Storage, time.Second*5) + return c } // GetStoreConfig returns the store config. @@ -93,16 +94,35 @@ func (mc *Cluster) GetOpts() sc.Config { return mc.PersistOptions } +// GetStorage returns the storage. +func (mc *Cluster) GetStorage() storage.Storage { + return mc.Storage +} + // GetAllocator returns the ID allocator. func (mc *Cluster) GetAllocator() id.Allocator { return mc.IDAllocator } -// GetStorage returns the storage. -func (mc *Cluster) GetStorage() storage.Storage { - return mc.Storage +// IsUnsafeRecovering returns if the cluster is in unsafe recovering. +func (mc *Cluster) IsUnsafeRecovering() bool { + return false +} + +// GetPersistOptions returns the persist options. +func (mc *Cluster) GetPersistOptions() *config.PersistOptions { + return mc.PersistOptions } +// UpdateRegionsLabelLevelStats updates the label level stats for the regions. +func (mc *Cluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) {} + +// IsSchedulerExisted checks if the scheduler with name is existed or not. +func (mc *Cluster) IsSchedulerExisted(name string) (bool, error) { return false, nil } + +// IsSchedulerDisabled checks if the scheduler with name is disabled or not. +func (mc *Cluster) IsSchedulerDisabled(name string) (bool, error) { return false, nil } + // ScanRegions scans region with start key, until number greater than limit. func (mc *Cluster) ScanRegions(startKey, endKey []byte, limit int) []*core.RegionInfo { return mc.ScanRange(startKey, endKey, limit) @@ -195,7 +215,7 @@ func (mc *Cluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { func (mc *Cluster) initRuleManager() { if mc.RuleManager == nil { - mc.RuleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), mc, mc.GetOpts()) + mc.RuleManager = placement.NewRuleManager(mc.GetStorage(), mc, mc.GetOpts()) mc.RuleManager.Initialize(int(mc.GetReplicationConfig().MaxReplicas), mc.GetReplicationConfig().LocationLabels) } } @@ -259,6 +279,22 @@ func (mc *Cluster) SetStoreBusy(storeID uint64, busy bool) { mc.PutStore(newStore) } +// BuryStore marks a store as tombstone in cluster. +func (mc *Cluster) BuryStore(storeID uint64, forceBury bool) error { + store := mc.GetStore(storeID) + if store.IsUp() { + if !forceBury { + return errs.ErrStoreIsUp.FastGenByArgs() + } else if !store.IsDisconnected() { + return errors.Errorf("The store %v is not offline nor disconnected", storeID) + } + } + + newStore := store.Clone(core.TombstoneStore()) + mc.PutStore(newStore) + return nil +} + // AddLeaderStore adds store with specified count of leader. func (mc *Cluster) AddLeaderStore(storeID uint64, leaderCount int, leaderSizes ...int64) { stats := &pdpb.StoreStats{} @@ -285,7 +321,13 @@ func (mc *Cluster) AddLeaderStore(storeID uint64, leaderCount int, leaderSizes . } // AddRegionStore adds store with specified count of region. -func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int) { +func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) { + var regionSize uint64 + if len(regionSizes) == 0 { + regionSize = uint64(int64(regionCount) * defaultRegionSize / units.MiB) + } else { + regionSize = regionSizes[0] + } stats := &pdpb.StoreStats{} stats.Capacity = defaultStoreCapacity stats.UsedSize = uint64(regionCount) * defaultRegionSize @@ -299,8 +341,8 @@ func (mc *Cluster) AddRegionStore(storeID uint64, regionCount int) { }}, core.SetStoreStats(stats), core.SetRegionCount(regionCount), - core.SetRegionSize(int64(regionCount)*defaultRegionSize/units.MiB), core.SetLastHeartbeatTS(time.Now()), + core.SetRegionSize(int64(regionSize)), ) mc.SetStoreLimit(storeID, storelimit.AddPeer, 60) mc.SetStoreLimit(storeID, storelimit.RemovePeer, 60) @@ -522,6 +564,11 @@ func (mc *Cluster) AddLeaderRegionWithWriteInfo( return items } +// DropCacheAllRegion removes all regions from the cache. +func (mc *Cluster) DropCacheAllRegion() { + mc.ResetRegionCache() +} + // UpdateStoreLeaderWeight updates store leader weight. func (mc *Cluster) UpdateStoreLeaderWeight(storeID uint64, weight float64) { store := mc.GetStore(storeID) @@ -822,10 +869,6 @@ func (mc *Cluster) AddSuspectRegions(ids ...uint64) { } } -// SetHotPendingInfluenceMetrics mock method -func (mc *Cluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { -} - // GetBasicCluster mock method func (mc *Cluster) GetBasicCluster() *core.BasicCluster { return mc.BasicCluster diff --git a/pkg/schedule/checker/checker_controller.go b/pkg/schedule/checker/checker_controller.go index c678c46eb2f..1aae4584a6f 100644 --- a/pkg/schedule/checker/checker_controller.go +++ b/pkg/schedule/checker/checker_controller.go @@ -22,7 +22,6 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/config" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/labeler" @@ -34,7 +33,7 @@ import ( // DefaultCacheSize is the default length of waiting list. const DefaultCacheSize = 1000 -var denyCheckersByLabelerCounter = schedule.LabelerEventCounter.WithLabelValues("checkers", "deny") +var denyCheckersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("checkers", "deny") // Controller is used to manage all checkers. type Controller struct { diff --git a/pkg/schedule/config/config.go b/pkg/schedule/config/config.go index 06dd3f31cfa..2c0842914c7 100644 --- a/pkg/schedule/config/config.go +++ b/pkg/schedule/config/config.go @@ -81,6 +81,7 @@ type Config interface { IsDebugMetricsEnabled() bool GetClusterVersion() *semver.Version GetStoreLimitVersion() string + IsDiagnosticAllowed() bool // for test purpose SetPlacementRuleEnabled(bool) SetSplitMergeInterval(time.Duration) diff --git a/server/cluster/coordinator.go b/pkg/schedule/coordinator.go similarity index 72% rename from server/cluster/coordinator.go rename to pkg/schedule/coordinator.go index 8eb80a07efe..e979bd9e324 100644 --- a/server/cluster/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedule import ( "bytes" @@ -29,9 +29,10 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/checker" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/hbstream" + "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/schedule/schedulers" @@ -50,6 +51,8 @@ const ( collectTimeout = 5 * time.Minute maxScheduleRetries = 10 maxLoadConfigRetries = 10 + // pushOperatorTickInterval is the interval try to push the operator. + pushOperatorTickInterval = 500 * time.Millisecond patrolScanRegionLimit = 128 // It takes about 14 minutes to iterate 1 million regions. // PluginLoad means action for load plugin @@ -60,68 +63,72 @@ const ( var ( // WithLabelValues is a heavy operation, define variable to avoid call it every time. - waitingListGauge = regionListGauge.WithLabelValues("waiting_list") - priorityListGauge = regionListGauge.WithLabelValues("priority_list") + waitingListGauge = regionListGauge.WithLabelValues("waiting_list") + priorityListGauge = regionListGauge.WithLabelValues("priority_list") + denySchedulersByLabelerCounter = labeler.LabelerEventCounter.WithLabelValues("schedulers", "deny") ) -// coordinator is used to manage all schedulers and checkers to decide if the region needs to be scheduled. -type coordinator struct { +// Coordinator is used to manage all schedulers and checkers to decide if the region needs to be scheduled. +type Coordinator struct { syncutil.RWMutex wg sync.WaitGroup ctx context.Context cancel context.CancelFunc - cluster *RaftCluster + cluster sche.ClusterInformer prepareChecker *prepareChecker checkers *checker.Controller - regionScatterer *schedule.RegionScatterer - regionSplitter *schedule.RegionSplitter + regionScatterer *RegionScatterer + regionSplitter *RegionSplitter schedulers map[string]*scheduleController opController *operator.Controller hbStreams *hbstream.HeartbeatStreams - pluginInterface *schedule.PluginInterface + pluginInterface *PluginInterface diagnosticManager *diagnosticManager } -// newCoordinator creates a new coordinator. -func newCoordinator(ctx context.Context, cluster *RaftCluster, hbStreams *hbstream.HeartbeatStreams) *coordinator { +// NewCoordinator creates a new Coordinator. +func NewCoordinator(ctx context.Context, cluster sche.ClusterInformer, hbStreams *hbstream.HeartbeatStreams) *Coordinator { ctx, cancel := context.WithCancel(ctx) opController := operator.NewController(ctx, cluster, hbStreams) schedulers := make(map[string]*scheduleController) - return &coordinator{ + return &Coordinator{ ctx: ctx, cancel: cancel, cluster: cluster, prepareChecker: newPrepareChecker(), - checkers: checker.NewController(ctx, cluster, cluster.opt, cluster.ruleManager, cluster.regionLabeler, opController), - regionScatterer: schedule.NewRegionScatterer(ctx, cluster, opController), - regionSplitter: schedule.NewRegionSplitter(cluster, schedule.NewSplitRegionsHandler(cluster, opController)), + checkers: checker.NewController(ctx, cluster, cluster.GetOpts(), cluster.GetRuleManager(), cluster.GetRegionLabeler(), opController), + regionScatterer: NewRegionScatterer(ctx, cluster, opController), + regionSplitter: NewRegionSplitter(cluster, NewSplitRegionsHandler(cluster, opController)), schedulers: schedulers, opController: opController, hbStreams: hbStreams, - pluginInterface: schedule.NewPluginInterface(), + pluginInterface: NewPluginInterface(), diagnosticManager: newDiagnosticManager(cluster), } } -func (c *coordinator) GetWaitingRegions() []*cache.Item { +// GetWaitingRegions returns the regions in the waiting list. +func (c *Coordinator) GetWaitingRegions() []*cache.Item { return c.checkers.GetWaitingRegions() } -func (c *coordinator) IsPendingRegion(region uint64) bool { +// IsPendingRegion returns if the region is in the pending list. +func (c *Coordinator) IsPendingRegion(region uint64) bool { return c.checkers.IsPendingRegion(region) } -// patrolRegions is used to scan regions. +// PatrolRegions is used to scan regions. // The checkers will check these regions to decide if they need to do some operations. -func (c *coordinator) patrolRegions() { +// The function is exposed for test purpose. +func (c *Coordinator) PatrolRegions() { defer logutil.LogPanic() defer c.wg.Done() timer := time.NewTimer(c.cluster.GetOpts().GetPatrolRegionInterval()) defer timer.Stop() - log.Info("coordinator starts patrol regions") + log.Info("Coordinator starts patrol regions") start := time.Now() var ( key []byte @@ -135,7 +142,7 @@ func (c *coordinator) patrolRegions() { log.Info("patrol regions has been stopped") return } - if c.cluster.GetUnsafeRecoveryController().IsRunning() { + if c.cluster.IsUnsafeRecovering() { // Skip patrolling regions during unsafe recovery. continue } @@ -152,7 +159,7 @@ func (c *coordinator) patrolRegions() { continue } // Updates the label level isolation statistics. - c.cluster.updateRegionsLabelLevelStats(regions) + c.cluster.UpdateRegionsLabelLevelStats(regions) if len(key) == 0 { patrolCheckRegionsGauge.Set(time.Since(start).Seconds()) start = time.Now() @@ -163,7 +170,7 @@ func (c *coordinator) patrolRegions() { } } -func (c *coordinator) checkRegions(startKey []byte) (key []byte, regions []*core.RegionInfo) { +func (c *Coordinator) checkRegions(startKey []byte) (key []byte, regions []*core.RegionInfo) { regions = c.cluster.ScanRegions(startKey, nil, patrolScanRegionLimit) if len(regions) == 0 { // Resets the scan key. @@ -178,14 +185,14 @@ func (c *coordinator) checkRegions(startKey []byte) (key []byte, regions []*core return } -func (c *coordinator) checkSuspectRegions() { +func (c *Coordinator) checkSuspectRegions() { for _, id := range c.checkers.GetSuspectRegions() { region := c.cluster.GetRegion(id) c.tryAddOperators(region) } } -func (c *coordinator) checkWaitingRegions() { +func (c *Coordinator) checkWaitingRegions() { items := c.checkers.GetWaitingRegions() waitingListGauge.Set(float64(len(items))) for _, item := range items { @@ -195,7 +202,7 @@ func (c *coordinator) checkWaitingRegions() { } // checkPriorityRegions checks priority regions -func (c *coordinator) checkPriorityRegions() { +func (c *Coordinator) checkPriorityRegions() { items := c.checkers.GetPriorityRegions() removes := make([]uint64, 0) priorityListGauge.Set(float64(len(items))) @@ -222,10 +229,10 @@ func (c *coordinator) checkPriorityRegions() { // checkSuspectRanges would pop one suspect key range group // The regions of new version key range and old version key range would be placed into // the suspect regions map -func (c *coordinator) checkSuspectRanges() { +func (c *Coordinator) checkSuspectRanges() { defer logutil.LogPanic() defer c.wg.Done() - log.Info("coordinator begins to check suspect key ranges") + log.Info("Coordinator begins to check suspect key ranges") ticker := time.NewTicker(checkSuspectRangesInterval) defer ticker.Stop() for { @@ -259,7 +266,7 @@ func (c *coordinator) checkSuspectRanges() { } } -func (c *coordinator) tryAddOperators(region *core.RegionInfo) { +func (c *Coordinator) tryAddOperators(region *core.RegionInfo) { if region == nil { // the region could be recent split, continue to wait. return @@ -285,12 +292,12 @@ func (c *coordinator) tryAddOperators(region *core.RegionInfo) { } // drivePushOperator is used to push the unfinished operator to the executor. -func (c *coordinator) drivePushOperator() { +func (c *Coordinator) drivePushOperator() { defer logutil.LogPanic() defer c.wg.Done() - log.Info("coordinator begins to actively drive push operator") - ticker := time.NewTicker(operator.PushOperatorTickInterval) + log.Info("Coordinator begins to actively drive push operator") + ticker := time.NewTicker(pushOperatorTickInterval) defer ticker.Stop() for { select { @@ -303,44 +310,46 @@ func (c *coordinator) drivePushOperator() { } } -func (c *coordinator) runUntilStop() { - c.run() +// RunUntilStop runs the coordinator until receiving the stop signal. +func (c *Coordinator) RunUntilStop() { + c.Run() <-c.ctx.Done() - log.Info("coordinator is stopping") + log.Info("Coordinator is stopping") c.wg.Wait() - log.Info("coordinator has been stopped") + log.Info("Coordinator has been stopped") } -func (c *coordinator) run() { +// Run starts coordinator. +func (c *Coordinator) Run() { ticker := time.NewTicker(runSchedulerCheckInterval) failpoint.Inject("changeCoordinatorTicker", func() { ticker = time.NewTicker(100 * time.Millisecond) }) defer ticker.Stop() - log.Info("coordinator starts to collect cluster information") + log.Info("Coordinator starts to collect cluster information") for { - if c.shouldRun() { - log.Info("coordinator has finished cluster information preparation") + if c.ShouldRun() { + log.Info("Coordinator has finished cluster information preparation") break } select { case <-ticker.C: case <-c.ctx.Done(): - log.Info("coordinator stops running") + log.Info("Coordinator stops running") return } } - log.Info("coordinator starts to run schedulers") + log.Info("Coordinator starts to run schedulers") var ( scheduleNames []string configs []string err error ) for i := 0; i < maxLoadConfigRetries; i++ { - scheduleNames, configs, err = c.cluster.storage.LoadAllScheduleConfig() + scheduleNames, configs, err = c.cluster.GetStorage().LoadAllScheduleConfig() select { case <-c.ctx.Done(): - log.Info("coordinator stops running") + log.Info("Coordinator stops running") return default: } @@ -353,7 +362,7 @@ func (c *coordinator) run() { log.Fatal("cannot load schedulers' config", errs.ZapError(err)) } - scheduleCfg := c.cluster.opt.GetScheduleConfig().Clone() + scheduleCfg := c.cluster.GetPersistOptions().GetScheduleConfig().Clone() // The new way to create scheduler with the independent configuration. for i, name := range scheduleNames { data := configs[i] @@ -373,13 +382,13 @@ func (c *coordinator) run() { log.Info("skip create scheduler with independent configuration", zap.String("scheduler-name", name), zap.String("scheduler-type", cfg.Type), zap.Strings("scheduler-args", cfg.Args)) continue } - s, err := schedulers.CreateScheduler(cfg.Type, c.opController, c.cluster.storage, schedulers.ConfigJSONDecoder([]byte(data))) + s, err := schedulers.CreateScheduler(cfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigJSONDecoder([]byte(data))) if err != nil { log.Error("can not create scheduler with independent configuration", zap.String("scheduler-name", name), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) continue } log.Info("create scheduler with independent configuration", zap.String("scheduler-name", s.GetName())) - if err = c.addScheduler(s); err != nil { + if err = c.AddScheduler(s); err != nil { log.Error("can not add scheduler with independent configuration", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", cfg.Args), errs.ZapError(err)) } } @@ -394,14 +403,14 @@ func (c *coordinator) run() { continue } - s, err := schedulers.CreateScheduler(schedulerCfg.Type, c.opController, c.cluster.storage, schedulers.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args)) + s, err := schedulers.CreateScheduler(schedulerCfg.Type, c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerCfg.Type, schedulerCfg.Args)) if err != nil { log.Error("can not create scheduler", zap.String("scheduler-type", schedulerCfg.Type), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) continue } log.Info("create scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args)) - if err = c.addScheduler(s, schedulerCfg.Args...); err != nil && !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { + if err = c.AddScheduler(s, schedulerCfg.Args...); err != nil && !errors.ErrorEqual(err, errs.ErrSchedulerExisted.FastGenByArgs()) { log.Error("can not add scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", schedulerCfg.Args), errs.ZapError(err)) } else { // Only records the valid scheduler config. @@ -412,21 +421,21 @@ func (c *coordinator) run() { // Removes the invalid scheduler config and persist. scheduleCfg.Schedulers = scheduleCfg.Schedulers[:k] - c.cluster.opt.SetScheduleConfig(scheduleCfg) - if err := c.cluster.opt.Persist(c.cluster.storage); err != nil { + c.cluster.GetPersistOptions().SetScheduleConfig(scheduleCfg) + if err := c.cluster.GetPersistOptions().Persist(c.cluster.GetStorage()); err != nil { log.Error("cannot persist schedule config", errs.ZapError(err)) } c.wg.Add(3) // Starts to patrol regions. - go c.patrolRegions() + go c.PatrolRegions() // Checks suspect key ranges go c.checkSuspectRanges() go c.drivePushOperator() } // LoadPlugin load user plugin -func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { +func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { log.Info("load plugin", zap.String("plugin-path", pluginPath)) // get func: SchedulerType from plugin SchedulerType, err := c.pluginInterface.GetFunction(pluginPath, "SchedulerType") @@ -443,13 +452,13 @@ func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { } schedulerArgs := SchedulerArgs.(func() []string) // create and add user scheduler - s, err := schedulers.CreateScheduler(schedulerType(), c.opController, c.cluster.storage, schedulers.ConfigSliceDecoder(schedulerType(), schedulerArgs())) + s, err := schedulers.CreateScheduler(schedulerType(), c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(schedulerType(), schedulerArgs())) if err != nil { log.Error("can not create scheduler", zap.String("scheduler-type", schedulerType()), errs.ZapError(err)) return } log.Info("create scheduler", zap.String("scheduler-name", s.GetName())) - if err = c.addScheduler(s); err != nil { + if err = c.AddScheduler(s); err != nil { log.Error("can't add scheduler", zap.String("scheduler-name", s.GetName()), errs.ZapError(err)) return } @@ -458,7 +467,7 @@ func (c *coordinator) LoadPlugin(pluginPath string, ch chan string) { go c.waitPluginUnload(pluginPath, s.GetName(), ch) } -func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan string) { +func (c *Coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan string) { defer logutil.LogPanic() defer c.wg.Done() // Get signal from channel which means user unload the plugin @@ -466,7 +475,7 @@ func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan select { case action := <-ch: if action == PluginUnload { - err := c.removeScheduler(schedulerName) + err := c.RemoveScheduler(schedulerName) if err != nil { log.Error("can not remove scheduler", zap.String("scheduler-name", schedulerName), errs.ZapError(err)) } else { @@ -483,11 +492,13 @@ func (c *coordinator) waitPluginUnload(pluginPath, schedulerName string, ch chan } } -func (c *coordinator) stop() { +// Stop stops the coordinator. +func (c *Coordinator) Stop() { c.cancel() } -func (c *coordinator) getHotRegionsByType(typ statistics.RWType) *statistics.StoreHotPeersInfos { +// GetHotRegionsByType gets hot regions' statistics by RWType. +func (c *Coordinator) GetHotRegionsByType(typ statistics.RWType) *statistics.StoreHotPeersInfos { isTraceFlow := c.cluster.GetOpts().IsTraceRegionFlow() storeLoads := c.cluster.GetStoresLoads() stores := c.cluster.GetStores() @@ -513,7 +524,13 @@ func (c *coordinator) getHotRegionsByType(typ statistics.RWType) *statistics.Sto return infos } -func (c *coordinator) getSchedulers() []string { +// GetWaitGroup returns the wait group. Only for test purpose. +func (c *Coordinator) GetWaitGroup() *sync.WaitGroup { + return &c.wg +} + +// GetSchedulers returns all names of schedulers. +func (c *Coordinator) GetSchedulers() []string { c.RLock() defer c.RUnlock() names := make([]string, 0, len(c.schedulers)) @@ -523,7 +540,8 @@ func (c *coordinator) getSchedulers() []string { return names } -func (c *coordinator) getSchedulerHandlers() map[string]http.Handler { +// GetSchedulerHandlers returns all handlers of schedulers. +func (c *Coordinator) GetSchedulerHandlers() map[string]http.Handler { c.RLock() defer c.RUnlock() handlers := make(map[string]http.Handler, len(c.schedulers)) @@ -533,25 +551,28 @@ func (c *coordinator) getSchedulerHandlers() map[string]http.Handler { return handlers } -func (c *coordinator) collectSchedulerMetrics() { +// CollectSchedulerMetrics collects metrics of all schedulers. +func (c *Coordinator) CollectSchedulerMetrics() { c.RLock() defer c.RUnlock() for _, s := range c.schedulers { var allowScheduler float64 // If the scheduler is not allowed to schedule, it will disappear in Grafana panel. // See issue #1341. - if !s.IsPaused() && !s.cluster.GetUnsafeRecoveryController().IsRunning() { + if !s.IsPaused() && !s.cluster.IsUnsafeRecovering() { allowScheduler = 1 } - schedulerStatusGauge.WithLabelValues(s.GetName(), "allow").Set(allowScheduler) + schedulerStatusGauge.WithLabelValues(s.Scheduler.GetName(), "allow").Set(allowScheduler) } } -func (c *coordinator) resetSchedulerMetrics() { +// ResetSchedulerMetrics resets metrics of all schedulers. +func (c *Coordinator) ResetSchedulerMetrics() { schedulerStatusGauge.Reset() } -func (c *coordinator) collectHotSpotMetrics() { +// CollectHotSpotMetrics collects hot spot metrics. +func (c *Coordinator) CollectHotSpotMetrics() { stores := c.cluster.GetStores() // Collects hot write region metrics. collectHotMetrics(c.cluster, stores, statistics.Write) @@ -559,7 +580,7 @@ func (c *coordinator) collectHotSpotMetrics() { collectHotMetrics(c.cluster, stores, statistics.Read) } -func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ statistics.RWType) { +func collectHotMetrics(cluster sche.ClusterInformer, stores []*core.StoreInfo, typ statistics.RWType) { var ( kind string regionStats map[uint64][]*statistics.HotPeerStat @@ -608,22 +629,25 @@ func collectHotMetrics(cluster *RaftCluster, stores []*core.StoreInfo, typ stati if !hasHotLeader && !hasHotPeer { statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, _ statistics.RegionStatKind) { - hotPendingSum.DeleteLabelValues(storeLabel, rwTy.String(), statistics.DimToString(dim)) + schedulers.HotPendingSum.DeleteLabelValues(storeLabel, rwTy.String(), statistics.DimToString(dim)) }) } } } -func (c *coordinator) resetHotSpotMetrics() { +// ResetHotSpotMetrics resets hot spot metrics. +func (c *Coordinator) ResetHotSpotMetrics() { hotSpotStatusGauge.Reset() - hotPendingSum.Reset() + schedulers.HotPendingSum.Reset() } -func (c *coordinator) shouldRun() bool { +// ShouldRun returns true if the coordinator should run. +func (c *Coordinator) ShouldRun() bool { return c.prepareChecker.check(c.cluster.GetBasicCluster()) } -func (c *coordinator) addScheduler(scheduler schedulers.Scheduler, args ...string) error { +// AddScheduler adds a scheduler. +func (c *Coordinator) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { c.Lock() defer c.Unlock() @@ -631,19 +655,20 @@ func (c *coordinator) addScheduler(scheduler schedulers.Scheduler, args ...strin return errs.ErrSchedulerExisted.FastGenByArgs() } - s := newScheduleController(c, scheduler) - if err := s.Prepare(c.cluster); err != nil { + s := NewScheduleController(c, scheduler) + if err := s.Scheduler.Prepare(c.cluster); err != nil { return err } c.wg.Add(1) go c.runScheduler(s) - c.schedulers[s.GetName()] = s - c.cluster.opt.AddSchedulerCfg(s.GetType(), args) + c.schedulers[s.Scheduler.GetName()] = s + c.cluster.GetPersistOptions().AddSchedulerCfg(s.Scheduler.GetType(), args) return nil } -func (c *coordinator) removeScheduler(name string) error { +// RemoveScheduler removes a scheduler by name. +func (c *Coordinator) RemoveScheduler(name string) error { c.Lock() defer c.Unlock() if c.cluster == nil { @@ -654,18 +679,18 @@ func (c *coordinator) removeScheduler(name string) error { return errs.ErrSchedulerNotFound.FastGenByArgs() } - opt := c.cluster.opt + opt := c.cluster.GetPersistOptions() if err := c.removeOptScheduler(opt, name); err != nil { log.Error("can not remove scheduler", zap.String("scheduler-name", name), errs.ZapError(err)) return err } - if err := opt.Persist(c.cluster.storage); err != nil { + if err := opt.Persist(c.cluster.GetStorage()); err != nil { log.Error("the option can not persist scheduler config", errs.ZapError(err)) return err } - if err := c.cluster.storage.RemoveScheduleConfig(name); err != nil { + if err := c.cluster.GetStorage().RemoveScheduleConfig(name); err != nil { log.Error("can not remove the scheduler config", errs.ZapError(err)) return err } @@ -677,7 +702,7 @@ func (c *coordinator) removeScheduler(name string) error { return nil } -func (c *coordinator) removeOptScheduler(o *config.PersistOptions, name string) error { +func (c *Coordinator) removeOptScheduler(o *config.PersistOptions, name string) error { v := o.GetScheduleConfig().Clone() for i, schedulerCfg := range v.Schedulers { // To create a temporary scheduler is just used to get scheduler's name @@ -700,7 +725,8 @@ func (c *coordinator) removeOptScheduler(o *config.PersistOptions, name string) return nil } -func (c *coordinator) pauseOrResumeScheduler(name string, t int64) error { +// PauseOrResumeScheduler pauses or resumes a scheduler by name. +func (c *Coordinator) PauseOrResumeScheduler(name string, t int64) error { c.Lock() defer c.Unlock() if c.cluster == nil { @@ -731,8 +757,8 @@ func (c *coordinator) pauseOrResumeScheduler(name string, t int64) error { return err } -// isSchedulerAllowed returns whether a scheduler is allowed to schedule, a scheduler is not allowed to schedule if it is paused or blocked by unsafe recovery. -func (c *coordinator) isSchedulerAllowed(name string) (bool, error) { +// IsSchedulerAllowed returns whether a scheduler is allowed to schedule, a scheduler is not allowed to schedule if it is paused or blocked by unsafe recovery. +func (c *Coordinator) IsSchedulerAllowed(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -745,7 +771,8 @@ func (c *coordinator) isSchedulerAllowed(name string) (bool, error) { return s.AllowSchedule(false), nil } -func (c *coordinator) isSchedulerPaused(name string) (bool, error) { +// IsSchedulerPaused returns whether a scheduler is paused. +func (c *Coordinator) IsSchedulerPaused(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -758,7 +785,8 @@ func (c *coordinator) isSchedulerPaused(name string) (bool, error) { return s.IsPaused(), nil } -func (c *coordinator) isSchedulerDisabled(name string) (bool, error) { +// IsSchedulerDisabled returns whether a scheduler is disabled. +func (c *Coordinator) IsSchedulerDisabled(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -768,8 +796,8 @@ func (c *coordinator) isSchedulerDisabled(name string) (bool, error) { if !ok { return false, errs.ErrSchedulerNotFound.FastGenByArgs() } - t := s.GetType() - scheduleConfig := c.cluster.GetScheduleConfig() + t := s.Scheduler.GetType() + scheduleConfig := c.cluster.GetPersistOptions().GetScheduleConfig() for _, s := range scheduleConfig.Schedulers { if t == s.Type { return s.Disable, nil @@ -778,7 +806,8 @@ func (c *coordinator) isSchedulerDisabled(name string) (bool, error) { return false, nil } -func (c *coordinator) isSchedulerExisted(name string) (bool, error) { +// IsSchedulerExisted returns whether a scheduler is existed. +func (c *Coordinator) IsSchedulerExisted(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -791,10 +820,10 @@ func (c *coordinator) isSchedulerExisted(name string) (bool, error) { return true, nil } -func (c *coordinator) runScheduler(s *scheduleController) { +func (c *Coordinator) runScheduler(s *scheduleController) { defer logutil.LogPanic() defer c.wg.Done() - defer s.Cleanup(c.cluster) + defer s.Scheduler.Cleanup(c.cluster) timer := time.NewTimer(s.GetInterval()) defer timer.Stop() @@ -808,19 +837,20 @@ func (c *coordinator) runScheduler(s *scheduleController) { } if op := s.Schedule(diagnosable); len(op) > 0 { added := c.opController.AddWaitingOperator(op...) - log.Debug("add operator", zap.Int("added", added), zap.Int("total", len(op)), zap.String("scheduler", s.GetName())) + log.Debug("add operator", zap.Int("added", added), zap.Int("total", len(op)), zap.String("scheduler", s.Scheduler.GetName())) } case <-s.Ctx().Done(): log.Info("scheduler has been stopped", - zap.String("scheduler-name", s.GetName()), + zap.String("scheduler-name", s.Scheduler.GetName()), errs.ZapError(s.Ctx().Err())) return } } } -func (c *coordinator) pauseOrResumeChecker(name string, t int64) error { +// PauseOrResumeChecker pauses or resumes a checker by name. +func (c *Coordinator) PauseOrResumeChecker(name string, t int64) error { c.Lock() defer c.Unlock() if c.cluster == nil { @@ -834,7 +864,8 @@ func (c *coordinator) pauseOrResumeChecker(name string, t int64) error { return nil } -func (c *coordinator) isCheckerPaused(name string) (bool, error) { +// IsCheckerPaused returns whether a checker is paused. +func (c *Coordinator) IsCheckerPaused(name string) (bool, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -847,14 +878,60 @@ func (c *coordinator) isCheckerPaused(name string) (bool, error) { return p.IsPaused(), nil } -func (c *coordinator) GetDiagnosticResult(name string) (*DiagnosticResult, error) { +// GetRegionScatterer returns the region scatterer. +func (c *Coordinator) GetRegionScatterer() *RegionScatterer { + return c.regionScatterer +} + +// GetRegionSplitter returns the region splitter. +func (c *Coordinator) GetRegionSplitter() *RegionSplitter { + return c.regionSplitter +} + +// GetOperatorController returns the operator controller. +func (c *Coordinator) GetOperatorController() *operator.Controller { + return c.opController +} + +// GetCheckerController returns the checker controller. +func (c *Coordinator) GetCheckerController() *checker.Controller { + return c.checkers +} + +// GetMergeChecker returns the merge checker. +func (c *Coordinator) GetMergeChecker() *checker.MergeChecker { + return c.checkers.GetMergeChecker() +} + +// GetRuleChecker returns the rule checker. +func (c *Coordinator) GetRuleChecker() *checker.RuleChecker { + return c.checkers.GetRuleChecker() +} + +// GetPrepareChecker returns the prepare checker. +func (c *Coordinator) GetPrepareChecker() *prepareChecker { + return c.prepareChecker +} + +// GetHeartbeatStreams returns the heartbeat streams. Only for test purpose. +func (c *Coordinator) GetHeartbeatStreams() *hbstream.HeartbeatStreams { + return c.hbStreams +} + +// GetCluster returns the cluster. Only for test purpose. +func (c *Coordinator) GetCluster() sche.ClusterInformer { + return c.cluster +} + +// GetDiagnosticResult returns the diagnostic result. +func (c *Coordinator) GetDiagnosticResult(name string) (*DiagnosticResult, error) { return c.diagnosticManager.getDiagnosticResult(name) } // scheduleController is used to manage a scheduler to schedulers. type scheduleController struct { schedulers.Scheduler - cluster *RaftCluster + cluster sche.ClusterInformer opController *operator.Controller nextInterval time.Duration ctx context.Context @@ -864,8 +941,8 @@ type scheduleController struct { diagnosticRecorder *diagnosticRecorder } -// newScheduleController creates a new scheduleController. -func newScheduleController(c *coordinator, s schedulers.Scheduler) *scheduleController { +// NewScheduleController creates a new scheduleController. +func NewScheduleController(c *Coordinator, s schedulers.Scheduler) *scheduleController { ctx, cancel := context.WithCancel(c.ctx) return &scheduleController{ Scheduler: s, @@ -939,6 +1016,11 @@ func (s *scheduleController) GetInterval() time.Duration { return s.nextInterval } +// SetInterval sets the interval of scheduling for a scheduler. for test purpose. +func (s *scheduleController) SetInterval(interval time.Duration) { + s.nextInterval = interval +} + // AllowSchedule returns if a scheduler is allowed to schedulers. func (s *scheduleController) AllowSchedule(diagnosable bool) bool { if !s.Scheduler.IsScheduleAllowed(s.cluster) { @@ -947,7 +1029,7 @@ func (s *scheduleController) AllowSchedule(diagnosable bool) bool { } return false } - if s.IsPaused() || s.cluster.GetUnsafeRecoveryController().IsRunning() { + if s.IsPaused() || s.cluster.IsUnsafeRecovering() { if diagnosable { s.diagnosticRecorder.setResultFromStatus(paused) } @@ -962,23 +1044,24 @@ func (s *scheduleController) IsPaused() bool { return time.Now().Unix() < delayUntil } -// GetPausedSchedulerDelayAt returns paused timestamp of a paused scheduler -func (s *scheduleController) GetDelayAt() int64 { +// getDelayAt returns paused timestamp of a paused scheduler +func (s *scheduleController) getDelayAt() int64 { if s.IsPaused() { return atomic.LoadInt64(&s.delayAt) } return 0 } -// GetPausedSchedulerDelayUntil returns resume timestamp of a paused scheduler -func (s *scheduleController) GetDelayUntil() int64 { +// getDelayUntil returns resume timestamp of a paused scheduler +func (s *scheduleController) getDelayUntil() int64 { if s.IsPaused() { return atomic.LoadInt64(&s.delayUntil) } return 0 } -func (c *coordinator) getPausedSchedulerDelayAt(name string) (int64, error) { +// GetPausedSchedulerDelayAt returns paused timestamp of a paused scheduler +func (c *Coordinator) GetPausedSchedulerDelayAt(name string) (int64, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -988,10 +1071,11 @@ func (c *coordinator) getPausedSchedulerDelayAt(name string) (int64, error) { if !ok { return -1, errs.ErrSchedulerNotFound.FastGenByArgs() } - return s.GetDelayAt(), nil + return s.getDelayAt(), nil } -func (c *coordinator) getPausedSchedulerDelayUntil(name string) (int64, error) { +// GetPausedSchedulerDelayUntil returns the delay time until the scheduler is paused. +func (c *Coordinator) GetPausedSchedulerDelayUntil(name string) (int64, error) { c.RLock() defer c.RUnlock() if c.cluster == nil { @@ -1001,11 +1085,11 @@ func (c *coordinator) getPausedSchedulerDelayUntil(name string) (int64, error) { if !ok { return -1, errs.ErrSchedulerNotFound.FastGenByArgs() } - return s.GetDelayUntil(), nil + return s.getDelayUntil(), nil } // CheckTransferWitnessLeader determines if transfer leader is required, then sends to the scheduler if needed -func (c *coordinator) CheckTransferWitnessLeader(region *core.RegionInfo) { +func (c *Coordinator) CheckTransferWitnessLeader(region *core.RegionInfo) { if core.NeedTransferWitnessLeader(region) { c.RLock() s, ok := c.schedulers[schedulers.TransferWitnessLeaderName] @@ -1019,3 +1103,22 @@ func (c *coordinator) CheckTransferWitnessLeader(region *core.RegionInfo) { } } } + +// cacheCluster include cache info to improve the performance. +type cacheCluster struct { + sche.ClusterInformer + stores []*core.StoreInfo +} + +// GetStores returns store infos from cache +func (c *cacheCluster) GetStores() []*core.StoreInfo { + return c.stores +} + +// newCacheCluster constructor for cache +func newCacheCluster(c sche.ClusterInformer) *cacheCluster { + return &cacheCluster{ + ClusterInformer: c, + stores: c.GetStores(), + } +} diff --git a/pkg/schedule/core/cluster_informer.go b/pkg/schedule/core/cluster_informer.go index 8588f184469..345cdeb74a9 100644 --- a/pkg/schedule/core/cluster_informer.go +++ b/pkg/schedule/core/cluster_informer.go @@ -17,12 +17,13 @@ package core import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/id" - "github.com/tikv/pd/pkg/schedule/config" + sc "github.com/tikv/pd/pkg/schedule/config" "github.com/tikv/pd/pkg/schedule/labeler" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/server/config" ) // ClusterInformer provides the necessary information of a cluster. @@ -33,14 +34,18 @@ type ClusterInformer interface { buckets.BucketStatInformer GetBasicCluster() *core.BasicCluster - GetStoreConfig() config.StoreConfig + GetStoreConfig() sc.StoreConfig GetAllocator() id.Allocator GetRegionLabeler() *labeler.RegionLabeler GetStorage() storage.Storage RemoveScheduler(name string) error AddSuspectRegions(ids ...uint64) - SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) RecordOpStepWithTTL(regionID uint64) + UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) + IsSchedulerExisted(name string) (bool, error) + IsSchedulerDisabled(name string) (bool, error) + GetPersistOptions() *config.PersistOptions + IsUnsafeRecovering() bool } // RegionHealthCluster is an aggregate interface that wraps multiple interfaces @@ -49,6 +54,6 @@ type RegionHealthCluster interface { core.StoreSetController core.RegionSetInformer - GetOpts() config.Config + GetOpts() sc.Config GetRuleManager() *placement.RuleManager } diff --git a/server/cluster/diagnostic_manager.go b/pkg/schedule/diagnostic_manager.go similarity index 94% rename from server/cluster/diagnostic_manager.go rename to pkg/schedule/diagnostic_manager.go index dc6ae042995..c68999f6cdd 100644 --- a/server/cluster/diagnostic_manager.go +++ b/pkg/schedule/diagnostic_manager.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedule import ( "fmt" @@ -22,6 +22,7 @@ import ( "github.com/tikv/pd/pkg/cache" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/movingaverage" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" "github.com/tikv/pd/pkg/schedule/schedulers" @@ -53,11 +54,11 @@ var DiagnosableSummaryFunc = map[string]plan.Summary{ } type diagnosticManager struct { - cluster *RaftCluster + cluster sche.ClusterInformer recorders map[string]*diagnosticRecorder } -func newDiagnosticManager(cluster *RaftCluster) *diagnosticManager { +func newDiagnosticManager(cluster sche.ClusterInformer) *diagnosticManager { recorders := make(map[string]*diagnosticRecorder) for name := range DiagnosableSummaryFunc { recorders[name] = newDiagnosticRecorder(name, cluster) @@ -69,7 +70,7 @@ func newDiagnosticManager(cluster *RaftCluster) *diagnosticManager { } func (d *diagnosticManager) getDiagnosticResult(name string) (*DiagnosticResult, error) { - if !d.cluster.opt.IsDiagnosticAllowed() { + if !d.cluster.GetOpts().IsDiagnosticAllowed() { return nil, errs.ErrDiagnosticDisabled } @@ -99,12 +100,12 @@ func (d *diagnosticManager) getRecorder(name string) *diagnosticRecorder { // diagnosticRecorder is used to manage diagnostic for one scheduler. type diagnosticRecorder struct { schedulerName string - cluster *RaftCluster + cluster sche.ClusterInformer summaryFunc plan.Summary results *cache.FIFO } -func newDiagnosticRecorder(name string, cluster *RaftCluster) *diagnosticRecorder { +func newDiagnosticRecorder(name string, cluster sche.ClusterInformer) *diagnosticRecorder { summaryFunc, ok := DiagnosableSummaryFunc[name] if !ok { log.Error("can't find summary function", zap.String("scheduler-name", name)) @@ -122,7 +123,7 @@ func (d *diagnosticRecorder) isAllowed() bool { if d == nil { return false } - return d.cluster.opt.IsDiagnosticAllowed() + return d.cluster.GetOpts().IsDiagnosticAllowed() } func (d *diagnosticRecorder) getLastResult() *DiagnosticResult { diff --git a/pkg/schedule/labeler/metrics.go b/pkg/schedule/labeler/metrics.go new file mode 100644 index 00000000000..31148259c60 --- /dev/null +++ b/pkg/schedule/labeler/metrics.go @@ -0,0 +1,30 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package labeler + +import "github.com/prometheus/client_golang/prometheus" + +// LabelerEventCounter is a counter of the scheduler labeler system. +var LabelerEventCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pd", + Subsystem: "schedule", + Name: "labeler_event_counter", + Help: "Counter of the scheduler label.", + }, []string{"type", "event"}) + +func init() { + prometheus.MustRegister(LabelerEventCounter) +} diff --git a/pkg/schedule/metrics.go b/pkg/schedule/metrics.go index 74f368daf77..9f45554ccac 100644 --- a/pkg/schedule/metrics.go +++ b/pkg/schedule/metrics.go @@ -33,18 +33,44 @@ var ( Help: "Counter of the distribution in scatter.", }, []string{"store", "is_leader", "engine"}) - // LabelerEventCounter is a counter of the scheduler labeler system. - LabelerEventCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ + hotSpotStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ Namespace: "pd", - Subsystem: "schedule", - Name: "labeler_event_counter", - Help: "Counter of the scheduler label.", - }, []string{"type", "event"}) + Subsystem: "hotspot", + Name: "status", + Help: "Status of the hotspot.", + }, []string{"address", "store", "type"}) + + schedulerStatusGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "status", + Help: "Status of the scheduler.", + }, []string{"kind", "type"}) + + regionListGauge = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "checker", + Name: "region_list", + Help: "Number of region in waiting list", + }, []string{"type"}) + + patrolCheckRegionsGauge = prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "checker", + Name: "patrol_regions_time", + Help: "Time spent of patrol checks region.", + }) ) func init() { prometheus.MustRegister(scatterCounter) prometheus.MustRegister(scatterDistributionCounter) - prometheus.MustRegister(LabelerEventCounter) + prometheus.MustRegister(schedulerStatusGauge) + prometheus.MustRegister(hotSpotStatusGauge) + prometheus.MustRegister(regionListGauge) + prometheus.MustRegister(patrolCheckRegionsGauge) } diff --git a/pkg/schedule/operator/operator_controller.go b/pkg/schedule/operator/operator_controller.go index 9561420f804..ab7a39ac032 100644 --- a/pkg/schedule/operator/operator_controller.go +++ b/pkg/schedule/operator/operator_controller.go @@ -46,8 +46,6 @@ const ( var ( slowNotifyInterval = 5 * time.Second fastNotifyInterval = 2 * time.Second - // PushOperatorTickInterval is the interval try to push the - PushOperatorTickInterval = 500 * time.Millisecond // StoreBalanceBaseTime represents the base time of balance rate. StoreBalanceBaseTime float64 = 60 // FastOperatorFinishTime min finish time, if finish duration less than it,op will be pushed to fast operator queue diff --git a/server/cluster/prepare_checker.go b/pkg/schedule/prepare_checker.go similarity index 84% rename from server/cluster/prepare_checker.go rename to pkg/schedule/prepare_checker.go index 39d4cab1f91..6a8acbcf9f5 100644 --- a/server/cluster/prepare_checker.go +++ b/pkg/schedule/prepare_checker.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package schedule import ( "time" @@ -65,7 +65,7 @@ func (checker *prepareChecker) check(c *core.BasicCluster) bool { return true } -func (checker *prepareChecker) collect(region *core.RegionInfo) { +func (checker *prepareChecker) Collect(region *core.RegionInfo) { checker.Lock() defer checker.Unlock() for _, p := range region.GetPeers() { @@ -74,8 +74,22 @@ func (checker *prepareChecker) collect(region *core.RegionInfo) { checker.sum++ } -func (checker *prepareChecker) isPrepared() bool { +func (checker *prepareChecker) IsPrepared() bool { checker.RLock() defer checker.RUnlock() return checker.prepared } + +// for test purpose +func (checker *prepareChecker) SetPrepared() { + checker.Lock() + defer checker.Unlock() + checker.prepared = true +} + +// for test purpose +func (checker *prepareChecker) GetSum() int { + checker.RLock() + defer checker.RUnlock() + return checker.sum +} diff --git a/pkg/schedule/schedulers/balance_region.go b/pkg/schedule/schedulers/balance_region.go index ab28f1d8af9..43e05863729 100644 --- a/pkg/schedule/schedulers/balance_region.go +++ b/pkg/schedule/schedulers/balance_region.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/schedule" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" @@ -153,7 +152,7 @@ func (s *balanceRegionScheduler) Schedule(cluster sche.ClusterInformer, dryRun b replicaFilter := filter.NewRegionReplicatedFilter(cluster) baseRegionFilters := []filter.RegionFilter{downFilter, replicaFilter, snapshotFilter} switch cluster.(type) { - case *schedule.RangeCluster: + case *rangeCluster: // allow empty region to be scheduled in range cluster default: baseRegionFilters = append(baseRegionFilters, filter.NewRegionEmptyFilter(cluster)) diff --git a/pkg/schedule/schedulers/hot_region.go b/pkg/schedule/schedulers/hot_region.go index b4402b2c079..af8f5310f49 100644 --- a/pkg/schedule/schedulers/hot_region.go +++ b/pkg/schedule/schedulers/hot_region.go @@ -120,7 +120,7 @@ func newBaseHotScheduler(opController *operator.Controller) *baseHotScheduler { // each store, only update read or write load detail func (h *baseHotScheduler) prepareForBalance(rw statistics.RWType, cluster sche.ClusterInformer) { h.stInfos = statistics.SummaryStoreInfos(cluster.GetStores()) - h.summaryPendingInfluence(cluster) + h.summaryPendingInfluence() h.storesLoads = cluster.GetStoresLoads() isTraceRegionFlow := cluster.GetOpts().IsTraceRegionFlow() @@ -157,7 +157,7 @@ func (h *baseHotScheduler) prepareForBalance(rw statistics.RWType, cluster sche. // summaryPendingInfluence calculate the summary of pending Influence for each store // and clean the region from regionInfluence if they have ended operator. // It makes each dim rate or count become `weight` times to the origin value. -func (h *baseHotScheduler) summaryPendingInfluence(cluster sche.ClusterInformer) { +func (h *baseHotScheduler) summaryPendingInfluence() { for id, p := range h.regionPendings { from := h.stInfos[p.from] to := h.stInfos[p.to] @@ -180,12 +180,17 @@ func (h *baseHotScheduler) summaryPendingInfluence(cluster sche.ClusterInformer) storeLabel := strconv.FormatUint(storeID, 10) if infl := info.PendingSum; infl != nil { statistics.ForeachRegionStats(func(rwTy statistics.RWType, dim int, kind statistics.RegionStatKind) { - cluster.SetHotPendingInfluenceMetrics(storeLabel, rwTy.String(), statistics.DimToString(dim), infl.Loads[kind]) + setHotPendingInfluenceMetrics(storeLabel, rwTy.String(), statistics.DimToString(dim), infl.Loads[kind]) }) } } } +// setHotPendingInfluenceMetrics sets pending influence in hot scheduler. +func setHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { + HotPendingSum.WithLabelValues(storeLabel, rwTy, dim).Set(load) +} + func (h *baseHotScheduler) randomRWType() statistics.RWType { return h.types[h.r.Int()%len(h.types)] } diff --git a/pkg/schedule/schedulers/hot_region_test.go b/pkg/schedule/schedulers/hot_region_test.go index d6e129c1678..3bb6df6b273 100644 --- a/pkg/schedule/schedulers/hot_region_test.go +++ b/pkg/schedule/schedulers/hot_region_test.go @@ -165,7 +165,7 @@ func checkGCPendingOpInfos(re *require.Assertions, enablePlacementRules bool) { } } - hb.summaryPendingInfluence(tc) // Calling this function will GC. + hb.summaryPendingInfluence() // Calling this function will GC. for i := range opInfluenceCreators { for j, typ := range typs { @@ -1895,7 +1895,7 @@ func TestInfluenceByRWType(t *testing.T) { op := ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence(tc) + hb.(*hotScheduler).summaryPendingInfluence() stInfos := hb.(*hotScheduler).stInfos re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -0.5*units.MiB)) re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteBytes], -0.5*units.MiB)) @@ -1920,7 +1920,7 @@ func TestInfluenceByRWType(t *testing.T) { op = ops[0] re.NotNil(op) - hb.(*hotScheduler).summaryPendingInfluence(tc) + hb.(*hotScheduler).summaryPendingInfluence() stInfos = hb.(*hotScheduler).stInfos // assert read/write influence is the sum of write peer and write leader re.True(nearlyAbout(stInfos[1].PendingSum.Loads[statistics.RegionWriteKeys], -1.2*units.MiB)) diff --git a/pkg/schedule/schedulers/metrics.go b/pkg/schedule/schedulers/metrics.go index 47512450ae5..e0ff8f7e87e 100644 --- a/pkg/schedule/schedulers/metrics.go +++ b/pkg/schedule/schedulers/metrics.go @@ -157,6 +157,15 @@ var storeSlowTrendMiscGauge = prometheus.NewGaugeVec( Help: "Store trend internal uncatelogued values", }, []string{"type"}) +// HotPendingSum is the sum of pending influence in hot region scheduler. +var HotPendingSum = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pd", + Subsystem: "scheduler", + Name: "hot_pending_sum", + Help: "Pending influence sum of store in hot region scheduler.", + }, []string{"store", "rw", "dim"}) + func init() { prometheus.MustRegister(schedulerCounter) prometheus.MustRegister(schedulerStatus) @@ -175,4 +184,5 @@ func init() { prometheus.MustRegister(storeSlowTrendEvictedStatusGauge) prometheus.MustRegister(storeSlowTrendActionStatusGauge) prometheus.MustRegister(storeSlowTrendMiscGauge) + prometheus.MustRegister(HotPendingSum) } diff --git a/pkg/schedule/range_cluster.go b/pkg/schedule/schedulers/range_cluster.go similarity index 81% rename from pkg/schedule/range_cluster.go rename to pkg/schedule/schedulers/range_cluster.go index 841d14982e4..3f81f4e59ca 100644 --- a/pkg/schedule/range_cluster.go +++ b/pkg/schedule/schedulers/range_cluster.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package schedule +package schedulers import ( "github.com/docker/go-units" @@ -20,28 +20,28 @@ import ( sche "github.com/tikv/pd/pkg/schedule/core" ) -// RangeCluster isolates the cluster by range. -type RangeCluster struct { +// rangeCluster isolates the cluster by range. +type rangeCluster struct { sche.ClusterInformer subCluster *core.BasicCluster // Collect all regions belong to the range. tolerantSizeRatio float64 } -// GenRangeCluster gets a range cluster by specifying start key and end key. -// The cluster can only know the regions within [startKey, endKey]. -func GenRangeCluster(cluster sche.ClusterInformer, startKey, endKey []byte) *RangeCluster { +// genRangeCluster gets a range cluster by specifying start key and end key. +// The cluster can only know the regions within [startKey, endKey). +func genRangeCluster(cluster sche.ClusterInformer, startKey, endKey []byte) *rangeCluster { subCluster := core.NewBasicCluster() for _, r := range cluster.ScanRegions(startKey, endKey, -1) { origin, overlaps, rangeChanged := subCluster.SetRegion(r) subCluster.UpdateSubTree(r, origin, overlaps, rangeChanged) } - return &RangeCluster{ + return &rangeCluster{ ClusterInformer: cluster, subCluster: subCluster, } } -func (r *RangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { +func (r *rangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { id := s.GetID() used := float64(s.GetUsedSize()) / units.MiB @@ -69,7 +69,7 @@ func (r *RangeCluster) updateStoreInfo(s *core.StoreInfo) *core.StoreInfo { } // GetStore searches for a store by ID. -func (r *RangeCluster) GetStore(id uint64) *core.StoreInfo { +func (r *rangeCluster) GetStore(id uint64) *core.StoreInfo { s := r.ClusterInformer.GetStore(id) if s == nil { return nil @@ -78,7 +78,7 @@ func (r *RangeCluster) GetStore(id uint64) *core.StoreInfo { } // GetStores returns all Stores in the cluster. -func (r *RangeCluster) GetStores() []*core.StoreInfo { +func (r *rangeCluster) GetStores() []*core.StoreInfo { stores := r.ClusterInformer.GetStores() newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { @@ -88,12 +88,12 @@ func (r *RangeCluster) GetStores() []*core.StoreInfo { } // SetTolerantSizeRatio sets the tolerant size ratio. -func (r *RangeCluster) SetTolerantSizeRatio(ratio float64) { +func (r *rangeCluster) SetTolerantSizeRatio(ratio float64) { r.tolerantSizeRatio = ratio } // GetTolerantSizeRatio gets the tolerant size ratio. -func (r *RangeCluster) GetTolerantSizeRatio() float64 { +func (r *rangeCluster) GetTolerantSizeRatio() float64 { if r.tolerantSizeRatio != 0 { return r.tolerantSizeRatio } @@ -101,22 +101,22 @@ func (r *RangeCluster) GetTolerantSizeRatio() float64 { } // RandFollowerRegions returns a random region that has a follower on the store. -func (r *RangeCluster) RandFollowerRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { +func (r *rangeCluster) RandFollowerRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { return r.subCluster.RandFollowerRegions(storeID, ranges) } // RandLeaderRegions returns a random region that has leader on the store. -func (r *RangeCluster) RandLeaderRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { +func (r *rangeCluster) RandLeaderRegions(storeID uint64, ranges []core.KeyRange) []*core.RegionInfo { return r.subCluster.RandLeaderRegions(storeID, ranges) } // GetAverageRegionSize returns the average region approximate size. -func (r *RangeCluster) GetAverageRegionSize() int64 { +func (r *rangeCluster) GetAverageRegionSize() int64 { return r.subCluster.GetAverageRegionSize() } // GetRegionStores returns all stores that contains the region's peer. -func (r *RangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInfo { +func (r *rangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInfo { stores := r.ClusterInformer.GetRegionStores(region) newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { @@ -126,7 +126,7 @@ func (r *RangeCluster) GetRegionStores(region *core.RegionInfo) []*core.StoreInf } // GetFollowerStores returns all stores that contains the region's follower peer. -func (r *RangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreInfo { +func (r *rangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreInfo { stores := r.ClusterInformer.GetFollowerStores(region) newStores := make([]*core.StoreInfo, 0, len(stores)) for _, s := range stores { @@ -136,7 +136,7 @@ func (r *RangeCluster) GetFollowerStores(region *core.RegionInfo) []*core.StoreI } // GetLeaderStore returns all stores that contains the region's leader peer. -func (r *RangeCluster) GetLeaderStore(region *core.RegionInfo) *core.StoreInfo { +func (r *rangeCluster) GetLeaderStore(region *core.RegionInfo) *core.StoreInfo { s := r.ClusterInformer.GetLeaderStore(region) if s != nil { return r.updateStoreInfo(s) diff --git a/pkg/schedule/schedulers/scatter_range.go b/pkg/schedule/schedulers/scatter_range.go index 2892bd46c66..6f9e48398f6 100644 --- a/pkg/schedule/schedulers/scatter_range.go +++ b/pkg/schedule/schedulers/scatter_range.go @@ -22,7 +22,6 @@ import ( "github.com/pingcap/errors" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/plan" @@ -192,7 +191,7 @@ func (l *scatterRangeScheduler) allowBalanceRegion(cluster sche.ClusterInformer) func (l *scatterRangeScheduler) Schedule(cluster sche.ClusterInformer, dryRun bool) ([]*operator.Operator, []plan.Plan) { scatterRangeCounter.Inc() // isolate a new cluster according to the key range - c := schedule.GenRangeCluster(cluster, l.config.GetStartKey(), l.config.GetEndKey()) + c := genRangeCluster(cluster, l.config.GetStartKey(), l.config.GetEndKey()) c.SetTolerantSizeRatio(2) if l.allowBalanceLeader(cluster) { ops, _ := l.balanceLeader.Schedule(c, false) diff --git a/pkg/schedule/schedulers/utils.go b/pkg/schedule/schedulers/utils.go index fa636d30d2f..45c3ada135f 100644 --- a/pkg/schedule/schedulers/utils.go +++ b/pkg/schedule/schedulers/utils.go @@ -23,7 +23,6 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/core/constant" "github.com/tikv/pd/pkg/errs" - "github.com/tikv/pd/pkg/schedule" sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -185,7 +184,7 @@ func (p *solver) getTolerantResource() int64 { func adjustTolerantRatio(cluster sche.ClusterInformer, kind constant.ScheduleKind) float64 { var tolerantSizeRatio float64 switch c := cluster.(type) { - case *schedule.RangeCluster: + case *rangeCluster: // range cluster use a separate configuration tolerantSizeRatio = c.GetTolerantSizeRatio() default: diff --git a/server/cluster/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go similarity index 84% rename from server/cluster/unsafe_recovery_controller.go rename to pkg/unsaferecovery/unsafe_recovery_controller.go index d54a331048e..1eb960059d6 100644 --- a/server/cluster/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +package unsaferecovery import ( "bytes" @@ -31,24 +31,26 @@ import ( "github.com/tikv/pd/pkg/codec" "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/id" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/syncutil" "go.uber.org/zap" ) -type unsafeRecoveryStage int +// stage is the stage of unsafe recovery. +type stage int const ( storeRequestInterval = time.Second * 40 ) -// Stage transition graph: for more details, please check `unsafeRecoveryController.HandleStoreHeartbeat()` +// Stage transition graph: for more details, please check `Controller.HandleStoreHeartbeat()` // // +-----------+ +-----------+ // +-----------+ | | | | -// | | | collect | | tombstone | -// | idle |------>| Report |-----+---->| tiflash |-----+ -// | | | | | | learner | | +// | | | Collect | | Tombstone | +// | Idle |------>| Report |-----+---->| Tiflash |-----+ +// | | | | | | Learner | | // +-----------+ +-----------+ | | | | // | +-----------+ | // | | | @@ -56,7 +58,7 @@ const ( // | v | // | +-----------+ | // | | | | -// | | force | | +// | | Force | | // | | LeaderFor |-----+ // | |CommitMerge| | // | | | | @@ -66,8 +68,8 @@ const ( // | v | // | +-----------+ | +-----------+ // | | | | | | +-----------+ -// | | force | | | exitForce | | | -// | | Leader |-----+---->| Leader |------->| failed | +// | | Force | | | exitForce | | | +// | | Leader |-----+---->| Leader |------->| Failed | // | | | | | | | | // | +-----------+ | +-----------+ +-----------+ // | | | @@ -75,7 +77,7 @@ const ( // | v | // | +-----------+ | // | | | | -// | | demote | | +// | | Demote | | // +-----| Voter |-----| // | | | // +-----------+ | @@ -84,28 +86,37 @@ const ( // v | // +-----------+ +-----------+ | // +-----------+ | | | | | -// | | | exitForce | | create | | -// | finished |<------| Leader |<----------| Region |-----+ +// | | | ExitForce | | Create | | +// | Finished |<------| Leader |<----------| Region |-----+ // | | | | | | // +-----------+ +-----------+ +-----------+ const ( - idle unsafeRecoveryStage = iota - collectReport - tombstoneTiFlashLearner - forceLeaderForCommitMerge - forceLeader - demoteFailedVoter - createEmptyRegion - exitForceLeader - finished - failed + Idle stage = iota + CollectReport + TombstoneTiFlashLearner + ForceLeaderForCommitMerge + ForceLeader + DemoteFailedVoter + CreateEmptyRegion + ExitForceLeader + Finished + Failed ) -type unsafeRecoveryController struct { +type cluster interface { + core.StoreSetInformer + + DropCacheAllRegion() + GetAllocator() id.Allocator + BuryStore(storeID uint64, forceBury bool) error +} + +// Controller is used to control the unsafe recovery process. +type Controller struct { syncutil.RWMutex - cluster *RaftCluster - stage unsafeRecoveryStage + cluster cluster + stage stage // the round of recovery, which is an increasing number to identify the reports of each round step uint64 failedStores map[uint64]struct{} @@ -120,8 +131,9 @@ type unsafeRecoveryController struct { storeRecoveryPlans map[uint64]*pdpb.RecoveryPlan // accumulated output for the whole recovery process - output []StageOutput - affectedTableIDs map[int64]struct{} + output []StageOutput + // exposed to the outside for testing + AffectedTableIDs map[int64]struct{} affectedMetaRegions map[uint64]struct{} err error } @@ -134,16 +146,17 @@ type StageOutput struct { Details []string `json:"details,omitempty"` } -func newUnsafeRecoveryController(cluster *RaftCluster) *unsafeRecoveryController { - u := &unsafeRecoveryController{ +// NewController creates a new Controller. +func NewController(cluster cluster) *Controller { + u := &Controller{ cluster: cluster, } u.reset() return u } -func (u *unsafeRecoveryController) reset() { - u.stage = idle +func (u *Controller) reset() { + u.stage = Idle u.step = 0 u.failedStores = make(map[uint64]struct{}) u.storeReports = make(map[uint64]*pdpb.StoreReport) @@ -151,25 +164,25 @@ func (u *unsafeRecoveryController) reset() { u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) u.output = make([]StageOutput, 0) - u.affectedTableIDs = make(map[int64]struct{}, 0) + u.AffectedTableIDs = make(map[int64]struct{}, 0) u.affectedMetaRegions = make(map[uint64]struct{}, 0) u.err = nil } // IsRunning returns whether there is ongoing unsafe recovery process. If yes, further unsafe // recovery requests, schedulers, checkers, AskSplit and AskBatchSplit requests are blocked. -func (u *unsafeRecoveryController) IsRunning() bool { +func (u *Controller) IsRunning() bool { u.RLock() defer u.RUnlock() return u.isRunningLocked() } -func (u *unsafeRecoveryController) isRunningLocked() bool { - return u.stage != idle && u.stage != finished && u.stage != failed +func (u *Controller) isRunningLocked() bool { + return u.stage != Idle && u.stage != Finished && u.stage != Failed } -// RemoveFailedStores removes failed stores from the cluster. -func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64, autoDetect bool) error { +// RemoveFailedStores removes Failed stores from the cluster. +func (u *Controller) RemoveFailedStores(failedStores map[uint64]struct{}, timeout uint64, autoDetect bool) error { u.Lock() defer u.Unlock() @@ -213,29 +226,29 @@ func (u *unsafeRecoveryController) RemoveFailedStores(failedStores map[uint64]st u.timeout = time.Now().Add(time.Duration(timeout) * time.Second) u.failedStores = failedStores u.autoDetect = autoDetect - u.changeStage(collectReport) + u.changeStage(CollectReport) return nil } // Show returns the current status of ongoing unsafe recover operation. -func (u *unsafeRecoveryController) Show() []StageOutput { +func (u *Controller) Show() []StageOutput { u.Lock() defer u.Unlock() - if u.stage == idle { + if u.stage == Idle { return []StageOutput{{Info: "No on-going recovery."}} } if err := u.checkTimeout(); err != nil { - u.HandleErr(err) + u.handleErr(err) } status := u.output - if u.stage != finished && u.stage != failed { + if u.stage != Finished && u.stage != Failed { status = append(status, u.getReportStatus()) } return status } -func (u *unsafeRecoveryController) getReportStatus() StageOutput { +func (u *Controller) getReportStatus() StageOutput { var status StageOutput status.Time = time.Now().Format("2006-01-02 15:04:05.000") if u.numStoresReported != len(u.storeReports) { @@ -262,8 +275,8 @@ func (u *unsafeRecoveryController) getReportStatus() StageOutput { return status } -func (u *unsafeRecoveryController) checkTimeout() error { - if u.stage == finished || u.stage == failed { +func (u *Controller) checkTimeout() error { + if u.stage == Finished || u.stage == Failed { return nil } @@ -273,32 +286,33 @@ func (u *unsafeRecoveryController) checkTimeout() error { return nil } -func (u *unsafeRecoveryController) HandleErr(err error) bool { +// handleErr handles the error occurred during the unsafe recovery process. +func (u *Controller) handleErr(err error) bool { // Keep the earliest error. if u.err == nil { u.err = err } - if u.stage == exitForceLeader { - // We already tried to exit force leader, and it still failed. - // We turn into failed stage directly. TiKV will step down force leader + if u.stage == ExitForceLeader { + // We already tried to exit force leader, and it still Failed. + // We turn into Failed stage directly. TiKV will step down force leader // automatically after being for a long time. - u.changeStage(failed) + u.changeStage(Failed) return true } // When encountering an error for the first time, we will try to exit force - // leader before turning into failed stage to avoid the leaking force leaders + // leader before turning into Failed stage to avoid the leaking force leaders // blocks reads and writes. u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) u.timeout = time.Now().Add(storeRequestInterval * 2) // empty recovery plan would trigger exit force leader - u.changeStage(exitForceLeader) + u.changeStage(ExitForceLeader) return false } // HandleStoreHeartbeat handles the store heartbeat requests and checks whether the stores need to // send detailed report back. -func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { +func (u *Controller) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { u.Lock() defer u.Unlock() @@ -312,7 +326,7 @@ func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHea return false, err } - allCollected, err := u.collectReport(heartbeat) + allCollected, err := u.CollectReport(heartbeat) if err != nil { return false, err } @@ -328,13 +342,13 @@ func (u *unsafeRecoveryController) HandleStoreHeartbeat(heartbeat *pdpb.StoreHea return false, nil }() - if done || (err != nil && u.HandleErr(err)) { + if done || (err != nil && u.handleErr(err)) { return } u.dispatchPlan(heartbeat, resp) } -func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generatePlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { // clean up previous plan u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) @@ -345,57 +359,57 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe var err error for { switch stage { - case collectReport: + case CollectReport: fallthrough - case tombstoneTiFlashLearner: + case TombstoneTiFlashLearner: if hasPlan, err = u.generateTombstoneTiFlashLearnerPlan(newestRegionTree, peersMap); hasPlan && err == nil { - u.changeStage(tombstoneTiFlashLearner) + u.changeStage(TombstoneTiFlashLearner) break } if err != nil { break } fallthrough - case forceLeaderForCommitMerge: + case ForceLeaderForCommitMerge: if hasPlan, err = u.generateForceLeaderPlan(newestRegionTree, peersMap, true); hasPlan && err == nil { - u.changeStage(forceLeaderForCommitMerge) + u.changeStage(ForceLeaderForCommitMerge) break } if err != nil { break } fallthrough - case forceLeader: + case ForceLeader: if hasPlan, err = u.generateForceLeaderPlan(newestRegionTree, peersMap, false); hasPlan && err == nil { - u.changeStage(forceLeader) + u.changeStage(ForceLeader) break } if err != nil { break } fallthrough - case demoteFailedVoter: + case DemoteFailedVoter: if hasPlan = u.generateDemoteFailedVoterPlan(newestRegionTree, peersMap); hasPlan { - u.changeStage(demoteFailedVoter) + u.changeStage(DemoteFailedVoter) break } else if !reCheck { reCheck = true - stage = tombstoneTiFlashLearner + stage = TombstoneTiFlashLearner continue } fallthrough - case createEmptyRegion: + case CreateEmptyRegion: if hasPlan, err = u.generateCreateEmptyRegionPlan(newestRegionTree, peersMap); hasPlan && err == nil { - u.changeStage(createEmptyRegion) + u.changeStage(CreateEmptyRegion) break } if err != nil { break } fallthrough - case exitForceLeader: + case ExitForceLeader: if hasPlan = u.generateExitForceLeaderPlan(); hasPlan { - u.changeStage(exitForceLeader) + u.changeStage(ExitForceLeader) } default: panic("unreachable") @@ -405,9 +419,9 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe if err == nil && !hasPlan { if u.err != nil { - u.changeStage(failed) + u.changeStage(Failed) } else { - u.changeStage(finished) + u.changeStage(Finished) } return true, nil } @@ -415,7 +429,7 @@ func (u *unsafeRecoveryController) generatePlan(newestRegionTree *regionTree, pe } // It dispatches recovery plan if any. -func (u *unsafeRecoveryController) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { +func (u *Controller) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRequest, resp *pdpb.StoreHeartbeatResponse) { storeID := heartbeat.Stats.StoreId now := time.Now() @@ -436,11 +450,11 @@ func (u *unsafeRecoveryController) dispatchPlan(heartbeat *pdpb.StoreHeartbeatRe } } -// It collects and checks if store reports have been fully collected. -func (u *unsafeRecoveryController) collectReport(heartbeat *pdpb.StoreHeartbeatRequest) (bool, error) { +// CollectReport collects and checks if store reports have been fully collected. +func (u *Controller) CollectReport(heartbeat *pdpb.StoreHeartbeatRequest) (bool, error) { storeID := heartbeat.Stats.StoreId if _, isFailedStore := u.failedStores[storeID]; isFailedStore { - return false, errors.Errorf("Receive heartbeat from failed store %d", storeID) + return false, errors.Errorf("Receive heartbeat from Failed store %d", storeID) } if heartbeat.StoreReport == nil { @@ -467,25 +481,25 @@ func (u *unsafeRecoveryController) collectReport(heartbeat *pdpb.StoreHeartbeatR return false, nil } -// Gets the stage of the current unsafe recovery. -func (u *unsafeRecoveryController) GetStage() unsafeRecoveryStage { +// GetStage gets the stage of the current unsafe recovery. +func (u *Controller) GetStage() stage { u.RLock() defer u.RUnlock() return u.stage } -func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { +func (u *Controller) changeStage(stage stage) { u.stage = stage var output StageOutput output.Time = time.Now().Format("2006-01-02 15:04:05.000") switch u.stage { - case idle: - case collectReport: + case Idle: + case CollectReport: // TODO: clean up existing operators output.Info = "Unsafe recovery enters collect report stage" if u.autoDetect { - output.Details = append(output.Details, "auto detect mode with no specified failed stores") + output.Details = append(output.Details, "auto detect mode with no specified Failed stores") } else { stores := "" count := 0 @@ -496,40 +510,40 @@ func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { stores += ", " } } - output.Details = append(output.Details, fmt.Sprintf("failed stores %s", stores)) + output.Details = append(output.Details, fmt.Sprintf("Failed stores %s", stores)) } - case tombstoneTiFlashLearner: + case TombstoneTiFlashLearner: output.Info = "Unsafe recovery enters tombstone TiFlash learner stage" output.Actions = u.getTombstoneTiFlashLearnerDigest() - case forceLeaderForCommitMerge: + case ForceLeaderForCommitMerge: output.Info = "Unsafe recovery enters force leader for commit merge stage" output.Actions = u.getForceLeaderPlanDigest() - case forceLeader: + case ForceLeader: output.Info = "Unsafe recovery enters force leader stage" output.Actions = u.getForceLeaderPlanDigest() - case demoteFailedVoter: - output.Info = "Unsafe recovery enters demote failed voter stage" + case DemoteFailedVoter: + output.Info = "Unsafe recovery enters demote Failed voter stage" output.Actions = u.getDemoteFailedVoterPlanDigest() - case createEmptyRegion: + case CreateEmptyRegion: output.Info = "Unsafe recovery enters create empty region stage" output.Actions = u.getCreateEmptyRegionPlanDigest() - case exitForceLeader: + case ExitForceLeader: output.Info = "Unsafe recovery enters exit force leader stage" if u.err != nil { output.Details = append(output.Details, fmt.Sprintf("triggered by error: %v", u.err.Error())) } - case finished: + case Finished: if u.step > 1 { // == 1 means no operation has done, no need to invalid cache u.cluster.DropCacheAllRegion() } - output.Info = "Unsafe recovery finished" + output.Info = "Unsafe recovery Finished" output.Details = u.getAffectedTableDigest() u.storePlanExpires = make(map[uint64]time.Time) u.storeRecoveryPlans = make(map[uint64]*pdpb.RecoveryPlan) - case failed: - output.Info = fmt.Sprintf("Unsafe recovery failed: %v", u.err) + case Failed: + output.Info = fmt.Sprintf("Unsafe recovery Failed: %v", u.err) output.Details = u.getAffectedTableDigest() if u.numStoresReported != len(u.storeReports) { // in collecting reports, print out which stores haven't reported yet @@ -556,7 +570,7 @@ func (u *unsafeRecoveryController) changeStage(stage unsafeRecoveryStage) { u.step += 1 } -func (u *unsafeRecoveryController) getForceLeaderPlanDigest() map[string][]string { +func (u *Controller) getForceLeaderPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { forceLeaders := plan.GetForceLeader() @@ -574,7 +588,7 @@ func (u *unsafeRecoveryController) getForceLeaderPlanDigest() map[string][]strin return outputs } -func (u *unsafeRecoveryController) getDemoteFailedVoterPlanDigest() map[string][]string { +func (u *Controller) getDemoteFailedVoterPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if len(plan.GetDemotes()) == 0 && len(plan.GetTombstones()) == 0 { @@ -596,7 +610,7 @@ func (u *unsafeRecoveryController) getDemoteFailedVoterPlanDigest() map[string][ return outputs } -func (u *unsafeRecoveryController) getTombstoneTiFlashLearnerDigest() map[string][]string { +func (u *Controller) getTombstoneTiFlashLearnerDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if len(plan.GetTombstones()) == 0 { @@ -611,7 +625,7 @@ func (u *unsafeRecoveryController) getTombstoneTiFlashLearnerDigest() map[string return outputs } -func (u *unsafeRecoveryController) getCreateEmptyRegionPlanDigest() map[string][]string { +func (u *Controller) getCreateEmptyRegionPlanDigest() map[string][]string { outputs := make(map[string][]string) for storeID, plan := range u.storeRecoveryPlans { if plan.GetCreates() == nil { @@ -630,7 +644,7 @@ func (u *unsafeRecoveryController) getCreateEmptyRegionPlanDigest() map[string][ return outputs } -func (u *unsafeRecoveryController) getAffectedTableDigest() []string { +func (u *Controller) getAffectedTableDigest() []string { var details []string if len(u.affectedMetaRegions) != 0 { regions := "" @@ -639,9 +653,9 @@ func (u *unsafeRecoveryController) getAffectedTableDigest() []string { } details = append(details, "affected meta regions: "+strings.Trim(regions, ", ")) } - if len(u.affectedTableIDs) != 0 { + if len(u.AffectedTableIDs) != 0 { tables := "" - for t := range u.affectedTableIDs { + for t := range u.AffectedTableIDs { tables += fmt.Sprintf("%d, ", t) } details = append(details, "affected table ids: "+strings.Trim(tables, ", ")) @@ -649,16 +663,16 @@ func (u *unsafeRecoveryController) getAffectedTableDigest() []string { return details } -func (u *unsafeRecoveryController) recordAffectedRegion(region *metapb.Region) { +func (u *Controller) recordAffectedRegion(region *metapb.Region) { isMeta, tableID := codec.Key(region.StartKey).MetaOrTable() if isMeta { u.affectedMetaRegions[region.GetId()] = struct{}{} } else if tableID != 0 { - u.affectedTableIDs[tableID] = struct{}{} + u.AffectedTableIDs[tableID] = struct{}{} } } -func (u *unsafeRecoveryController) isFailed(peer *metapb.Peer) bool { +func (u *Controller) isFailed(peer *metapb.Peer) bool { _, isFailed := u.failedStores[peer.StoreId] _, isLive := u.storeReports[peer.StoreId] if isFailed || (u.autoDetect && !isLive) { @@ -667,7 +681,7 @@ func (u *unsafeRecoveryController) isFailed(peer *metapb.Peer) bool { return false } -func (u *unsafeRecoveryController) canElectLeader(region *metapb.Region, onlyIncoming bool) bool { +func (u *Controller) canElectLeader(region *metapb.Region, onlyIncoming bool) bool { hasQuorum := func(voters []*metapb.Peer) bool { numFailedVoters := 0 numLiveVoters := 0 @@ -698,7 +712,7 @@ func (u *unsafeRecoveryController) canElectLeader(region *metapb.Region, onlyInc return hasQuorum(incomingVoters) && (onlyIncoming || hasQuorum(outgoingVoters)) } -func (u *unsafeRecoveryController) getFailedPeers(region *metapb.Region) []*metapb.Peer { +func (u *Controller) getFailedPeers(region *metapb.Region) []*metapb.Peer { // if it can form a quorum after exiting the joint state, then no need to demotes any peer if u.canElectLeader(region, true) { return nil @@ -747,7 +761,7 @@ func (r *regionItem) IsEpochStale(other *regionItem) bool { return re.GetVersion() < oe.GetVersion() || (re.GetVersion() == oe.GetVersion() && re.GetConfVer() < oe.GetConfVer()) } -func (r *regionItem) IsRaftStale(origin *regionItem, u *unsafeRecoveryController) bool { +func (r *regionItem) IsRaftStale(origin *regionItem, u *Controller) bool { cmps := []func(a, b *regionItem) int{ func(a, b *regionItem) int { return int(a.report.GetRaftState().GetHardState().GetTerm()) - int(b.report.GetRaftState().GetHardState().GetTerm()) @@ -877,14 +891,14 @@ func (t *regionTree) insert(item *regionItem) (bool, error) { return true, nil } -func (u *unsafeRecoveryController) getRecoveryPlan(storeID uint64) *pdpb.RecoveryPlan { +func (u *Controller) getRecoveryPlan(storeID uint64) *pdpb.RecoveryPlan { if _, exists := u.storeRecoveryPlans[storeID]; !exists { u.storeRecoveryPlans[storeID] = &pdpb.RecoveryPlan{} } return u.storeRecoveryPlans[storeID] } -func (u *unsafeRecoveryController) buildUpFromReports() (*regionTree, map[uint64][]*regionItem, error) { +func (u *Controller) buildUpFromReports() (*regionTree, map[uint64][]*regionItem, error) { peersMap := make(map[uint64][]*regionItem) // Go through all the peer reports to build up the newest region tree for storeID, storeReport := range u.storeReports { @@ -925,7 +939,7 @@ func (u *unsafeRecoveryController) buildUpFromReports() (*regionTree, map[uint64 return newestRegionTree, peersMap, nil } -func (u *unsafeRecoveryController) selectLeader(peersMap map[uint64][]*regionItem, region *metapb.Region) *regionItem { +func (u *Controller) selectLeader(peersMap map[uint64][]*regionItem, region *metapb.Region) *regionItem { var leader *regionItem for _, peer := range peersMap[region.GetId()] { if leader == nil || leader.IsRaftStale(peer, u) { @@ -935,7 +949,7 @@ func (u *unsafeRecoveryController) selectLeader(peersMap map[uint64][]*regionIte return leader } -func (u *unsafeRecoveryController) generateTombstoneTiFlashLearnerPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generateTombstoneTiFlashLearnerPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { if u.err != nil { return false, nil } @@ -964,7 +978,7 @@ func (u *unsafeRecoveryController) generateTombstoneTiFlashLearnerPlan(newestReg return hasPlan, err } -func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem, forCommitMerge bool) (bool, error) { +func (u *Controller) generateForceLeaderPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem, forCommitMerge bool) (bool, error) { if u.err != nil { return false, nil } @@ -981,7 +995,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg var err error // Check the regions in newest Region Tree to see if it can still elect leader - // considering the failed stores + // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { report := item.report region := item.Region() @@ -1014,7 +1028,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg } } if u.autoDetect { - // For auto detect, the failedStores is empty. So need to add the detected failed store to the list + // For auto detect, the failedStores is empty. So need to add the detected Failed store to the list for _, peer := range u.getFailedPeers(leader.Region()) { found := false for _, store := range storeRecoveryPlan.ForceLeader.FailedStores { @@ -1049,7 +1063,7 @@ func (u *unsafeRecoveryController) generateForceLeaderPlan(newestRegionTree *reg return hasPlan, err } -func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) bool { +func (u *Controller) generateDemoteFailedVoterPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) bool { if u.err != nil { return false } @@ -1067,7 +1081,7 @@ func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTre } // Check the regions in newest Region Tree to see if it can still elect leader - // considering the failed stores + // considering the Failed stores newestRegionTree.tree.Ascend(func(item *regionItem) bool { region := item.Region() if !u.canElectLeader(region, false) { @@ -1107,7 +1121,7 @@ func (u *unsafeRecoveryController) generateDemoteFailedVoterPlan(newestRegionTre return hasPlan } -func (u *unsafeRecoveryController) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { +func (u *Controller) generateCreateEmptyRegionPlan(newestRegionTree *regionTree, peersMap map[uint64][]*regionItem) (bool, error) { if u.err != nil { return false, nil } @@ -1214,7 +1228,7 @@ func (u *unsafeRecoveryController) generateCreateEmptyRegionPlan(newestRegionTre return hasPlan, nil } -func (u *unsafeRecoveryController) generateExitForceLeaderPlan() bool { +func (u *Controller) generateExitForceLeaderPlan() bool { hasPlan := false for storeID, storeReport := range u.storeReports { for _, peerReport := range storeReport.PeerReports { diff --git a/server/cluster/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go similarity index 83% rename from server/cluster/unsafe_recovery_controller_test.go rename to pkg/unsaferecovery/unsafe_recovery_controller_test.go index 6ef20fdb6b7..5cb33bd8eab 100644 --- a/server/cluster/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cluster +// change the package to avoid import cycle +package unsaferecovery import ( "context" + "fmt" "testing" "time" @@ -26,9 +28,10 @@ import ( "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/codec" "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/mock/mockid" + "github.com/tikv/pd/pkg/mock/mockcluster" + "github.com/tikv/pd/pkg/mock/mockconfig" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/hbstream" - "github.com/tikv/pd/pkg/storage" ) func newStoreHeartbeat(storeID uint64, report *pdpb.StoreReport) *pdpb.StoreHeartbeatRequest { @@ -162,7 +165,7 @@ func applyRecoveryPlan(re *require.Assertions, storeID uint64, storeReports map[ } } -func advanceUntilFinished(re *require.Assertions, recoveryController *unsafeRecoveryController, reports map[uint64]*pdpb.StoreReport) { +func advanceUntilFinished(re *require.Assertions, recoveryController *Controller, reports map[uint64]*pdpb.StoreReport) { retry := 0 for { @@ -173,10 +176,10 @@ func advanceUntilFinished(re *require.Assertions, recoveryController *unsafeReco recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - if recoveryController.GetStage() == finished { + if recoveryController.GetStage() == Finished { break - } else if recoveryController.GetStage() == failed { - panic("failed to recovery") + } else if recoveryController.GetStage() == Failed { + panic("Failed to recovery") } else if retry >= 10 { panic("retry timeout") } @@ -189,14 +192,14 @@ func TestFinished(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -214,7 +217,7 @@ func TestFinished(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) for storeID := range reports { req := newStoreHeartbeat(storeID, nil) resp := &pdpb.StoreHeartbeatResponse{} @@ -240,7 +243,7 @@ func TestFinished(t *testing.T) { re.NotNil(resp.RecoveryPlan.ForceLeader.FailedStores) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -251,7 +254,7 @@ func TestFinished(t *testing.T) { re.Len(resp.RecoveryPlan.Demotes, 1) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) req.StoreReport = report @@ -261,7 +264,7 @@ func TestFinished(t *testing.T) { // remove the two failed peers applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestFailed(t *testing.T) { @@ -269,14 +272,14 @@ func TestFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -294,7 +297,7 @@ func TestFailed(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // require peer report for storeID := range reports { req := newStoreHeartbeat(storeID, nil) @@ -319,7 +322,7 @@ func TestFailed(t *testing.T) { re.NotNil(resp.RecoveryPlan.ForceLeader.FailedStores) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -330,13 +333,13 @@ func TestFailed(t *testing.T) { re.Len(resp.RecoveryPlan.Demotes, 1) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) - // received heartbeat from failed store, abort + // received heartbeat from Failed store, abort req := newStoreHeartbeat(2, nil) resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -354,7 +357,7 @@ func TestFailed(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(failed, recoveryController.GetStage()) + re.Equal(Failed, recoveryController.GetStage()) } func TestForceLeaderFail(t *testing.T) { @@ -362,14 +365,14 @@ func TestForceLeaderFail(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(4, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 3: {}, 4: {}, @@ -414,7 +417,7 @@ func TestForceLeaderFail(t *testing.T) { resp2 := &pdpb.StoreHeartbeatResponse{} req2.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) recoveryController.HandleStoreHeartbeat(req1, resp1) // force leader on store 1 succeed @@ -426,7 +429,7 @@ func TestForceLeaderFail(t *testing.T) { // force leader should retry on store 2 recoveryController.HandleStoreHeartbeat(req1, resp1) recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) recoveryController.HandleStoreHeartbeat(req1, resp1) // force leader succeed this time @@ -434,7 +437,7 @@ func TestForceLeaderFail(t *testing.T) { applyRecoveryPlan(re, 2, reports, resp2) recoveryController.HandleStoreHeartbeat(req1, resp1) recoveryController.HandleStoreHeartbeat(req2, resp2) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) } func TestAffectedTableID(t *testing.T) { @@ -442,14 +445,14 @@ func TestAffectedTableID(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -473,8 +476,8 @@ func TestAffectedTableID(t *testing.T) { advanceUntilFinished(re, recoveryController, reports) - re.Len(recoveryController.affectedTableIDs, 1) - _, exists := recoveryController.affectedTableIDs[6] + re.Len(recoveryController.AffectedTableIDs, 1) + _, exists := recoveryController.AffectedTableIDs[6] re.True(exists) } @@ -483,14 +486,14 @@ func TestForceLeaderForCommitMerge(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -529,7 +532,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { resp := &pdpb.StoreHeartbeatResponse{} req.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeaderForCommitMerge, recoveryController.GetStage()) + re.Equal(ForceLeaderForCommitMerge, recoveryController.GetStage()) // force leader on regions of commit merge first re.NotNil(resp.RecoveryPlan) @@ -540,7 +543,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) // force leader on the rest regions re.NotNil(resp.RecoveryPlan) @@ -551,7 +554,7 @@ func TestForceLeaderForCommitMerge(t *testing.T) { applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) } func TestAutoDetectMode(t *testing.T) { @@ -559,14 +562,14 @@ func TestAutoDetectMode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(1, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(nil, 60, true)) reports := map[uint64]*pdpb.StoreReport{ @@ -611,14 +614,14 @@ func TestOneLearner(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -666,17 +669,17 @@ func TestTiflashLearnerPeer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { if store.GetID() == 3 { store.GetMeta().Labels = []*metapb.StoreLabel{{Key: "engine", Value: "tiflash"}} } - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -841,14 +844,14 @@ func TestUninitializedPeer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -897,14 +900,14 @@ func TestJointState(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1090,14 +1093,14 @@ func TestExecutionTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1107,10 +1110,10 @@ func TestExecutionTimeout(t *testing.T) { req := newStoreHeartbeat(1, nil) resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) req.StoreReport = &pdpb.StoreReport{Step: 2} recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(failed, recoveryController.GetStage()) + re.Equal(Failed, recoveryController.GetStage()) output := recoveryController.Show() re.Equal(len(output), 3) @@ -1122,14 +1125,14 @@ func TestNoHeartbeatTimeout(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1137,7 +1140,7 @@ func TestNoHeartbeatTimeout(t *testing.T) { time.Sleep(time.Second) recoveryController.Show() - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) } func TestExitForceLeader(t *testing.T) { @@ -1145,14 +1148,14 @@ func TestExitForceLeader(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1183,7 +1186,7 @@ func TestExitForceLeader(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(exitForceLeader, recoveryController.GetStage()) + re.Equal(ExitForceLeader, recoveryController.GetStage()) for storeID, report := range reports { req := newStoreHeartbeat(storeID, report) @@ -1192,7 +1195,7 @@ func TestExitForceLeader(t *testing.T) { recoveryController.HandleStoreHeartbeat(req, resp) applyRecoveryPlan(re, storeID, reports, resp) } - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) expects := map[uint64]*pdpb.StoreReport{ 1: { @@ -1223,14 +1226,14 @@ func TestStep(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1255,22 +1258,22 @@ func TestStep(t *testing.T) { resp := &pdpb.StoreHeartbeatResponse{} recoveryController.HandleStoreHeartbeat(req, resp) // step is not set, ignore - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // valid store report req.StoreReport.Step = 1 recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) // duplicate report with same step, ignore recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(forceLeader, recoveryController.GetStage()) + re.Equal(ForceLeader, recoveryController.GetStage()) applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(demoteFailedVoter, recoveryController.GetStage()) + re.Equal(DemoteFailedVoter, recoveryController.GetStage()) applyRecoveryPlan(re, 1, reports, resp) recoveryController.HandleStoreHeartbeat(req, resp) - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestOnHealthyRegions(t *testing.T) { @@ -1278,14 +1281,14 @@ func TestOnHealthyRegions(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1323,7 +1326,7 @@ func TestOnHealthyRegions(t *testing.T) { {Id: 11, StoreId: 1}, {Id: 21, StoreId: 2}, {Id: 31, StoreId: 3}}}}}, }}, } - re.Equal(collectReport, recoveryController.GetStage()) + re.Equal(CollectReport, recoveryController.GetStage()) // require peer report for storeID := range reports { req := newStoreHeartbeat(storeID, nil) @@ -1346,7 +1349,7 @@ func TestOnHealthyRegions(t *testing.T) { applyRecoveryPlan(re, storeID, reports, resp) } // nothing to do, finish directly - re.Equal(finished, recoveryController.GetStage()) + re.Equal(Finished, recoveryController.GetStage()) } func TestCreateEmptyRegion(t *testing.T) { @@ -1354,14 +1357,14 @@ func TestCreateEmptyRegion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1463,14 +1466,14 @@ func TestRangeOverlap1(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1558,14 +1561,14 @@ func TestRangeOverlap2(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(5, "6.0.0") { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 4: {}, 5: {}, @@ -1652,17 +1655,16 @@ func TestRemoveFailedStores(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() stores := newTestStores(2, "5.3.0") stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now())) for _, store := range stores { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) - + recoveryController := NewController(cluster) // Store 3 doesn't exist, reject to remove. re.Error(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 1: {}, @@ -1673,8 +1675,8 @@ func TestRemoveFailedStores(t *testing.T) { 1: {}, }, 60, false)) re.True(cluster.GetStore(uint64(1)).IsRemoved()) - for _, s := range cluster.GetSchedulers() { - paused, err := cluster.IsSchedulerAllowed(s) + for _, s := range coordinator.GetSchedulers() { + paused, err := coordinator.IsSchedulerAllowed(s) if s != "split-bucket-scheduler" { re.NoError(err) re.True(paused) @@ -1688,34 +1690,26 @@ func TestRemoveFailedStores(t *testing.T) { }, 60, false)) } -func TestSplitPaused(t *testing.T) { +func TestRunning(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - recoveryController := newUnsafeRecoveryController(cluster) - cluster.Lock() - cluster.unsafeRecoveryController = recoveryController - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.Unlock() - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() stores := newTestStores(2, "5.3.0") stores[1] = stores[1].Clone(core.SetLastHeartbeatTS(time.Now())) for _, store := range stores { - re.NoError(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } failedStores := map[uint64]struct{}{ 1: {}, } + recoveryController := NewController(cluster) re.NoError(recoveryController.RemoveFailedStores(failedStores, 60, false)) - askSplitReq := &pdpb.AskSplitRequest{} - _, err := cluster.HandleAskSplit(askSplitReq) - re.Equal("[PD:unsaferecovery:ErrUnsafeRecoveryIsRunning]unsafe recovery is running", err.Error()) - askBatchSplitReq := &pdpb.AskBatchSplitRequest{} - _, err = cluster.HandleAskBatchSplit(askBatchSplitReq) - re.Equal("[PD:unsaferecovery:ErrUnsafeRecoveryIsRunning]unsafe recovery is running", err.Error()) + re.True(recoveryController.IsRunning()) } func TestEpochComparsion(t *testing.T) { @@ -1723,14 +1717,14 @@ func TestEpochComparsion(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - _, opt, _ := newTestScheduleConfig() - cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.meta.GetId(), cluster, true)) - cluster.coordinator.run() + opts := mockconfig.NewTestOptions() + cluster := mockcluster.NewCluster(ctx, opts) + coordinator := schedule.NewCoordinator(ctx, cluster, hbstream.NewTestHeartbeatStreams(ctx, cluster.ID, cluster, true)) + coordinator.Run() for _, store := range newTestStores(3, "6.0.0") { - re.Nil(cluster.PutStore(store.GetMeta())) + cluster.PutStore(store) } - recoveryController := newUnsafeRecoveryController(cluster) + recoveryController := NewController(cluster) re.Nil(recoveryController.RemoveFailedStores(map[uint64]struct{}{ 2: {}, 3: {}, @@ -1796,3 +1790,26 @@ func TestEpochComparsion(t *testing.T) { } } } + +// TODO: remove them +// Create n stores (0..n). +func newTestStores(n uint64, version string) []*core.StoreInfo { + stores := make([]*core.StoreInfo, 0, n) + for i := uint64(1); i <= n; i++ { + store := &metapb.Store{ + Id: i, + Address: fmt.Sprintf("127.0.0.1:%d", i), + StatusAddress: fmt.Sprintf("127.0.0.1:%d", i), + State: metapb.StoreState_Up, + Version: version, + DeployPath: getTestDeployPath(i), + NodeState: metapb.NodeState_Serving, + } + stores = append(stores, core.NewStoreInfo(store)) + } + return stores +} + +func getTestDeployPath(storeID uint64) string { + return fmt.Sprintf("test/store%d", storeID) +} diff --git a/server/api/diagnostic.go b/server/api/diagnostic.go index 6a90baebd94..7de60b82b10 100644 --- a/server/api/diagnostic.go +++ b/server/api/diagnostic.go @@ -19,8 +19,8 @@ import ( "github.com/gorilla/mux" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/unrolled/render" ) @@ -38,7 +38,7 @@ func newDiagnosticHandler(svr *server.Server, rd *render.Render) *diagnosticHand func (h *diagnosticHandler) GetDiagnosticResult(w http.ResponseWriter, r *http.Request) { name := mux.Vars(r)["name"] - if _, ok := cluster.DiagnosableSummaryFunc[name]; !ok { + if _, ok := schedule.DiagnosableSummaryFunc[name]; !ok { h.rd.JSON(w, http.StatusBadRequest, errs.ErrSchedulerUndiagnosable.FastGenByArgs(name).Error()) return } diff --git a/server/api/diagnostic_test.go b/server/api/diagnostic_test.go index 64e3d589607..fb5946f5f94 100644 --- a/server/api/diagnostic_test.go +++ b/server/api/diagnostic_test.go @@ -23,11 +23,11 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/tikv/pd/server/config" ) @@ -66,7 +66,7 @@ func (suite *diagnosticTestSuite) TearDownSuite() { func (suite *diagnosticTestSuite) checkStatus(status string, url string) { re := suite.Require() suite.Eventually(func() bool { - result := &cluster.DiagnosticResult{} + result := &schedule.DiagnosticResult{} err := tu.ReadGetJSON(re, testDialClient, url, result) suite.NoError(err) return result.Status == status @@ -95,7 +95,7 @@ func (suite *diagnosticTestSuite) TestSchedulerDiagnosticAPI() { suite.True(cfg.Schedule.EnableDiagnostic) balanceRegionURL := suite.urlPrefix + "/" + schedulers.BalanceRegionName - result := &cluster.DiagnosticResult{} + result := &schedule.DiagnosticResult{} err = tu.ReadGetJSON(re, testDialClient, balanceRegionURL, result) suite.NoError(err) suite.Equal("disabled", result.Status) diff --git a/server/api/plugin.go b/server/api/plugin.go index 192310cca7e..fd75cc6bb2b 100644 --- a/server/api/plugin.go +++ b/server/api/plugin.go @@ -19,9 +19,9 @@ import ( "net/http" "os" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" "github.com/unrolled/render" ) @@ -48,7 +48,7 @@ func newPluginHandler(handler *server.Handler, rd *render.Render) *pluginHandler // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /plugin [post] func (h *pluginHandler) LoadPlugin(w http.ResponseWriter, r *http.Request) { - h.processPluginCommand(w, r, cluster.PluginLoad) + h.processPluginCommand(w, r, schedule.PluginLoad) } // FIXME: details of input json body params @@ -62,7 +62,7 @@ func (h *pluginHandler) LoadPlugin(w http.ResponseWriter, r *http.Request) { // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /plugin [delete] func (h *pluginHandler) UnloadPlugin(w http.ResponseWriter, r *http.Request) { - h.processPluginCommand(w, r, cluster.PluginUnload) + h.processPluginCommand(w, r, schedule.PluginUnload) } func (h *pluginHandler) processPluginCommand(w http.ResponseWriter, r *http.Request, action string) { @@ -77,14 +77,14 @@ func (h *pluginHandler) processPluginCommand(w http.ResponseWriter, r *http.Requ } var err error switch action { - case cluster.PluginLoad: + case schedule.PluginLoad: err = h.PluginLoad(path) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } h.rd.JSON(w, http.StatusOK, "Load plugin successfully.") - case cluster.PluginUnload: + case schedule.PluginUnload: err = h.PluginUnload(path) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) diff --git a/server/api/unsafe_operation_test.go b/server/api/unsafe_operation_test.go index d83cba27d61..9a5b3887175 100644 --- a/server/api/unsafe_operation_test.go +++ b/server/api/unsafe_operation_test.go @@ -21,9 +21,9 @@ import ( "github.com/pingcap/kvproto/pkg/metapb" "github.com/stretchr/testify/suite" + "github.com/tikv/pd/pkg/unsaferecovery" tu "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/server" - "github.com/tikv/pd/server/cluster" ) type unsafeOperationTestSuite struct { @@ -80,7 +80,7 @@ func (suite *unsafeOperationTestSuite) TestRemoveFailedStores() { suite.NoError(err) // Test show - var output []cluster.StageOutput + var output []unsaferecovery.StageOutput err = tu.ReadGetJSON(re, testDialClient, suite.urlPrefix+"/remove-failed-stores/show", &output) suite.NoError(err) } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index a583a902bc7..b99eaed7466 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -51,6 +51,7 @@ import ( "github.com/tikv/pd/pkg/statistics/buckets" "github.com/tikv/pd/pkg/storage" "github.com/tikv/pd/pkg/storage/endpoint" + "github.com/tikv/pd/pkg/unsaferecovery" "github.com/tikv/pd/pkg/utils/etcdutil" "github.com/tikv/pd/pkg/utils/logutil" "github.com/tikv/pd/pkg/utils/netutil" @@ -74,8 +75,6 @@ var ( regionCacheMissCounter = bucketEventCounter.WithLabelValues("region_cache_miss") versionNotMatchCounter = bucketEventCounter.WithLabelValues("version_not_match") updateFailedCounter = bucketEventCounter.WithLabelValues("update_failed") - - denySchedulersByLabelerCounter = schedule.LabelerEventCounter.WithLabelValues("schedulers", "deny") ) // regionLabelGCInterval is the interval to run region-label's GC work. @@ -152,7 +151,7 @@ type RaftCluster struct { core *core.BasicCluster // cached cluster info opt *config.PersistOptions limiter *StoreLimiter - coordinator *coordinator + coordinator *schedule.Coordinator labelLevelStats *statistics.LabelStatistics regionStats *statistics.RegionStatistics hotStat *statistics.HotStat @@ -161,7 +160,7 @@ type RaftCluster struct { ruleManager *placement.RuleManager regionLabeler *labeler.RegionLabeler replicationMode *replication.ModeManager - unsafeRecoveryController *unsafeRecoveryController + unsafeRecoveryController *unsaferecovery.Controller progressManager *progress.Manager regionSyncer *syncer.RegionSyncer changedRegions chan *core.RegionInfo @@ -254,7 +253,7 @@ func (c *RaftCluster) InitCluster( c.progressManager = progress.NewManager() c.changedRegions = make(chan *core.RegionInfo, defaultChangedRegionsLimit) c.prevStoreLimit = make(map[uint64]map[storelimit.Type]float64) - c.unsafeRecoveryController = newUnsafeRecoveryController(c) + c.unsafeRecoveryController = unsaferecovery.NewController(c) c.keyspaceGroupManager = keyspaceGroupManager } @@ -301,7 +300,7 @@ func (c *RaftCluster) Start(s Server) error { return err } c.storeConfigManager = config.NewStoreConfigManager(c.httpClient) - c.coordinator = newCoordinator(c.ctx, cluster, s.GetHBStreams()) + c.coordinator = schedule.NewCoordinator(c.ctx, cluster, s.GetHBStreams()) c.regionStats = statistics.NewRegionStatistics(c.opt, c.ruleManager, c.storeConfigManager) c.limiter = NewStoreLimiter(s.GetPersistOptions()) c.externalTS, err = c.storage.LoadExternalTS() @@ -591,7 +590,7 @@ func (c *RaftCluster) runUpdateStoreStats() { func (c *RaftCluster) runCoordinator() { defer logutil.LogPanic() defer c.wg.Done() - c.coordinator.runUntilStop() + c.coordinator.RunUntilStop() } func (c *RaftCluster) syncRegions() { @@ -614,7 +613,7 @@ func (c *RaftCluster) Stop() { return } c.running = false - c.coordinator.stop() + c.coordinator.Stop() c.cancel() c.Unlock() @@ -640,40 +639,38 @@ func (c *RaftCluster) Context() context.Context { } // GetCoordinator returns the coordinator. -func (c *RaftCluster) GetCoordinator() *coordinator { +func (c *RaftCluster) GetCoordinator() *schedule.Coordinator { return c.coordinator } // GetOperatorController returns the operator controller. func (c *RaftCluster) GetOperatorController() *operator.Controller { - return c.coordinator.opController + return c.coordinator.GetOperatorController() } // SetPrepared set the prepare check to prepared. Only for test purpose. func (c *RaftCluster) SetPrepared() { - c.coordinator.prepareChecker.Lock() - defer c.coordinator.prepareChecker.Unlock() - c.coordinator.prepareChecker.prepared = true + c.coordinator.GetPrepareChecker().SetPrepared() } // GetRegionScatter returns the region scatter. func (c *RaftCluster) GetRegionScatter() *schedule.RegionScatterer { - return c.coordinator.regionScatterer + return c.coordinator.GetRegionScatterer() } // GetRegionSplitter returns the region splitter func (c *RaftCluster) GetRegionSplitter() *schedule.RegionSplitter { - return c.coordinator.regionSplitter + return c.coordinator.GetRegionSplitter() } // GetMergeChecker returns merge checker. func (c *RaftCluster) GetMergeChecker() *checker.MergeChecker { - return c.coordinator.checkers.GetMergeChecker() + return c.coordinator.GetMergeChecker() } // GetRuleChecker returns rule checker. func (c *RaftCluster) GetRuleChecker() *checker.RuleChecker { - return c.coordinator.checkers.GetRuleChecker() + return c.coordinator.GetRuleChecker() } // RecordOpStepWithTTL records OpStep with TTL @@ -683,57 +680,57 @@ func (c *RaftCluster) RecordOpStepWithTTL(regionID uint64) { // GetSchedulers gets all schedulers. func (c *RaftCluster) GetSchedulers() []string { - return c.coordinator.getSchedulers() + return c.coordinator.GetSchedulers() } // GetSchedulerHandlers gets all scheduler handlers. func (c *RaftCluster) GetSchedulerHandlers() map[string]http.Handler { - return c.coordinator.getSchedulerHandlers() + return c.coordinator.GetSchedulerHandlers() } // AddScheduler adds a scheduler. func (c *RaftCluster) AddScheduler(scheduler schedulers.Scheduler, args ...string) error { - return c.coordinator.addScheduler(scheduler, args...) + return c.coordinator.AddScheduler(scheduler, args...) } // RemoveScheduler removes a scheduler. func (c *RaftCluster) RemoveScheduler(name string) error { - return c.coordinator.removeScheduler(name) + return c.coordinator.RemoveScheduler(name) } // PauseOrResumeScheduler pauses or resumes a scheduler. func (c *RaftCluster) PauseOrResumeScheduler(name string, t int64) error { - return c.coordinator.pauseOrResumeScheduler(name, t) + return c.coordinator.PauseOrResumeScheduler(name, t) } // IsSchedulerPaused checks if a scheduler is paused. func (c *RaftCluster) IsSchedulerPaused(name string) (bool, error) { - return c.coordinator.isSchedulerPaused(name) + return c.coordinator.IsSchedulerPaused(name) } // IsSchedulerDisabled checks if a scheduler is disabled. func (c *RaftCluster) IsSchedulerDisabled(name string) (bool, error) { - return c.coordinator.isSchedulerDisabled(name) + return c.coordinator.IsSchedulerDisabled(name) } // IsSchedulerAllowed checks if a scheduler is allowed. func (c *RaftCluster) IsSchedulerAllowed(name string) (bool, error) { - return c.coordinator.isSchedulerAllowed(name) + return c.coordinator.IsSchedulerAllowed(name) } // IsSchedulerExisted checks if a scheduler is existed. func (c *RaftCluster) IsSchedulerExisted(name string) (bool, error) { - return c.coordinator.isSchedulerExisted(name) + return c.coordinator.IsSchedulerExisted(name) } // PauseOrResumeChecker pauses or resumes checker. func (c *RaftCluster) PauseOrResumeChecker(name string, t int64) error { - return c.coordinator.pauseOrResumeChecker(name, t) + return c.coordinator.PauseOrResumeChecker(name, t) } // IsCheckerPaused returns if checker is paused func (c *RaftCluster) IsCheckerPaused(name string) (bool, error) { - return c.coordinator.isCheckerPaused(name) + return c.coordinator.IsCheckerPaused(name) } // GetAllocator returns cluster's id allocator. @@ -763,24 +760,20 @@ func (c *RaftCluster) GetRegionLabeler() *labeler.RegionLabeler { // GetStorage returns the storage. func (c *RaftCluster) GetStorage() storage.Storage { - c.RLock() - defer c.RUnlock() return c.storage } -// SetStorage set the storage for test purpose. -func (c *RaftCluster) SetStorage(s storage.Storage) { - c.Lock() - defer c.Unlock() - c.storage = s -} - // GetOpts returns cluster's configuration. // There is no need a lock since it won't changed. func (c *RaftCluster) GetOpts() sc.Config { return c.opt } +// GetPersistOptions returns cluster's configuration. +func (c *RaftCluster) GetPersistOptions() *config.PersistOptions { + return c.opt +} + // GetScheduleConfig returns scheduling configurations. func (c *RaftCluster) GetScheduleConfig() *config.ScheduleConfig { return c.opt.GetScheduleConfig() @@ -808,12 +801,12 @@ func (c *RaftCluster) SetPDServerConfig(cfg *config.PDServerConfig) { // AddSuspectRegions adds regions to suspect list. func (c *RaftCluster) AddSuspectRegions(regionIDs ...uint64) { - c.coordinator.checkers.AddSuspectRegions(regionIDs...) + c.coordinator.GetCheckerController().AddSuspectRegions(regionIDs...) } // GetSuspectRegions gets all suspect regions. func (c *RaftCluster) GetSuspectRegions() []uint64 { - return c.coordinator.checkers.GetSuspectRegions() + return c.coordinator.GetCheckerController().GetSuspectRegions() } // GetHotStat gets hot stat for test. @@ -823,31 +816,36 @@ func (c *RaftCluster) GetHotStat() *statistics.HotStat { // RemoveSuspectRegion removes region from suspect list. func (c *RaftCluster) RemoveSuspectRegion(id uint64) { - c.coordinator.checkers.RemoveSuspectRegion(id) + c.coordinator.GetCheckerController().RemoveSuspectRegion(id) } // GetUnsafeRecoveryController returns the unsafe recovery controller. -func (c *RaftCluster) GetUnsafeRecoveryController() *unsafeRecoveryController { +func (c *RaftCluster) GetUnsafeRecoveryController() *unsaferecovery.Controller { return c.unsafeRecoveryController } +// IsUnsafeRecovering returns if the cluster is in unsafe recovering. +func (c *RaftCluster) IsUnsafeRecovering() bool { + return c.unsafeRecoveryController.IsRunning() +} + // AddSuspectKeyRange adds the key range with the its ruleID as the key // The instance of each keyRange is like following format: // [2][]byte: start key/end key func (c *RaftCluster) AddSuspectKeyRange(start, end []byte) { - c.coordinator.checkers.AddSuspectKeyRange(start, end) + c.coordinator.GetCheckerController().AddSuspectKeyRange(start, end) } // PopOneSuspectKeyRange gets one suspect keyRange group. // it would return value and true if pop success, or return empty [][2][]byte and false // if suspectKeyRanges couldn't pop keyRange group. func (c *RaftCluster) PopOneSuspectKeyRange() ([2][]byte, bool) { - return c.coordinator.checkers.PopOneSuspectKeyRange() + return c.coordinator.GetCheckerController().PopOneSuspectKeyRange() } // ClearSuspectKeyRanges clears the suspect keyRanges, only for unit test func (c *RaftCluster) ClearSuspectKeyRanges() { - c.coordinator.checkers.ClearSuspectKeyRanges() + c.coordinator.GetCheckerController().ClearSuspectKeyRanges() } // HandleStoreHeartbeat updates the store status. @@ -998,7 +996,7 @@ func (c *RaftCluster) processReportBuckets(buckets *metapb.Buckets) error { // IsPrepared return true if the prepare checker is ready. func (c *RaftCluster) IsPrepared() bool { - return c.coordinator.prepareChecker.isPrepared() + return c.coordinator.GetPrepareChecker().IsPrepared() } var regionGuide = core.GenerateRegionGuideFunc(true) @@ -1067,7 +1065,7 @@ func (c *RaftCluster) processRegionHeartbeat(region *core.RegionInfo) error { } if !c.IsPrepared() && isNew { - c.coordinator.prepareChecker.collect(region) + c.coordinator.GetPrepareChecker().Collect(region) } if c.storage != nil { @@ -1484,7 +1482,7 @@ func (c *RaftCluster) getEvictLeaderStores() (evictStores []uint64) { if c.coordinator == nil { return nil } - handler, ok := c.coordinator.getSchedulerHandlers()[schedulers.EvictLeaderName] + handler, ok := c.coordinator.GetSchedulerHandlers()[schedulers.EvictLeaderName] if !ok { return } @@ -2067,11 +2065,6 @@ func (c *RaftCluster) deleteStore(store *core.StoreInfo) error { return nil } -// SetHotPendingInfluenceMetrics sets pending influence in hot scheduler. -func (c *RaftCluster) SetHotPendingInfluenceMetrics(storeLabel, rwTy, dim string, load float64) { - hotPendingSum.WithLabelValues(storeLabel, rwTy, dim).Set(load) -} - func (c *RaftCluster) collectMetrics() { statsMap := statistics.NewStoreStatisticsMap(c.opt, c.storeConfigManager.GetStoreConfig()) stores := c.GetStores() @@ -2080,8 +2073,8 @@ func (c *RaftCluster) collectMetrics() { } statsMap.Collect() - c.coordinator.collectSchedulerMetrics() - c.coordinator.collectHotSpotMetrics() + c.coordinator.CollectSchedulerMetrics() + c.coordinator.CollectHotSpotMetrics() c.collectClusterMetrics() c.collectHealthStatus() } @@ -2090,8 +2083,8 @@ func (c *RaftCluster) resetMetrics() { statsMap := statistics.NewStoreStatisticsMap(c.opt, c.storeConfigManager.GetStoreConfig()) statsMap.Reset() - c.coordinator.resetSchedulerMetrics() - c.coordinator.resetHotSpotMetrics() + c.coordinator.ResetSchedulerMetrics() + c.coordinator.ResetHotSpotMetrics() c.resetClusterMetrics() c.resetHealthStatus() c.resetProgressIndicator() @@ -2159,7 +2152,8 @@ func (c *RaftCluster) GetOfflineRegionStatsByType(typ statistics.RegionStatistic return c.regionStats.GetOfflineRegionStatsByType(typ) } -func (c *RaftCluster) updateRegionsLabelLevelStats(regions []*core.RegionInfo) { +// UpdateRegionsLabelLevelStats updates the status of the region label level by types. +func (c *RaftCluster) UpdateRegionsLabelLevelStats(regions []*core.RegionInfo) { for _, region := range regions { c.labelLevelStats.Observe(region, c.getStoresWithoutLabelLocked(region, core.EngineKey, core.EngineTiFlash), c.opt.GetLocationLabels()) } @@ -2321,7 +2315,7 @@ func (c *RaftCluster) putRegion(region *core.RegionInfo) error { // GetHotWriteRegions gets hot write regions' info. func (c *RaftCluster) GetHotWriteRegions(storeIDs ...uint64) *statistics.StoreHotPeersInfos { - hotWriteRegions := c.coordinator.getHotRegionsByType(statistics.Write) + hotWriteRegions := c.coordinator.GetHotRegionsByType(statistics.Write) if len(storeIDs) > 0 && hotWriteRegions != nil { hotWriteRegions = getHotRegionsByStoreIDs(hotWriteRegions, storeIDs...) } @@ -2330,7 +2324,7 @@ func (c *RaftCluster) GetHotWriteRegions(storeIDs ...uint64) *statistics.StoreHo // GetHotReadRegions gets hot read regions' info. func (c *RaftCluster) GetHotReadRegions(storeIDs ...uint64) *statistics.StoreHotPeersInfos { - hotReadRegions := c.coordinator.getHotRegionsByType(statistics.Read) + hotReadRegions := c.coordinator.GetHotRegionsByType(statistics.Read) if len(storeIDs) > 0 && hotReadRegions != nil { hotReadRegions = getHotRegionsByStoreIDs(hotReadRegions, storeIDs...) } @@ -2706,31 +2700,12 @@ func IsClientURL(addr string, etcdClient *clientv3.Client) bool { return false } -// cacheCluster include cache info to improve the performance. -type cacheCluster struct { - *RaftCluster - stores []*core.StoreInfo -} - -// GetStores returns store infos from cache -func (c *cacheCluster) GetStores() []*core.StoreInfo { - return c.stores -} - -// newCacheCluster constructor for cache -func newCacheCluster(c *RaftCluster) *cacheCluster { - return &cacheCluster{ - RaftCluster: c, - stores: c.GetStores(), - } -} - // GetPausedSchedulerDelayAt returns DelayAt of a paused scheduler func (c *RaftCluster) GetPausedSchedulerDelayAt(name string) (int64, error) { - return c.coordinator.getPausedSchedulerDelayAt(name) + return c.coordinator.GetPausedSchedulerDelayAt(name) } // GetPausedSchedulerDelayUntil returns DelayUntil of a paused scheduler func (c *RaftCluster) GetPausedSchedulerDelayUntil(name string) (int64, error) { - return c.coordinator.getPausedSchedulerDelayUntil(name) + return c.coordinator.GetPausedSchedulerDelayUntil(name) } diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index ca67f1052b3..00272990dc5 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -16,6 +16,7 @@ package cluster import ( "context" + "encoding/json" "fmt" "math" "math/rand" @@ -23,22 +24,33 @@ import ( "testing" "time" + "github.com/docker/go-units" "github.com/pingcap/errors" "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/eraftpb" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/core/constant" + "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/id" + "github.com/tikv/pd/pkg/mock/mockhbstream" "github.com/tikv/pd/pkg/mock/mockid" "github.com/tikv/pd/pkg/progress" + "github.com/tikv/pd/pkg/schedule" + sche "github.com/tikv/pd/pkg/schedule/core" "github.com/tikv/pd/pkg/schedule/filter" + "github.com/tikv/pd/pkg/schedule/hbstream" "github.com/tikv/pd/pkg/schedule/labeler" + "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/schedule/schedulers" "github.com/tikv/pd/pkg/statistics" "github.com/tikv/pd/pkg/storage" + "github.com/tikv/pd/pkg/utils/operatorutil" + "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" "github.com/tikv/pd/pkg/versioninfo" "github.com/tikv/pd/server/config" @@ -224,7 +236,7 @@ func TestSetOfflineStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -290,7 +302,7 @@ func TestSetOfflineWithReplica(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "2.0.0") { @@ -329,7 +341,7 @@ func TestSetOfflineStoreWithEvictLeader(t *testing.T) { re.NoError(err) opt.SetMaxReplicas(1) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 3 stores. for _, store := range newTestStores(3, "2.0.0") { @@ -375,7 +387,7 @@ func TestReuseAddress(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "2.0.0") { re.NoError(cluster.PutStore(store.GetMeta())) @@ -421,7 +433,7 @@ func TestUpStore(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -466,7 +478,7 @@ func TestRemovingProcess(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.SetPrepared() // Put 5 stores. @@ -524,7 +536,7 @@ func TestDeleteStoreUpdatesClusterVersion(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -584,7 +596,7 @@ func TestRegionHeartbeatHotStat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) newTestStores(4, "2.0.0") peers := []*metapb.Peer{ { @@ -646,7 +658,7 @@ func TestBucketHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // case1: region is not exist buckets := &metapb.Buckets{ @@ -705,7 +717,7 @@ func TestRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) n, np := uint64(3), uint64(3) cluster.wg.Add(1) go cluster.runUpdateStoreStats() @@ -940,7 +952,7 @@ func TestRegionFlowChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} processRegions := func(regions []*core.RegionInfo) { for _, r := range regions { @@ -965,7 +977,7 @@ func TestRegionSizeChanged(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) region := newTestRegions(1, 3, 3)[0] cluster.opt.GetMaxMergeRegionKeys() @@ -1008,7 +1020,7 @@ func TestConcurrentReportBucket(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} heartbeatRegions(re, cluster, regions) @@ -1038,7 +1050,7 @@ func TestConcurrentRegionHeartbeat(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} regions = core.SplitRegions(regions) @@ -1116,7 +1128,7 @@ func TestRegionLabelIsolationLevel(t *testing.T) { r := core.NewRegionInfo(region, peers[0]) re.NoError(cluster.putRegion(r)) - cluster.updateRegionsLabelLevelStats([]*core.RegionInfo{r}) + cluster.UpdateRegionsLabelLevelStats([]*core.RegionInfo{r}) counter := cluster.labelLevelStats.GetLabelCounter() re.Equal(0, counter["none"]) re.Equal(1, counter["zone"]) @@ -1158,7 +1170,7 @@ func TestHeartbeatSplit(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // 1: [nil, nil) region1 := core.NewRegionInfo(&metapb.Region{Id: 1, RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, nil) @@ -1202,7 +1214,7 @@ func TestRegionSplitAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) regions := []*core.RegionInfo{core.NewTestRegionInfo(1, 1, []byte{}, []byte{})} @@ -1240,7 +1252,7 @@ func TestOfflineAndMerge(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.ruleManager = placement.NewRuleManager(storage.NewStorageWithMemoryBackend(), cluster, cluster.GetOpts()) if opt.IsPlacementRulesEnabled() { err := cluster.ruleManager.Initialize(opt.GetMaxReplicas(), opt.GetLocationLabels()) @@ -1249,7 +1261,7 @@ func TestOfflineAndMerge(t *testing.T) { } } cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) // Put 4 stores. for _, store := range newTestStores(4, "5.0.0") { @@ -1354,7 +1366,7 @@ func TestUpdateStorePendingPeerCount(t *testing.T) { _, opt, err := newTestScheduleConfig() re.NoError(err) tc := newTestCluster(ctx, opt) - tc.RaftCluster.coordinator = newCoordinator(ctx, tc.RaftCluster, nil) + tc.RaftCluster.coordinator = schedule.NewCoordinator(ctx, tc.RaftCluster, nil) stores := newTestStores(5, "2.0.0") for _, s := range stores { re.NoError(tc.putStoreLocked(s)) @@ -1502,7 +1514,7 @@ func TestCalculateStoreSize1(t *testing.T) { cfg.EnablePlacementRules = true opt.SetReplicationConfig(cfg) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) // Put 10 stores. @@ -1585,7 +1597,7 @@ func TestCalculateStoreSize2(t *testing.T) { opt.SetReplicationConfig(cfg) opt.SetMaxReplicas(3) cluster := newTestRaftCluster(ctx, mockid.NewIDAllocator(), opt, storage.NewStorageWithMemoryBackend(), core.NewBasicCluster()) - cluster.coordinator = newCoordinator(ctx, cluster, nil) + cluster.coordinator = schedule.NewCoordinator(ctx, cluster, nil) cluster.regionStats = statistics.NewRegionStatistics(cluster.GetOpts(), cluster.ruleManager, cluster.storeConfigManager) // Put 10 stores. @@ -2139,3 +2151,1375 @@ func checkStaleRegion(origin *metapb.Region, region *metapb.Region) error { return nil } + +func newTestOperator(regionID uint64, regionEpoch *metapb.RegionEpoch, kind operator.OpKind, steps ...operator.OpStep) *operator.Operator { + return operator.NewTestOperator(regionID, regionEpoch, kind, steps...) +} + +func (c *testCluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { + id, err := c.GetAllocator().Alloc() + if err != nil { + return nil, err + } + return &metapb.Peer{Id: id, StoreId: storeID}, nil +} + +func (c *testCluster) addRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) error { + var regionSize uint64 + if len(regionSizes) == 0 { + regionSize = uint64(regionCount) * 10 + } else { + regionSize = regionSizes[0] + } + + stats := &pdpb.StoreStats{} + stats.Capacity = 100 * units.GiB + stats.UsedSize = regionSize * units.MiB + stats.Available = stats.Capacity - stats.UsedSize + newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, + core.SetStoreStats(stats), + core.SetRegionCount(regionCount), + core.SetRegionSize(int64(regionSize)), + core.SetLastHeartbeatTS(time.Now()), + ) + + c.SetStoreLimit(storeID, storelimit.AddPeer, 60) + c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) addLeaderRegion(regionID uint64, leaderStoreID uint64, followerStoreIDs ...uint64) error { + region := newTestRegionMeta(regionID) + leader, _ := c.AllocPeer(leaderStoreID) + region.Peers = []*metapb.Peer{leader} + for _, followerStoreID := range followerStoreIDs { + peer, _ := c.AllocPeer(followerStoreID) + region.Peers = append(region.Peers, peer) + } + regionInfo := core.NewRegionInfo(region, leader, core.SetApproximateSize(10), core.SetApproximateKeys(10)) + return c.putRegion(regionInfo) +} + +func (c *testCluster) updateLeaderCount(storeID uint64, leaderCount int) error { + store := c.GetStore(storeID) + newStore := store.Clone( + core.SetLeaderCount(leaderCount), + core.SetLeaderSize(int64(leaderCount)*10), + ) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) addLeaderStore(storeID uint64, leaderCount int) error { + stats := &pdpb.StoreStats{} + newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, + core.SetStoreStats(stats), + core.SetLeaderCount(leaderCount), + core.SetLeaderSize(int64(leaderCount)*10), + core.SetLastHeartbeatTS(time.Now()), + ) + + c.SetStoreLimit(storeID, storelimit.AddPeer, 60) + c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) setStoreDown(storeID uint64) error { + store := c.GetStore(storeID) + newStore := store.Clone( + core.UpStore(), + core.SetLastHeartbeatTS(typeutil.ZeroTime), + ) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) setStoreOffline(storeID uint64) error { + store := c.GetStore(storeID) + newStore := store.Clone(core.OfflineStore(false)) + c.Lock() + defer c.Unlock() + return c.putStoreLocked(newStore) +} + +func (c *testCluster) LoadRegion(regionID uint64, followerStoreIDs ...uint64) error { + // regions load from etcd will have no leader + region := newTestRegionMeta(regionID) + region.Peers = []*metapb.Peer{} + for _, id := range followerStoreIDs { + peer, _ := c.AllocPeer(id) + region.Peers = append(region.Peers, peer) + } + return c.putRegion(core.NewRegionInfo(region, nil)) +} + +func TestBasic(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + + re.NoError(tc.addLeaderRegion(1, 1)) + + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) + re.Equal(op1.RegionID(), oc.GetOperator(1).RegionID()) + + // Region 1 already has an operator, cannot add another one. + op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op2) + re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) + + // Remove the operator manually, then we can add a new operator. + re.True(oc.RemoveOperator(op1)) + op3 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op3) + re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) + re.Equal(op3.RegionID(), oc.GetOperator(1).RegionID()) +} + +func TestDispatch(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + co.GetPrepareChecker().SetPrepared() + // Transfer peer from store 4 to store 1. + re.NoError(tc.addRegionStore(4, 40)) + re.NoError(tc.addRegionStore(3, 30)) + re.NoError(tc.addRegionStore(2, 20)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + + // Transfer leader from store 4 to store 2. + re.NoError(tc.updateLeaderCount(4, 50)) + re.NoError(tc.updateLeaderCount(3, 50)) + re.NoError(tc.updateLeaderCount(2, 20)) + re.NoError(tc.updateLeaderCount(1, 10)) + re.NoError(tc.addLeaderRegion(2, 4, 3, 2)) + + go co.RunUntilStop() + + // Wait for schedule and turn off balance. + waitOperator(re, co, 1) + operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) + re.NoError(co.RemoveScheduler(schedulers.BalanceRegionName)) + waitOperator(re, co, 2) + operatorutil.CheckTransferLeader(re, co.GetOperatorController().GetOperator(2), operator.OpKind(0), 4, 2) + re.NoError(co.RemoveScheduler(schedulers.BalanceLeaderName)) + + stream := mockhbstream.NewHeartbeatStream() + + // Transfer peer. + region := tc.GetRegion(1).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitRemovePeer(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Transfer leader. + region = tc.GetRegion(2).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + waitTransferLeader(re, stream, region, 2) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func dispatchHeartbeat(co *schedule.Coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { + co.GetHeartbeatStreams().BindStream(region.GetLeader().GetStoreId(), stream) + if err := co.GetCluster().(*RaftCluster).putRegion(region.Clone()); err != nil { + return err + } + co.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat) + return nil +} + +func TestCollectMetricsConcurrent(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, func(tc *testCluster) { + tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) + }, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + // Make sure there are no problem when concurrent write and read + var wg sync.WaitGroup + count := 10 + wg.Add(count + 1) + for i := 0; i <= count; i++ { + go func(i int) { + defer wg.Done() + for j := 0; j < 1000; j++ { + re.NoError(tc.addRegionStore(uint64(i%5), rand.Intn(200))) + } + }(i) + } + for i := 0; i < 1000; i++ { + co.CollectHotSpotMetrics() + co.CollectSchedulerMetrics() + co.GetCluster().(*RaftCluster).collectClusterMetrics() + } + co.ResetHotSpotMetrics() + co.ResetSchedulerMetrics() + co.GetCluster().(*RaftCluster).resetClusterMetrics() + wg.Wait() +} + +func TestCollectMetrics(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, func(tc *testCluster) { + tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) + }, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + count := 10 + for i := 0; i <= count; i++ { + for k := 0; k < 200; k++ { + item := &statistics.HotPeerStat{ + StoreID: uint64(i % 5), + RegionID: uint64(i*1000 + k), + Loads: []float64{10, 20, 30}, + HotDegree: 10, + AntiCount: statistics.HotRegionAntiCount, // for write + } + tc.hotStat.HotCache.Update(item, statistics.Write) + } + } + for i := 0; i < 1000; i++ { + co.CollectHotSpotMetrics() + co.CollectSchedulerMetrics() + co.GetCluster().(*RaftCluster).collectClusterMetrics() + } + stores := co.GetCluster().GetStores() + regionStats := co.GetCluster().RegionWriteStats() + status1 := statistics.CollectHotPeerInfos(stores, regionStats) + status2 := statistics.GetHotStatus(stores, co.GetCluster().GetStoresLoads(), regionStats, statistics.Write, co.GetCluster().GetOpts().IsTraceRegionFlow()) + for _, s := range status2.AsLeader { + s.Stats = nil + } + for _, s := range status2.AsPeer { + s.Stats = nil + } + re.Equal(status1, status2) + co.ResetHotSpotMetrics() + co.ResetSchedulerMetrics() + co.GetCluster().(*RaftCluster).resetClusterMetrics() +} + +func prepare(setCfg func(*config.ScheduleConfig), setTc func(*testCluster), run func(*schedule.Coordinator), re *require.Assertions) (*testCluster, *schedule.Coordinator, func()) { + ctx, cancel := context.WithCancel(context.Background()) + cfg, opt, err := newTestScheduleConfig() + re.NoError(err) + if setCfg != nil { + setCfg(cfg) + } + tc := newTestCluster(ctx, opt) + hbStreams := hbstream.NewTestHeartbeatStreams(ctx, tc.meta.GetId(), tc, true /* need to run */) + if setTc != nil { + setTc(tc) + } + co := schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + if run != nil { + run(co) + } + return tc, co, func() { + co.Stop() + co.GetWaitGroup().Wait() + hbStreams.Close() + cancel() + } +} + +func checkRegionAndOperator(re *require.Assertions, tc *testCluster, co *schedule.Coordinator, regionID uint64, expectAddOperator int) { + ops := co.GetCheckerController().CheckRegion(tc.GetRegion(regionID)) + if ops == nil { + re.Equal(0, expectAddOperator) + } else { + re.Equal(expectAddOperator, co.GetOperatorController().AddWaitingOperator(ops...)) + } +} + +func TestCheckRegion(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(nil, nil, nil, re) + hbStreams, opt := co.GetHeartbeatStreams(), tc.opt + defer cleanup() + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addLeaderRegion(1, 2, 3)) + checkRegionAndOperator(re, tc, co, 1, 1) + operatorutil.CheckAddPeer(re, co.GetOperatorController().GetOperator(1), operator.OpReplica, 1) + checkRegionAndOperator(re, tc, co, 1, 0) + + r := tc.GetRegion(1) + p := &metapb.Peer{Id: 1, StoreId: 1, Role: metapb.PeerRole_Learner} + r = r.Clone( + core.WithAddPeer(p), + core.WithPendingPeers(append(r.GetPendingPeers(), p)), + ) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 0) + + tc = newTestCluster(ctx, opt) + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 0) + r = r.Clone(core.WithPendingPeers(nil)) + re.NoError(tc.putRegion(r)) + checkRegionAndOperator(re, tc, co, 1, 1) + op := co.GetOperatorController().GetOperator(1) + re.Equal(1, op.Len()) + re.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) + checkRegionAndOperator(re, tc, co, 1, 0) +} + +func TestCheckRegionWithScheduleDeny(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(4, 4)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addLeaderRegion(1, 2, 3)) + region := tc.GetRegion(1) + re.NotNil(region) + // test with label schedule=deny + labelerManager := tc.GetRegionLabeler() + labelerManager.SetLabelRule(&labeler.LabelRule{ + ID: "schedulelabel", + Labels: []labeler.RegionLabel{{Key: "schedule", Value: "deny"}}, + RuleType: labeler.KeyRange, + Data: []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}}, + }) + + // should allow to do rule checker + re.True(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 1, 1) + + // should not allow to merge + tc.opt.SetSplitMergeInterval(time.Duration(0)) + re.NoError(tc.addLeaderRegion(2, 2, 3, 4)) + re.NoError(tc.addLeaderRegion(3, 2, 3, 4)) + region = tc.GetRegion(2) + re.True(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 2, 0) + // delete label rule, should allow to do merge + labelerManager.DeleteLabelRule("schedulelabel") + re.False(labelerManager.ScheduleDisabled(region)) + checkRegionAndOperator(re, tc, co, 2, 2) +} + +func TestCheckerIsBusy(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + cfg.ReplicaScheduleLimit = 0 // ensure replica checker is busy + cfg.MergeScheduleLimit = 10 + }, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 0)) + num := 1 + typeutil.MaxUint64(tc.opt.GetReplicaScheduleLimit(), tc.opt.GetMergeScheduleLimit()) + var operatorKinds = []operator.OpKind{ + operator.OpReplica, operator.OpRegion | operator.OpMerge, + } + for i, operatorKind := range operatorKinds { + for j := uint64(0); j < num; j++ { + regionID := j + uint64(i+1)*num + re.NoError(tc.addLeaderRegion(regionID, 1)) + switch operatorKind { + case operator.OpReplica: + op := newTestOperator(regionID, tc.GetRegion(regionID).GetRegionEpoch(), operatorKind) + re.Equal(1, co.GetOperatorController().AddWaitingOperator(op)) + case operator.OpRegion | operator.OpMerge: + if regionID%2 == 1 { + ops, err := operator.CreateMergeRegionOperator("merge-region", co.GetCluster(), tc.GetRegion(regionID), tc.GetRegion(regionID-1), operator.OpMerge) + re.NoError(err) + re.Len(ops, co.GetOperatorController().AddWaitingOperator(ops...)) + } + } + } + } + checkRegionAndOperator(re, tc, co, num, 0) +} + +func TestReplica(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + // Turn off balance. + cfg.LeaderScheduleLimit = 0 + cfg.RegionScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addRegionStore(4, 4)) + + stream := mockhbstream.NewHeartbeatStream() + + // Add peer to store 1. + re.NoError(tc.addLeaderRegion(1, 2, 3)) + region := tc.GetRegion(1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Peer in store 3 is down, remove peer in store 3 and add peer to store 4. + re.NoError(tc.setStoreDown(3)) + downPeer := &pdpb.PeerStats{ + Peer: region.GetStorePeer(3), + DownSeconds: 24 * 60 * 60, + } + region = region.Clone( + core.WithDownPeers(append(region.GetDownPeers(), downPeer)), + ) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 4) + region = region.Clone(core.WithDownPeers(nil)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Remove peer from store 4. + re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4)) + region = tc.GetRegion(2) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitRemovePeer(re, stream, region, 4) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + + // Remove offline peer directly when it's pending. + re.NoError(tc.addLeaderRegion(3, 1, 2, 3)) + re.NoError(tc.setStoreOffline(3)) + region = tc.GetRegion(3) + region = region.Clone(core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func TestCheckCache(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + // Turn off replica scheduling. + cfg.ReplicaScheduleLimit = 0 + }, nil, nil, re) + defer cleanup() + + re.NoError(tc.addRegionStore(1, 0)) + re.NoError(tc.addRegionStore(2, 0)) + re.NoError(tc.addRegionStore(3, 0)) + + // Add a peer with two replicas. + re.NoError(tc.addLeaderRegion(1, 2, 3)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/break-patrol", `return`)) + + // case 1: operator cannot be created due to replica-schedule-limit restriction + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(co.GetCheckerController().GetWaitingRegions(), 1) + + // cancel the replica-schedule-limit restriction + cfg := tc.GetScheduleConfig() + cfg.ReplicaScheduleLimit = 10 + tc.SetScheduleConfig(cfg) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + oc := co.GetOperatorController() + re.Len(oc.GetOperators(), 1) + re.Empty(co.GetCheckerController().GetWaitingRegions()) + + // case 2: operator cannot be created due to store limit restriction + oc.RemoveOperator(oc.GetOperator(1)) + tc.SetStoreLimit(1, storelimit.AddPeer, 0) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(co.GetCheckerController().GetWaitingRegions(), 1) + + // cancel the store limit restriction + tc.SetStoreLimit(1, storelimit.AddPeer, 10) + time.Sleep(time.Second) + co.GetWaitGroup().Add(1) + co.PatrolRegions() + re.Len(oc.GetOperators(), 1) + re.Empty(co.GetCheckerController().GetWaitingRegions()) + + co.GetWaitGroup().Wait() + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/break-patrol")) +} + +func TestPeerState(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + // Transfer peer from store 4 to store 1. + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addRegionStore(2, 10)) + re.NoError(tc.addRegionStore(3, 10)) + re.NoError(tc.addRegionStore(4, 40)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + + stream := mockhbstream.NewHeartbeatStream() + + // Wait for schedule. + waitOperator(re, co, 1) + operatorutil.CheckTransferPeer(re, co.GetOperatorController().GetOperator(1), operator.OpKind(0), 4, 1) + + region := tc.GetRegion(1).Clone() + + // Add new peer. + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 1) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 1) + + // If the new peer is pending, the operator will not finish. + region = region.Clone(core.WithPendingPeers(append(region.GetPendingPeers(), region.GetStorePeer(1)))) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) + re.NotNil(co.GetOperatorController().GetOperator(region.GetID())) + + // The new peer is not pending now, the operator will finish. + // And we will proceed to remove peer in store 4. + region = region.Clone(core.WithPendingPeers(nil)) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitRemovePeer(re, stream, region, 4) + re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) + region = tc.GetRegion(1).Clone() + re.NoError(dispatchHeartbeat(co, region, stream)) + waitNoResponse(re, stream) +} + +func TestShouldRun(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + tc.RaftCluster.coordinator = co + defer cleanup() + + re.NoError(tc.addLeaderStore(1, 5)) + re.NoError(tc.addLeaderStore(2, 2)) + re.NoError(tc.addLeaderStore(3, 0)) + re.NoError(tc.addLeaderStore(4, 0)) + re.NoError(tc.LoadRegion(1, 1, 2, 3)) + re.NoError(tc.LoadRegion(2, 1, 2, 3)) + re.NoError(tc.LoadRegion(3, 1, 2, 3)) + re.NoError(tc.LoadRegion(4, 1, 2, 3)) + re.NoError(tc.LoadRegion(5, 1, 2, 3)) + re.NoError(tc.LoadRegion(6, 2, 1, 4)) + re.NoError(tc.LoadRegion(7, 2, 1, 4)) + re.False(co.ShouldRun()) + re.Equal(2, tc.GetStoreRegionCount(4)) + + testCases := []struct { + regionID uint64 + ShouldRun bool + }{ + {1, false}, + {2, false}, + {3, false}, + {4, false}, + {5, false}, + // store4 needs Collect two region + {6, false}, + {7, true}, + } + + for _, testCase := range testCases { + r := tc.GetRegion(testCase.regionID) + nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + re.NoError(tc.processRegionHeartbeat(nr)) + re.Equal(testCase.ShouldRun, co.ShouldRun()) + } + nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} + newRegion := core.NewRegionInfo(nr, nil) + re.Error(tc.processRegionHeartbeat(newRegion)) + re.Equal(7, co.GetPrepareChecker().GetSum()) +} + +func TestShouldRunWithNonLeaderRegions(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + tc.RaftCluster.coordinator = co + defer cleanup() + + re.NoError(tc.addLeaderStore(1, 10)) + re.NoError(tc.addLeaderStore(2, 0)) + re.NoError(tc.addLeaderStore(3, 0)) + for i := 0; i < 10; i++ { + re.NoError(tc.LoadRegion(uint64(i+1), 1, 2, 3)) + } + re.False(co.ShouldRun()) + re.Equal(10, tc.GetStoreRegionCount(1)) + + testCases := []struct { + regionID uint64 + ShouldRun bool + }{ + {1, false}, + {2, false}, + {3, false}, + {4, false}, + {5, false}, + {6, false}, + {7, false}, + {8, false}, + {9, true}, + } + + for _, testCase := range testCases { + r := tc.GetRegion(testCase.regionID) + nr := r.Clone(core.WithLeader(r.GetPeers()[0])) + re.NoError(tc.processRegionHeartbeat(nr)) + re.Equal(testCase.ShouldRun, co.ShouldRun()) + } + nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} + newRegion := core.NewRegionInfo(nr, nil) + re.Error(tc.processRegionHeartbeat(newRegion)) + re.Equal(9, co.GetPrepareChecker().GetSum()) + + // Now, after server is prepared, there exist some regions with no leader. + re.Equal(uint64(0), tc.GetRegion(10).GetLeader().GetStoreId()) +} + +func TestAddScheduler(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + + re.Len(co.GetSchedulers(), len(config.DefaultSchedulers)) + re.NoError(co.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(co.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(co.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.Empty(co.GetSchedulers()) + + stream := mockhbstream.NewHeartbeatStream() + + // Add stores 1,2,3 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + re.NoError(tc.addLeaderStore(3, 1)) + // Add regions 1 with leader in store 1 and followers in stores 2,3 + re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) + // Add regions 2 with leader in store 2 and followers in stores 1,3 + re.NoError(tc.addLeaderRegion(2, 2, 1, 3)) + // Add regions 3 with leader in store 3 and followers in stores 1,2 + re.NoError(tc.addLeaderRegion(3, 3, 1, 2)) + + oc := co.GetOperatorController() + + // test ConfigJSONDecoder create + bl, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + re.NoError(err) + conf, err := bl.EncodeConfig() + re.NoError(err) + data := make(map[string]interface{}) + err = json.Unmarshal(conf, &data) + re.NoError(err) + batch := data["batch"].(float64) + re.Equal(4, int(batch)) + + gls, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"})) + re.NoError(err) + re.NotNil(co.AddScheduler(gls)) + re.NotNil(co.RemoveScheduler(gls.GetName())) + + gls, err = schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) + re.NoError(err) + re.NoError(co.AddScheduler(gls)) + + hb, err := schedulers.CreateScheduler(schedulers.HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) + re.NoError(err) + conf, err = hb.EncodeConfig() + re.NoError(err) + data = make(map[string]interface{}) + re.NoError(json.Unmarshal(conf, &data)) + re.Contains(data, "enable-for-tiflash") + re.Equal("true", data["enable-for-tiflash"].(string)) + + // Transfer all leaders to store 1. + waitOperator(re, co, 2) + region2 := tc.GetRegion(2) + re.NoError(dispatchHeartbeat(co, region2, stream)) + region2 = waitTransferLeader(re, stream, region2, 1) + re.NoError(dispatchHeartbeat(co, region2, stream)) + waitNoResponse(re, stream) + + waitOperator(re, co, 3) + region3 := tc.GetRegion(3) + re.NoError(dispatchHeartbeat(co, region3, stream)) + region3 = waitTransferLeader(re, stream, region3, 1) + re.NoError(dispatchHeartbeat(co, region3, stream)) + waitNoResponse(re, stream) +} + +func TestPersistScheduler(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + defaultCount := len(config.DefaultSchedulers) + // Add stores 1,2 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + + re.Len(co.GetSchedulers(), defaultCount) + oc := co.GetOperatorController() + storage := tc.RaftCluster.storage + + gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) + re.NoError(err) + re.NoError(co.AddScheduler(gls1, "1")) + evict, err := schedulers.CreateScheduler(schedulers.EvictLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, []string{"2"})) + re.NoError(err) + re.NoError(co.AddScheduler(evict, "2")) + re.Len(co.GetSchedulers(), defaultCount+2) + sches, _, err := storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, defaultCount+2) + + // remove 5 schedulers + re.NoError(co.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(co.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(co.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + re.Len(co.GetSchedulers(), defaultCount-3) + re.NoError(co.GetCluster().GetPersistOptions().Persist(storage)) + co.Stop() + co.GetWaitGroup().Wait() + // make a new coordinator for testing + // whether the schedulers added or removed in dynamic way are recorded in opt + _, newOpt, err := newTestScheduleConfig() + re.NoError(err) + _, err = schedulers.CreateScheduler(schedulers.ShuffleRegionType, oc, storage, schedulers.ConfigJSONDecoder([]byte("null"))) + re.NoError(err) + // suppose we add a new default enable scheduler + config.DefaultSchedulers = append(config.DefaultSchedulers, config.SchedulerConfig{Type: "shuffle-region"}) + defer func() { + config.DefaultSchedulers = config.DefaultSchedulers[:len(config.DefaultSchedulers)-1] + }() + re.Len(newOpt.GetSchedulers(), defaultCount) + re.NoError(newOpt.Reload(storage)) + // only remains 3 items with independent config. + sches, _, err = storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, 3) + + // option have 6 items because the default scheduler do not remove. + re.Len(newOpt.GetSchedulers(), defaultCount+3) + re.NoError(newOpt.Persist(storage)) + tc.RaftCluster.opt = newOpt + + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + re.Len(co.GetSchedulers(), 3) + co.Stop() + co.GetWaitGroup().Wait() + // suppose restart PD again + _, newOpt, err = newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(storage)) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + re.Len(co.GetSchedulers(), 3) + bls, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + re.NoError(co.AddScheduler(bls)) + brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + re.NoError(co.AddScheduler(brs)) + re.Len(co.GetSchedulers(), defaultCount) + + // the scheduler option should contain 6 items + // the `hot scheduler` are disabled + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount+3) + re.NoError(co.RemoveScheduler(schedulers.GrantLeaderName)) + // the scheduler that is not enable by default will be completely deleted + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount+2) + re.Len(co.GetSchedulers(), 4) + re.NoError(co.GetCluster().GetPersistOptions().Persist(co.GetCluster().GetStorage())) + co.Stop() + co.GetWaitGroup().Wait() + _, newOpt, err = newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(co.GetCluster().GetStorage())) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + + co.Run() + re.Len(co.GetSchedulers(), defaultCount-1) + re.NoError(co.RemoveScheduler(schedulers.EvictLeaderName)) + re.Len(co.GetSchedulers(), defaultCount-2) +} + +func TestRemoveScheduler(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + cfg.ReplicaScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + + // Add stores 1,2 + re.NoError(tc.addLeaderStore(1, 1)) + re.NoError(tc.addLeaderStore(2, 1)) + defaultCount := len(config.DefaultSchedulers) + + re.Len(co.GetSchedulers(), defaultCount) + oc := co.GetOperatorController() + storage := tc.RaftCluster.storage + + gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) + re.NoError(err) + re.NoError(co.AddScheduler(gls1, "1")) + re.Len(co.GetSchedulers(), defaultCount+1) + sches, _, err := storage.LoadAllScheduleConfig() + re.NoError(err) + re.Len(sches, defaultCount+1) + + // remove all schedulers + re.NoError(co.RemoveScheduler(schedulers.BalanceLeaderName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceRegionName)) + re.NoError(co.RemoveScheduler(schedulers.HotRegionName)) + re.NoError(co.RemoveScheduler(schedulers.GrantLeaderName)) + re.NoError(co.RemoveScheduler(schedulers.BalanceWitnessName)) + re.NoError(co.RemoveScheduler(schedulers.TransferWitnessLeaderName)) + // all removed + sches, _, err = storage.LoadAllScheduleConfig() + re.NoError(err) + re.Empty(sches) + re.Empty(co.GetSchedulers()) + re.NoError(co.GetCluster().GetPersistOptions().Persist(co.GetCluster().GetStorage())) + co.Stop() + co.GetWaitGroup().Wait() + + // suppose restart PD again + _, newOpt, err := newTestScheduleConfig() + re.NoError(err) + re.NoError(newOpt.Reload(tc.storage)) + tc.RaftCluster.opt = newOpt + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.Run() + re.Empty(co.GetSchedulers()) + // the option remains default scheduler + re.Len(co.GetCluster().GetPersistOptions().GetSchedulers(), defaultCount) + co.Stop() + co.GetWaitGroup().Wait() +} + +func TestRestart(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + // Turn off balance, we test add replica only. + cfg.LeaderScheduleLimit = 0 + cfg.RegionScheduleLimit = 0 + }, nil, func(co *schedule.Coordinator) { co.Run() }, re) + hbStreams := co.GetHeartbeatStreams() + defer cleanup() + + // Add 3 stores (1, 2, 3) and a region with 1 replica on store 1. + re.NoError(tc.addRegionStore(1, 1)) + re.NoError(tc.addRegionStore(2, 2)) + re.NoError(tc.addRegionStore(3, 3)) + re.NoError(tc.addLeaderRegion(1, 1)) + region := tc.GetRegion(1) + co.GetPrepareChecker().Collect(region) + + // Add 1 replica on store 2. + stream := mockhbstream.NewHeartbeatStream() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 2) + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitPromoteLearner(re, stream, region, 2) + co.Stop() + co.GetWaitGroup().Wait() + + // Recreate coordinator then add another replica on store 3. + co = schedule.NewCoordinator(ctx, tc.RaftCluster, hbStreams) + co.GetPrepareChecker().Collect(region) + co.Run() + re.NoError(dispatchHeartbeat(co, region, stream)) + region = waitAddLearner(re, stream, region, 3) + re.NoError(dispatchHeartbeat(co, region, stream)) + waitPromoteLearner(re, stream, region, 3) +} + +func TestPauseScheduler(t *testing.T) { + re := require.New(t) + + _, co, cleanup := prepare(nil, nil, func(co *schedule.Coordinator) { co.Run() }, re) + defer cleanup() + _, err := co.IsSchedulerAllowed("test") + re.Error(err) + co.PauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) + paused, _ := co.IsSchedulerPaused(schedulers.BalanceLeaderName) + re.True(paused) + pausedAt, err := co.GetPausedSchedulerDelayAt(schedulers.BalanceLeaderName) + re.NoError(err) + resumeAt, err := co.GetPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) + re.NoError(err) + re.Equal(int64(60), resumeAt-pausedAt) + allowed, _ := co.IsSchedulerAllowed(schedulers.BalanceLeaderName) + re.False(allowed) +} + +func BenchmarkPatrolRegion(b *testing.B) { + re := require.New(b) + + mergeLimit := uint64(4100) + regionNum := 10000 + + tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { + cfg.MergeScheduleLimit = mergeLimit + }, nil, nil, re) + defer cleanup() + + tc.opt.SetSplitMergeInterval(time.Duration(0)) + for i := 1; i < 4; i++ { + if err := tc.addRegionStore(uint64(i), regionNum, 96); err != nil { + return + } + } + for i := 0; i < regionNum; i++ { + if err := tc.addLeaderRegion(uint64(i), 1, 2, 3); err != nil { + return + } + } + + listen := make(chan int) + go func() { + oc := co.GetOperatorController() + listen <- 0 + for { + if oc.OperatorCount(operator.OpMerge) == mergeLimit { + co.Stop() + return + } + } + }() + <-listen + + co.GetWaitGroup().Add(1) + b.ResetTimer() + co.PatrolRegions() +} + +func waitOperator(re *require.Assertions, co *schedule.Coordinator, regionID uint64) { + testutil.Eventually(re, func() bool { + return co.GetOperatorController().GetOperator(regionID) != nil + }) +} + +func TestOperatorCount(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) + re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) + + re.NoError(tc.addLeaderRegion(1, 1)) + re.NoError(tc.addLeaderRegion(2, 2)) + { + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 1:leader + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + oc.AddWaitingOperator(op2) + re.Equal(uint64(2), oc.OperatorCount(operator.OpLeader)) // 1:leader, 2:leader + re.True(oc.RemoveOperator(op1)) + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 2:leader + } + + { + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) + oc.AddWaitingOperator(op1) + re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) // 1:region 2:leader + re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion) + op2.SetPriorityLevel(constant.High) + oc.AddWaitingOperator(op2) + re.Equal(uint64(2), oc.OperatorCount(operator.OpRegion)) // 1:region 2:region + re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) + } +} + +func TestStoreOverloaded(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + opt := tc.GetOpts() + re.NoError(tc.addRegionStore(4, 100)) + re.NoError(tc.addRegionStore(3, 100)) + re.NoError(tc.addRegionStore(2, 100)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + start := time.Now() + { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Len(ops, 1) + op1 := ops[0] + re.NotNil(op1) + re.True(oc.AddOperator(op1)) + re.True(oc.RemoveOperator(op1)) + } + for { + time.Sleep(time.Millisecond * 10) + ops, _ := lb.Schedule(tc, false /* dryRun */) + if time.Since(start) > time.Second { + break + } + re.Empty(ops) + } + + // reset all stores' limit + // scheduling one time needs 1/10 seconds + opt.SetAllStoresLimit(storelimit.AddPeer, 600) + opt.SetAllStoresLimit(storelimit.RemovePeer, 600) + time.Sleep(time.Second) + for i := 0; i < 10; i++ { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Len(ops, 1) + op := ops[0] + re.True(oc.AddOperator(op)) + re.True(oc.RemoveOperator(op)) + } + // sleep 1 seconds to make sure that the token is filled up + time.Sleep(time.Second) + for i := 0; i < 100; i++ { + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Greater(len(ops), 0) + } +} + +func TestStoreOverloadedWithReplace(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) + re.NoError(err) + + re.NoError(tc.addRegionStore(4, 100)) + re.NoError(tc.addRegionStore(3, 100)) + re.NoError(tc.addRegionStore(2, 100)) + re.NoError(tc.addRegionStore(1, 10)) + re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) + re.NoError(tc.addLeaderRegion(2, 1, 3, 4)) + region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + region = tc.GetRegion(2).Clone(core.SetApproximateSize(60)) + tc.putRegion(region) + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 1}) + re.True(oc.AddOperator(op1)) + op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 2}) + op2.SetPriorityLevel(constant.High) + re.True(oc.AddOperator(op2)) + op3 := newTestOperator(1, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 3}) + re.False(oc.AddOperator(op3)) + ops, _ := lb.Schedule(tc, false /* dryRun */) + re.Empty(ops) + // sleep 2 seconds to make sure that token is filled up + time.Sleep(2 * time.Second) + ops, _ = lb.Schedule(tc, false /* dryRun */) + re.Greater(len(ops), 0) +} + +func TestDownStoreLimit(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + rc := co.GetCheckerController().GetRuleChecker() + + tc.addRegionStore(1, 100) + tc.addRegionStore(2, 100) + tc.addRegionStore(3, 100) + tc.addLeaderRegion(1, 1, 2, 3) + + region := tc.GetRegion(1) + tc.setStoreDown(1) + tc.SetStoreLimit(1, storelimit.RemovePeer, 1) + + region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ + { + Peer: region.GetStorePeer(1), + DownSeconds: 24 * 60 * 60, + }, + }), core.SetApproximateSize(1)) + tc.putRegion(region) + for i := uint64(1); i < 20; i++ { + tc.addRegionStore(i+3, 100) + op := rc.Check(region) + re.NotNil(op) + re.True(oc.AddOperator(op)) + oc.RemoveOperator(op) + } + + region = region.Clone(core.SetApproximateSize(100)) + tc.putRegion(region) + for i := uint64(20); i < 25; i++ { + tc.addRegionStore(i+3, 100) + op := rc.Check(region) + re.NotNil(op) + re.True(oc.AddOperator(op)) + oc.RemoveOperator(op) + } +} + +// FIXME: remove after move into schedulers package +type mockLimitScheduler struct { + schedulers.Scheduler + limit uint64 + counter *operator.Controller + kind operator.OpKind +} + +func (s *mockLimitScheduler) IsScheduleAllowed(cluster sche.ClusterInformer) bool { + return s.counter.OperatorCount(s.kind) < s.limit +} + +func TestController(t *testing.T) { + re := require.New(t) + + tc, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + oc := co.GetOperatorController() + + re.NoError(tc.addLeaderRegion(1, 1)) + re.NoError(tc.addLeaderRegion(2, 2)) + scheduler, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + lb := &mockLimitScheduler{ + Scheduler: scheduler, + counter: oc, + kind: operator.OpLeader, + } + + sc := schedule.NewScheduleController(co, lb) + + for i := schedulers.MinScheduleInterval; sc.GetInterval() != schedulers.MaxScheduleInterval; i = sc.GetNextInterval(i) { + re.Equal(i, sc.GetInterval()) + re.Empty(sc.Schedule(false)) + } + // limit = 2 + lb.limit = 2 + // count = 0 + { + re.True(sc.AllowSchedule(false)) + op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op1)) + // count = 1 + re.True(sc.AllowSchedule(false)) + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op2)) + // count = 2 + re.False(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op1)) + // count = 1 + re.True(sc.AllowSchedule(false)) + } + + op11 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) + // add a PriorityKind operator will remove old operator + { + op3 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpHotRegion) + op3.SetPriorityLevel(constant.High) + re.Equal(1, oc.AddWaitingOperator(op11)) + re.False(sc.AllowSchedule(false)) + re.Equal(1, oc.AddWaitingOperator(op3)) + re.True(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op3)) + } + + // add a admin operator will remove old operator + { + op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op2)) + re.False(sc.AllowSchedule(false)) + op4 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpAdmin) + op4.SetPriorityLevel(constant.High) + re.Equal(1, oc.AddWaitingOperator(op4)) + re.True(sc.AllowSchedule(false)) + re.True(oc.RemoveOperator(op4)) + } + + // test wrong region id. + { + op5 := newTestOperator(3, &metapb.RegionEpoch{}, operator.OpHotRegion) + re.Equal(0, oc.AddWaitingOperator(op5)) + } + + // test wrong region epoch. + re.True(oc.RemoveOperator(op11)) + epoch := &metapb.RegionEpoch{ + Version: tc.GetRegion(1).GetRegionEpoch().GetVersion() + 1, + ConfVer: tc.GetRegion(1).GetRegionEpoch().GetConfVer(), + } + { + op6 := newTestOperator(1, epoch, operator.OpLeader) + re.Equal(0, oc.AddWaitingOperator(op6)) + } + epoch.Version-- + { + op6 := newTestOperator(1, epoch, operator.OpLeader) + re.Equal(1, oc.AddWaitingOperator(op6)) + re.True(oc.RemoveOperator(op6)) + } +} + +func TestInterval(t *testing.T) { + re := require.New(t) + + _, co, cleanup := prepare(nil, nil, nil, re) + defer cleanup() + + lb, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, co.GetOperatorController(), storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) + re.NoError(err) + sc := schedule.NewScheduleController(co, lb) + + // If no operator for x seconds, the next check should be in x/2 seconds. + idleSeconds := []int{5, 10, 20, 30, 60} + for _, n := range idleSeconds { + sc.SetInterval(schedulers.MinScheduleInterval) + for totalSleep := time.Duration(0); totalSleep <= time.Second*time.Duration(n); totalSleep += sc.GetInterval() { + re.Empty(sc.Schedule(false)) + } + re.Less(sc.GetInterval(), time.Second*time.Duration(n/2)) + } +} + +func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddLearnerNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + return region.Clone( + core.WithAddPeer(res.GetChangePeer().GetPeer()), + core.WithIncConfVer(), + ) +} + +func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + // Remove learner than add voter. + return region.Clone( + core.WithRemoveStorePeer(storeID), + core.WithAddPeer(res.GetChangePeer().GetPeer()), + ) +} + +func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + return res.GetRegionId() == region.GetID() && + res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_RemoveNode && + res.GetChangePeer().GetPeer().GetStoreId() == storeID + } + return false + }) + return region.Clone( + core.WithRemoveStorePeer(storeID), + core.WithIncConfVer(), + ) +} + +func waitTransferLeader(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { + var res *pdpb.RegionHeartbeatResponse + testutil.Eventually(re, func() bool { + if res = stream.Recv(); res != nil { + if res.GetRegionId() == region.GetID() { + for _, peer := range append(res.GetTransferLeader().GetPeers(), res.GetTransferLeader().GetPeer()) { + if peer.GetStoreId() == storeID { + return true + } + } + } + } + return false + }) + return region.Clone( + core.WithLeader(region.GetStorePeer(storeID)), + ) +} + +func waitNoResponse(re *require.Assertions, stream mockhbstream.HeartbeatStream) { + testutil.Eventually(re, func() bool { + res := stream.Recv() + return res == nil + }) +} diff --git a/server/cluster/cluster_worker.go b/server/cluster/cluster_worker.go index edae92fe494..06f5c1725aa 100644 --- a/server/cluster/cluster_worker.go +++ b/server/cluster/cluster_worker.go @@ -37,7 +37,7 @@ func (c *RaftCluster) HandleRegionHeartbeat(region *core.RegionInfo) error { return err } - c.coordinator.opController.Dispatch(region, operator.DispatchFromHeartBeat) + c.coordinator.GetOperatorController().Dispatch(region, operator.DispatchFromHeartBeat) return nil } diff --git a/server/cluster/coordinator_test.go b/server/cluster/coordinator_test.go deleted file mode 100644 index 2f2a5055e0b..00000000000 --- a/server/cluster/coordinator_test.go +++ /dev/null @@ -1,1461 +0,0 @@ -// Copyright 2016 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cluster - -import ( - "context" - "encoding/json" - "math/rand" - "sync" - "testing" - "time" - - "github.com/docker/go-units" - "github.com/pingcap/failpoint" - "github.com/pingcap/kvproto/pkg/eraftpb" - "github.com/pingcap/kvproto/pkg/metapb" - "github.com/pingcap/kvproto/pkg/pdpb" - "github.com/stretchr/testify/require" - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/core/constant" - "github.com/tikv/pd/pkg/core/storelimit" - "github.com/tikv/pd/pkg/mock/mockhbstream" - sche "github.com/tikv/pd/pkg/schedule/core" - "github.com/tikv/pd/pkg/schedule/hbstream" - "github.com/tikv/pd/pkg/schedule/labeler" - "github.com/tikv/pd/pkg/schedule/operator" - "github.com/tikv/pd/pkg/schedule/schedulers" - "github.com/tikv/pd/pkg/statistics" - "github.com/tikv/pd/pkg/storage" - "github.com/tikv/pd/pkg/utils/operatorutil" - "github.com/tikv/pd/pkg/utils/testutil" - "github.com/tikv/pd/pkg/utils/typeutil" - "github.com/tikv/pd/server/config" -) - -func newTestOperator(regionID uint64, regionEpoch *metapb.RegionEpoch, kind operator.OpKind, steps ...operator.OpStep) *operator.Operator { - return operator.NewTestOperator(regionID, regionEpoch, kind, steps...) -} - -func (c *testCluster) AllocPeer(storeID uint64) (*metapb.Peer, error) { - id, err := c.GetAllocator().Alloc() - if err != nil { - return nil, err - } - return &metapb.Peer{Id: id, StoreId: storeID}, nil -} - -func (c *testCluster) addRegionStore(storeID uint64, regionCount int, regionSizes ...uint64) error { - var regionSize uint64 - if len(regionSizes) == 0 { - regionSize = uint64(regionCount) * 10 - } else { - regionSize = regionSizes[0] - } - - stats := &pdpb.StoreStats{} - stats.Capacity = 100 * units.GiB - stats.UsedSize = regionSize * units.MiB - stats.Available = stats.Capacity - stats.UsedSize - newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, - core.SetStoreStats(stats), - core.SetRegionCount(regionCount), - core.SetRegionSize(int64(regionSize)), - core.SetLastHeartbeatTS(time.Now()), - ) - - c.SetStoreLimit(storeID, storelimit.AddPeer, 60) - c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) addLeaderRegion(regionID uint64, leaderStoreID uint64, followerStoreIDs ...uint64) error { - region := newTestRegionMeta(regionID) - leader, _ := c.AllocPeer(leaderStoreID) - region.Peers = []*metapb.Peer{leader} - for _, followerStoreID := range followerStoreIDs { - peer, _ := c.AllocPeer(followerStoreID) - region.Peers = append(region.Peers, peer) - } - regionInfo := core.NewRegionInfo(region, leader, core.SetApproximateSize(10), core.SetApproximateKeys(10)) - return c.putRegion(regionInfo) -} - -func (c *testCluster) updateLeaderCount(storeID uint64, leaderCount int) error { - store := c.GetStore(storeID) - newStore := store.Clone( - core.SetLeaderCount(leaderCount), - core.SetLeaderSize(int64(leaderCount)*10), - ) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) addLeaderStore(storeID uint64, leaderCount int) error { - stats := &pdpb.StoreStats{} - newStore := core.NewStoreInfo(&metapb.Store{Id: storeID}, - core.SetStoreStats(stats), - core.SetLeaderCount(leaderCount), - core.SetLeaderSize(int64(leaderCount)*10), - core.SetLastHeartbeatTS(time.Now()), - ) - - c.SetStoreLimit(storeID, storelimit.AddPeer, 60) - c.SetStoreLimit(storeID, storelimit.RemovePeer, 60) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) setStoreDown(storeID uint64) error { - store := c.GetStore(storeID) - newStore := store.Clone( - core.UpStore(), - core.SetLastHeartbeatTS(typeutil.ZeroTime), - ) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) setStoreOffline(storeID uint64) error { - store := c.GetStore(storeID) - newStore := store.Clone(core.OfflineStore(false)) - c.Lock() - defer c.Unlock() - return c.putStoreLocked(newStore) -} - -func (c *testCluster) LoadRegion(regionID uint64, followerStoreIDs ...uint64) error { - // regions load from etcd will have no leader - region := newTestRegionMeta(regionID) - region.Peers = []*metapb.Peer{} - for _, id := range followerStoreIDs { - peer, _ := c.AllocPeer(id) - region.Peers = append(region.Peers, peer) - } - return c.putRegion(core.NewRegionInfo(region, nil)) -} - -func TestBasic(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - - re.NoError(tc.addLeaderRegion(1, 1)) - - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) - re.Equal(op1.RegionID(), oc.GetOperator(1).RegionID()) - - // Region 1 already has an operator, cannot add another one. - op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op2) - re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) - - // Remove the operator manually, then we can add a new operator. - re.True(oc.RemoveOperator(op1)) - op3 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op3) - re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) - re.Equal(op3.RegionID(), oc.GetOperator(1).RegionID()) -} - -func TestDispatch(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - co.prepareChecker.prepared = true - // Transfer peer from store 4 to store 1. - re.NoError(tc.addRegionStore(4, 40)) - re.NoError(tc.addRegionStore(3, 30)) - re.NoError(tc.addRegionStore(2, 20)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - - // Transfer leader from store 4 to store 2. - re.NoError(tc.updateLeaderCount(4, 50)) - re.NoError(tc.updateLeaderCount(3, 50)) - re.NoError(tc.updateLeaderCount(2, 20)) - re.NoError(tc.updateLeaderCount(1, 10)) - re.NoError(tc.addLeaderRegion(2, 4, 3, 2)) - - go co.runUntilStop() - - // Wait for schedule and turn off balance. - waitOperator(re, co, 1) - operatorutil.CheckTransferPeer(re, co.opController.GetOperator(1), operator.OpKind(0), 4, 1) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - waitOperator(re, co, 2) - operatorutil.CheckTransferLeader(re, co.opController.GetOperator(2), operator.OpKind(0), 4, 2) - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - - stream := mockhbstream.NewHeartbeatStream() - - // Transfer peer. - region := tc.GetRegion(1).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitRemovePeer(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Transfer leader. - region = tc.GetRegion(2).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - waitTransferLeader(re, stream, region, 2) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func dispatchHeartbeat(co *coordinator, region *core.RegionInfo, stream hbstream.HeartbeatStream) error { - co.hbStreams.BindStream(region.GetLeader().GetStoreId(), stream) - if err := co.cluster.putRegion(region.Clone()); err != nil { - return err - } - co.opController.Dispatch(region, operator.DispatchFromHeartBeat) - return nil -} - -func TestCollectMetricsConcurrent(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, func(tc *testCluster) { - tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) - }, func(co *coordinator) { co.run() }, re) - defer cleanup() - - // Make sure there are no problem when concurrent write and read - var wg sync.WaitGroup - count := 10 - wg.Add(count + 1) - for i := 0; i <= count; i++ { - go func(i int) { - defer wg.Done() - for j := 0; j < 1000; j++ { - re.NoError(tc.addRegionStore(uint64(i%5), rand.Intn(200))) - } - }(i) - } - for i := 0; i < 1000; i++ { - co.collectHotSpotMetrics() - co.collectSchedulerMetrics() - co.cluster.collectClusterMetrics() - } - co.resetHotSpotMetrics() - co.resetSchedulerMetrics() - co.cluster.resetClusterMetrics() - wg.Wait() -} - -func TestCollectMetrics(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, func(tc *testCluster) { - tc.regionStats = statistics.NewRegionStatistics(tc.GetOpts(), nil, tc.storeConfigManager) - }, func(co *coordinator) { co.run() }, re) - defer cleanup() - count := 10 - for i := 0; i <= count; i++ { - for k := 0; k < 200; k++ { - item := &statistics.HotPeerStat{ - StoreID: uint64(i % 5), - RegionID: uint64(i*1000 + k), - Loads: []float64{10, 20, 30}, - HotDegree: 10, - AntiCount: statistics.HotRegionAntiCount, // for write - } - tc.hotStat.HotCache.Update(item, statistics.Write) - } - } - for i := 0; i < 1000; i++ { - co.collectHotSpotMetrics() - co.collectSchedulerMetrics() - co.cluster.collectClusterMetrics() - } - stores := co.cluster.GetStores() - regionStats := co.cluster.RegionWriteStats() - status1 := statistics.CollectHotPeerInfos(stores, regionStats) - status2 := statistics.GetHotStatus(stores, co.cluster.GetStoresLoads(), regionStats, statistics.Write, co.cluster.GetOpts().IsTraceRegionFlow()) - for _, s := range status2.AsLeader { - s.Stats = nil - } - for _, s := range status2.AsPeer { - s.Stats = nil - } - re.Equal(status1, status2) - co.resetHotSpotMetrics() - co.resetSchedulerMetrics() - co.cluster.resetClusterMetrics() -} - -func prepare(setCfg func(*config.ScheduleConfig), setTc func(*testCluster), run func(*coordinator), re *require.Assertions) (*testCluster, *coordinator, func()) { - ctx, cancel := context.WithCancel(context.Background()) - cfg, opt, err := newTestScheduleConfig() - re.NoError(err) - if setCfg != nil { - setCfg(cfg) - } - tc := newTestCluster(ctx, opt) - hbStreams := hbstream.NewTestHeartbeatStreams(ctx, tc.meta.GetId(), tc, true /* need to run */) - if setTc != nil { - setTc(tc) - } - co := newCoordinator(ctx, tc.RaftCluster, hbStreams) - if run != nil { - run(co) - } - return tc, co, func() { - co.stop() - co.wg.Wait() - hbStreams.Close() - cancel() - } -} - -func checkRegionAndOperator(re *require.Assertions, tc *testCluster, co *coordinator, regionID uint64, expectAddOperator int) { - ops := co.checkers.CheckRegion(tc.GetRegion(regionID)) - if ops == nil { - re.Equal(0, expectAddOperator) - } else { - re.Equal(expectAddOperator, co.opController.AddWaitingOperator(ops...)) - } -} - -func TestCheckRegion(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(nil, nil, nil, re) - hbStreams, opt := co.hbStreams, tc.opt - defer cleanup() - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addLeaderRegion(1, 2, 3)) - checkRegionAndOperator(re, tc, co, 1, 1) - operatorutil.CheckAddPeer(re, co.opController.GetOperator(1), operator.OpReplica, 1) - checkRegionAndOperator(re, tc, co, 1, 0) - - r := tc.GetRegion(1) - p := &metapb.Peer{Id: 1, StoreId: 1, Role: metapb.PeerRole_Learner} - r = r.Clone( - core.WithAddPeer(p), - core.WithPendingPeers(append(r.GetPendingPeers(), p)), - ) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 0) - - tc = newTestCluster(ctx, opt) - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 0) - r = r.Clone(core.WithPendingPeers(nil)) - re.NoError(tc.putRegion(r)) - checkRegionAndOperator(re, tc, co, 1, 1) - op := co.opController.GetOperator(1) - re.Equal(1, op.Len()) - re.Equal(uint64(1), op.Step(0).(operator.PromoteLearner).ToStore) - checkRegionAndOperator(re, tc, co, 1, 0) -} - -func TestCheckRegionWithScheduleDeny(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - - defer cleanup() - - re.NoError(tc.addRegionStore(4, 4)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addLeaderRegion(1, 2, 3)) - region := tc.GetRegion(1) - re.NotNil(region) - // test with label schedule=deny - labelerManager := tc.GetRegionLabeler() - labelerManager.SetLabelRule(&labeler.LabelRule{ - ID: "schedulelabel", - Labels: []labeler.RegionLabel{{Key: "schedule", Value: "deny"}}, - RuleType: labeler.KeyRange, - Data: []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}}, - }) - - // should allow to do rule checker - re.True(labelerManager.ScheduleDisabled(region)) - checkRegionAndOperator(re, tc, co, 1, 1) - - // should not allow to merge - tc.opt.SetSplitMergeInterval(time.Duration(0)) - - re.NoError(tc.addLeaderRegion(2, 2, 3, 4)) - re.NoError(tc.addLeaderRegion(3, 2, 3, 4)) - region = tc.GetRegion(2) - re.True(labelerManager.ScheduleDisabled(region)) - checkRegionAndOperator(re, tc, co, 2, 0) - - // delete label rule, should allow to do merge - labelerManager.DeleteLabelRule("schedulelabel") - re.False(labelerManager.ScheduleDisabled(region)) - checkRegionAndOperator(re, tc, co, 2, 2) -} - -func TestCheckerIsBusy(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.ReplicaScheduleLimit = 0 // ensure replica checker is busy - cfg.MergeScheduleLimit = 10 - }, nil, nil, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 0)) - num := 1 + typeutil.MaxUint64(tc.opt.GetReplicaScheduleLimit(), tc.opt.GetMergeScheduleLimit()) - var operatorKinds = []operator.OpKind{ - operator.OpReplica, operator.OpRegion | operator.OpMerge, - } - for i, operatorKind := range operatorKinds { - for j := uint64(0); j < num; j++ { - regionID := j + uint64(i+1)*num - re.NoError(tc.addLeaderRegion(regionID, 1)) - switch operatorKind { - case operator.OpReplica: - op := newTestOperator(regionID, tc.GetRegion(regionID).GetRegionEpoch(), operatorKind) - re.Equal(1, co.opController.AddWaitingOperator(op)) - case operator.OpRegion | operator.OpMerge: - if regionID%2 == 1 { - ops, err := operator.CreateMergeRegionOperator("merge-region", co.cluster, tc.GetRegion(regionID), tc.GetRegion(regionID-1), operator.OpMerge) - re.NoError(err) - re.Len(ops, co.opController.AddWaitingOperator(ops...)) - } - } - } - } - checkRegionAndOperator(re, tc, co, num, 0) -} - -func TestReplica(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off balance. - cfg.LeaderScheduleLimit = 0 - cfg.RegionScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addRegionStore(4, 4)) - - stream := mockhbstream.NewHeartbeatStream() - - // Add peer to store 1. - re.NoError(tc.addLeaderRegion(1, 2, 3)) - region := tc.GetRegion(1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Peer in store 3 is down, remove peer in store 3 and add peer to store 4. - re.NoError(tc.setStoreDown(3)) - downPeer := &pdpb.PeerStats{ - Peer: region.GetStorePeer(3), - DownSeconds: 24 * 60 * 60, - } - region = region.Clone( - core.WithDownPeers(append(region.GetDownPeers(), downPeer)), - ) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 4) - region = region.Clone(core.WithDownPeers(nil)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Remove peer from store 4. - re.NoError(tc.addLeaderRegion(2, 1, 2, 3, 4)) - region = tc.GetRegion(2) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitRemovePeer(re, stream, region, 4) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - - // Remove offline peer directly when it's pending. - re.NoError(tc.addLeaderRegion(3, 1, 2, 3)) - re.NoError(tc.setStoreOffline(3)) - region = tc.GetRegion(3) - region = region.Clone(core.WithPendingPeers([]*metapb.Peer{region.GetStorePeer(3)})) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func TestCheckCache(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off replica scheduling. - cfg.ReplicaScheduleLimit = 0 - }, nil, nil, re) - defer cleanup() - - re.NoError(tc.addRegionStore(1, 0)) - re.NoError(tc.addRegionStore(2, 0)) - re.NoError(tc.addRegionStore(3, 0)) - - // Add a peer with two replicas. - re.NoError(tc.addLeaderRegion(1, 2, 3)) - re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/break-patrol", `return`)) - - // case 1: operator cannot be created due to replica-schedule-limit restriction - co.wg.Add(1) - co.patrolRegions() - re.Len(co.checkers.GetWaitingRegions(), 1) - - // cancel the replica-schedule-limit restriction - cfg := tc.GetScheduleConfig() - cfg.ReplicaScheduleLimit = 10 - tc.SetScheduleConfig(cfg) - co.wg.Add(1) - co.patrolRegions() - oc := co.opController - re.Len(oc.GetOperators(), 1) - re.Empty(co.checkers.GetWaitingRegions()) - - // case 2: operator cannot be created due to store limit restriction - oc.RemoveOperator(oc.GetOperator(1)) - tc.SetStoreLimit(1, storelimit.AddPeer, 0) - co.wg.Add(1) - co.patrolRegions() - re.Len(co.checkers.GetWaitingRegions(), 1) - - // cancel the store limit restriction - tc.SetStoreLimit(1, storelimit.AddPeer, 10) - time.Sleep(time.Second) - co.wg.Add(1) - co.patrolRegions() - re.Len(oc.GetOperators(), 1) - re.Empty(co.checkers.GetWaitingRegions()) - - co.wg.Wait() - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/break-patrol")) -} - -func TestPeerState(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - // Transfer peer from store 4 to store 1. - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addRegionStore(2, 10)) - re.NoError(tc.addRegionStore(3, 10)) - re.NoError(tc.addRegionStore(4, 40)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - - stream := mockhbstream.NewHeartbeatStream() - - // Wait for schedulers. - waitOperator(re, co, 1) - operatorutil.CheckTransferPeer(re, co.opController.GetOperator(1), operator.OpKind(0), 4, 1) - - region := tc.GetRegion(1).Clone() - - // Add new peer. - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 1) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 1) - - // If the new peer is pending, the operator will not finish. - region = region.Clone(core.WithPendingPeers(append(region.GetPendingPeers(), region.GetStorePeer(1)))) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) - re.NotNil(co.opController.GetOperator(region.GetID())) - - // The new peer is not pending now, the operator will finish. - // And we will proceed to remove peer in store 4. - region = region.Clone(core.WithPendingPeers(nil)) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitRemovePeer(re, stream, region, 4) - re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) - region = tc.GetRegion(1).Clone() - re.NoError(dispatchHeartbeat(co, region, stream)) - waitNoResponse(re, stream) -} - -func TestShouldRun(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - tc.RaftCluster.coordinator = co - defer cleanup() - - re.NoError(tc.addLeaderStore(1, 5)) - re.NoError(tc.addLeaderStore(2, 2)) - re.NoError(tc.addLeaderStore(3, 0)) - re.NoError(tc.addLeaderStore(4, 0)) - re.NoError(tc.LoadRegion(1, 1, 2, 3)) - re.NoError(tc.LoadRegion(2, 1, 2, 3)) - re.NoError(tc.LoadRegion(3, 1, 2, 3)) - re.NoError(tc.LoadRegion(4, 1, 2, 3)) - re.NoError(tc.LoadRegion(5, 1, 2, 3)) - re.NoError(tc.LoadRegion(6, 2, 1, 4)) - re.NoError(tc.LoadRegion(7, 2, 1, 4)) - re.False(co.shouldRun()) - re.Equal(2, tc.GetStoreRegionCount(4)) - - testCases := []struct { - regionID uint64 - shouldRun bool - }{ - {1, false}, - {2, false}, - {3, false}, - {4, false}, - {5, false}, - // store4 needs collect two region - {6, false}, - {7, true}, - } - - for _, testCase := range testCases { - r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) - re.NoError(tc.processRegionHeartbeat(nr)) - re.Equal(testCase.shouldRun, co.shouldRun()) - } - nr := &metapb.Region{Id: 6, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) - re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(7, co.prepareChecker.sum) -} - -func TestShouldRunWithNonLeaderRegions(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - tc.RaftCluster.coordinator = co - defer cleanup() - - re.NoError(tc.addLeaderStore(1, 10)) - re.NoError(tc.addLeaderStore(2, 0)) - re.NoError(tc.addLeaderStore(3, 0)) - for i := 0; i < 10; i++ { - re.NoError(tc.LoadRegion(uint64(i+1), 1, 2, 3)) - } - re.False(co.shouldRun()) - re.Equal(10, tc.GetStoreRegionCount(1)) - - testCases := []struct { - regionID uint64 - shouldRun bool - }{ - {1, false}, - {2, false}, - {3, false}, - {4, false}, - {5, false}, - {6, false}, - {7, false}, - {8, false}, - {9, true}, - } - - for _, testCase := range testCases { - r := tc.GetRegion(testCase.regionID) - nr := r.Clone(core.WithLeader(r.GetPeers()[0])) - re.NoError(tc.processRegionHeartbeat(nr)) - re.Equal(testCase.shouldRun, co.shouldRun()) - } - nr := &metapb.Region{Id: 9, Peers: []*metapb.Peer{}} - newRegion := core.NewRegionInfo(nr, nil) - re.Error(tc.processRegionHeartbeat(newRegion)) - re.Equal(9, co.prepareChecker.sum) - - // Now, after server is prepared, there exist some regions with no leader. - re.Equal(uint64(0), tc.GetRegion(10).GetLeader().GetStoreId()) -} - -func TestAddScheduler(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - - re.Len(co.schedulers, len(config.DefaultSchedulers)) - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - re.Empty(co.schedulers) - - stream := mockhbstream.NewHeartbeatStream() - - // Add stores 1,2,3 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - re.NoError(tc.addLeaderStore(3, 1)) - // Add regions 1 with leader in store 1 and followers in stores 2,3 - re.NoError(tc.addLeaderRegion(1, 1, 2, 3)) - // Add regions 2 with leader in store 2 and followers in stores 1,3 - re.NoError(tc.addLeaderRegion(2, 2, 1, 3)) - // Add regions 3 with leader in store 3 and followers in stores 1,2 - re.NoError(tc.addLeaderRegion(3, 3, 1, 2)) - - oc := co.opController - - // test ConfigJSONDecoder create - bl, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) - re.NoError(err) - conf, err := bl.EncodeConfig() - re.NoError(err) - data := make(map[string]interface{}) - err = json.Unmarshal(conf, &data) - re.NoError(err) - batch := data["batch"].(float64) - re.Equal(4, int(batch)) - - gls, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"0"})) - re.NoError(err) - re.NotNil(co.addScheduler(gls)) - re.NotNil(co.removeScheduler(gls.GetName())) - - gls, err = schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls)) - - hb, err := schedulers.CreateScheduler(schedulers.HotRegionType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigJSONDecoder([]byte("{}"))) - re.NoError(err) - conf, err = hb.EncodeConfig() - re.NoError(err) - data = make(map[string]interface{}) - re.NoError(json.Unmarshal(conf, &data)) - re.Contains(data, "enable-for-tiflash") - re.Equal("true", data["enable-for-tiflash"].(string)) - - // Transfer all leaders to store 1. - waitOperator(re, co, 2) - region2 := tc.GetRegion(2) - re.NoError(dispatchHeartbeat(co, region2, stream)) - region2 = waitTransferLeader(re, stream, region2, 1) - re.NoError(dispatchHeartbeat(co, region2, stream)) - waitNoResponse(re, stream) - - waitOperator(re, co, 3) - region3 := tc.GetRegion(3) - re.NoError(dispatchHeartbeat(co, region3, stream)) - region3 = waitTransferLeader(re, stream, region3, 1) - re.NoError(dispatchHeartbeat(co, region3, stream)) - waitNoResponse(re, stream) -} - -func TestPersistScheduler(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - defaultCount := len(config.DefaultSchedulers) - // Add stores 1,2 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - - re.Len(co.schedulers, defaultCount) - oc := co.opController - storage := tc.RaftCluster.storage - - gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls1, "1")) - evict, err := schedulers.CreateScheduler(schedulers.EvictLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.EvictLeaderType, []string{"2"})) - re.NoError(err) - re.NoError(co.addScheduler(evict, "2")) - re.Len(co.schedulers, defaultCount+2) - sches, _, err := storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, defaultCount+2) - - // remove 5 schedulers - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - re.Len(co.schedulers, defaultCount-3) - re.NoError(co.cluster.opt.Persist(storage)) - co.stop() - co.wg.Wait() - // make a new coordinator for testing - // whether the schedulers added or removed in dynamic way are recorded in opt - _, newOpt, err := newTestScheduleConfig() - re.NoError(err) - _, err = schedulers.CreateScheduler(schedulers.ShuffleRegionType, oc, storage, schedulers.ConfigJSONDecoder([]byte("null"))) - re.NoError(err) - // suppose we add a new default enable scheduler - config.DefaultSchedulers = append(config.DefaultSchedulers, config.SchedulerConfig{Type: "shuffle-region"}) - defer func() { - config.DefaultSchedulers = config.DefaultSchedulers[:len(config.DefaultSchedulers)-1] - }() - re.Len(newOpt.GetSchedulers(), defaultCount) - re.NoError(newOpt.Reload(storage)) - // only remains 3 items with independent config. - sches, _, err = storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, 3) - - // option have 6 items because the default scheduler do not remove. - re.Len(newOpt.GetSchedulers(), defaultCount+3) - re.NoError(newOpt.Persist(storage)) - tc.RaftCluster.opt = newOpt - - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Len(co.schedulers, 3) - co.stop() - co.wg.Wait() - // suppose restart PD again - _, newOpt, err = newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Len(co.schedulers, 3) - bls, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - re.NoError(co.addScheduler(bls)) - brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - re.NoError(co.addScheduler(brs)) - re.Len(co.schedulers, defaultCount) - - // the scheduler option should contain 6 items - // the `hot scheduler` are disabled - re.Len(co.cluster.opt.GetSchedulers(), defaultCount+3) - re.NoError(co.removeScheduler(schedulers.GrantLeaderName)) - // the scheduler that is not enable by default will be completely deleted - re.Len(co.cluster.opt.GetSchedulers(), defaultCount+2) - re.Len(co.schedulers, 4) - re.NoError(co.cluster.opt.Persist(co.cluster.storage)) - co.stop() - co.wg.Wait() - _, newOpt, err = newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(co.cluster.storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - - co.run() - re.Len(co.schedulers, defaultCount-1) - re.NoError(co.removeScheduler(schedulers.EvictLeaderName)) - re.Len(co.schedulers, defaultCount-2) -} - -func TestDenyScheduler(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, func(co *coordinator) { - labelerManager := co.cluster.GetRegionLabeler() - labelerManager.SetLabelRule(&labeler.LabelRule{ - ID: "schedulelabel", - Labels: []labeler.RegionLabel{{Key: "schedule", Value: "deny"}}, - RuleType: labeler.KeyRange, - Data: []interface{}{map[string]interface{}{"start_key": "", "end_key": ""}}, - }) - co.run() - }, re) - defer cleanup() - - re.Len(co.schedulers, len(config.DefaultSchedulers)) - - // Transfer peer from store 4 to store 1 if not set deny. - re.NoError(tc.addRegionStore(4, 40)) - re.NoError(tc.addRegionStore(3, 30)) - re.NoError(tc.addRegionStore(2, 20)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - - // Transfer leader from store 4 to store 2 if not set deny. - re.NoError(tc.updateLeaderCount(4, 1000)) - re.NoError(tc.updateLeaderCount(3, 50)) - re.NoError(tc.updateLeaderCount(2, 20)) - re.NoError(tc.updateLeaderCount(1, 10)) - re.NoError(tc.addLeaderRegion(2, 4, 3, 2)) - - // there should no balance leader/region operator - for i := 0; i < 10; i++ { - re.Nil(co.opController.GetOperator(1)) - re.Nil(co.opController.GetOperator(2)) - time.Sleep(10 * time.Millisecond) - } -} - -func TestRemoveScheduler(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.ReplicaScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - - // Add stores 1,2 - re.NoError(tc.addLeaderStore(1, 1)) - re.NoError(tc.addLeaderStore(2, 1)) - defaultCount := len(config.DefaultSchedulers) - - re.Len(co.schedulers, defaultCount) - oc := co.opController - storage := tc.RaftCluster.storage - - gls1, err := schedulers.CreateScheduler(schedulers.GrantLeaderType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.GrantLeaderType, []string{"1"})) - re.NoError(err) - re.NoError(co.addScheduler(gls1, "1")) - re.Len(co.schedulers, defaultCount+1) - sches, _, err := storage.LoadAllScheduleConfig() - re.NoError(err) - re.Len(sches, defaultCount+1) - - // remove all schedulers - re.NoError(co.removeScheduler(schedulers.BalanceLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceRegionName)) - re.NoError(co.removeScheduler(schedulers.HotRegionName)) - re.NoError(co.removeScheduler(schedulers.GrantLeaderName)) - re.NoError(co.removeScheduler(schedulers.BalanceWitnessName)) - re.NoError(co.removeScheduler(schedulers.TransferWitnessLeaderName)) - // all removed - sches, _, err = storage.LoadAllScheduleConfig() - re.NoError(err) - re.Empty(sches) - re.Empty(co.schedulers) - re.NoError(co.cluster.opt.Persist(co.cluster.storage)) - co.stop() - co.wg.Wait() - - // suppose restart PD again - _, newOpt, err := newTestScheduleConfig() - re.NoError(err) - re.NoError(newOpt.Reload(tc.storage)) - tc.RaftCluster.opt = newOpt - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.run() - re.Empty(co.schedulers) - // the option remains default scheduler - - re.Len(co.cluster.opt.GetSchedulers(), defaultCount) - co.stop() - co.wg.Wait() -} - -func TestRestart(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - // Turn off balance, we test add replica only. - cfg.LeaderScheduleLimit = 0 - cfg.RegionScheduleLimit = 0 - }, nil, func(co *coordinator) { co.run() }, re) - hbStreams := co.hbStreams - defer cleanup() - - // Add 3 stores (1, 2, 3) and a region with 1 replica on store 1. - re.NoError(tc.addRegionStore(1, 1)) - re.NoError(tc.addRegionStore(2, 2)) - re.NoError(tc.addRegionStore(3, 3)) - re.NoError(tc.addLeaderRegion(1, 1)) - region := tc.GetRegion(1) - co.prepareChecker.collect(region) - - // Add 1 replica on store 2. - stream := mockhbstream.NewHeartbeatStream() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 2) - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitPromoteLearner(re, stream, region, 2) - co.stop() - co.wg.Wait() - - // Recreate coordinator then add another replica on store 3. - co = newCoordinator(ctx, tc.RaftCluster, hbStreams) - co.prepareChecker.collect(region) - co.run() - re.NoError(dispatchHeartbeat(co, region, stream)) - region = waitAddLearner(re, stream, region, 3) - re.NoError(dispatchHeartbeat(co, region, stream)) - waitPromoteLearner(re, stream, region, 3) -} - -func TestPauseScheduler(t *testing.T) { - re := require.New(t) - - _, co, cleanup := prepare(nil, nil, func(co *coordinator) { co.run() }, re) - defer cleanup() - _, err := co.isSchedulerAllowed("test") - re.Error(err) - co.pauseOrResumeScheduler(schedulers.BalanceLeaderName, 60) - paused, _ := co.isSchedulerPaused(schedulers.BalanceLeaderName) - re.True(paused) - pausedAt, err := co.getPausedSchedulerDelayAt(schedulers.BalanceLeaderName) - re.NoError(err) - resumeAt, err := co.getPausedSchedulerDelayUntil(schedulers.BalanceLeaderName) - re.NoError(err) - re.Equal(int64(60), resumeAt-pausedAt) - allowed, _ := co.isSchedulerAllowed(schedulers.BalanceLeaderName) - re.False(allowed) -} - -func BenchmarkPatrolRegion(b *testing.B) { - re := require.New(b) - - mergeLimit := uint64(4100) - regionNum := 10000 - - tc, co, cleanup := prepare(func(cfg *config.ScheduleConfig) { - cfg.MergeScheduleLimit = mergeLimit - }, nil, nil, re) - defer cleanup() - - tc.opt.SetSplitMergeInterval(time.Duration(0)) - for i := 1; i < 4; i++ { - if err := tc.addRegionStore(uint64(i), regionNum, 96); err != nil { - return - } - } - for i := 0; i < regionNum; i++ { - if err := tc.addLeaderRegion(uint64(i), 1, 2, 3); err != nil { - return - } - } - - listen := make(chan int) - go func() { - oc := co.opController - listen <- 0 - for { - if oc.OperatorCount(operator.OpMerge) == mergeLimit { - co.cancel() - return - } - } - }() - <-listen - - co.wg.Add(1) - b.ResetTimer() - co.patrolRegions() -} - -func waitOperator(re *require.Assertions, co *coordinator, regionID uint64) { - testutil.Eventually(re, func() bool { - return co.opController.GetOperator(regionID) != nil - }) -} - -func TestOperatorCount(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) - re.Equal(uint64(0), oc.OperatorCount(operator.OpRegion)) - - re.NoError(tc.addLeaderRegion(1, 1)) - re.NoError(tc.addLeaderRegion(2, 2)) - { - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 1:leader - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - oc.AddWaitingOperator(op2) - re.Equal(uint64(2), oc.OperatorCount(operator.OpLeader)) // 1:leader, 2:leader - re.True(oc.RemoveOperator(op1)) - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) // 2:leader - } - - { - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion) - oc.AddWaitingOperator(op1) - re.Equal(uint64(1), oc.OperatorCount(operator.OpRegion)) // 1:region 2:leader - re.Equal(uint64(1), oc.OperatorCount(operator.OpLeader)) - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion) - op2.SetPriorityLevel(constant.High) - oc.AddWaitingOperator(op2) - re.Equal(uint64(2), oc.OperatorCount(operator.OpRegion)) // 1:region 2:region - re.Equal(uint64(0), oc.OperatorCount(operator.OpLeader)) - } -} - -func TestStoreOverloaded(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - opt := tc.GetOpts() - re.NoError(tc.addRegionStore(4, 100)) - re.NoError(tc.addRegionStore(3, 100)) - re.NoError(tc.addRegionStore(2, 100)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - start := time.Now() - { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Len(ops, 1) - op1 := ops[0] - re.NotNil(op1) - re.True(oc.AddOperator(op1)) - re.True(oc.RemoveOperator(op1)) - } - for { - time.Sleep(time.Millisecond * 10) - ops, _ := lb.Schedule(tc, false /* dryRun */) - if time.Since(start) > time.Second { - break - } - re.Empty(ops) - } - - // reset all stores' limit - // scheduling one time needs 1/10 seconds - opt.SetAllStoresLimit(storelimit.AddPeer, 600) - opt.SetAllStoresLimit(storelimit.RemovePeer, 600) - time.Sleep(time.Second) - for i := 0; i < 10; i++ { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Len(ops, 1) - op := ops[0] - re.True(oc.AddOperator(op)) - re.True(oc.RemoveOperator(op)) - } - // sleep 1 seconds to make sure that the token is filled up - time.Sleep(time.Second) - for i := 0; i < 100; i++ { - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) - } -} - -func TestStoreOverloadedWithReplace(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - lb, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, tc.storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""})) - re.NoError(err) - - re.NoError(tc.addRegionStore(4, 100)) - re.NoError(tc.addRegionStore(3, 100)) - re.NoError(tc.addRegionStore(2, 100)) - re.NoError(tc.addRegionStore(1, 10)) - re.NoError(tc.addLeaderRegion(1, 2, 3, 4)) - re.NoError(tc.addLeaderRegion(2, 1, 3, 4)) - region := tc.GetRegion(1).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - region = tc.GetRegion(2).Clone(core.SetApproximateSize(60)) - tc.putRegion(region) - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 1}) - re.True(oc.AddOperator(op1)) - op2 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 2}) - op2.SetPriorityLevel(constant.High) - re.True(oc.AddOperator(op2)) - op3 := newTestOperator(1, tc.GetRegion(2).GetRegionEpoch(), operator.OpRegion, operator.AddPeer{ToStore: 1, PeerID: 3}) - re.False(oc.AddOperator(op3)) - ops, _ := lb.Schedule(tc, false /* dryRun */) - re.Empty(ops) - // sleep 2 seconds to make sure that token is filled up - time.Sleep(2 * time.Second) - ops, _ = lb.Schedule(tc, false /* dryRun */) - re.Greater(len(ops), 0) -} - -func TestDownStoreLimit(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - rc := co.checkers.GetRuleChecker() - - tc.addRegionStore(1, 100) - tc.addRegionStore(2, 100) - tc.addRegionStore(3, 100) - tc.addLeaderRegion(1, 1, 2, 3) - - region := tc.GetRegion(1) - tc.setStoreDown(1) - tc.SetStoreLimit(1, storelimit.RemovePeer, 1) - - region = region.Clone(core.WithDownPeers([]*pdpb.PeerStats{ - { - Peer: region.GetStorePeer(1), - DownSeconds: 24 * 60 * 60, - }, - }), core.SetApproximateSize(1)) - tc.putRegion(region) - for i := uint64(1); i < 20; i++ { - tc.addRegionStore(i+3, 100) - op := rc.Check(region) - re.NotNil(op) - re.True(oc.AddOperator(op)) - oc.RemoveOperator(op) - } - - region = region.Clone(core.SetApproximateSize(100)) - tc.putRegion(region) - for i := uint64(20); i < 25; i++ { - tc.addRegionStore(i+3, 100) - op := rc.Check(region) - re.NotNil(op) - re.True(oc.AddOperator(op)) - oc.RemoveOperator(op) - } -} - -// FIXME: remove after move into schedulers package -type mockLimitScheduler struct { - schedulers.Scheduler - limit uint64 - counter *operator.Controller - kind operator.OpKind -} - -func (s *mockLimitScheduler) IsScheduleAllowed(cluster sche.ClusterInformer) bool { - return s.counter.OperatorCount(s.kind) < s.limit -} - -func TestController(t *testing.T) { - re := require.New(t) - - tc, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - oc := co.opController - - re.NoError(tc.addLeaderRegion(1, 1)) - re.NoError(tc.addLeaderRegion(2, 2)) - scheduler, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, oc, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - lb := &mockLimitScheduler{ - Scheduler: scheduler, - counter: oc, - kind: operator.OpLeader, - } - - sc := newScheduleController(co, lb) - - for i := schedulers.MinScheduleInterval; sc.GetInterval() != schedulers.MaxScheduleInterval; i = sc.GetNextInterval(i) { - re.Equal(i, sc.GetInterval()) - re.Empty(sc.Schedule(false)) - } - // limit = 2 - lb.limit = 2 - // count = 0 - { - re.True(sc.AllowSchedule(false)) - op1 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op1)) - // count = 1 - re.True(sc.AllowSchedule(false)) - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op2)) - // count = 2 - re.False(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op1)) - // count = 1 - re.True(sc.AllowSchedule(false)) - } - - op11 := newTestOperator(1, tc.GetRegion(1).GetRegionEpoch(), operator.OpLeader) - // add a PriorityKind operator will remove old operator - { - op3 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpHotRegion) - op3.SetPriorityLevel(constant.High) - re.Equal(1, oc.AddWaitingOperator(op11)) - re.False(sc.AllowSchedule(false)) - re.Equal(1, oc.AddWaitingOperator(op3)) - re.True(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op3)) - } - - // add a admin operator will remove old operator - { - op2 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op2)) - re.False(sc.AllowSchedule(false)) - op4 := newTestOperator(2, tc.GetRegion(2).GetRegionEpoch(), operator.OpAdmin) - op4.SetPriorityLevel(constant.High) - re.Equal(1, oc.AddWaitingOperator(op4)) - re.True(sc.AllowSchedule(false)) - re.True(oc.RemoveOperator(op4)) - } - - // test wrong region id. - { - op5 := newTestOperator(3, &metapb.RegionEpoch{}, operator.OpHotRegion) - re.Equal(0, oc.AddWaitingOperator(op5)) - } - - // test wrong region epoch. - re.True(oc.RemoveOperator(op11)) - epoch := &metapb.RegionEpoch{ - Version: tc.GetRegion(1).GetRegionEpoch().GetVersion() + 1, - ConfVer: tc.GetRegion(1).GetRegionEpoch().GetConfVer(), - } - { - op6 := newTestOperator(1, epoch, operator.OpLeader) - re.Equal(0, oc.AddWaitingOperator(op6)) - } - epoch.Version-- - { - op6 := newTestOperator(1, epoch, operator.OpLeader) - re.Equal(1, oc.AddWaitingOperator(op6)) - re.True(oc.RemoveOperator(op6)) - } -} - -func TestInterval(t *testing.T) { - re := require.New(t) - - _, co, cleanup := prepare(nil, nil, nil, re) - defer cleanup() - - lb, err := schedulers.CreateScheduler(schedulers.BalanceLeaderType, co.opController, storage.NewStorageWithMemoryBackend(), schedulers.ConfigSliceDecoder(schedulers.BalanceLeaderType, []string{"", ""})) - re.NoError(err) - sc := newScheduleController(co, lb) - - // If no operator for x seconds, the next check should be in x/2 seconds. - idleSeconds := []int{5, 10, 20, 30, 60} - for _, n := range idleSeconds { - sc.nextInterval = schedulers.MinScheduleInterval - for totalSleep := time.Duration(0); totalSleep <= time.Second*time.Duration(n); totalSleep += sc.GetInterval() { - re.Empty(sc.Schedule(false)) - } - re.Less(sc.GetInterval(), time.Second*time.Duration(n/2)) - } -} - -func waitAddLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddLearnerNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - return region.Clone( - core.WithAddPeer(res.GetChangePeer().GetPeer()), - core.WithIncConfVer(), - ) -} - -func waitPromoteLearner(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_AddNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - // Remove learner than add voter. - return region.Clone( - core.WithRemoveStorePeer(storeID), - core.WithAddPeer(res.GetChangePeer().GetPeer()), - ) -} - -func waitRemovePeer(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - return res.GetRegionId() == region.GetID() && - res.GetChangePeer().GetChangeType() == eraftpb.ConfChangeType_RemoveNode && - res.GetChangePeer().GetPeer().GetStoreId() == storeID - } - return false - }) - return region.Clone( - core.WithRemoveStorePeer(storeID), - core.WithIncConfVer(), - ) -} - -func waitTransferLeader(re *require.Assertions, stream mockhbstream.HeartbeatStream, region *core.RegionInfo, storeID uint64) *core.RegionInfo { - var res *pdpb.RegionHeartbeatResponse - testutil.Eventually(re, func() bool { - if res = stream.Recv(); res != nil { - if res.GetRegionId() == region.GetID() { - for _, peer := range append(res.GetTransferLeader().GetPeers(), res.GetTransferLeader().GetPeer()) { - if peer.GetStoreId() == storeID { - return true - } - } - } - } - return false - }) - return region.Clone( - core.WithLeader(region.GetStorePeer(storeID)), - ) -} - -func waitNoResponse(re *require.Assertions, stream mockhbstream.HeartbeatStream) { - testutil.Eventually(re, func() bool { - res := stream.Recv() - return res == nil - }) -} diff --git a/server/cluster/metrics.go b/server/cluster/metrics.go index 8c0bceb94ca..e43fe595f70 100644 --- a/server/cluster/metrics.go +++ b/server/cluster/metrics.go @@ -41,38 +41,6 @@ var ( Help: "Counter of the bucket event", }, []string{"event"}) - schedulerStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "status", - Help: "Status of the scheduler.", - }, []string{"kind", "type"}) - - hotSpotStatusGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "hotspot", - Name: "status", - Help: "Status of the hotspot.", - }, []string{"address", "store", "type"}) - - hotPendingSum = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "hot_pending_sum", - Help: "Pending influence sum of store in hot region scheduler.", - }, []string{"store", "rw", "dim"}) - - patrolCheckRegionsGauge = prometheus.NewGauge( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "checker", - Name: "patrol_regions_time", - Help: "Time spent of patrol checks region.", - }) - updateStoreStatsGauge = prometheus.NewGauge( prometheus.GaugeOpts{ Namespace: "pd", @@ -96,14 +64,6 @@ var ( Help: "Current state of the cluster", }, []string{"state"}) - regionListGauge = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: "pd", - Subsystem: "checker", - Name: "region_list", - Help: "Number of region in waiting list", - }, []string{"type"}) - storesProgressGauge = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: "pd", @@ -140,12 +100,8 @@ var ( func init() { prometheus.MustRegister(regionEventCounter) prometheus.MustRegister(healthStatusGauge) - prometheus.MustRegister(schedulerStatusGauge) - prometheus.MustRegister(hotSpotStatusGauge) - prometheus.MustRegister(patrolCheckRegionsGauge) prometheus.MustRegister(clusterStateCPUGauge) prometheus.MustRegister(clusterStateCurrent) - prometheus.MustRegister(regionListGauge) prometheus.MustRegister(bucketEventCounter) prometheus.MustRegister(storesProgressGauge) prometheus.MustRegister(storesSpeedGauge) diff --git a/server/handler.go b/server/handler.go index 8426d2eda5c..be730d205f1 100644 --- a/server/handler.go +++ b/server/handler.go @@ -33,6 +33,7 @@ import ( "github.com/tikv/pd/pkg/core/storelimit" "github.com/tikv/pd/pkg/encryption" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/schedule" "github.com/tikv/pd/pkg/schedule/filter" "github.com/tikv/pd/pkg/schedule/operator" "github.com/tikv/pd/pkg/schedule/placement" @@ -982,7 +983,7 @@ func (h *Handler) PluginUnload(pluginPath string) error { h.pluginChMapLock.Lock() defer h.pluginChMapLock.Unlock() if ch, ok := h.pluginChMap[pluginPath]; ok { - ch <- cluster.PluginUnload + ch <- schedule.PluginUnload return nil } return ErrPluginNotFound(pluginPath) diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 1ab458fa742..06ed6e0a65f 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -626,7 +626,6 @@ func TestConcurrentHandleRegion(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1", "127.0.1.1:2"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) stores := make([]*metapb.Store, 0, len(storeAddrs)) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { @@ -1056,7 +1055,6 @@ func TestOfflineStoreLimit(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { storeID, err := id.Alloc() @@ -1148,7 +1146,6 @@ func TestUpgradeStoreLimit(t *testing.T) { bootstrapCluster(re, clusterID, grpcPDClient) rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) store := newMetaStore(1, "127.0.1.1:0", "4.0.0", metapb.StoreState_Up, "test/store1") resp, err := putStore(grpcPDClient, clusterID, store) re.NoError(err) @@ -1208,7 +1205,6 @@ func TestStaleTermHeartbeat(t *testing.T) { storeAddrs := []string{"127.0.1.1:0", "127.0.1.1:1", "127.0.1.1:2"} rc := leaderServer.GetRaftCluster() re.NotNil(rc) - rc.SetStorage(storage.NewStorageWithMemoryBackend()) peers := make([]*metapb.Peer, 0, len(storeAddrs)) id := leaderServer.GetAllocator() for _, addr := range storeAddrs { diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index aa9f76d157d..0b50de77147 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -202,7 +202,7 @@ func TestPrepareChecker(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/changeCoordinatorTicker", `return(true)`)) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker", `return(true)`)) cluster, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, serverName string) { conf.PDServerCfg.UseRegionStorage = true }) defer cluster.Destroy() re.NoError(err) @@ -243,7 +243,7 @@ func TestPrepareChecker(t *testing.T) { } time.Sleep(time.Second) re.True(rc.IsPrepared()) - re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/changeCoordinatorTicker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker")) } func initRegions(regionLen int) []*core.RegionInfo { diff --git a/tools/pd-ctl/pdctl/command/plugin_command.go b/tools/pd-ctl/pdctl/command/plugin_command.go index f6ae7a97159..a713c8fd063 100644 --- a/tools/pd-ctl/pdctl/command/plugin_command.go +++ b/tools/pd-ctl/pdctl/command/plugin_command.go @@ -20,7 +20,7 @@ import ( "net/http" "github.com/spf13/cobra" - "github.com/tikv/pd/server/cluster" + "github.com/tikv/pd/pkg/schedule" ) var ( @@ -59,11 +59,11 @@ func NewUnloadPluginCommand() *cobra.Command { } func loadPluginCommandFunc(cmd *cobra.Command, args []string) { - sendPluginCommand(cmd, cluster.PluginLoad, args) + sendPluginCommand(cmd, schedule.PluginLoad, args) } func unloadPluginCommandFunc(cmd *cobra.Command, args []string) { - sendPluginCommand(cmd, cluster.PluginUnload, args) + sendPluginCommand(cmd, schedule.PluginUnload, args) } func sendPluginCommand(cmd *cobra.Command, action string, args []string) { @@ -80,9 +80,9 @@ func sendPluginCommand(cmd *cobra.Command, action string, args []string) { return } switch action { - case cluster.PluginLoad: + case schedule.PluginLoad: _, err = doRequest(cmd, pluginPrefix, http.MethodPost, http.Header{"Content-Type": {"application/json"}}, WithBody(bytes.NewBuffer(reqData))) - case cluster.PluginUnload: + case schedule.PluginUnload: _, err = doRequest(cmd, pluginPrefix, http.MethodDelete, http.Header{"Content-Type": {"application/json"}}, WithBody(bytes.NewBuffer(reqData))) default: cmd.Printf("Unknown action %s\n", action)