-
Notifications
You must be signed in to change notification settings - Fork 727
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
resource_manager: implement the independent primary election #6086
Changes from 1 commit
1dad889
e7d2bfa
2f3d845
e8d9c05
841a20a
3e15ec4
8c495a5
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,6 +17,7 @@ package server | |
import ( | ||
"context" | ||
"crypto/tls" | ||
"fmt" | ||
"net" | ||
"net/http" | ||
"net/url" | ||
|
@@ -38,6 +39,7 @@ import ( | |
"github.com/tikv/pd/pkg/member" | ||
"github.com/tikv/pd/pkg/utils/etcdutil" | ||
"github.com/tikv/pd/pkg/utils/logutil" | ||
"github.com/tikv/pd/pkg/utils/memberutil" | ||
"github.com/tikv/pd/pkg/utils/metricutil" | ||
"github.com/tikv/pd/pkg/versioninfo" | ||
"go.etcd.io/etcd/clientv3" | ||
|
@@ -47,13 +49,17 @@ import ( | |
) | ||
|
||
const ( | ||
leaderTickInterval = 50 * time.Millisecond | ||
tcpNetworkStr = "tcp" | ||
tcpNetworkStr = "tcp" | ||
// resourceManagerKeyspaceGroupPrimaryElectionPrefix defines the key prefix for keyspace group primary election. | ||
// The entire key is in the format of "/pd/<cluster-id>/microservice/resource_manager/keyspace-group-XXXXX/primary" | ||
JmPotato marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// in which XXXXX is 5 digits integer with leading zeros. For now we use 0 as the default cluster id. | ||
resourceManagerKeyspaceGroupPrimaryElectionPrefix = "/pd/0/microservice/resource_manager/keyspace-group-" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Will the resource manager have the concept of a keyspace group? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, in the assumption of the future, we may separate the resource manager service into several to serve different keyspace groups.
JmPotato marked this conversation as resolved.
Show resolved
Hide resolved
|
||
// defaultGRPCGracefulStopTimeout is the default timeout to wait for grpc server to gracefully stop | ||
defaultGRPCGracefulStopTimeout = 5 * time.Second | ||
// defaultHTTPGracefulShutdownTimeout is the default timeout to wait for http server to gracefully shutdown | ||
defaultHTTPGracefulShutdownTimeout = 5 * time.Second | ||
defaultLeaseInSeconds = 3 | ||
leaderTickInterval = 50 * time.Millisecond | ||
) | ||
|
||
// Server is the resource manager server, and it implements bs.Server. | ||
|
@@ -66,10 +72,8 @@ type Server struct { | |
serverLoopCancel func() | ||
serverLoopWg sync.WaitGroup | ||
|
||
cfg *Config | ||
name string | ||
backendURLs []url.URL | ||
listenURL *url.URL | ||
cfg *Config | ||
name string | ||
|
||
// for the primary election of resource manager | ||
participant *member.Participant | ||
|
@@ -205,6 +209,7 @@ func (s *Server) Close() { | |
log.Info("closing resource manager server ...") | ||
s.serviceRegister.Deregister() | ||
s.muxListener.Close() | ||
s.serverLoopCancel() | ||
s.serverLoopWg.Wait() | ||
|
||
if s.etcdClient != nil { | ||
|
@@ -242,8 +247,7 @@ func (s *Server) AddStartCallback(callbacks ...func()) { | |
|
||
// IsServing returns whether the server is the leader, if there is embedded etcd, or the primary otherwise. | ||
func (s *Server) IsServing() bool { | ||
// TODO: implement this function with primary. | ||
return atomic.LoadInt64(&s.isServing) == 1 | ||
return s.participant.IsLeader() | ||
} | ||
|
||
// IsClosed checks if the server loop is closed | ||
|
@@ -265,8 +269,7 @@ func (s *Server) initClient() error { | |
if err != nil { | ||
return err | ||
} | ||
s.backendURLs = []url.URL(u) | ||
s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, s.backendURLs) | ||
s.etcdClient, s.httpClient, err = etcdutil.CreateClients(tlsConfig, []url.URL(u)) | ||
return err | ||
} | ||
|
||
|
@@ -347,11 +350,23 @@ func (s *Server) startGRPCAndHTTPServers(l net.Listener) { | |
|
||
// GetPrimary returns the primary member. | ||
func (s *Server) GetPrimary() bs.MemberProvider { | ||
// TODO: implement this function with primary. | ||
return nil | ||
return s.participant.GetLeader() | ||
} | ||
|
||
func (s *Server) startServer() error { | ||
// The independent Resource Manager service still reuses PD version info since PD and Resource Manager are just | ||
// different service modes provided by the same pd-server binary | ||
serverInfo.WithLabelValues(versioninfo.PDReleaseVersion, versioninfo.PDGitHash).Set(float64(time.Now().Unix())) | ||
|
||
uniqueName := s.cfg.ListenAddr | ||
uniqueID := memberutil.GenerateUniqueID(uniqueName) | ||
log.Info("joining primary election", zap.String("participant-name", uniqueName), zap.Uint64("participant-id", uniqueID)) | ||
s.participant = member.NewParticipant(s.etcdClient, uniqueID) | ||
s.participant.InitInfo(uniqueName, resourceManagerKeyspaceGroupPrimaryElectionPrefix+fmt.Sprintf("%05d", 0), "primary", "keyspace group primary election") | ||
s.participant.SetMemberDeployPath(s.participant.ID()) | ||
s.participant.SetMemberBinaryVersion(s.participant.ID(), versioninfo.PDReleaseVersion) | ||
s.participant.SetMemberGitHash(s.participant.ID(), versioninfo.PDGitHash) | ||
|
||
s.service = &Service{ | ||
ctx: s.ctx, | ||
manager: NewManager[*Server](s), | ||
|
@@ -361,14 +376,10 @@ func (s *Server) startServer() error { | |
if err != nil { | ||
return err | ||
} | ||
s.listenURL, err = url.Parse(s.cfg.ListenAddr) | ||
if err != nil { | ||
return err | ||
} | ||
if tlsConfig != nil { | ||
s.muxListener, err = tls.Listen(tcpNetworkStr, s.listenURL.Host, tlsConfig) | ||
s.muxListener, err = tls.Listen(tcpNetworkStr, s.cfg.ListenAddr, tlsConfig) | ||
} else { | ||
s.muxListener, err = net.Listen(tcpNetworkStr, s.listenURL.Host) | ||
s.muxListener, err = net.Listen(tcpNetworkStr, s.cfg.ListenAddr) | ||
} | ||
if err != nil { | ||
return err | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,60 @@ | ||
// Copyright 2023 TiKV Project Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package server | ||
|
||
import ( | ||
"context" | ||
"os" | ||
|
||
"github.com/pingcap/log" | ||
"github.com/spf13/cobra" | ||
"github.com/stretchr/testify/require" | ||
"github.com/tikv/pd/pkg/utils/logutil" | ||
) | ||
|
||
// CleanupFunc closes test resource manager server(s) and deletes any files left behind. | ||
type CleanupFunc func() | ||
|
||
// NewTestServer creates a resource manager server for testing. | ||
func NewTestServer(ctx context.Context, re *require.Assertions, cfg *Config) (*Server, CleanupFunc, error) { | ||
// New zap logger | ||
err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) | ||
re.NoError(err) | ||
log.ReplaceGlobals(cfg.Logger, cfg.LogProps) | ||
// Flushing any buffered log entries | ||
defer log.Sync() | ||
|
||
s := NewServer(ctx, cfg) | ||
if err = s.Run(); err != nil { | ||
return nil, nil, err | ||
} | ||
|
||
cleanup := func() { | ||
s.Close() | ||
os.RemoveAll(cfg.DataDir) | ||
} | ||
return s, cleanup, nil | ||
} | ||
|
||
// NewTestDefaultConfig creates a new default config for testing. | ||
func NewTestDefaultConfig() (*Config, error) { | ||
cmd := &cobra.Command{ | ||
Use: "resource_manager", | ||
Short: "Run the resource manager service", | ||
} | ||
cfg := NewConfig() | ||
flagSet := cmd.Flags() | ||
return cfg, cfg.Parse(flagSet) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
// Copyright 2023 TiKV Project Authors. | ||
// | ||
// Licensed under the Apache License, Version 2.0 (the "License"); | ||
// you may not use this file except in compliance with the License. | ||
// You may obtain a copy of the License at | ||
// | ||
// http://www.apache.org/licenses/LICENSE-2.0 | ||
// | ||
// Unless required by applicable law or agreed to in writing, software | ||
// distributed under the License is distributed on an "AS IS" BASIS, | ||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
// See the License for the specific language governing permissions and | ||
// limitations under the License. | ||
|
||
package memberutil | ||
|
||
import ( | ||
"crypto/sha256" | ||
"encoding/binary" | ||
) | ||
|
||
// GenerateUniqueID generates a unique ID based on the given seed. | ||
// This is used to generate a unique ID for a member. | ||
func GenerateUniqueID(seed string) uint64 { | ||
hash := sha256.Sum256([]byte(seed)) | ||
return binary.LittleEndian.Uint64(hash[:8]) | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
maybe conflict with tso's
defaultListenAddr
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
BTW, I think 2379 is better.