From 7e725fa6e09e3cf34985066f99a7bbc24211f393 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 4 Dec 2023 11:04:30 +0800 Subject: [PATCH 01/16] add service client Signed-off-by: Cabinfever_B --- client/client.go | 4 - client/go.mod | 13 +- client/go.sum | 24 ++- client/grpcutil/grpcutil.go | 57 ++++++ client/pd_service_discovery.go | 247 ++++++++++++++++++++++- client/pd_service_discovery_test.go | 290 ++++++++++++++++++++++++++++ client/testutil/check_env.go | 21 ++ client/testutil/check_env_linux.go | 41 ++++ client/testutil/leak.go | 4 + client/testutil/tempurl.go | 65 +++++++ tests/integrations/client/go.mod | 10 +- tests/integrations/client/go.sum | 20 +- tests/integrations/mcs/go.mod | 10 +- tests/integrations/mcs/go.sum | 20 +- tests/integrations/tso/go.mod | 10 +- tests/integrations/tso/go.sum | 20 +- tools/pd-api-bench/go.mod | 10 +- tools/pd-api-bench/go.sum | 20 +- tools/pd-tso-bench/go.mod | 8 +- tools/pd-tso-bench/go.sum | 20 +- 20 files changed, 821 insertions(+), 93 deletions(-) create mode 100644 client/pd_service_discovery_test.go create mode 100644 client/testutil/check_env.go create mode 100644 client/testutil/check_env_linux.go create mode 100644 client/testutil/tempurl.go diff --git a/client/client.go b/client/client.go index b320be6d3d5..d509a7ec7da 100644 --- a/client/client.go +++ b/client/client.go @@ -939,10 +939,6 @@ func handleRegionResponse(res *pdpb.GetRegionResponse) *Region { return r } -func isNetworkError(code codes.Code) bool { - return code == codes.Unavailable || code == codes.DeadlineExceeded -} - func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs []string, opts ...GetRegionOption) (*Region, error) { if span := opentracing.SpanFromContext(ctx); span != nil { span = opentracing.StartSpan("pdclient.GetRegionFromMember", opentracing.ChildOf(span.Context())) diff --git a/client/go.mod b/client/go.mod index 948a5c22b14..b1652f570bc 100644 --- a/client/go.mod +++ b/client/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/BurntSushi/toml v0.3.1 + github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 github.com/cloudfoundry/gosigar v1.3.6 github.com/gogo/protobuf v1.3.2 github.com/opentracing/opentracing-go v1.2.0 @@ -19,6 +20,8 @@ require ( google.golang.org/grpc v1.54.0 ) +require google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -33,11 +36,11 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/client/go.sum b/client/go.sum index 921c4671581..4d4e9f396da 100644 --- a/client/go.sum +++ b/client/go.sum @@ -13,6 +13,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -176,8 +178,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -201,14 +203,14 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -225,10 +227,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -237,8 +241,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index fe149e76ecc..22009a2ca1f 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -36,6 +36,8 @@ const ( dialTimeout = 3 * time.Second // ForwardMetadataKey is used to record the forwarded host of PD. ForwardMetadataKey = "pd-forwarded-host" + // FollowerHandleMetadataKey is used to mark the permit of follower handle. + FollowerHandleMetadataKey = "pd-allow-follower-handle" ) // GetClientConn returns a gRPC client connection. @@ -75,6 +77,61 @@ func BuildForwardContext(ctx context.Context, addr string) context.Context { return metadata.NewOutgoingContext(ctx, md) } +// GetForwardedHostInClientSide returns the forwarded host in metadata. +// Only used for test. +func GetForwardedHostInClientSide(ctx context.Context) string { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + return "" + } + if t, ok := md[ForwardMetadataKey]; ok { + return t[0] + } + return "" +} + +// GetForwardedHostInServerSide returns the forwarded host in metadata. +// Only used for test. +func GetForwardedHostInServerSide(ctx context.Context) string { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "" + } + if t, ok := md[ForwardMetadataKey]; ok { + return t[0] + } + return "" +} + +// BuildFollowerHandleContext creates a context with follower handle metadata information. +// It is used in client side. +func BuildFollowerHandleContext(ctx context.Context) context.Context { + md := metadata.Pairs(FollowerHandleMetadataKey, "") + return metadata.NewOutgoingContext(ctx, md) +} + +// GetFollowerHandleEnableInClientSide returns the enable follower handle in metadata. +// Only used for test. +func GetFollowerHandleEnableInClientSide(ctx context.Context) bool { + md, ok := metadata.FromOutgoingContext(ctx) + if !ok { + return false + } + _, ok = md[FollowerHandleMetadataKey] + return ok +} + +// GetFollowerHandleEnableInServerSide returns the enable follower handle in metadata. +// Only used for test. +func GetFollowerHandleEnableInServerSide(ctx context.Context) bool { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return false + } + _, ok = md[FollowerHandleMetadataKey] + return ok +} + // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr. // Returns the old one if's already existed in the clientConns; otherwise creates a new one and returns it. func GetOrCreateGRPCConn(ctx context.Context, clientConns *sync.Map, addr string, tlsCfg *tlsutil.TLSConfig, opt ...grpc.DialOption) (*grpc.ClientConn, error) { diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index b75276adbe9..7e185a0da87 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -33,6 +33,9 @@ import ( "github.com/tikv/pd/client/tlsutil" "go.uber.org/zap" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) const ( @@ -74,8 +77,8 @@ type ServiceDiscovery interface { // GetServingAddr returns the serving endpoint which is the leader in a quorum-based cluster // or the primary in a primary/secondary configured cluster. GetServingAddr() string - // GetBackupAddrs gets the addresses of the current reachable and healthy backup service - // endpoints randomly. Backup service endpoints are followers in a quorum-based cluster or + // GetBackupAddrs gets the addresses of the current reachable backup service + // endpoints. Backup service endpoints are followers in a quorum-based cluster or // secondaries in a primary/secondary configured cluster. GetBackupAddrs() []string // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr @@ -97,6 +100,246 @@ type ServiceDiscovery interface { AddServiceAddrsSwitchedCallback(callbacks ...func()) } +// ServiceClient is an interface that defines a set of operations for a raw PD gRPC client to specific PD server. +type ServiceClient interface { + // GetAddress returns the address information of the PD server. + GetAddress() string + // GetClientConn returns the gRPC connection of the service client + GetClientConn() *grpc.ClientConn + // BuildGRPCContext builds a context object with a gRPC context. + // ctx: the original context object. + // mustLeader: whether must send to leader. + BuildGRPCContext(ctx context.Context, mustLeader bool) context.Context + // IsLeader returns whether the target PD server is leader. + IsLeader() bool + // Available returns if the network or other availability for the current service client is available. + Available() bool + // CheckAvailable checks the status about network connection or other availability of the current service client + CheckAvailable() + // CheckNetworkAvailable checks if the network connection for the current service client is available + CheckNetworkAvailable(context.Context) + // RespToErr checks if client need to retry based on the PD server error response. + RespToErr(*pdpb.Error, error) bool +} + +var _ ServiceClient = (*pdServiceClient)(nil) +var _ ServiceClient = (*pdServiceAPIClient)(nil) + +type pdServiceClient struct { + addr string + conn *grpc.ClientConn + isLeader bool + leaderAddr string + + networkFailure atomic.Bool +} + +func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) ServiceClient { + return &pdServiceClient{ + addr: addr, + conn: conn, + isLeader: isLeader, + leaderAddr: leaderAddr, + } +} + +// GetAddress implements ServiceClient. +func (c *pdServiceClient) GetAddress() string { + if c == nil { + return "" + } + return c.addr +} + +// BuildGRPCContext implements ServiceClient. +func (c *pdServiceClient) BuildGRPCContext(ctx context.Context, toLeader bool) context.Context { + if c == nil { + return ctx + } + if c.IsLeader() { + return ctx + } + if toLeader { + return grpcutil.BuildForwardContext(ctx, c.leaderAddr) + } + return grpcutil.BuildFollowerHandleContext(ctx) +} + +// IsLeader implements ServiceClient. +func (c *pdServiceClient) IsLeader() bool { + if c == nil { + return false + } + return c.isLeader +} + +// NetworkAvailable implements ServiceClient. +func (c *pdServiceClient) Available() bool { + if c == nil { + return false + } + return !c.networkFailure.Load() +} + +// CheckAvailable implements ServiceClient. +func (c *pdServiceClient) CheckAvailable() {} + +// CheckNetworkAvailable implements ServiceClient. +func (c *pdServiceClient) CheckNetworkAvailable(ctx context.Context) { + if c == nil || c.conn == nil { + return + } + healthCli := healthpb.NewHealthClient(c.conn) + resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) + failpoint.Inject("unreachableNetwork1", func() { + resp = nil + err = status.New(codes.Unavailable, "unavailable").Err() + }) + rpcErr, ok := status.FromError(err) + if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { + c.networkFailure.Store(true) + } else { + c.networkFailure.Store(false) + } +} + +func isNetworkError(code codes.Code) bool { + return code == codes.Unavailable || code == codes.DeadlineExceeded +} + +// GetClientConn implements ServiceClient. +func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { + if c == nil { + return nil + } + return c.conn +} + +// RespToErr implements ServiceClient. +func (c *pdServiceClient) RespToErr(pdErr *pdpb.Error, err error) bool { + if c.isLeader { + return false + } + // NOTE: maybe we can mark network unavailable here. + return false +} + +type errFn func(pdErr *pdpb.Error) bool + +func regionAPIErrorFn(pdErr *pdpb.Error) bool { + return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND +} + +// pdServiceAPIClient is a specific API client for PD service. +// It extends the pdServiceClient and adds additional fields for managing availability +type pdServiceAPIClient struct { + ServiceClient + fn errFn + + unavailable atomic.Bool + unavailableUntil atomic.Value +} + +func newPDServiceAPIClient(client ServiceClient, f errFn) ServiceClient { + return &pdServiceAPIClient{ + ServiceClient: client, + fn: f, + } +} + +// Available implements ServiceClient. +func (c *pdServiceAPIClient) Available() bool { + return c.ServiceClient.Available() && !c.unavailable.Load() +} + +// CheckkAvailable implements ServiceClient. +func (c *pdServiceAPIClient) CheckAvailable() { + if !c.unavailable.Load() { + return + } + until := c.unavailableUntil.Load().(time.Time) + if time.Now().After(until) { + c.unavailable.Store(false) + } +} + +// RespToErr implements ServiceClient. +func (c *pdServiceAPIClient) RespToErr(pdErr *pdpb.Error, err error) bool { + if c.IsLeader() { + return false + } + if err == nil && pdErr == nil { + return false + } + if c.fn(pdErr) && c.unavailable.CompareAndSwap(false, true) { + c.unavailableUntil.Store(time.Now().Add(time.Second * 10)) + failpoint.Inject("fastCheckAvailable", func() { + c.unavailableUntil.Store(time.Now().Add(time.Millisecond * 100)) + }) + } + return true +} + +// pdServiceBalancerNode is a balancer node for PD service. +// It extends the pdServiceClient and adds additional fields for the next polling client in the chain. +type pdServiceBalancerNode struct { + ServiceClient + next *pdServiceBalancerNode +} + +// pdServiceBalancer is a load balancer for PD service clients. +// It is used to balance the request to all servers and manage the connections to multiple PD service nodes. +type pdServiceBalancer struct { + mu sync.Mutex + now *pdServiceBalancerNode + totalNode int +} + +func (c *pdServiceBalancer) set(clients []ServiceClient) { + c.mu.Lock() + defer c.mu.Unlock() + if len(clients) == 0 { + return + } + c.totalNode = len(clients) + head := &pdServiceBalancerNode{ + ServiceClient: clients[0], + } + head.next = head + last := head + for i := 1; i < c.totalNode; i++ { + next := &pdServiceBalancerNode{ + ServiceClient: clients[i], + next: head, + } + head = next + last.next = head + } + c.now = head +} + +func (c *pdServiceBalancer) next() { + c.now = c.now.next +} + +func (c *pdServiceBalancer) get() (ret ServiceClient) { + c.mu.Lock() + i := 0 + defer c.mu.Unlock() + if c.now == nil { + return nil + } + for ; i < c.totalNode; i++ { + if c.now.Available() { + ret = c.now + c.next() + return + } + c.next() + } + return +} + type updateKeyspaceIDFunc func() error type tsoLocalServAddrsUpdatedFunc func(map[string]string) error type tsoGlobalServAddrUpdatedFunc func(string) error diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go new file mode 100644 index 00000000000..18db21c428e --- /dev/null +++ b/client/pd_service_discovery_test.go @@ -0,0 +1,290 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pd + +import ( + "context" + "errors" + "log" + "net" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/pingcap/failpoint" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/stretchr/testify/suite" + "github.com/tikv/pd/client/grpcutil" + "github.com/tikv/pd/client/testutil" + "google.golang.org/grpc" + pb "google.golang.org/grpc/examples/helloworld/helloworld" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +type testGRPCServer struct { + pb.UnimplementedGreeterServer + isLeader bool + leaderAddr string + leaderConn *grpc.ClientConn + handleCount atomic.Int32 + forwardCount atomic.Int32 +} + +// SayHello implements helloworld.GreeterServer +func (s *testGRPCServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { + if !s.isLeader { + if !grpcutil.GetFollowerHandleEnableInServerSide(ctx) { + if addr := grpcutil.GetForwardedHostInServerSide(ctx); addr == s.leaderAddr { + s.forwardCount.Add(1) + return pb.NewGreeterClient(s.leaderConn).SayHello(ctx, in) + } + return nil, errors.New("not leader") + } + } + s.handleCount.Add(1) + return &pb.HelloReply{Message: "Hello " + in.GetName()}, nil +} + +func (s *testGRPCServer) resetCount() { + s.handleCount.Store(0) + s.forwardCount.Store(0) +} + +func (s *testGRPCServer) getHandleCount() int32 { + return s.handleCount.Load() +} + +func (s *testGRPCServer) getForwardCount() int32 { + return s.forwardCount.Load() +} + +type testServer struct { + server *testGRPCServer + grpcServer *grpc.Server + addr string +} + +func newTestServer(isLeader bool) *testServer { + addr := testutil.Alloc() + u, err := url.Parse(addr) + if err != nil { + return nil + } + grpcServer := grpc.NewServer() + server := &testGRPCServer{ + isLeader: isLeader, + } + pb.RegisterGreeterServer(grpcServer, server) + hsrv := health.NewServer() + hsrv.SetServingStatus("", healthpb.HealthCheckResponse_SERVING) + healthpb.RegisterHealthServer(grpcServer, hsrv) + return &testServer{ + server: server, + grpcServer: grpcServer, + addr: u.Host, + } +} + +func (s *testServer) run() { + lis, err := net.Listen("tcp", s.addr) + if err != nil { + log.Fatalf("failed to serve: %v", err) + } + if err := s.grpcServer.Serve(lis); err != nil { + log.Fatalf("failed to serve: %v", err) + } +} + +type serviceClientTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + + leaderServer *testServer + followerServer *testServer + + leaderClient ServiceClient + followerClient ServiceClient +} + +func TestServiceClientClientTestSuite(t *testing.T) { + suite.Run(t, new(serviceClientTestSuite)) +} + +func (suite *serviceClientTestSuite) SetupSuite() { + suite.ctx, suite.clean = context.WithCancel(context.Background()) + + suite.leaderServer = newTestServer(true) + suite.followerServer = newTestServer(false) + go suite.leaderServer.run() + go suite.followerServer.run() + for i := 0; i < 10; i++ { + leaderConn, err1 := grpc.Dial(suite.leaderServer.addr, grpc.WithInsecure()) //nolint + followerConn, err2 := grpc.Dial(suite.followerServer.addr, grpc.WithInsecure()) //nolint + if err1 == nil && err2 == nil { + suite.followerClient = newPDServiceClient(suite.followerServer.addr, suite.leaderServer.addr, followerConn, false) + suite.leaderClient = newPDServiceClient(suite.leaderServer.addr, suite.leaderServer.addr, leaderConn, true) + suite.followerServer.server.leaderConn = suite.leaderClient.GetClientConn() + suite.followerServer.server.leaderAddr = suite.leaderClient.GetAddress() + return + } + time.Sleep(50 * time.Millisecond) + } + suite.NotNil(suite.leaderClient) +} + +func (suite *serviceClientTestSuite) TearDownTest() { + suite.leaderServer.grpcServer.GracefulStop() + suite.followerServer.grpcServer.GracefulStop() + suite.clean() +} + +func (suite *serviceClientTestSuite) TestServiceClient() { + re := suite.Require() + leaderAddress := suite.leaderServer.addr + followerAddress := suite.followerServer.addr + + follower := suite.followerClient + leader := suite.leaderClient + + re.Equal(follower.GetAddress(), followerAddress) + re.Equal(leader.GetAddress(), leaderAddress) + + re.True(follower.Available()) + re.True(leader.Available()) + + re.False(follower.IsLeader()) + re.True(leader.IsLeader()) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) + follower.CheckNetworkAvailable(suite.ctx) + leader.CheckNetworkAvailable(suite.ctx) + re.False(follower.Available()) + re.False(leader.Available()) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + + follower.CheckNetworkAvailable(suite.ctx) + leader.CheckNetworkAvailable(suite.ctx) + re.True(follower.Available()) + re.True(leader.Available()) + + followerConn := follower.GetClientConn() + leaderConn := leader.GetClientConn() + re.NotNil(followerConn) + re.NotNil(leaderConn) + + resp, err := pb.NewGreeterClient(followerConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal(resp.GetMessage(), "Hello pd") + resp, err = pb.NewGreeterClient(leaderConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal(resp.GetMessage(), "Hello pd") + + re.False(follower.RespToErr(nil, nil)) + re.False(leader.RespToErr(nil, nil)) + + ctx1 := context.WithoutCancel(suite.ctx) + ctx1 = follower.BuildGRPCContext(ctx1, false) + re.True(grpcutil.GetFollowerHandleEnableInClientSide(ctx1)) + re.Len(grpcutil.GetForwardedHostInClientSide(ctx1), 0) + ctx2 := context.WithoutCancel(suite.ctx) + ctx2 = follower.BuildGRPCContext(ctx2, true) + re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx2)) + re.Equal(grpcutil.GetForwardedHostInClientSide(ctx2), leaderAddress) + + ctx3 := context.WithoutCancel(suite.ctx) + ctx3 = leader.BuildGRPCContext(ctx3, false) + re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx3)) + re.Len(grpcutil.GetForwardedHostInClientSide(ctx3), 0) + ctx4 := context.WithoutCancel(suite.ctx) + ctx4 = leader.BuildGRPCContext(ctx4, true) + re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx4)) + re.Len(grpcutil.GetForwardedHostInClientSide(ctx4), 0) + + followerAPIClient := newPDServiceAPIClient(follower, regionAPIErrorFn) + leaderAPIClient := newPDServiceAPIClient(leader, regionAPIErrorFn) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/fastCheckAvailable", "return(true)")) + + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + pdErr1 := &pdpb.Error{ + Type: pdpb.ErrorType_UNKNOWN, + } + pdErr2 := &pdpb.Error{ + Type: pdpb.ErrorType_REGION_NOT_FOUND, + } + err = errors.New("error") + re.True(followerAPIClient.RespToErr(pdErr1, nil)) + re.False(leaderAPIClient.RespToErr(pdErr1, nil)) + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + + re.True(followerAPIClient.RespToErr(pdErr2, nil)) + re.False(leaderAPIClient.RespToErr(pdErr2, nil)) + re.False(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + followerAPIClient.CheckAvailable() + leaderAPIClient.CheckAvailable() + re.False(followerAPIClient.Available()) + time.Sleep(time.Millisecond * 100) + followerAPIClient.CheckAvailable() + re.True(followerAPIClient.Available()) + + re.True(followerAPIClient.RespToErr(nil, err)) + re.False(leaderAPIClient.RespToErr(nil, err)) + re.True(followerAPIClient.Available()) + re.True(leaderAPIClient.Available()) + + re.NoError(failpoint.Disable("github.com/tikv/pd/client/fastCheckAvailable")) +} + +func (suite *serviceClientTestSuite) TestServiceClientBalancer() { + re := suite.Require() + follower := suite.followerClient + leader := suite.leaderClient + b := &pdServiceBalancer{} + b.set([]ServiceClient{leader, follower}) + re.Equal(b.totalNode, 2) + + for i := 0; i < 10; i++ { + client := b.get() + ctx := client.BuildGRPCContext(suite.ctx, false) + conn := client.GetClientConn() + re.NotNil(conn) + resp, err := pb.NewGreeterClient(conn).SayHello(ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal(resp.GetMessage(), "Hello pd") + } + re.Equal(suite.leaderServer.server.getHandleCount(), int32(5)) + re.Equal(suite.followerServer.server.getHandleCount(), int32(5)) + suite.followerServer.server.resetCount() + suite.leaderServer.server.resetCount() + + for i := 0; i < 10; i++ { + client := b.get() + ctx := client.BuildGRPCContext(suite.ctx, true) + conn := client.GetClientConn() + re.NotNil(conn) + resp, err := pb.NewGreeterClient(conn).SayHello(ctx, &pb.HelloRequest{Name: "pd"}) + re.NoError(err) + re.Equal(resp.GetMessage(), "Hello pd") + } + re.Equal(suite.leaderServer.server.getHandleCount(), int32(10)) + re.Equal(suite.followerServer.server.getHandleCount(), int32(0)) + re.Equal(suite.followerServer.server.getForwardCount(), int32(5)) +} diff --git a/client/testutil/check_env.go b/client/testutil/check_env.go new file mode 100644 index 00000000000..d73b9133c5f --- /dev/null +++ b/client/testutil/check_env.go @@ -0,0 +1,21 @@ +// Copyright 2020 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build !linux +// +build !linux + +package testutil + +func environmentCheck(addr string) bool { + return true +} diff --git a/client/testutil/check_env_linux.go b/client/testutil/check_env_linux.go new file mode 100644 index 00000000000..625ae8ac201 --- /dev/null +++ b/client/testutil/check_env_linux.go @@ -0,0 +1,41 @@ +// Copyright 2020 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +//go:build linux +// +build linux + +package testutil + +import ( + "github.com/cakturk/go-netstat/netstat" + "github.com/pingcap/log" +) + +func environmentCheck(addr string) bool { + valid, err := checkAddr(addr[len("http://"):]) + if err != nil { + log.Error("check port status failed", err) + return false + } + return valid +} + +func checkAddr(addr string) (bool, error) { + tabs, err := netstat.TCPSocks(func(s *netstat.SockTabEntry) bool { + return s.RemoteAddr.String() == addr || s.LocalAddr.String() == addr + }) + if err != nil { + return false, err + } + return len(tabs) < 1, nil +} diff --git a/client/testutil/leak.go b/client/testutil/leak.go index 28b5baae60f..10345725171 100644 --- a/client/testutil/leak.go +++ b/client/testutil/leak.go @@ -23,4 +23,8 @@ var LeakOptions = []goleak.Option{ goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).createTransport"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).resetTransport"), goleak.IgnoreTopFunction("google.golang.org/grpc.(*Server).handleRawConn"), + // TODO: remove the below options once we fixed the http connection leak problems + goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*controlBuffer).get"), + goleak.IgnoreTopFunction("google.golang.org/grpc/internal/transport.(*http2Server).keepalive"), } diff --git a/client/testutil/tempurl.go b/client/testutil/tempurl.go new file mode 100644 index 00000000000..ac8f29fa345 --- /dev/null +++ b/client/testutil/tempurl.go @@ -0,0 +1,65 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package testutil + +import ( + "fmt" + "net" + "sync" + "time" + + "github.com/pingcap/log" + "go.uber.org/zap" +) + +var ( + testAddrMutex sync.Mutex + testAddrMap = make(map[string]struct{}) +) + +// Alloc allocates a local URL for testing. +func Alloc() string { + for i := 0; i < 10; i++ { + if u := tryAllocTestURL(); u != "" { + return u + } + time.Sleep(time.Second) + } + log.Fatal("failed to alloc test URL") + return "" +} + +func tryAllocTestURL() string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Fatal("listen failed", zap.Error(err)) + } + addr := fmt.Sprintf("http://%s", l.Addr()) + err = l.Close() + if err != nil { + log.Fatal("close failed", zap.Error(err)) + } + + testAddrMutex.Lock() + defer testAddrMutex.Unlock() + if _, ok := testAddrMap[addr]; ok { + return "" + } + if !environmentCheck(addr) { + return "" + } + testAddrMap[addr] = struct{}{} + return addr +} diff --git a/tests/integrations/client/go.mod b/tests/integrations/client/go.mod index 799901ff2e3..27a5b092c73 100644 --- a/tests/integrations/client/go.mod +++ b/tests/integrations/client/go.mod @@ -165,19 +165,19 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/client/go.sum b/tests/integrations/client/go.sum index e13da5d8375..d832d59d3b4 100644 --- a/tests/integrations/client/go.sum +++ b/tests/integrations/client/go.sum @@ -637,8 +637,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -684,8 +684,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -740,8 +740,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -750,8 +750,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -815,8 +815,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/integrations/mcs/go.mod b/tests/integrations/mcs/go.mod index 75d70e3cf06..81647deeedd 100644 --- a/tests/integrations/mcs/go.mod +++ b/tests/integrations/mcs/go.mod @@ -165,19 +165,19 @@ require ( go.uber.org/fx v1.12.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/mcs/go.sum b/tests/integrations/mcs/go.sum index dfead54afe1..4639ac8820a 100644 --- a/tests/integrations/mcs/go.sum +++ b/tests/integrations/mcs/go.sum @@ -641,8 +641,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -688,8 +688,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -744,8 +744,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -754,8 +754,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -818,8 +818,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/integrations/tso/go.mod b/tests/integrations/tso/go.mod index 309ea9dbc4d..b02ef7d03cb 100644 --- a/tests/integrations/tso/go.mod +++ b/tests/integrations/tso/go.mod @@ -166,19 +166,19 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.24.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect golang.org/x/image v0.5.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tests/integrations/tso/go.sum b/tests/integrations/tso/go.sum index 94fbde2ad57..a7588d3e78d 100644 --- a/tests/integrations/tso/go.sum +++ b/tests/integrations/tso/go.sum @@ -636,8 +636,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -683,8 +683,8 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -739,8 +739,8 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -749,8 +749,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= @@ -813,8 +813,8 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/pd-api-bench/go.mod b/tools/pd-api-bench/go.mod index 8050f433e8b..3cf3f9634ba 100644 --- a/tools/pd-api-bench/go.mod +++ b/tools/pd-api-bench/go.mod @@ -98,14 +98,14 @@ require ( go.uber.org/goleak v1.1.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.14.0 // indirect + golang.org/x/crypto v0.15.0 // indirect golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.1.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tools/pd-api-bench/go.sum b/tools/pd-api-bench/go.sum index 1e40c511586..605b8cf3b5f 100644 --- a/tools/pd-api-bench/go.sum +++ b/tools/pd-api-bench/go.sum @@ -367,8 +367,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= +golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4 h1:QLureRX3moex6NVu/Lr4MGakp9FdA7sBHGBmvRW7NaM= golang.org/x/exp v0.0.0-20230711005742-c3f37128e5a4/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= @@ -397,8 +397,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -434,16 +434,16 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -487,8 +487,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/pd-tso-bench/go.mod b/tools/pd-tso-bench/go.mod index 8d4b3d18a31..c8fe68aab44 100644 --- a/tools/pd-tso-bench/go.mod +++ b/tools/pd-tso-bench/go.mod @@ -27,11 +27,11 @@ require ( github.com/prometheus/procfs v0.6.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/tools/pd-tso-bench/go.sum b/tools/pd-tso-bench/go.sum index 15ba2923695..31875879c2d 100644 --- a/tools/pd-tso-bench/go.sum +++ b/tools/pd-tso-bench/go.sum @@ -12,6 +12,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -159,8 +161,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -180,13 +182,13 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -207,6 +209,8 @@ google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= +google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5/go.mod h1:5av8LiY5jU2KRcrX+SHtvLHnaOpPJ7gzWStBurgHlqY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -215,8 +219,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 889c0eb3df3eea3d0b5fa5cb18110bd954e9131f Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 4 Dec 2023 11:18:25 +0800 Subject: [PATCH 02/16] add service client Signed-off-by: Cabinfever_B --- client/testutil/check_env_linux.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/testutil/check_env_linux.go b/client/testutil/check_env_linux.go index 625ae8ac201..be70e78a6be 100644 --- a/client/testutil/check_env_linux.go +++ b/client/testutil/check_env_linux.go @@ -19,12 +19,13 @@ package testutil import ( "github.com/cakturk/go-netstat/netstat" "github.com/pingcap/log" + "go.uber.org/zap" ) func environmentCheck(addr string) bool { valid, err := checkAddr(addr[len("http://"):]) if err != nil { - log.Error("check port status failed", err) + log.Error("check port status failed", zap.Error(err)) return false } return valid From 631579390b045072b573a1248abee9736b0bbb3d Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 4 Dec 2023 13:47:09 +0800 Subject: [PATCH 03/16] address comment Signed-off-by: Cabinfever_B --- client/grpcutil/grpcutil.go | 52 +++++++++-------------------- client/pd_service_discovery_test.go | 28 ++++++++-------- 2 files changed, 29 insertions(+), 51 deletions(-) diff --git a/client/grpcutil/grpcutil.go b/client/grpcutil/grpcutil.go index 22009a2ca1f..48a25186fdf 100644 --- a/client/grpcutil/grpcutil.go +++ b/client/grpcutil/grpcutil.go @@ -77,30 +77,11 @@ func BuildForwardContext(ctx context.Context, addr string) context.Context { return metadata.NewOutgoingContext(ctx, md) } -// GetForwardedHostInClientSide returns the forwarded host in metadata. +// GetForwardedHost returns the forwarded host in metadata. // Only used for test. -func GetForwardedHostInClientSide(ctx context.Context) string { - md, ok := metadata.FromOutgoingContext(ctx) - if !ok { - return "" - } - if t, ok := md[ForwardMetadataKey]; ok { - return t[0] - } - return "" -} - -// GetForwardedHostInServerSide returns the forwarded host in metadata. -// Only used for test. -func GetForwardedHostInServerSide(ctx context.Context) string { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return "" - } - if t, ok := md[ForwardMetadataKey]; ok { - return t[0] - } - return "" +func GetForwardedHost(ctx context.Context, f func(context.Context) (metadata.MD, bool)) string { + v, _ := getValueFromMetadata(ctx, ForwardMetadataKey, f) + return v } // BuildFollowerHandleContext creates a context with follower handle metadata information. @@ -110,26 +91,23 @@ func BuildFollowerHandleContext(ctx context.Context) context.Context { return metadata.NewOutgoingContext(ctx, md) } -// GetFollowerHandleEnableInClientSide returns the enable follower handle in metadata. +// GetFollowerHandleEnable returns the forwarded host in metadata. // Only used for test. -func GetFollowerHandleEnableInClientSide(ctx context.Context) bool { - md, ok := metadata.FromOutgoingContext(ctx) - if !ok { - return false - } - _, ok = md[FollowerHandleMetadataKey] +func GetFollowerHandleEnable(ctx context.Context, f func(context.Context) (metadata.MD, bool)) bool { + _, ok := getValueFromMetadata(ctx, FollowerHandleMetadataKey, f) return ok } -// GetFollowerHandleEnableInServerSide returns the enable follower handle in metadata. -// Only used for test. -func GetFollowerHandleEnableInServerSide(ctx context.Context) bool { - md, ok := metadata.FromIncomingContext(ctx) +func getValueFromMetadata(ctx context.Context, key string, f func(context.Context) (metadata.MD, bool)) (string, bool) { + md, ok := f(ctx) if !ok { - return false + return "", false } - _, ok = md[FollowerHandleMetadataKey] - return ok + vs, ok := md[key] + if !ok { + return "", false + } + return vs[0], true } // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr. diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go index 18db21c428e..09219b4f8c2 100644 --- a/client/pd_service_discovery_test.go +++ b/client/pd_service_discovery_test.go @@ -33,6 +33,7 @@ import ( pb "google.golang.org/grpc/examples/helloworld/helloworld" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/metadata" ) type testGRPCServer struct { @@ -47,8 +48,8 @@ type testGRPCServer struct { // SayHello implements helloworld.GreeterServer func (s *testGRPCServer) SayHello(ctx context.Context, in *pb.HelloRequest) (*pb.HelloReply, error) { if !s.isLeader { - if !grpcutil.GetFollowerHandleEnableInServerSide(ctx) { - if addr := grpcutil.GetForwardedHostInServerSide(ctx); addr == s.leaderAddr { + if !grpcutil.GetFollowerHandleEnable(ctx, metadata.FromIncomingContext) { + if addr := grpcutil.GetForwardedHost(ctx, metadata.FromIncomingContext); addr == s.leaderAddr { s.forwardCount.Add(1) return pb.NewGreeterClient(s.leaderConn).SayHello(ctx, in) } @@ -187,10 +188,9 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.NotNil(followerConn) re.NotNil(leaderConn) - resp, err := pb.NewGreeterClient(followerConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) - re.NoError(err) - re.Equal(resp.GetMessage(), "Hello pd") - resp, err = pb.NewGreeterClient(leaderConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + _, err := pb.NewGreeterClient(followerConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) + re.ErrorContains(err, "not leader") + resp, err := pb.NewGreeterClient(leaderConn).SayHello(suite.ctx, &pb.HelloRequest{Name: "pd"}) re.NoError(err) re.Equal(resp.GetMessage(), "Hello pd") @@ -199,21 +199,21 @@ func (suite *serviceClientTestSuite) TestServiceClient() { ctx1 := context.WithoutCancel(suite.ctx) ctx1 = follower.BuildGRPCContext(ctx1, false) - re.True(grpcutil.GetFollowerHandleEnableInClientSide(ctx1)) - re.Len(grpcutil.GetForwardedHostInClientSide(ctx1), 0) + re.True(grpcutil.GetFollowerHandleEnable(ctx1, metadata.FromOutgoingContext)) + re.Len(grpcutil.GetForwardedHost(ctx1, metadata.FromOutgoingContext), 0) ctx2 := context.WithoutCancel(suite.ctx) ctx2 = follower.BuildGRPCContext(ctx2, true) - re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx2)) - re.Equal(grpcutil.GetForwardedHostInClientSide(ctx2), leaderAddress) + re.False(grpcutil.GetFollowerHandleEnable(ctx2, metadata.FromOutgoingContext)) + re.Equal(grpcutil.GetForwardedHost(ctx2, metadata.FromOutgoingContext), leaderAddress) ctx3 := context.WithoutCancel(suite.ctx) ctx3 = leader.BuildGRPCContext(ctx3, false) - re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx3)) - re.Len(grpcutil.GetForwardedHostInClientSide(ctx3), 0) + re.False(grpcutil.GetFollowerHandleEnable(ctx3, metadata.FromOutgoingContext)) + re.Len(grpcutil.GetForwardedHost(ctx3, metadata.FromOutgoingContext), 0) ctx4 := context.WithoutCancel(suite.ctx) ctx4 = leader.BuildGRPCContext(ctx4, true) - re.False(grpcutil.GetFollowerHandleEnableInClientSide(ctx4)) - re.Len(grpcutil.GetForwardedHostInClientSide(ctx4), 0) + re.False(grpcutil.GetFollowerHandleEnable(ctx4, metadata.FromOutgoingContext)) + re.Len(grpcutil.GetForwardedHost(ctx4, metadata.FromOutgoingContext), 0) followerAPIClient := newPDServiceAPIClient(follower, regionAPIErrorFn) leaderAPIClient := newPDServiceAPIClient(leader, regionAPIErrorFn) From 6535c6e9c3e9dc085734b19b29f418f7cb7d2a8f Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 5 Dec 2023 09:12:53 +0800 Subject: [PATCH 04/16] address comment Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 19 ++++++++----------- client/pd_service_discovery_test.go | 16 ++++++++-------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 7e185a0da87..4a91d847866 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -118,8 +118,9 @@ type ServiceClient interface { CheckAvailable() // CheckNetworkAvailable checks if the network connection for the current service client is available CheckNetworkAvailable(context.Context) - // RespToErr checks if client need to retry based on the PD server error response. - RespToErr(*pdpb.Error, error) bool + // NeedRetry checks if client need to retry based on the PD server error response. + // And It will mark the client as unavailable if the pd error shows the follower can't handle request. + NeedRetry(*pdpb.Error, error) bool } var _ ServiceClient = (*pdServiceClient)(nil) @@ -215,13 +216,9 @@ func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { return c.conn } -// RespToErr implements ServiceClient. -func (c *pdServiceClient) RespToErr(pdErr *pdpb.Error, err error) bool { - if c.isLeader { - return false - } - // NOTE: maybe we can mark network unavailable here. - return false +// NeedRetry implements ServiceClient. +func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { + return !c.IsLeader() } type errFn func(pdErr *pdpb.Error) bool @@ -263,8 +260,8 @@ func (c *pdServiceAPIClient) CheckAvailable() { } } -// RespToErr implements ServiceClient. -func (c *pdServiceAPIClient) RespToErr(pdErr *pdpb.Error, err error) bool { +// NeedRetry implements ServiceClient. +func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { if c.IsLeader() { return false } diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go index 09219b4f8c2..092c69b9b11 100644 --- a/client/pd_service_discovery_test.go +++ b/client/pd_service_discovery_test.go @@ -194,8 +194,8 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.NoError(err) re.Equal(resp.GetMessage(), "Hello pd") - re.False(follower.RespToErr(nil, nil)) - re.False(leader.RespToErr(nil, nil)) + re.False(follower.NeedRetry(nil, nil)) + re.False(leader.NeedRetry(nil, nil)) ctx1 := context.WithoutCancel(suite.ctx) ctx1 = follower.BuildGRPCContext(ctx1, false) @@ -229,13 +229,13 @@ func (suite *serviceClientTestSuite) TestServiceClient() { Type: pdpb.ErrorType_REGION_NOT_FOUND, } err = errors.New("error") - re.True(followerAPIClient.RespToErr(pdErr1, nil)) - re.False(leaderAPIClient.RespToErr(pdErr1, nil)) + re.True(followerAPIClient.NeedRetry(pdErr1, nil)) + re.False(leaderAPIClient.NeedRetry(pdErr1, nil)) re.True(followerAPIClient.Available()) re.True(leaderAPIClient.Available()) - re.True(followerAPIClient.RespToErr(pdErr2, nil)) - re.False(leaderAPIClient.RespToErr(pdErr2, nil)) + re.True(followerAPIClient.NeedRetry(pdErr2, nil)) + re.False(leaderAPIClient.NeedRetry(pdErr2, nil)) re.False(followerAPIClient.Available()) re.True(leaderAPIClient.Available()) followerAPIClient.CheckAvailable() @@ -245,8 +245,8 @@ func (suite *serviceClientTestSuite) TestServiceClient() { followerAPIClient.CheckAvailable() re.True(followerAPIClient.Available()) - re.True(followerAPIClient.RespToErr(nil, err)) - re.False(leaderAPIClient.RespToErr(nil, err)) + re.True(followerAPIClient.NeedRetry(nil, err)) + re.False(leaderAPIClient.NeedRetry(nil, err)) re.True(followerAPIClient.Available()) re.True(leaderAPIClient.Available()) From 423717ce750295c4939e7b174fbed593ed6efec6 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 5 Dec 2023 09:30:51 +0800 Subject: [PATCH 05/16] address comment Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 17 ++++------------- client/pd_service_discovery_test.go | 18 +++++++++--------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 4a91d847866..d8b90de89c7 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -114,10 +114,6 @@ type ServiceClient interface { IsLeader() bool // Available returns if the network or other availability for the current service client is available. Available() bool - // CheckAvailable checks the status about network connection or other availability of the current service client - CheckAvailable() - // CheckNetworkAvailable checks if the network connection for the current service client is available - CheckNetworkAvailable(context.Context) // NeedRetry checks if client need to retry based on the PD server error response. // And It will mark the client as unavailable if the pd error shows the follower can't handle request. NeedRetry(*pdpb.Error, error) bool @@ -135,7 +131,7 @@ type pdServiceClient struct { networkFailure atomic.Bool } -func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) ServiceClient { +func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) *pdServiceClient { return &pdServiceClient{ addr: addr, conn: conn, @@ -182,11 +178,7 @@ func (c *pdServiceClient) Available() bool { return !c.networkFailure.Load() } -// CheckAvailable implements ServiceClient. -func (c *pdServiceClient) CheckAvailable() {} - -// CheckNetworkAvailable implements ServiceClient. -func (c *pdServiceClient) CheckNetworkAvailable(ctx context.Context) { +func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { if c == nil || c.conn == nil { return } @@ -237,7 +229,7 @@ type pdServiceAPIClient struct { unavailableUntil atomic.Value } -func newPDServiceAPIClient(client ServiceClient, f errFn) ServiceClient { +func newPDServiceAPIClient(client ServiceClient, f errFn) *pdServiceAPIClient { return &pdServiceAPIClient{ ServiceClient: client, fn: f, @@ -249,8 +241,7 @@ func (c *pdServiceAPIClient) Available() bool { return c.ServiceClient.Available() && !c.unavailable.Load() } -// CheckkAvailable implements ServiceClient. -func (c *pdServiceAPIClient) CheckAvailable() { +func (c *pdServiceAPIClient) markAsAvailable() { if !c.unavailable.Load() { return } diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go index 092c69b9b11..03745f621be 100644 --- a/client/pd_service_discovery_test.go +++ b/client/pd_service_discovery_test.go @@ -118,8 +118,8 @@ type serviceClientTestSuite struct { leaderServer *testServer followerServer *testServer - leaderClient ServiceClient - followerClient ServiceClient + leaderClient *pdServiceClient + followerClient *pdServiceClient } func TestServiceClientClientTestSuite(t *testing.T) { @@ -172,14 +172,14 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.True(leader.IsLeader()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) - follower.CheckNetworkAvailable(suite.ctx) - leader.CheckNetworkAvailable(suite.ctx) + follower.checkNetworkAvailable(suite.ctx) + leader.checkNetworkAvailable(suite.ctx) re.False(follower.Available()) re.False(leader.Available()) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) - follower.CheckNetworkAvailable(suite.ctx) - leader.CheckNetworkAvailable(suite.ctx) + follower.checkNetworkAvailable(suite.ctx) + leader.checkNetworkAvailable(suite.ctx) re.True(follower.Available()) re.True(leader.Available()) @@ -238,11 +238,11 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.False(leaderAPIClient.NeedRetry(pdErr2, nil)) re.False(followerAPIClient.Available()) re.True(leaderAPIClient.Available()) - followerAPIClient.CheckAvailable() - leaderAPIClient.CheckAvailable() + followerAPIClient.markAsAvailable() + leaderAPIClient.markAsAvailable() re.False(followerAPIClient.Available()) time.Sleep(time.Millisecond * 100) - followerAPIClient.CheckAvailable() + followerAPIClient.markAsAvailable() re.True(followerAPIClient.Available()) re.True(followerAPIClient.NeedRetry(nil, err)) From 0891cf1eb41cd00a482206f4d4c3e778c90d230d Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 5 Dec 2023 09:51:16 +0800 Subject: [PATCH 06/16] address comment Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 5 ++++- client/pd_service_discovery_test.go | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index d8b90de89c7..e43faaa9725 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -210,7 +210,10 @@ func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { // NeedRetry implements ServiceClient. func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { - return !c.IsLeader() + if c.IsLeader() { + return false + } + return !(err == nil && pdErr == nil) } type errFn func(pdErr *pdpb.Error) bool diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go index 03745f621be..9cc0055aac3 100644 --- a/client/pd_service_discovery_test.go +++ b/client/pd_service_discovery_test.go @@ -149,6 +149,11 @@ func (suite *serviceClientTestSuite) SetupSuite() { } func (suite *serviceClientTestSuite) TearDownTest() { + suite.leaderServer.server.resetCount() + suite.followerServer.server.resetCount() +} + +func (suite *serviceClientTestSuite) TearDownSuite() { suite.leaderServer.grpcServer.GracefulStop() suite.followerServer.grpcServer.GracefulStop() suite.clean() From 739deba67ad187e7d9ca8350632a09b9c95641c4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 5 Dec 2023 17:36:37 +0800 Subject: [PATCH 07/16] impl follower region Signed-off-by: Cabinfever_B --- client/client.go | 183 +++++++---------- client/pd_service_discovery.go | 248 +++++++++++++++++++---- client/pd_service_discovery_test.go | 4 +- client/tso_service_discovery.go | 5 + pkg/utils/grpcutil/grpcutil.go | 15 +- server/forward.go | 6 +- server/grpc_service.go | 144 ++++++++++--- tests/integrations/client/client_test.go | 128 +++++++++++- 8 files changed, 546 insertions(+), 187 deletions(-) diff --git a/client/client.go b/client/client.go index 0ae362d06a8..b0d7498a7df 100644 --- a/client/client.go +++ b/client/client.go @@ -17,29 +17,22 @@ package pd import ( "context" "fmt" - "math/rand" "runtime/trace" "strings" "sync" - "sync/atomic" "time" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/client/tlsutil" "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" ) const ( @@ -217,9 +210,6 @@ func WithAllowFollowerHandle() GetRegionOption { return func(op *GetRegionOp) { op.allowFollowerHandle = true } } -// LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. -var LeaderHealthCheckInterval = time.Second - var ( // errUnmatchedClusterID is returned when found a PD with a different cluster ID. errUnmatchedClusterID = errors.New("[pd] unmatched cluster id") @@ -308,7 +298,7 @@ func (k *serviceModeKeeper) close() { type client struct { keyspaceID uint32 svrUrls []string - pdSvcDiscovery ServiceDiscovery + pdSvcDiscovery *pdServiceDiscovery tokenDispatcher *tokenDispatcher // For service mode switching. @@ -316,7 +306,6 @@ type client struct { // For internal usage. updateTokenConnectionCh chan struct{} - leaderNetworkFailure int32 ctx context.Context cancel context.CancelFunc @@ -514,7 +503,7 @@ func newClientWithKeyspaceName( return err } // c.keyspaceID is the source of truth for keyspace id. - c.pdSvcDiscovery.(*pdServiceDiscovery).SetKeyspaceID(c.keyspaceID) + c.pdSvcDiscovery.SetKeyspaceID(c.keyspaceID) return nil } @@ -575,10 +564,6 @@ func (c *client) setup() error { // Create dispatchers c.createTokenDispatcher() - - // Start the daemons. - c.wg.Add(1) - go c.leaderCheckLoop() return nil } @@ -719,46 +704,6 @@ func (c *client) UpdateOption(option DynamicOption, value interface{}) error { return nil } -func (c *client) leaderCheckLoop() { - defer c.wg.Done() - - leaderCheckLoopCtx, leaderCheckLoopCancel := context.WithCancel(c.ctx) - defer leaderCheckLoopCancel() - - ticker := time.NewTicker(LeaderHealthCheckInterval) - defer ticker.Stop() - - for { - select { - case <-c.ctx.Done(): - return - case <-ticker.C: - c.checkLeaderHealth(leaderCheckLoopCtx) - } - } -} - -func (c *client) checkLeaderHealth(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, c.option.timeout) - defer cancel() - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - healthCli := healthpb.NewHealthClient(client) - resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() - }) - rpcErr, ok := status.FromError(err) - if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(0)) - } - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } -} - func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { start := time.Now() defer func() { cmdDurationGetAllMembers.Observe(time.Since(start).Seconds()) }() @@ -778,50 +723,31 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return resp.GetMembers(), nil } -// leaderClient gets the client of current PD leader. -func (c *client) leaderClient() pdpb.PDClient { - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - return pdpb.NewPDClient(client) +// getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns +// follower pd client and the context which holds forward information. +func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { + serviceClient := c.pdSvcDiscovery.GetServiceClient() + if serviceClient == nil { + return nil, ctx } - return nil + return pdpb.NewPDClient(serviceClient.GetClientConn()), serviceClient.BuildGRPCContext(ctx, true) } -// backupClientConn gets a grpc client connection of the current reachable and healthy -// backup service endpoints randomly. Backup service endpoints are followers in a -// quorum-based cluster or secondaries in a primary/secondary configured cluster. -func (c *client) backupClientConn() (*grpc.ClientConn, string) { - addrs := c.pdSvcDiscovery.GetBackupAddrs() - if len(addrs) < 1 { - return nil, "" - } - var ( - cc *grpc.ClientConn - err error - ) - for i := 0; i < len(addrs); i++ { - addr := addrs[rand.Intn(len(addrs))] - if cc, err = c.pdSvcDiscovery.GetOrCreateGRPCConn(addr); err != nil { - continue - } - healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) - resp, err := healthpb.NewHealthClient(cc).Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - return cc, addr +// getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns +// follower pd client and the context which holds forward information. +func (c *client) getRegionAPIClientAndContext(ctx context.Context, allowFollower bool) (ServiceClient, context.Context) { + var serviceClient ServiceClient + if allowFollower { + serviceClient = c.pdSvcDiscovery.getServiceClientByKind(regionAPIKind) + if serviceClient != nil { + return serviceClient, serviceClient.BuildGRPCContext(ctx, !allowFollower) } } - return nil, "" -} - -func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { - if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { - backupClientConn, addr := c.backupClientConn() - if backupClientConn != nil { - log.Debug("[pd] use follower client", zap.String("addr", addr)) - return pdpb.NewPDClient(backupClientConn), grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - } + serviceClient = c.pdSvcDiscovery.GetServiceClient() + if serviceClient == nil { + return nil, ctx } - return c.leaderClient(), ctx + return serviceClient, serviceClient.BuildGRPCContext(ctx, !allowFollower) } func (c *client) GetTSAsync(ctx context.Context) TSFuture { @@ -976,6 +902,7 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOpt start := time.Now() defer func() { cmdDurationGetRegion.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() options := &GetRegionOp{} for _, opt := range opts { @@ -986,13 +913,19 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOpt RegionKey: key, NeedBuckets: options.needBuckets, } - protoClient, ctx := c.getClientAndContext(ctx) - if protoClient == nil { - cancel() + serviceClient, cctx := c.getRegionAPIClientAndContext(ctx, options.allowFollowerHandle && c.option.getEnableFollowerHandle()) + fmt.Println(serviceClient.GetAddress(), serviceClient.IsLeader()) + if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := protoClient.GetRegion(ctx, req) - cancel() + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegion(cctx, req) + if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { + protoClient, cctx := c.getClientAndContext(ctx) + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + resp, err = protoClient.GetRegion(cctx, req) + } if err = c.respForErr(cmdFailDurationGetRegion, start, err, resp.GetHeader()); err != nil { return nil, err @@ -1008,6 +941,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegio start := time.Now() defer func() { cmdDurationGetPrevRegion.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() options := &GetRegionOp{} for _, opt := range opts { @@ -1018,13 +952,18 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...GetRegio RegionKey: key, NeedBuckets: options.needBuckets, } - protoClient, ctx := c.getClientAndContext(ctx) - if protoClient == nil { - cancel() + serviceClient, cctx := c.getRegionAPIClientAndContext(ctx, options.allowFollowerHandle && c.option.getEnableFollowerHandle()) + if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := protoClient.GetPrevRegion(ctx, req) - cancel() + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetPrevRegion(cctx, req) + if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { + protoClient, cctx := c.getClientAndContext(ctx) + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + resp, err = protoClient.GetPrevRegion(cctx, req) + } if err = c.respForErr(cmdFailDurationGetPrevRegion, start, err, resp.GetHeader()); err != nil { return nil, err @@ -1040,6 +979,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get start := time.Now() defer func() { cmdDurationGetRegionByID.Observe(time.Since(start).Seconds()) }() ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() options := &GetRegionOp{} for _, opt := range opts { @@ -1050,13 +990,18 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...Get RegionId: regionID, NeedBuckets: options.needBuckets, } - protoClient, ctx := c.getClientAndContext(ctx) - if protoClient == nil { - cancel() + serviceClient, cctx := c.getRegionAPIClientAndContext(ctx, options.allowFollowerHandle && c.option.getEnableFollowerHandle()) + if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := protoClient.GetRegionByID(ctx, req) - cancel() + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegionByID(cctx, req) + if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { + protoClient, cctx := c.getClientAndContext(ctx) + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + resp, err = protoClient.GetRegionByID(cctx, req) + } if err = c.respForErr(cmdFailedDurationGetRegionByID, start, err, resp.GetHeader()); err != nil { return nil, err @@ -1078,18 +1023,28 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, scanCtx, cancel = context.WithTimeout(ctx, c.option.timeout) defer cancel() } + options := &GetRegionOp{} + for _, opt := range opts { + opt(options) + } req := &pdpb.ScanRegionsRequest{ Header: c.requestHeader(), StartKey: key, EndKey: endKey, Limit: int32(limit), } - protoClient, scanCtx := c.getClientAndContext(scanCtx) - if protoClient == nil { - cancel() + serviceClient, cctx := c.getRegionAPIClientAndContext(scanCtx, options.allowFollowerHandle && c.option.getEnableFollowerHandle()) + if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := protoClient.ScanRegions(scanCtx, req) + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).ScanRegions(cctx, req) + if !serviceClient.IsLeader() && err != nil || resp.Header.GetError() != nil { + protoClient, cctx := c.getClientAndContext(scanCtx) + if protoClient == nil { + return nil, errs.ErrClientGetProtoClient + } + resp, err = protoClient.ScanRegions(cctx, req) + } if err = c.respForErr(cmdFailedDurationScanRegions, start, err, resp.GetHeader()); err != nil { return nil, err diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index e43faaa9725..85221c639bd 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -46,6 +46,17 @@ const ( updateMemberBackOffBaseTime = 100 * time.Millisecond ) +// MemberHealthCheckInterval might be changed in the unit to shorten the testing time. +var MemberHealthCheckInterval = time.Second + +type apiKind int + +const ( + defaultAPIKind apiKind = iota + regionAPIKind + apiKindCount +) + type serviceType int const ( @@ -81,6 +92,9 @@ type ServiceDiscovery interface { // endpoints. Backup service endpoints are followers in a quorum-based cluster or // secondaries in a primary/secondary configured cluster. GetBackupAddrs() []string + // GetServiceClient tries to get the leader/primary ServiceClient. + // If the leader ServiceClient meets network problem, it returns a follower/secondary ServiceClient. + GetServiceClient() ServiceClient // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr GetOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) // ScheduleCheckMemberChanged is used to trigger a check to see if there is any membership change @@ -184,9 +198,11 @@ func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { } healthCli := healthpb.NewHealthClient(c.conn) resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() + failpoint.Inject("unreachableNetwork1", func(val failpoint.Value) { + if val, ok := val.(string); (ok && val == c.GetAddress()) || !ok { + resp = nil + err = status.New(codes.Unavailable, "unavailable").Err() + } }) rpcErr, ok := status.FromError(err) if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { @@ -218,6 +234,10 @@ func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { type errFn func(pdErr *pdpb.Error) bool +func emptyErrorFn(pdErr *pdpb.Error) bool { + return false +} + func regionAPIErrorFn(pdErr *pdpb.Error) bool { return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND } @@ -244,6 +264,7 @@ func (c *pdServiceAPIClient) Available() bool { return c.ServiceClient.Available() && !c.unavailable.Load() } +// markAsAvailable is used to try to mark the client as available if unavailable status is expired. func (c *pdServiceAPIClient) markAsAvailable() { if !c.unavailable.Load() { return @@ -274,7 +295,7 @@ func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { // pdServiceBalancerNode is a balancer node for PD service. // It extends the pdServiceClient and adds additional fields for the next polling client in the chain. type pdServiceBalancerNode struct { - ServiceClient + *pdServiceAPIClient next *pdServiceBalancerNode } @@ -284,6 +305,13 @@ type pdServiceBalancer struct { mu sync.Mutex now *pdServiceBalancerNode totalNode int + errFn errFn +} + +func newPDServiceBalancer(fn errFn) *pdServiceBalancer { + return &pdServiceBalancer{ + errFn: fn, + } } func (c *pdServiceBalancer) set(clients []ServiceClient) { @@ -294,14 +322,14 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { } c.totalNode = len(clients) head := &pdServiceBalancerNode{ - ServiceClient: clients[0], + pdServiceAPIClient: newPDServiceAPIClient(clients[0], c.errFn), } head.next = head last := head for i := 1; i < c.totalNode; i++ { next := &pdServiceBalancerNode{ - ServiceClient: clients[i], - next: head, + pdServiceAPIClient: newPDServiceAPIClient(clients[i], c.errFn), + next: head, } head = next last.next = head @@ -309,6 +337,15 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { c.now = head } +func (c *pdServiceBalancer) check() { + c.mu.Lock() + defer c.mu.Unlock() + for i := 0; i < c.totalNode; i++ { + c.now.markAsAvailable() + c.next() + } +} + func (c *pdServiceBalancer) next() { c.now = c.now.next } @@ -353,9 +390,12 @@ type pdServiceDiscovery struct { urls atomic.Value // Store as []string // PD leader URL - leader atomic.Value // Store as string + leader atomic.Value // Store as pdServiceClient // PD follower URLs - followers atomic.Value // Store as []string + followers sync.Map // Store as map[string]pdServiceClient + apiCandidateNodes [apiKindCount]*pdServiceBalancer + // PD follower URLs. Only for tso. + followerAddresses atomic.Value // Store as []string clusterID uint64 // addr -> a gRPC connection @@ -403,6 +443,7 @@ func newPDServiceDiscovery( ctx: ctx, cancel: cancel, wg: wg, + apiCandidateNodes: [2]*pdServiceBalancer{newPDServiceBalancer(emptyErrorFn), newPDServiceBalancer(regionAPIErrorFn)}, serviceModeUpdateCb: serviceModeUpdateCb, updateKeyspaceIDCb: updateKeyspaceIDCb, keyspaceID: keyspaceID, @@ -440,9 +481,10 @@ func (c *pdServiceDiscovery) Init() error { log.Warn("[pd] failed to check service mode and will check later", zap.Error(err)) } - c.wg.Add(2) + c.wg.Add(3) go c.updateMemberLoop() go c.updateServiceModeLoop() + go c.memberHealthCheckLoop() c.isInitialized = true return nil @@ -520,6 +562,47 @@ func (c *pdServiceDiscovery) updateServiceModeLoop() { } } +func (c *pdServiceDiscovery) memberHealthCheckLoop() { + defer c.wg.Done() + + memberCheckLoopCtx, memberCheckLoopCancel := context.WithCancel(c.ctx) + defer memberCheckLoopCancel() + + ticker := time.NewTicker(MemberHealthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + c.checkLeaderHealth(memberCheckLoopCtx) + c.checkFollowerHealth(memberCheckLoopCtx) + } + } +} + +func (c *pdServiceDiscovery) checkLeaderHealth(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() + leader := c.getLeaderServiceClient() + leader.checkNetworkAvailable(ctx) +} + +func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { + c.followers.Range(func(key, value any) bool { + // To ensure that the leader's healthy check is not delayed, shorten the duration. + ctx, cancel := context.WithTimeout(ctx, MemberHealthCheckInterval/3) + defer cancel() + serviceClient := value.(*pdServiceClient) + serviceClient.checkNetworkAvailable(ctx) + return true + }) + for _, balancer := range c.apiCandidateNodes { + balancer.check() + } +} + // Close releases all resources. func (c *pdServiceDiscovery) Close() { c.closeOnce.Do(func() { @@ -607,12 +690,45 @@ func (c *pdServiceDiscovery) GetServingAddr() string { return c.getLeaderAddr() } -// GetBackupAddrs gets the addresses of the current reachable and healthy followers -// in a quorum-based cluster. +// GetBackupAddrs gets the addresses of the current reachable followers +// in a quorum-based cluster. Used for tso currently. func (c *pdServiceDiscovery) GetBackupAddrs() []string { return c.getFollowerAddrs() } +// getLeaderServiceClient returns the leader ServiceClient. +func (c *pdServiceDiscovery) getLeaderServiceClient() *pdServiceClient { + leader := c.leader.Load() + if leader == nil { + return nil + } + return leader.(*pdServiceClient) +} + +// getServiceClientByKind returns ServiceClient of the specific kind. +func (c *pdServiceDiscovery) getServiceClientByKind(kind apiKind) ServiceClient { + client := c.apiCandidateNodes[kind].get() + if client == nil { + return nil + } + return client +} + +// GetServiceClient returns the leader/primary ServiceClient if it is healthy. +func (c *pdServiceDiscovery) GetServiceClient() ServiceClient { + leaderClient := c.getLeaderServiceClient() + if c.option.enableForwarding && !leaderClient.Available() { + if followerClient := c.getServiceClientByKind(defaultAPIKind); followerClient != nil { + log.Debug("[pd] use follower client", zap.String("addr", followerClient.GetAddress())) + return followerClient + } + } + if leaderClient == nil { + return nil + } + return leaderClient +} + // ScheduleCheckMemberChanged is used to check if there is any membership // change among the leader and the followers. func (c *pdServiceDiscovery) ScheduleCheckMemberChanged() { @@ -658,16 +774,12 @@ func (c *pdServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGlo // getLeaderAddr returns the leader address. func (c *pdServiceDiscovery) getLeaderAddr() string { - leaderAddr := c.leader.Load() - if leaderAddr == nil { - return "" - } - return leaderAddr.(string) + return c.getLeaderServiceClient().GetAddress() } // getFollowerAddrs returns the follower address. func (c *pdServiceDiscovery) getFollowerAddrs() []string { - followerAddrs := c.followers.Load() + followerAddrs := c.followerAddresses.Load() if followerAddrs == nil { return []string{} } @@ -765,8 +877,7 @@ func (c *pdServiceDiscovery) updateMember() error { } c.updateURLs(members.GetMembers()) - c.updateFollowers(members.GetMembers(), members.GetLeader()) - if err := c.switchLeader(members.GetLeader().GetClientUrls()); err != nil { + if err := c.updateServiceClient(members.GetMembers(), members.GetLeader()); err != nil { return err } @@ -838,42 +949,107 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls)) } -func (c *pdServiceDiscovery) switchLeader(addrs []string) error { +func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { // FIXME: How to safely compare leader urls? For now, only allows one client url. addr := addrs[0] - oldLeader := c.getLeaderAddr() - if addr == oldLeader { - return nil + oldLeader := c.getLeaderServiceClient() + if addr == oldLeader.GetAddress() && oldLeader.GetClientConn() != nil { + return false, nil } - - if _, err := c.GetOrCreateGRPCConn(addr); err != nil { - log.Warn("[pd] failed to connect leader", zap.String("leader", addr), errs.ZapError(err)) + newConn, err := c.GetOrCreateGRPCConn(addr) + // If gRPC connect is created successfully or leader is new, still saves. + if addr != oldLeader.GetAddress() || newConn != nil { + // Set PD leader and Global TSO Allocator (which is also the PD leader) + leaderClient := newPDServiceClient(addr, addr, newConn, true) + c.leader.Store(leaderClient) + } + if err != nil { + return true, err } - // Set PD leader and Global TSO Allocator (which is also the PD leader) - c.leader.Store(addr) // Run callbacks if c.tsoGlobalAllocLeaderUpdatedCb != nil { if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil { - return err + return true, err } } for _, cb := range c.leaderSwitchedCbs { cb() } - log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader)) - return nil + log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress())) + return true, nil } -func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) { - var addrs []string +func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) (changed bool) { + followers := make(map[string]*pdServiceClient) + c.followers.Range(func(key, value any) bool { + followers[key.(string)] = value.(*pdServiceClient) + return true + }) + var followerAddrs []string for _, member := range members { if member.GetMemberId() != leader.GetMemberId() { if len(member.GetClientUrls()) > 0 { - addrs = append(addrs, member.GetClientUrls()...) + followerAddrs = append(followerAddrs, member.GetClientUrls()...) + + // FIXME: How to safely compare urls(also for leader)? For now, only allows one client url. + addr := member.GetClientUrls()[0] + if client, ok := c.followers.Load(addr); ok { + if client.(*pdServiceClient).GetClientConn() == nil { + conn, err := c.GetOrCreateGRPCConn(addr) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + continue + } + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + c.followers.Store(addr, follower) + changed = true + } + delete(followers, addr) + } else { + changed = true + conn, err := c.GetOrCreateGRPCConn(addr) + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + follower.networkFailure.Store(true) + } + c.followers.LoadOrStore(addr, follower) + } } } } - c.followers.Store(addrs) + if len(followers) > 0 { + changed = true + for key := range followers { + c.followers.Delete(key) + } + } + c.followerAddresses.Store(followerAddrs) + return +} + +func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader *pdpb.Member) error { + leaderChanged, err := c.switchLeader(leader.GetClientUrls()) + followerChanged := c.updateFollowers(members, leader) + // don't need to recreate balancer if no changess. + if !followerChanged && !leaderChanged { + return err + } + // If error is not nil, still updates candidates. + clients := make([]ServiceClient, 0) + c.followers.Range(func(_, value any) bool { + clients = append(clients, value.(*pdServiceClient)) + return true + }) + leaderClient := c.getLeaderServiceClient() + if leaderClient != nil { + clients = append(clients, leaderClient) + } + // create candidate services for all kinds of request. + for i := 0; i < int(apiKindCount); i++ { + c.apiCandidateNodes[i].set(clients) + } + return err } func (c *pdServiceDiscovery) switchTSOAllocatorLeaders(allocatorMap map[string]*pdpb.Member) error { diff --git a/client/pd_service_discovery_test.go b/client/pd_service_discovery_test.go index 9cc0055aac3..3709a858504 100644 --- a/client/pd_service_discovery_test.go +++ b/client/pd_service_discovery_test.go @@ -262,8 +262,10 @@ func (suite *serviceClientTestSuite) TestServiceClientBalancer() { re := suite.Require() follower := suite.followerClient leader := suite.leaderClient + followerAPIClient := newPDServiceAPIClient(follower, emptyErrorFn) + leaderAPIClient := newPDServiceAPIClient(leader, emptyErrorFn) b := &pdServiceBalancer{} - b.set([]ServiceClient{leader, follower}) + b.set([]ServiceClient{leaderAPIClient, followerAPIClient}) re.Equal(b.totalNode, 2) for i := 0; i < 10; i++ { diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index 5f14c406797..7caaacd0dfe 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -373,6 +373,11 @@ func (c *tsoServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGl c.globalAllocPrimariesUpdatedCb = callback } +// GetServiceClient implements ServiceDiscovery +func (c *tsoServiceDiscovery) GetServiceClient() ServiceClient { + return c.apiSvcDiscovery.GetServiceClient() +} + // getPrimaryAddr returns the primary address. func (c *tsoServiceDiscovery) getPrimaryAddr() string { c.keyspaceGroupSD.RLock() diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index a001ec4bd03..7ab81a50a6b 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -38,6 +38,8 @@ import ( const ( // ForwardMetadataKey is used to record the forwarded host of PD. ForwardMetadataKey = "pd-forwarded-host" + // FollowerHandleMetadataKey is used to mark the permit of follower handle. + FollowerHandleMetadataKey = "pd-allow-follower-handle" ) // TLSConfig is the configuration for supporting tls. @@ -162,7 +164,7 @@ func ResetForwardContext(ctx context.Context) context.Context { func GetForwardedHost(ctx context.Context) string { md, ok := metadata.FromIncomingContext(ctx) if !ok { - log.Debug("failed to get forwarding metadata") + log.Debug("failed to get gRPC incoming metadata when getting forwarded host") return "" } if t, ok := md[ForwardMetadataKey]; ok { @@ -171,6 +173,17 @@ func GetForwardedHost(ctx context.Context) string { return "" } +// GetEnableFollowerHandle returns the follower host in metadata. +func GetEnableFollowerHandle(ctx context.Context) bool { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + log.Debug("failed to get gRPC incoming metadata when checking follower handle is enabled") + return false + } + _, ok = md[FollowerHandleMetadataKey] + return ok +} + func establish(ctx context.Context, addr string, tlsConfig *TLSConfig, do ...grpc.DialOption) (*grpc.ClientConn, error) { tlsCfg, err := tlsConfig.ToTLSConfig() if err != nil { diff --git a/server/forward.go b/server/forward.go index e765d442539..65750fcd4be 100644 --- a/server/forward.go +++ b/server/forward.go @@ -384,16 +384,16 @@ func (s *GrpcServer) getForwardedHost(ctx, streamCtx context.Context, serviceNam return forwardedHost, nil } -func (s *GrpcServer) isLocalRequest(forwardedHost string) bool { +func (s *GrpcServer) isLocalRequest(host string) bool { failpoint.Inject("useForwardRequest", func() { failpoint.Return(false) }) - if forwardedHost == "" { + if host == "" { return true } memberAddrs := s.GetMember().Member().GetClientUrls() for _, addr := range memberAddrs { - if addr == forwardedHost { + if addr == host { return true } } diff --git a/server/grpc_service.go b/server/grpc_service.go index bb20fcbe484..afe0f7901bf 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -75,6 +75,7 @@ var ( ErrMaxCountTSOProxyRoutinesExceeded = status.Errorf(codes.ResourceExhausted, "max count of concurrent tso proxy routines exceeded") ErrTSOProxyRecvFromClientTimeout = status.Errorf(codes.DeadlineExceeded, "tso proxy timeout when receiving from client; stream closed by server") ErrEtcdNotStarted = status.Errorf(codes.Unavailable, "server is started, but etcd not started") + ErrFollowerHandlingNotAllowed = status.Errorf(codes.Unavailable, "not leader and follower handling not allowed") ) // GrpcServer wraps Server to provide grpc service. @@ -221,6 +222,11 @@ type request interface { type forwardFn func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) func (s *GrpcServer) unaryMiddleware(ctx context.Context, req request, fn forwardFn) (rsp interface{}, err error) { + return s.unaryFollowerMiddleware(ctx, req, fn, nil) +} + +// unaryFollowerMiddleware adds the check of followers enable compared to unaryMiddleware. +func (s *GrpcServer) unaryFollowerMiddleware(ctx context.Context, req request, fn forwardFn, allowFollower *bool) (rsp interface{}, err error) { failpoint.Inject("customTimeout", func() { time.Sleep(5 * time.Second) }) @@ -233,7 +239,7 @@ func (s *GrpcServer) unaryMiddleware(ctx context.Context, req request, fn forwar ctx = grpcutil.ResetForwardContext(ctx) return fn(ctx, client) } - if err := s.validateRequest(req.GetHeader()); err != nil { + if err := s.validateRoleInRequest(ctx, req.GetHeader(), allowFollower); err != nil { return nil, err } return nil, nil @@ -1281,22 +1287,39 @@ func (s *GrpcServer) GetRegion(ctx context.Context, request *pdpb.GetRegionReque fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { return pdpb.NewPDClient(client).GetRegion(ctx, request) } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + followerHandle := new(bool) + if rsp, err := s.unaryFollowerMiddleware(ctx, request, fn, followerHandle); err != nil { return nil, err } else if rsp != nil { return rsp.(*pdpb.GetRegionResponse), nil } - - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.GetRegionResponse{Header: s.notBootstrappedHeader()}, nil - } - region := rc.GetRegionByKey(request.GetRegionKey()) - if region == nil { - return &pdpb.GetRegionResponse{Header: s.header()}, nil + var rc *cluster.RaftCluster + var region *core.RegionInfo + if *followerHandle { + rc = s.cluster + if !s.cluster.GetRegionSyncer().IsRunning() { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } + region = rc.GetRegionByKey(request.GetRegionKey()) + if region == nil { + log.Warn("follower get region nil", zap.String("key", string(request.GetRegionKey()))) + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } + } else { + rc = s.GetRaftCluster() + if rc == nil { + return &pdpb.GetRegionResponse{Header: s.notBootstrappedHeader()}, nil + } + region = rc.GetRegionByKey(request.GetRegionKey()) + if region == nil { + log.Warn("leader get region nil", zap.String("key", string(request.GetRegionKey()))) + return &pdpb.GetRegionResponse{Header: s.header()}, nil + } } + var buckets *metapb.Buckets - if rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { + // FIXME: If the bucket is disabled dynamically, the bucket information is returned unexpectedly + if !*followerHandle && rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { buckets = region.GetBuckets() } return &pdpb.GetRegionResponse{ @@ -1325,23 +1348,37 @@ func (s *GrpcServer) GetPrevRegion(ctx context.Context, request *pdpb.GetRegionR fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { return pdpb.NewPDClient(client).GetPrevRegion(ctx, request) } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + followerHandle := new(bool) + if rsp, err := s.unaryFollowerMiddleware(ctx, request, fn, followerHandle); err != nil { return nil, err } else if rsp != nil { return rsp.(*pdpb.GetRegionResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.GetRegionResponse{Header: s.notBootstrappedHeader()}, nil + var rc *cluster.RaftCluster + if *followerHandle { + // no need to check running status + rc = s.cluster + if !s.cluster.GetRegionSyncer().IsRunning() { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } + } else { + rc = s.GetRaftCluster() + if rc == nil { + return &pdpb.GetRegionResponse{Header: s.notBootstrappedHeader()}, nil + } } region := rc.GetPrevRegionByKey(request.GetRegionKey()) if region == nil { + if *followerHandle { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } return &pdpb.GetRegionResponse{Header: s.header()}, nil } var buckets *metapb.Buckets - if rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { + // FIXME: If the bucket is disabled dynamically, the bucket information is returned unexpectedly + if !*followerHandle && rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { buckets = region.GetBuckets() } return &pdpb.GetRegionResponse{ @@ -1370,22 +1407,39 @@ func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionB fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { return pdpb.NewPDClient(client).GetRegionByID(ctx, request) } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + followerHandle := new(bool) + if rsp, err := s.unaryFollowerMiddleware(ctx, request, fn, followerHandle); err != nil { return nil, err } else if rsp != nil { return rsp.(*pdpb.GetRegionResponse), err } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.GetRegionResponse{Header: s.notBootstrappedHeader()}, nil + var rc *cluster.RaftCluster + if *followerHandle { + rc = s.cluster + if !s.cluster.GetRegionSyncer().IsRunning() { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } + } else { + rc = s.GetRaftCluster() + if rc == nil { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } } region := rc.GetRegion(request.GetRegionId()) + failpoint.Inject("followerHandleError", func() { + if *followerHandle { + region = nil + } + }) if region == nil { + if *followerHandle { + return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil + } return &pdpb.GetRegionResponse{Header: s.header()}, nil } var buckets *metapb.Buckets - if rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { + if !*followerHandle && rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() { buckets = region.GetBuckets() } return &pdpb.GetRegionResponse{ @@ -1414,17 +1468,29 @@ func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsR fn := func(ctx context.Context, client *grpc.ClientConn) (interface{}, error) { return pdpb.NewPDClient(client).ScanRegions(ctx, request) } - if rsp, err := s.unaryMiddleware(ctx, request, fn); err != nil { + followerHandle := new(bool) + if rsp, err := s.unaryFollowerMiddleware(ctx, request, fn, followerHandle); err != nil { return nil, err } else if rsp != nil { return rsp.(*pdpb.ScanRegionsResponse), nil } - rc := s.GetRaftCluster() - if rc == nil { - return &pdpb.ScanRegionsResponse{Header: s.notBootstrappedHeader()}, nil + var rc *cluster.RaftCluster + if *followerHandle { + rc = s.cluster + if !s.cluster.GetRegionSyncer().IsRunning() { + return &pdpb.ScanRegionsResponse{Header: s.regionNotFound()}, nil + } + } else { + rc = s.GetRaftCluster() + if rc == nil { + return &pdpb.ScanRegionsResponse{Header: s.notBootstrappedHeader()}, nil + } } regions := rc.ScanRegions(request.GetStartKey(), request.GetEndKey(), int(request.GetLimit())) + if *followerHandle && len(regions) == 0 { + return &pdpb.ScanRegionsResponse{Header: s.regionNotFound()}, nil + } resp := &pdpb.ScanRegionsResponse{Header: s.header()} for _, r := range regions { leader := r.GetLeader() @@ -1942,10 +2008,25 @@ func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorR } // validateRequest checks if Server is leader and clusterID is matched. -// TODO: Call it in gRPC interceptor. func (s *GrpcServer) validateRequest(header *pdpb.RequestHeader) error { - if s.IsClosed() || !s.member.IsLeader() { - return ErrNotLeader + return s.validateRoleInRequest(context.TODO(), header, nil) +} + +// validateRoleRequest checks if Server is leader when disallow follower-handle and clusterID is matched. +// TODO: Call it in gRPC interceptor. +func (s *GrpcServer) validateRoleInRequest(ctx context.Context, header *pdpb.RequestHeader, allowFollower *bool) error { + if s.IsClosed() { + return ErrNotStarted + } + if !s.member.IsLeader() { + if allowFollower == nil { + return ErrNotLeader + } + if !grpcutil.GetEnableFollowerHandle(ctx) { + // TODO: change the error code + return ErrFollowerHandlingNotAllowed + } + *allowFollower = true } if header.GetClusterId() != s.clusterID { return status.Errorf(codes.FailedPrecondition, "mismatch cluster id, need %d but got %d", s.clusterID, header.GetClusterId()) @@ -1996,6 +2077,13 @@ func (s *GrpcServer) invalidValue(msg string) *pdpb.ResponseHeader { }) } +func (s *GrpcServer) regionNotFound() *pdpb.ResponseHeader { + return s.errorHeader(&pdpb.Error{ + Type: pdpb.ErrorType_REGION_NOT_FOUND, + Message: "region not found", + }) +} + func (s *GrpcServer) convertHeader(header *schedulingpb.ResponseHeader) *pdpb.ResponseHeader { switch header.GetError().GetType() { case schedulingpb.ErrorType_UNKNOWN: diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index bb4d6851fd0..e93969cabd5 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -522,7 +522,7 @@ func TestGetRegionByFollowerForwarding(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond + pd.MemberHealthCheckInterval = 100 * time.Millisecond cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() @@ -548,7 +548,7 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond + pd.MemberHealthCheckInterval = 100 * time.Millisecond cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() @@ -579,7 +579,7 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond + pd.MemberHealthCheckInterval = 100 * time.Millisecond cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() @@ -614,7 +614,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond + pd.MemberHealthCheckInterval = 100 * time.Millisecond cluster, err := tests.NewTestCluster(ctx, 3) re.NoError(err) defer cluster.Destroy() @@ -704,6 +704,126 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { }) } +func TestGetRegionFromFollower(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pd.MemberHealthCheckInterval = 100 * time.Millisecond + cluster, err := tests.NewTestCluster(ctx, 3) + re.NoError(err) + defer cluster.Destroy() + + endpoints := runServer(re, cluster) + cli := setupCli(re, ctx, endpoints) + cli.UpdateOption(pd.EnableFollowerHandle, true) + re.NotEmpty(cluster.WaitLeader()) + leader := cluster.GetLeaderServer() + testutil.Eventually(re, func() bool { + ret := true + for _, s := range cluster.GetServers() { + if s.IsLeader() { + continue + } + if !s.GetServer().DirectlyGetRaftCluster().GetRegionSyncer().IsRunning() { + ret = false + } + } + return ret + }) + // follower have no region + cnt := 0 + for i := 0; i < 100; i++ { + resp, err := cli.GetRegion(ctx, []byte("a"), pd.WithAllowFollowerHandle()) + if err == nil && resp != nil { + cnt++ + } + re.Equal(resp.Meta.Id, uint64(2)) + } + re.Equal(cnt, 100) + + // send heartbeat, so follower will have region. + grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) + testutil.Eventually(re, func() bool { + regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) + re.NoError(err) + regionID := regionIDAllocator.alloc() + region := &metapb.Region{ + Id: regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: peers, + } + req := &pdpb.RegionHeartbeatRequest{ + Header: newHeader(leader.GetServer()), + Region: region, + Leader: peers[0], + } + err = regionHeartbeat.Send(req) + re.NoError(err) + _, err = regionHeartbeat.Recv() + return err == nil + }) + + // because we can't check whether this request is processed by followers from response, + // we can disable forward and make network problem for leader. + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", leader.GetAddr()))) + time.Sleep(150 * time.Millisecond) + cnt = 0 + for i := 0; i < 100; i++ { + resp, err := cli.GetRegion(ctx, []byte("a"), pd.WithAllowFollowerHandle()) + if err == nil && resp != nil { + cnt++ + } + re.Equal(resp.Meta.Id, uint64(4)) + } + re.Equal(cnt, 100) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + + follower := cluster.GetServer(cluster.GetFollower()) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) + time.Sleep(100 * time.Millisecond) + cnt = 0 + for i := 0; i < 100; i++ { + resp, err := cli.GetRegion(ctx, []byte("a"), pd.WithAllowFollowerHandle()) + if err == nil && resp != nil { + cnt++ + } + re.Equal(resp.Meta.Id, uint64(4)) + } + re.Equal(cnt, 100) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + + // follower client failed will retry by leader service client. + re.NoError(failpoint.Enable("github.com/tikv/pd/server/followerHandleError", "return(true)")) + cnt = 0 + for i := 0; i < 100; i++ { + resp, err := cli.GetRegion(ctx, []byte("a"), pd.WithAllowFollowerHandle()) + if err == nil && resp != nil { + cnt++ + } + re.Equal(resp.Meta.Id, uint64(4)) + } + re.Equal(cnt, 100) + re.NoError(failpoint.Disable("github.com/tikv/pd/server/followerHandleError")) + + re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", leader.GetAddr()))) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/fastCheckAvailable", "return(true)")) + time.Sleep(100 * time.Millisecond) + cnt = 0 + for i := 0; i < 100; i++ { + resp, err := cli.GetRegion(ctx, []byte("a"), pd.WithAllowFollowerHandle()) + if err == nil && resp != nil { + cnt++ + } + re.Equal(resp.Meta.Id, uint64(4)) + } + re.Equal(cnt, 100) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/fastCheckAvailable")) +} + func checkTS(re *require.Assertions, cli pd.Client, lastTS uint64) uint64 { for i := 0; i < tsoRequestRound; i++ { physical, logical, err := cli.GetTS(context.TODO()) From 01a983b1833b334140156c73f69eb8344710648e Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Wed, 6 Dec 2023 10:04:02 +0800 Subject: [PATCH 08/16] impl follower region Signed-off-by: Cabinfever_B --- client/client.go | 1 - client/pd_service_discovery.go | 11 +- tests/integrations/client/client_test.go | 201 +++++++++++------------ 3 files changed, 99 insertions(+), 114 deletions(-) diff --git a/client/client.go b/client/client.go index b0d7498a7df..eff5afea402 100644 --- a/client/client.go +++ b/client/client.go @@ -914,7 +914,6 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...GetRegionOpt NeedBuckets: options.needBuckets, } serviceClient, cctx := c.getRegionAPIClientAndContext(ctx, options.allowFollowerHandle && c.option.getEnableFollowerHandle()) - fmt.Println(serviceClient.GetAddress(), serviceClient.IsLeader()) if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 85221c639bd..5d4faf8e6cb 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -146,12 +146,16 @@ type pdServiceClient struct { } func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) *pdServiceClient { - return &pdServiceClient{ + cli := &pdServiceClient{ addr: addr, conn: conn, isLeader: isLeader, leaderAddr: leaderAddr, } + if conn == nil { + cli.networkFailure.Store(true) + } + return cli } // GetAddress implements ServiceClient. @@ -963,9 +967,6 @@ func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { leaderClient := newPDServiceClient(addr, addr, newConn, true) c.leader.Store(leaderClient) } - if err != nil { - return true, err - } // Run callbacks if c.tsoGlobalAllocLeaderUpdatedCb != nil { if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil { @@ -976,7 +977,7 @@ func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { cb() } log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress())) - return true, nil + return true, err } func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) (changed bool) { diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index e93969cabd5..2188ab50c82 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -518,18 +518,69 @@ func TestCustomTimeout(t *testing.T) { re.Less(time.Since(start), 2*time.Second) } -func TestGetRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +type followerForwardAndHandleTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + + cluster *tests.TestCluster + endpoints []string + regionID uint64 +} + +func TestFollowerForwardAndHandleTestSuite(t *testing.T) { + suite.Run(t, new(followerForwardAndHandleTestSuite)) +} + +func (suite *followerForwardAndHandleTestSuite) SetupSuite() { + re := suite.Require() + suite.ctx, suite.clean = context.WithCancel(context.Background()) pd.MemberHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() + cluster, err := tests.NewTestCluster(suite.ctx, 3) + suite.NoError(err) + suite.cluster = cluster + suite.endpoints = runServer(re, cluster) + cluster.WaitLeader() + leader := cluster.GetLeaderServer() + grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) + suite.regionID = regionIDAllocator.alloc() + testutil.Eventually(re, func() bool { + regionHeartbeat, err := grpcPDClient.RegionHeartbeat(suite.ctx) + re.NoError(err) + region := &metapb.Region{ + Id: suite.regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: peers, + } + req := &pdpb.RegionHeartbeatRequest{ + Header: newHeader(leader.GetServer()), + Region: region, + Leader: peers[0], + } + err = regionHeartbeat.Send(req) + re.NoError(err) + _, err = regionHeartbeat.Recv() + return err == nil + }) +} - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) +func (suite *followerForwardAndHandleTestSuite) TearDownTest() { +} +func (suite *followerForwardAndHandleTestSuite) TearDownSuite() { + suite.cluster.Destroy() + suite.clean() +} + +func (suite *followerForwardAndHandleTestSuite) TestGetRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) time.Sleep(200 * time.Millisecond) r, err := cli.GetRegion(context.Background(), []byte("a")) @@ -544,17 +595,11 @@ func TestGetRegionByFollowerForwarding(t *testing.T) { } // case 1: unreachable -> normal -func TestGetTsoByFollowerForwarding1(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding1() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.MemberHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -564,7 +609,7 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) @@ -575,17 +620,11 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { } // case 2: unreachable -> leader transfer -> normal -func TestGetTsoByFollowerForwarding2(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding2() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.MemberHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -595,13 +634,13 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) - re.NoError(cluster.GetLeaderServer().ResignLeader()) - re.NotEmpty(cluster.WaitLeader()) + re.NoError(suite.cluster.GetLeaderServer().ResignLeader()) + re.NotEmpty(suite.cluster.WaitLeader()) lastTS = checkTS(re, cli, lastTS) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) @@ -610,45 +649,18 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { } // case 3: network partition between client and follower A -> transfer leader to follower A -> normal -func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoAndRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.MemberHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - endpoints := runServer(re, cluster) - re.NotEmpty(cluster.WaitLeader()) + cluster := suite.cluster leader := cluster.GetLeaderServer() - grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) - testutil.Eventually(re, func() bool { - regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) - re.NoError(err) - regionID := regionIDAllocator.alloc() - region := &metapb.Region{ - Id: regionID, - RegionEpoch: &metapb.RegionEpoch{ - ConfVer: 1, - Version: 1, - }, - Peers: peers, - } - req := &pdpb.RegionHeartbeatRequest{ - Header: newHeader(leader.GetServer()), - Region: region, - Leader: peers[0], - } - err = regionHeartbeat.Send(req) - re.NoError(err) - _, err = regionHeartbeat.Recv() - return err == nil - }) + follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) var lastTS uint64 testutil.Eventually(re, func() bool { physical, logical, err := cli.GetTS(context.TODO()) @@ -656,7 +668,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -672,7 +684,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -691,7 +703,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -704,17 +716,13 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { }) } -func TestGetRegionFromFollower(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.MemberHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints) + cluster := suite.cluster + cli := setupCli(re, ctx, suite.endpoints) cli.UpdateOption(pd.EnableFollowerHandle, true) re.NotEmpty(cluster.WaitLeader()) leader := cluster.GetLeaderServer() @@ -737,35 +745,10 @@ func TestGetRegionFromFollower(t *testing.T) { if err == nil && resp != nil { cnt++ } - re.Equal(resp.Meta.Id, uint64(2)) + re.Equal(resp.Meta.Id, suite.regionID) } re.Equal(cnt, 100) - // send heartbeat, so follower will have region. - grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) - testutil.Eventually(re, func() bool { - regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) - re.NoError(err) - regionID := regionIDAllocator.alloc() - region := &metapb.Region{ - Id: regionID, - RegionEpoch: &metapb.RegionEpoch{ - ConfVer: 1, - Version: 1, - }, - Peers: peers, - } - req := &pdpb.RegionHeartbeatRequest{ - Header: newHeader(leader.GetServer()), - Region: region, - Leader: peers[0], - } - err = regionHeartbeat.Send(req) - re.NoError(err) - _, err = regionHeartbeat.Recv() - return err == nil - }) - // because we can't check whether this request is processed by followers from response, // we can disable forward and make network problem for leader. re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", leader.GetAddr()))) @@ -776,11 +759,12 @@ func TestGetRegionFromFollower(t *testing.T) { if err == nil && resp != nil { cnt++ } - re.Equal(resp.Meta.Id, uint64(4)) + re.Equal(resp.Meta.Id, suite.regionID) } re.Equal(cnt, 100) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) + // make network problem for follower. follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) time.Sleep(100 * time.Millisecond) @@ -790,7 +774,7 @@ func TestGetRegionFromFollower(t *testing.T) { if err == nil && resp != nil { cnt++ } - re.Equal(resp.Meta.Id, uint64(4)) + re.Equal(resp.Meta.Id, suite.regionID) } re.Equal(cnt, 100) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) @@ -803,11 +787,12 @@ func TestGetRegionFromFollower(t *testing.T) { if err == nil && resp != nil { cnt++ } - re.Equal(resp.Meta.Id, uint64(4)) + re.Equal(resp.Meta.Id, suite.regionID) } re.Equal(cnt, 100) re.NoError(failpoint.Disable("github.com/tikv/pd/server/followerHandleError")) + // test after being healthy re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", fmt.Sprintf("return(\"%s\")", leader.GetAddr()))) re.NoError(failpoint.Enable("github.com/tikv/pd/client/fastCheckAvailable", "return(true)")) time.Sleep(100 * time.Millisecond) @@ -817,7 +802,7 @@ func TestGetRegionFromFollower(t *testing.T) { if err == nil && resp != nil { cnt++ } - re.Equal(resp.Meta.Id, uint64(4)) + re.Equal(resp.Meta.Id, suite.regionID) } re.Equal(cnt, 100) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) From 87efe031548d3f9817035f5e904b71c82cd0dcaa Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 22 Dec 2023 12:06:57 +0800 Subject: [PATCH 09/16] merge master Signed-off-by: Cabinfever_B --- client/client.go | 8 ++++---- client/go.mod | 4 +--- client/go.sum | 4 ++-- tests/integrations/client/client_test.go | 12 ++++++------ 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/client/client.go b/client/client.go index eff5afea402..df153608c05 100644 --- a/client/client.go +++ b/client/client.go @@ -730,7 +730,7 @@ func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, contex if serviceClient == nil { return nil, ctx } - return pdpb.NewPDClient(serviceClient.GetClientConn()), serviceClient.BuildGRPCContext(ctx, true) + return pdpb.NewPDClient(serviceClient.GetClientConn()), serviceClient.BuildGRPCTargetContext(ctx, true) } // getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns @@ -740,14 +740,14 @@ func (c *client) getRegionAPIClientAndContext(ctx context.Context, allowFollower if allowFollower { serviceClient = c.pdSvcDiscovery.getServiceClientByKind(regionAPIKind) if serviceClient != nil { - return serviceClient, serviceClient.BuildGRPCContext(ctx, !allowFollower) + return serviceClient, serviceClient.BuildGRPCTargetContext(ctx, !allowFollower) } } serviceClient = c.pdSvcDiscovery.GetServiceClient() if serviceClient == nil { return nil, ctx } - return serviceClient, serviceClient.BuildGRPCContext(ctx, !allowFollower) + return serviceClient, serviceClient.BuildGRPCTargetContext(ctx, !allowFollower) } func (c *client) GetTSAsync(ctx context.Context) TSFuture { @@ -1037,7 +1037,7 @@ func (c *client) ScanRegions(ctx context.Context, key, endKey []byte, limit int, return nil, errs.ErrClientGetProtoClient } resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).ScanRegions(cctx, req) - if !serviceClient.IsLeader() && err != nil || resp.Header.GetError() != nil { + if !serviceClient.IsConnectedToLeader() && err != nil || resp.Header.GetError() != nil { protoClient, cctx := c.getClientAndContext(scanCtx) if protoClient == nil { return nil, errs.ErrClientGetProtoClient diff --git a/client/go.mod b/client/go.mod index a0cb86aa19f..2a203490700 100644 --- a/client/go.mod +++ b/client/go.mod @@ -21,8 +21,6 @@ require ( google.golang.org/grpc v1.54.0 ) -require google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect - require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -39,7 +37,7 @@ require ( golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/client/go.sum b/client/go.sum index 4d4e9f396da..d6a2d6bce63 100644 --- a/client/go.sum +++ b/client/go.sum @@ -227,8 +227,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index ce8254f4a50..ed394f6fad4 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -537,7 +537,7 @@ func (suite *followerForwardAndHandleTestSuite) SetupSuite() { suite.ctx, suite.clean = context.WithCancel(context.Background()) pd.MemberHealthCheckInterval = 100 * time.Millisecond cluster, err := tests.NewTestCluster(suite.ctx, 3) - suite.NoError(err) + re.NoError(err) suite.cluster = cluster suite.endpoints = runServer(re, cluster) cluster.WaitLeader() @@ -747,7 +747,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { } re.Equal(resp.Meta.Id, suite.regionID) } - re.Equal(cnt, 100) + re.Equal(100, cnt) // because we can't check whether this request is processed by followers from response, // we can disable forward and make network problem for leader. @@ -761,7 +761,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { } re.Equal(resp.Meta.Id, suite.regionID) } - re.Equal(cnt, 100) + re.Equal(100, cnt) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) // make network problem for follower. @@ -776,7 +776,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { } re.Equal(resp.Meta.Id, suite.regionID) } - re.Equal(cnt, 100) + re.Equal(100, cnt) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) // follower client failed will retry by leader service client. @@ -789,7 +789,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { } re.Equal(resp.Meta.Id, suite.regionID) } - re.Equal(cnt, 100) + re.Equal(100, cnt) re.NoError(failpoint.Disable("github.com/tikv/pd/server/followerHandleError")) // test after being healthy @@ -804,7 +804,7 @@ func (suite *followerForwardAndHandleTestSuite) TestGetRegionFromFollower() { } re.Equal(resp.Meta.Id, suite.regionID) } - re.Equal(cnt, 100) + re.Equal(100, cnt) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork1")) re.NoError(failpoint.Disable("github.com/tikv/pd/client/fastCheckAvailable")) } From eac3646790be9669a496ef4f6569c79837ebd923 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 22 Dec 2023 13:19:22 +0800 Subject: [PATCH 10/16] merge master Signed-off-by: Cabinfever_B --- client/go.mod | 2 +- client/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/go.mod b/client/go.mod index 2a203490700..a97a94a3e95 100644 --- a/client/go.mod +++ b/client/go.mod @@ -37,7 +37,7 @@ require ( golang.org/x/net v0.18.0 // indirect golang.org/x/sys v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 google.golang.org/protobuf v1.31.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/client/go.sum b/client/go.sum index d6a2d6bce63..4d4e9f396da 100644 --- a/client/go.sum +++ b/client/go.sum @@ -227,8 +227,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.54.0 h1:EhTqbhiYeixwWQtAEZAxmV9MGqcjEU2mFx52xCzNyag= google.golang.org/grpc v1.54.0/go.mod h1:PUSEXI6iWghWaB6lXM4knEgpJNu2qUcKfDtNci3EC2g= google.golang.org/grpc/examples v0.0.0-20230419000256-16651f60ddc5 h1:FjbWL/mGfyRQNxjagfT1chiHL1569WEA/OGH0ZIzGcI= From ac675a933a86ddd81340b72bdcc9862c9402c15d Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 22 Dec 2023 14:17:34 +0800 Subject: [PATCH 11/16] merge master Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index e775ff6258f..dda3298e9ec 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -170,10 +170,7 @@ func (c *pdServiceClient) GetAddress() string { // BuildGRPCTargetContext implements ServiceClient. func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { - if c == nil { - return ctx - } - if c.IsConnectedToLeader() { + if c == nil || c.IsConnectedToLeader() { return ctx } if toLeader { From 82657d407cf74b12e0bc7709a39bff1ee427e635 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 22 Dec 2023 16:15:29 +0800 Subject: [PATCH 12/16] use ServiceClient in pd discovery Signed-off-by: Cabinfever_B --- client/client.go | 103 +--------- client/pd_service_discovery.go | 247 +++++++++++++++++++---- client/tso_service_discovery.go | 5 + tests/integrations/client/client_test.go | 150 +++++++------- 4 files changed, 304 insertions(+), 201 deletions(-) diff --git a/client/client.go b/client/client.go index 0ae362d06a8..4e03e5e3507 100644 --- a/client/client.go +++ b/client/client.go @@ -17,29 +17,22 @@ package pd import ( "context" "fmt" - "math/rand" "runtime/trace" "strings" "sync" - "sync/atomic" "time" "github.com/opentracing/opentracing-go" "github.com/pingcap/errors" - "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" "github.com/prometheus/client_golang/prometheus" "github.com/tikv/pd/client/errs" - "github.com/tikv/pd/client/grpcutil" "github.com/tikv/pd/client/tlsutil" "github.com/tikv/pd/client/tsoutil" "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - healthpb "google.golang.org/grpc/health/grpc_health_v1" - "google.golang.org/grpc/status" ) const ( @@ -217,9 +210,6 @@ func WithAllowFollowerHandle() GetRegionOption { return func(op *GetRegionOp) { op.allowFollowerHandle = true } } -// LeaderHealthCheckInterval might be changed in the unit to shorten the testing time. -var LeaderHealthCheckInterval = time.Second - var ( // errUnmatchedClusterID is returned when found a PD with a different cluster ID. errUnmatchedClusterID = errors.New("[pd] unmatched cluster id") @@ -316,7 +306,6 @@ type client struct { // For internal usage. updateTokenConnectionCh chan struct{} - leaderNetworkFailure int32 ctx context.Context cancel context.CancelFunc @@ -575,10 +564,6 @@ func (c *client) setup() error { // Create dispatchers c.createTokenDispatcher() - - // Start the daemons. - c.wg.Add(1) - go c.leaderCheckLoop() return nil } @@ -719,46 +704,6 @@ func (c *client) UpdateOption(option DynamicOption, value interface{}) error { return nil } -func (c *client) leaderCheckLoop() { - defer c.wg.Done() - - leaderCheckLoopCtx, leaderCheckLoopCancel := context.WithCancel(c.ctx) - defer leaderCheckLoopCancel() - - ticker := time.NewTicker(LeaderHealthCheckInterval) - defer ticker.Stop() - - for { - select { - case <-c.ctx.Done(): - return - case <-ticker.C: - c.checkLeaderHealth(leaderCheckLoopCtx) - } - } -} - -func (c *client) checkLeaderHealth(ctx context.Context) { - ctx, cancel := context.WithTimeout(ctx, c.option.timeout) - defer cancel() - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - healthCli := healthpb.NewHealthClient(client) - resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() - }) - rpcErr, ok := status.FromError(err) - if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(0)) - } - } else { - atomic.StoreInt32(&(c.leaderNetworkFailure), int32(1)) - } -} - func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { start := time.Now() defer func() { cmdDurationGetAllMembers.Observe(time.Since(start).Seconds()) }() @@ -778,50 +723,14 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { return resp.GetMembers(), nil } -// leaderClient gets the client of current PD leader. -func (c *client) leaderClient() pdpb.PDClient { - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { - return pdpb.NewPDClient(client) - } - return nil -} - -// backupClientConn gets a grpc client connection of the current reachable and healthy -// backup service endpoints randomly. Backup service endpoints are followers in a -// quorum-based cluster or secondaries in a primary/secondary configured cluster. -func (c *client) backupClientConn() (*grpc.ClientConn, string) { - addrs := c.pdSvcDiscovery.GetBackupAddrs() - if len(addrs) < 1 { - return nil, "" - } - var ( - cc *grpc.ClientConn - err error - ) - for i := 0; i < len(addrs); i++ { - addr := addrs[rand.Intn(len(addrs))] - if cc, err = c.pdSvcDiscovery.GetOrCreateGRPCConn(addr); err != nil { - continue - } - healthCtx, healthCancel := context.WithTimeout(c.ctx, c.option.timeout) - resp, err := healthpb.NewHealthClient(cc).Check(healthCtx, &healthpb.HealthCheckRequest{Service: ""}) - healthCancel() - if err == nil && resp.GetStatus() == healthpb.HealthCheckResponse_SERVING { - return cc, addr - } - } - return nil, "" -} - +// getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns +// follower pd client and the context which holds forward information. func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { - if c.option.enableForwarding && atomic.LoadInt32(&c.leaderNetworkFailure) == 1 { - backupClientConn, addr := c.backupClientConn() - if backupClientConn != nil { - log.Debug("[pd] use follower client", zap.String("addr", addr)) - return pdpb.NewPDClient(backupClientConn), grpcutil.BuildForwardContext(ctx, c.GetLeaderAddr()) - } + serviceClient := c.pdSvcDiscovery.GetServiceClient() + if serviceClient == nil { + return nil, ctx } - return c.leaderClient(), ctx + return pdpb.NewPDClient(serviceClient.GetClientConn()), serviceClient.BuildGRPCTargetContext(ctx, true) } func (c *client) GetTSAsync(ctx context.Context) TSFuture { diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 33dda1ad282..9eca3200d48 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -46,6 +46,16 @@ const ( updateMemberBackOffBaseTime = 100 * time.Millisecond ) +// MemberHealthCheckInterval might be changed in the unit to shorten the testing time. +var MemberHealthCheckInterval = time.Second + +type apiKind int + +const ( + forwardAPIKind apiKind = iota + apiKindCount +) + type serviceType int const ( @@ -81,6 +91,10 @@ type ServiceDiscovery interface { // endpoints. Backup service endpoints are followers in a quorum-based cluster or // secondaries in a primary/secondary configured cluster. GetBackupAddrs() []string + // GetServiceClient tries to get the leader/primary ServiceClient. + // If the leader ServiceClient meets network problem, + // it returns a follower/secondary ServiceClient which can forward the request to leader. + GetServiceClient() ServiceClient // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given addr GetOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) // ScheduleCheckMemberChanged is used to trigger a check to see if there is any membership change @@ -134,12 +148,16 @@ type pdServiceClient struct { } func newPDServiceClient(addr, leaderAddr string, conn *grpc.ClientConn, isLeader bool) ServiceClient { - return &pdServiceClient{ + cli := &pdServiceClient{ addr: addr, conn: conn, isLeader: isLeader, leaderAddr: leaderAddr, } + if conn == nil { + cli.networkFailure.Store(true) + } + return cli } // GetAddress implements ServiceClient. @@ -150,7 +168,7 @@ func (c *pdServiceClient) GetAddress() string { return c.addr } -// BuildGRPCContext implements ServiceClient. +// BuildGRPCTargetContext implements ServiceClient. func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { if c == nil || c.isLeader { return ctx @@ -183,9 +201,11 @@ func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { } healthCli := healthpb.NewHealthClient(c.conn) resp, err := healthCli.Check(ctx, &healthpb.HealthCheckRequest{Service: ""}) - failpoint.Inject("unreachableNetwork1", func() { - resp = nil - err = status.New(codes.Unavailable, "unavailable").Err() + failpoint.Inject("unreachableNetwork1", func(val failpoint.Value) { + if val, ok := val.(string); (ok && val == c.GetAddress()) || !ok { + resp = nil + err = status.New(codes.Unavailable, "unavailable").Err() + } }) rpcErr, ok := status.FromError(err) if (ok && isNetworkError(rpcErr.Code())) || resp.GetStatus() != healthpb.HealthCheckResponse_SERVING { @@ -217,6 +237,10 @@ func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { type errFn func(pdErr *pdpb.Error) bool +func emptyErrorFn(pdErr *pdpb.Error) bool { + return false +} + func regionAPIErrorFn(pdErr *pdpb.Error) bool { return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND } @@ -243,6 +267,7 @@ func (c *pdServiceAPIClient) Available() bool { return c.ServiceClient.Available() && !c.unavailable.Load() } +// markAsAvailable is used to try to mark the client as available if unavailable status is expired. func (c *pdServiceAPIClient) markAsAvailable() { if !c.unavailable.Load() { return @@ -273,7 +298,7 @@ func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { // pdServiceBalancerNode is a balancer node for PD service. // It extends the pdServiceClient and adds additional fields for the next polling client in the chain. type pdServiceBalancerNode struct { - ServiceClient + *pdServiceAPIClient next *pdServiceBalancerNode } @@ -283,8 +308,14 @@ type pdServiceBalancer struct { mu sync.Mutex now *pdServiceBalancerNode totalNode int + errFn errFn } +func newPDServiceBalancer(fn errFn) *pdServiceBalancer { + return &pdServiceBalancer{ + errFn: fn, + } +} func (c *pdServiceBalancer) set(clients []ServiceClient) { c.mu.Lock() defer c.mu.Unlock() @@ -293,14 +324,14 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { } c.totalNode = len(clients) head := &pdServiceBalancerNode{ - ServiceClient: clients[0], + pdServiceAPIClient: newPDServiceAPIClient(clients[0], c.errFn).(*pdServiceAPIClient), } head.next = head last := head for i := 1; i < c.totalNode; i++ { next := &pdServiceBalancerNode{ - ServiceClient: clients[i], - next: head, + pdServiceAPIClient: newPDServiceAPIClient(clients[i], c.errFn).(*pdServiceAPIClient), + next: head, } head = next last.next = head @@ -308,6 +339,15 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { c.now = head } +func (c *pdServiceBalancer) check() { + c.mu.Lock() + defer c.mu.Unlock() + for i := 0; i < c.totalNode; i++ { + c.now.markAsAvailable() + c.next() + } +} + func (c *pdServiceBalancer) next() { c.now = c.now.next } @@ -352,9 +392,12 @@ type pdServiceDiscovery struct { urls atomic.Value // Store as []string // PD leader URL - leader atomic.Value // Store as string + leader atomic.Value // Store as pdServiceClient // PD follower URLs - followers atomic.Value // Store as []string + followers sync.Map // Store as map[string]pdServiceClient + apiCandidateNodes [apiKindCount]*pdServiceBalancer + // PD follower URLs. Only for tso. + followerAddresses atomic.Value // Store as []string clusterID uint64 // addr -> a gRPC connection @@ -402,6 +445,7 @@ func newPDServiceDiscovery( ctx: ctx, cancel: cancel, wg: wg, + apiCandidateNodes: [apiKindCount]*pdServiceBalancer{newPDServiceBalancer(emptyErrorFn)}, serviceModeUpdateCb: serviceModeUpdateCb, updateKeyspaceIDCb: updateKeyspaceIDCb, keyspaceID: keyspaceID, @@ -439,9 +483,10 @@ func (c *pdServiceDiscovery) Init() error { log.Warn("[pd] failed to check service mode and will check later", zap.Error(err)) } - c.wg.Add(2) + c.wg.Add(3) go c.updateMemberLoop() go c.updateServiceModeLoop() + go c.memberHealthCheckLoop() c.isInitialized = true return nil @@ -518,6 +563,46 @@ func (c *pdServiceDiscovery) updateServiceModeLoop() { } } } +func (c *pdServiceDiscovery) memberHealthCheckLoop() { + defer c.wg.Done() + + memberCheckLoopCtx, memberCheckLoopCancel := context.WithCancel(c.ctx) + defer memberCheckLoopCancel() + + ticker := time.NewTicker(MemberHealthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + c.checkLeaderHealth(memberCheckLoopCtx) + c.checkFollowerHealth(memberCheckLoopCtx) + } + } +} + +func (c *pdServiceDiscovery) checkLeaderHealth(ctx context.Context) { + ctx, cancel := context.WithTimeout(ctx, c.option.timeout) + defer cancel() + leader := c.getLeaderServiceClient() + leader.checkNetworkAvailable(ctx) +} + +func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { + c.followers.Range(func(key, value any) bool { + // To ensure that the leader's healthy check is not delayed, shorten the duration. + ctx, cancel := context.WithTimeout(ctx, MemberHealthCheckInterval/3) + defer cancel() + serviceClient := value.(*pdServiceClient) + serviceClient.checkNetworkAvailable(ctx) + return true + }) + for _, balancer := range c.apiCandidateNodes { + balancer.check() + } +} // Close releases all resources. func (c *pdServiceDiscovery) Close() { @@ -606,12 +691,45 @@ func (c *pdServiceDiscovery) GetServingAddr() string { return c.getLeaderAddr() } -// GetBackupAddrs gets the addresses of the current reachable and healthy followers -// in a quorum-based cluster. +// GetBackupAddrs gets the addresses of the current reachable followers +// in a quorum-based cluster. Used for tso currently. func (c *pdServiceDiscovery) GetBackupAddrs() []string { return c.getFollowerAddrs() } +// getLeaderServiceClient returns the leader ServiceClient. +func (c *pdServiceDiscovery) getLeaderServiceClient() *pdServiceClient { + leader := c.leader.Load() + if leader == nil { + return nil + } + return leader.(*pdServiceClient) +} + +// getServiceClientByKind returns ServiceClient of the specific kind. +func (c *pdServiceDiscovery) getServiceClientByKind(kind apiKind) ServiceClient { + client := c.apiCandidateNodes[kind].get() + if client == nil { + return nil + } + return client +} + +// GetServiceClient returns the leader/primary ServiceClient if it is healthy. +func (c *pdServiceDiscovery) GetServiceClient() ServiceClient { + leaderClient := c.getLeaderServiceClient() + if c.option.enableForwarding && !leaderClient.Available() { + if followerClient := c.getServiceClientByKind(forwardAPIKind); followerClient != nil { + log.Debug("[pd] use follower client", zap.String("addr", followerClient.GetAddress())) + return followerClient + } + } + if leaderClient == nil { + return nil + } + return leaderClient +} + // ScheduleCheckMemberChanged is used to check if there is any membership // change among the leader and the followers. func (c *pdServiceDiscovery) ScheduleCheckMemberChanged() { @@ -657,16 +775,12 @@ func (c *pdServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGlo // getLeaderAddr returns the leader address. func (c *pdServiceDiscovery) getLeaderAddr() string { - leaderAddr := c.leader.Load() - if leaderAddr == nil { - return "" - } - return leaderAddr.(string) + return c.getLeaderServiceClient().GetAddress() } // getFollowerAddrs returns the follower address. func (c *pdServiceDiscovery) getFollowerAddrs() []string { - followerAddrs := c.followers.Load() + followerAddrs := c.followerAddresses.Load() if followerAddrs == nil { return []string{} } @@ -764,8 +878,7 @@ func (c *pdServiceDiscovery) updateMember() error { } c.updateURLs(members.GetMembers()) - c.updateFollowers(members.GetMembers(), members.GetLeader()) - if err := c.switchLeader(members.GetLeader().GetClientUrls()); err != nil { + if err := c.updateServiceClient(members.GetMembers(), members.GetLeader()); err != nil { return err } @@ -837,42 +950,106 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls)) } -func (c *pdServiceDiscovery) switchLeader(addrs []string) error { +func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { // FIXME: How to safely compare leader urls? For now, only allows one client url. addr := addrs[0] - oldLeader := c.getLeaderAddr() - if addr == oldLeader { - return nil + oldLeader := c.getLeaderServiceClient() + if addr == oldLeader.GetAddress() && oldLeader.GetClientConn() != nil { + return false, nil } - if _, err := c.GetOrCreateGRPCConn(addr); err != nil { - log.Warn("[pd] failed to connect leader", zap.String("leader", addr), errs.ZapError(err)) + newConn, err := c.GetOrCreateGRPCConn(addr) + // If gRPC connect is created successfully or leader is new, still saves. + if addr != oldLeader.GetAddress() || newConn != nil { + // Set PD leader and Global TSO Allocator (which is also the PD leader) + leaderClient := newPDServiceClient(addr, addr, newConn, true) + c.leader.Store(leaderClient) } // Set PD leader and Global TSO Allocator (which is also the PD leader) c.leader.Store(addr) // Run callbacks if c.tsoGlobalAllocLeaderUpdatedCb != nil { if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil { - return err + return true, err } } for _, cb := range c.leaderSwitchedCbs { cb() } - log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader)) - return nil + log.Info("[pd] switch leader", zap.String("new-leader", addr), zap.String("old-leader", oldLeader.GetAddress())) + return true, err } -func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) { - var addrs []string +func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leader *pdpb.Member) (changed bool) { + followers := make(map[string]*pdServiceClient) + c.followers.Range(func(key, value any) bool { + followers[key.(string)] = value.(*pdServiceClient) + return true + }) + var followerAddrs []string for _, member := range members { if member.GetMemberId() != leader.GetMemberId() { if len(member.GetClientUrls()) > 0 { - addrs = append(addrs, member.GetClientUrls()...) + followerAddrs = append(followerAddrs, member.GetClientUrls()...) + + // FIXME: How to safely compare urls(also for leader)? For now, only allows one client url. + addr := member.GetClientUrls()[0] + if client, ok := c.followers.Load(addr); ok { + if client.(*pdServiceClient).GetClientConn() == nil { + conn, err := c.GetOrCreateGRPCConn(addr) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + continue + } + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + c.followers.Store(addr, follower) + changed = true + } + delete(followers, addr) + } else { + changed = true + conn, err := c.GetOrCreateGRPCConn(addr) + follower := newPDServiceClient(addr, leader.GetClientUrls()[0], conn, false) + if err != nil || conn == nil { + log.Warn("[pd] failed to connect follower", zap.String("follower", addr), errs.ZapError(err)) + } + c.followers.LoadOrStore(addr, follower) + } } } } - c.followers.Store(addrs) + if len(followers) > 0 { + changed = true + for key := range followers { + c.followers.Delete(key) + } + } + c.followerAddresses.Store(followerAddrs) + return +} + +func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader *pdpb.Member) error { + leaderChanged, err := c.switchLeader(leader.GetClientUrls()) + followerChanged := c.updateFollowers(members, leader) + // don't need to recreate balancer if no changess. + if !followerChanged && !leaderChanged { + return err + } + // If error is not nil, still updates candidates. + clients := make([]ServiceClient, 0) + c.followers.Range(func(_, value any) bool { + clients = append(clients, value.(*pdServiceClient)) + return true + }) + leaderClient := c.getLeaderServiceClient() + if leaderClient != nil { + clients = append(clients, leaderClient) + } + // create candidate services for all kinds of request. + for i := 0; i < int(apiKindCount); i++ { + c.apiCandidateNodes[i].set(clients) + } + return err } func (c *pdServiceDiscovery) switchTSOAllocatorLeaders(allocatorMap map[string]*pdpb.Member) error { diff --git a/client/tso_service_discovery.go b/client/tso_service_discovery.go index 5f14c406797..7caaacd0dfe 100644 --- a/client/tso_service_discovery.go +++ b/client/tso_service_discovery.go @@ -373,6 +373,11 @@ func (c *tsoServiceDiscovery) SetTSOGlobalServAddrUpdatedCallback(callback tsoGl c.globalAllocPrimariesUpdatedCb = callback } +// GetServiceClient implements ServiceDiscovery +func (c *tsoServiceDiscovery) GetServiceClient() ServiceClient { + return c.apiSvcDiscovery.GetServiceClient() +} + // getPrimaryAddr returns the primary address. func (c *tsoServiceDiscovery) getPrimaryAddr() string { c.keyspaceGroupSD.RLock() diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 1fd8d75dec4..e1e841342ad 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -518,18 +518,69 @@ func TestCustomTimeout(t *testing.T) { re.Less(time.Since(start), 2*time.Second) } -func TestGetRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) +type followerForwardAndHandleTestSuite struct { + suite.Suite + ctx context.Context + clean context.CancelFunc + + cluster *tests.TestCluster + endpoints []string + regionID uint64 +} + +func TestFollowerForwardAndHandleTestSuite(t *testing.T) { + suite.Run(t, new(followerForwardAndHandleTestSuite)) +} + +func (suite *followerForwardAndHandleTestSuite) SetupSuite() { + re := suite.Require() + suite.ctx, suite.clean = context.WithCancel(context.Background()) + pd.MemberHealthCheckInterval = 100 * time.Millisecond + cluster, err := tests.NewTestCluster(suite.ctx, 3) re.NoError(err) - defer cluster.Destroy() + suite.cluster = cluster + suite.endpoints = runServer(re, cluster) + cluster.WaitLeader() + leader := cluster.GetLeaderServer() + grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) + suite.regionID = regionIDAllocator.alloc() + testutil.Eventually(re, func() bool { + regionHeartbeat, err := grpcPDClient.RegionHeartbeat(suite.ctx) + re.NoError(err) + region := &metapb.Region{ + Id: suite.regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + Peers: peers, + } + req := &pdpb.RegionHeartbeatRequest{ + Header: newHeader(leader.GetServer()), + Region: region, + Leader: peers[0], + } + err = regionHeartbeat.Send(req) + re.NoError(err) + _, err = regionHeartbeat.Recv() + return err == nil + }) +} - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) +func (suite *followerForwardAndHandleTestSuite) TearDownTest() { +} + +func (suite *followerForwardAndHandleTestSuite) TearDownSuite() { + suite.cluster.Destroy() + suite.clean() +} + +func (suite *followerForwardAndHandleTestSuite) TestGetRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork1", "return(true)")) time.Sleep(200 * time.Millisecond) r, err := cli.GetRegion(context.Background(), []byte("a")) @@ -544,17 +595,11 @@ func TestGetRegionByFollowerForwarding(t *testing.T) { } // case 1: unreachable -> normal -func TestGetTsoByFollowerForwarding1(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding1() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -564,7 +609,7 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) @@ -575,17 +620,11 @@ func TestGetTsoByFollowerForwarding1(t *testing.T) { } // case 2: unreachable -> leader transfer -> normal -func TestGetTsoByFollowerForwarding2(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoByFollowerForwarding2() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - - endpoints := runServer(re, cluster) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) re.NoError(failpoint.Enable("github.com/tikv/pd/client/unreachableNetwork", "return(true)")) var lastTS uint64 @@ -595,13 +634,13 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) - re.NoError(cluster.GetLeaderServer().ResignLeader()) - re.NotEmpty(cluster.WaitLeader()) + re.NoError(suite.cluster.GetLeaderServer().ResignLeader()) + re.NotEmpty(suite.cluster.WaitLeader()) lastTS = checkTS(re, cli, lastTS) re.NoError(failpoint.Disable("github.com/tikv/pd/client/unreachableNetwork")) @@ -610,45 +649,18 @@ func TestGetTsoByFollowerForwarding2(t *testing.T) { } // case 3: network partition between client and follower A -> transfer leader to follower A -> normal -func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { - re := require.New(t) - ctx, cancel := context.WithCancel(context.Background()) +func (suite *followerForwardAndHandleTestSuite) TestGetTsoAndRegionByFollowerForwarding() { + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) defer cancel() - pd.LeaderHealthCheckInterval = 100 * time.Millisecond - cluster, err := tests.NewTestCluster(ctx, 3) - re.NoError(err) - defer cluster.Destroy() - endpoints := runServer(re, cluster) - re.NotEmpty(cluster.WaitLeader()) + cluster := suite.cluster leader := cluster.GetLeaderServer() - grpcPDClient := testutil.MustNewGrpcClient(re, leader.GetAddr()) - testutil.Eventually(re, func() bool { - regionHeartbeat, err := grpcPDClient.RegionHeartbeat(ctx) - re.NoError(err) - regionID := regionIDAllocator.alloc() - region := &metapb.Region{ - Id: regionID, - RegionEpoch: &metapb.RegionEpoch{ - ConfVer: 1, - Version: 1, - }, - Peers: peers, - } - req := &pdpb.RegionHeartbeatRequest{ - Header: newHeader(leader.GetServer()), - Region: region, - Leader: peers[0], - } - err = regionHeartbeat.Send(req) - re.NoError(err) - _, err = regionHeartbeat.Recv() - return err == nil - }) + follower := cluster.GetServer(cluster.GetFollower()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/grpcutil/unreachableNetwork2", fmt.Sprintf("return(\"%s\")", follower.GetAddr()))) - cli := setupCli(re, ctx, endpoints, pd.WithForwardingOption(true)) + cli := setupCli(re, ctx, suite.endpoints, pd.WithForwardingOption(true)) var lastTS uint64 testutil.Eventually(re, func() bool { physical, logical, err := cli.GetTS(context.TODO()) @@ -656,7 +668,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -672,7 +684,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) @@ -691,7 +703,7 @@ func TestGetTsoAndRegionByFollowerForwarding(t *testing.T) { lastTS = tsoutil.ComposeTS(physical, logical) return true } - t.Log(err) + suite.T().Log(err) return false }) lastTS = checkTS(re, cli, lastTS) From a2d7dfbb16a92459d356254bfbff98515ad6c7a4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Fri, 22 Dec 2023 17:14:31 +0800 Subject: [PATCH 13/16] use ServiceClient in pd discovery Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 9eca3200d48..7e549fb0326 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -965,8 +965,6 @@ func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { leaderClient := newPDServiceClient(addr, addr, newConn, true) c.leader.Store(leaderClient) } - // Set PD leader and Global TSO Allocator (which is also the PD leader) - c.leader.Store(addr) // Run callbacks if c.tsoGlobalAllocLeaderUpdatedCb != nil { if err := c.tsoGlobalAllocLeaderUpdatedCb(addr); err != nil { From b2c0b161310c4e62eed4ac32f603093e715a88c3 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Mon, 25 Dec 2023 09:19:06 +0800 Subject: [PATCH 14/16] address comment Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 7e549fb0326..f9bdf888b9f 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -187,7 +187,7 @@ func (c *pdServiceClient) IsConnectedToLeader() bool { return c.isLeader } -// NetworkAvailable implements ServiceClient. +// Available implements ServiceClient. func (c *pdServiceClient) Available() bool { if c == nil { return false From d5ca4c6a9e3496e33f064ed6ffd3637cdefd67f7 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Tue, 26 Dec 2023 17:02:23 +0800 Subject: [PATCH 15/16] merge master Signed-off-by: Cabinfever_B --- client/pd_service_discovery.go | 3 ++- pkg/utils/grpcutil/grpcutil.go | 4 ++-- server/grpc_service.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/client/pd_service_discovery.go b/client/pd_service_discovery.go index 9a163117583..efef2940adf 100644 --- a/client/pd_service_discovery.go +++ b/client/pd_service_discovery.go @@ -171,7 +171,7 @@ func (c *pdServiceClient) GetAddress() string { // BuildGRPCTargetContext implements ServiceClient. func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { - if c == nil || c.IsConnectedToLeader() { + if c == nil || c.isLeader { return ctx } if toLeader { @@ -959,6 +959,7 @@ func (c *pdServiceDiscovery) switchLeader(addrs []string) (bool, error) { if addr == oldLeader.GetAddress() && oldLeader.GetClientConn() != nil { return false, nil } + newConn, err := c.GetOrCreateGRPCConn(addr) // If gRPC connect is created successfully or leader is new, still saves. if addr != oldLeader.GetAddress() || newConn != nil { diff --git a/pkg/utils/grpcutil/grpcutil.go b/pkg/utils/grpcutil/grpcutil.go index d20b0eb3ee8..1bfb64868f3 100644 --- a/pkg/utils/grpcutil/grpcutil.go +++ b/pkg/utils/grpcutil/grpcutil.go @@ -184,8 +184,8 @@ func GetForwardedHost(ctx context.Context) string { return "" } -// GetEnableFollowerHandle returns the follower host in metadata. -func GetEnableFollowerHandle(ctx context.Context) bool { +// IsFollowerHandleEnabled returns the follower host in metadata. +func IsFollowerHandleEnabled(ctx context.Context) bool { md, ok := metadata.FromIncomingContext(ctx) if !ok { log.Debug("failed to get gRPC incoming metadata when checking follower handle is enabled") diff --git a/server/grpc_service.go b/server/grpc_service.go index b5ea4956ed6..014fca79ff3 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -2040,7 +2040,7 @@ func (s *GrpcServer) validateRoleInRequest(ctx context.Context, header *pdpb.Req if allowFollower == nil { return ErrNotLeader } - if !grpcutil.GetEnableFollowerHandle(ctx) { + if !grpcutil.IsFollowerHandleEnabled(ctx) { // TODO: change the error code return ErrFollowerHandlingNotAllowed } From c7423531bc257dda1895c9e047d50eb485e315c4 Mon Sep 17 00:00:00 2001 From: Cabinfever_B Date: Thu, 28 Dec 2023 15:23:51 +0800 Subject: [PATCH 16/16] address comment Signed-off-by: Cabinfever_B --- server/grpc_service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index 6d79d7a98bf..4174570379f 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1406,7 +1406,7 @@ func (s *GrpcServer) GetRegion(ctx context.Context, request *pdpb.GetRegionReque var region *core.RegionInfo if *followerHandle { rc = s.cluster - if !s.cluster.GetRegionSyncer().IsRunning() { + if !rc.GetRegionSyncer().IsRunning() { return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil } region = rc.GetRegionByKey(request.GetRegionKey()) @@ -1468,7 +1468,7 @@ func (s *GrpcServer) GetPrevRegion(ctx context.Context, request *pdpb.GetRegionR if *followerHandle { // no need to check running status rc = s.cluster - if !s.cluster.GetRegionSyncer().IsRunning() { + if !rc.GetRegionSyncer().IsRunning() { return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil } } else { @@ -1526,7 +1526,7 @@ func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionB var rc *cluster.RaftCluster if *followerHandle { rc = s.cluster - if !s.cluster.GetRegionSyncer().IsRunning() { + if !rc.GetRegionSyncer().IsRunning() { return &pdpb.GetRegionResponse{Header: s.regionNotFound()}, nil } } else { @@ -1587,7 +1587,7 @@ func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsR var rc *cluster.RaftCluster if *followerHandle { rc = s.cluster - if !s.cluster.GetRegionSyncer().IsRunning() { + if !rc.GetRegionSyncer().IsRunning() { return &pdpb.ScanRegionsResponse{Header: s.regionNotFound()}, nil } } else { @@ -2253,7 +2253,7 @@ func (s *GrpcServer) validateRequest(header *pdpb.RequestHeader) error { return s.validateRoleInRequest(context.TODO(), header, nil) } -// validateRoleRequest checks if Server is leader when disallow follower-handle and clusterID is matched. +// validateRoleInRequest checks if Server is leader when disallow follower-handle and clusterID is matched. // TODO: Call it in gRPC interceptor. func (s *GrpcServer) validateRoleInRequest(ctx context.Context, header *pdpb.RequestHeader, allowFollower *bool) error { if s.IsClosed() {