Skip to content
This repository has been archived by the owner on Apr 19, 2024. It is now read-only.

fix: mutex deadlocks in PeerClient #223

Merged
merged 23 commits into from
Feb 28, 2024
Merged

fix: mutex deadlocks in PeerClient #223

Changes from 1 commit
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
498acea
fix race issuers in PeerClient
miparnisari Feb 20, 2024
5c0e54e
undo changes in comments
miparnisari Feb 20, 2024
2b940a8
more fixes
miparnisari Feb 20, 2024
747d3cd
final fixes
miparnisari Feb 20, 2024
33098ee
Merge remote-tracking branch 'origin/master' into fix-peer-client
miparnisari Feb 25, 2024
53a1627
do not create 1 connection every time
miparnisari Feb 25, 2024
4b701fb
add comment
miparnisari Feb 25, 2024
0eccd37
remove comment
miparnisari Feb 25, 2024
42c8e72
fix lint
miparnisari Feb 25, 2024
27e91fd
add ctx back to shutdown
miparnisari Feb 25, 2024
dc64a09
move conn.Close out of goroutine
miparnisari Feb 25, 2024
045f826
fix lint
miparnisari Feb 25, 2024
9ce0eb6
oops
miparnisari Feb 25, 2024
e7eabb0
make it blocking
miparnisari Feb 25, 2024
f132176
put cap of 1 second to NewPeerClient connect
miparnisari Feb 25, 2024
f6102d9
undo put cap of 1 second to NewPeerClient connect
miparnisari Feb 26, 2024
e37a3fe
Merge remote-tracking branch 'origin/master' into fix-peer-client
miparnisari Feb 26, 2024
b218cbe
fix another leak
miparnisari Feb 26, 2024
fcb7b15
on peer shutdown, clear all errors
miparnisari Feb 28, 2024
6a53c43
add time.sleep to end of TestHealthCheck
miparnisari Feb 28, 2024
e4d42b7
use testutil.UntilPass
miparnisari Feb 28, 2024
795af06
check health of every instance
miparnisari Feb 28, 2024
db541c0
use cluster.GetAllPeers
miparnisari Feb 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 25 additions & 21 deletions functional_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,28 +36,29 @@ import (
"github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
json "google.golang.org/protobuf/encoding/protojson"
)

var allPeers = []guber.PeerInfo{
Baliedge marked this conversation as resolved.
Show resolved Hide resolved
{GRPCAddress: "127.0.0.1:9990", HTTPAddress: "127.0.0.1:9980", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9991", HTTPAddress: "127.0.0.1:9981", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9992", HTTPAddress: "127.0.0.1:9982", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9993", HTTPAddress: "127.0.0.1:9983", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9994", HTTPAddress: "127.0.0.1:9984", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9995", HTTPAddress: "127.0.0.1:9985", DataCenter: cluster.DataCenterNone},

// DataCenterOne
{GRPCAddress: "127.0.0.1:9890", HTTPAddress: "127.0.0.1:9880", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9891", HTTPAddress: "127.0.0.1:9881", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9892", HTTPAddress: "127.0.0.1:9882", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9893", HTTPAddress: "127.0.0.1:9883", DataCenter: cluster.DataCenterOne},
}

// Setup and shutdown the mock gubernator cluster for the entire test suite
func TestMain(m *testing.M) {
if err := cluster.StartWith([]guber.PeerInfo{
{GRPCAddress: "127.0.0.1:9990", HTTPAddress: "127.0.0.1:9980", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9991", HTTPAddress: "127.0.0.1:9981", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9992", HTTPAddress: "127.0.0.1:9982", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9993", HTTPAddress: "127.0.0.1:9983", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9994", HTTPAddress: "127.0.0.1:9984", DataCenter: cluster.DataCenterNone},
{GRPCAddress: "127.0.0.1:9995", HTTPAddress: "127.0.0.1:9985", DataCenter: cluster.DataCenterNone},

// DataCenterOne
{GRPCAddress: "127.0.0.1:9890", HTTPAddress: "127.0.0.1:9880", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9891", HTTPAddress: "127.0.0.1:9881", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9892", HTTPAddress: "127.0.0.1:9882", DataCenter: cluster.DataCenterOne},
{GRPCAddress: "127.0.0.1:9893", HTTPAddress: "127.0.0.1:9883", DataCenter: cluster.DataCenterOne},
}); err != nil {
if err := cluster.StartWith(allPeers); err != nil {
fmt.Println(err)
os.Exit(1)
}
Expand Down Expand Up @@ -1619,12 +1620,15 @@ func TestHealthCheck(t *testing.T) {
defer cancel()
require.NoError(t, cluster.Restart(ctx))

// wait for instances to come back online
testutil.UntilPass(t, 10, clock.Millisecond*300, func(t testutil.TestingT) {
healthResp, err = client.HealthCheck(context.Background(), &guber.HealthCheckReq{})
assert.Nil(t, err)
assert.Equal(t, "healthy", healthResp.GetStatus())
})
// wait for every peer instance to come back online
for _, peer := range allPeers {
peerClient, err := guber.DialV1Server(peer.GRPCAddress, nil)
require.NoError(t, err)
testutil.UntilPass(t, 10, clock.Millisecond*300, func(t testutil.TestingT) {
healthResp, err = peerClient.HealthCheck(context.Background(), &guber.HealthCheckReq{})
assert.Equal(t, "healthy", healthResp.GetStatus())
})
}
}

func TestLeakyBucketDivBug(t *testing.T) {
Expand Down
Loading