Skip to content

Commit

Permalink
Skipping flaky tests (celo-org#1848)
Browse files Browse the repository at this point in the history
This PR comments/skips tests, builds and benchmarks that are known to be flaky.

Unit tests:
- TestAnnounceGossipQueryMsg celo-org#1849
- TestEthClient/TestCallContract celo-org#1850
- TestGethClient/TestAccessList celo-org#1851

Benchmarks:
- BenchmarkNet1000Txs celo-org#1852

Circleci builds:
- end-to-end-cip35-eth-compatibility-test celo-org#1853


While creating this PR, the istanbul-e2e-coverage failed once, then worked when it was re-run. I'm not skipping this one right now since I'd like to test it a bit more. Most likely it is because the parent commit has flaky tests, which are run for coverage report.

codecov/project is failing because we are removing a few tests. It is expected.
  • Loading branch information
hbandura authored Feb 18, 2022
1 parent 300e836 commit c78a51f
Show file tree
Hide file tree
Showing 6 changed files with 19 additions and 12 deletions.
9 changes: 5 additions & 4 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -545,10 +545,11 @@ workflows:
requires:
- checkout-monorepo
- build-geth
- end-to-end-cip35-eth-compatibility-test:
requires:
- checkout-monorepo
- build-geth
# Flaky!
# - end-to-end-cip35-eth-compatibility-test:
# requires:
# - checkout-monorepo
# - build-geth
- end-to-end-replica-test:
requires:
- checkout-monorepo
Expand Down
1 change: 1 addition & 0 deletions consensus/istanbul/backend/announce_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
// This test function will test the announce message generator and handler.
// It will also test the gossip query generator and handler.
func TestAnnounceGossipQueryMsg(t *testing.T) {
t.Skip() // Flaky
// Create three backends
numValidators := 3
genesisCfg, nodeKeys := getGenesisAndKeys(numValidators, true)
Expand Down
1 change: 1 addition & 0 deletions e2e_test/e2e_bench_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func BenchmarkNet100EmptyBlocks(b *testing.B) {
}

func BenchmarkNet1000Txs(b *testing.B) {
b.Skip() // Flaky
// Seed the random number generator so that the generated numbers are
// different on each run.
rand.Seed(time.Now().UnixNano())
Expand Down
4 changes: 2 additions & 2 deletions e2e_test/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ func TestRPCDynamicTxGasPriceWithState(t *testing.T) {
network, shutdown, err := test.NewNetwork(ac, gc, ec)
require.NoError(t, err)
defer shutdown()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*90)
defer cancel()

accounts := test.Accounts(ac.DeveloperAccounts(), gc.ChainConfig())
Expand Down Expand Up @@ -306,7 +306,7 @@ func TestRPCDynamicTxGasPriceWithoutState(t *testing.T) {
network, shutdown, err := test.NewNetwork(ac, gc, ec)
require.NoError(t, err)
defer shutdown()
ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
ctx, cancel := context.WithTimeout(context.Background(), time.Second*90)
defer cancel()

accounts := test.Accounts(ac.DeveloperAccounts(), gc.ChainConfig())
Expand Down
8 changes: 5 additions & 3 deletions ethclient/ethclient_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -260,9 +260,10 @@ func TestEthClient(t *testing.T) {
"TestStatusFunctions": {
func(t *testing.T) { testStatusFunctions(t, client) },
},
"TestCallContract": {
func(t *testing.T) { testCallContract(t, client) },
},
// Flaky
// "TestCallContract": {
// func(t *testing.T) { testCallContract(t, client) },
// },
"TestAtFunctions": {
func(t *testing.T) { testAtFunctions(t, client) },
},
Expand Down Expand Up @@ -476,6 +477,7 @@ func testStatusFunctions(t *testing.T, client *rpc.Client) {
}
}

// nolint:deadcode
func testCallContract(t *testing.T, client *rpc.Client) {
ec := NewClient(client)

Expand Down
8 changes: 5 additions & 3 deletions ethclient/gethclient/gethclient_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,10 @@ func TestGethClient(t *testing.T) {
tests := map[string]struct {
test func(t *testing.T)
}{
"TestAccessList": {
func(t *testing.T) { testAccessList(t, client) },
},
// Flaky
// "TestAccessList": {
// func(t *testing.T) { testAccessList(t, client) },
// },
"TestGetProof": {
func(t *testing.T) { testGetProof(t, client) },
},
Expand Down Expand Up @@ -130,6 +131,7 @@ func TestGethClient(t *testing.T) {
}
}

// nolint:deadcode
func testAccessList(t *testing.T, client *rpc.Client) {
ec := New(client)
// Test transfer
Expand Down

0 comments on commit c78a51f

Please sign in to comment.