Address test flakiness with injectAsymmetricDrops (#35)

* Bump up log level ERROR in SharedResources * Log decide view change in MembershipService * Reduce logging in Retries implementation * Reduce logging in Retries implementation * Increase test maxTries * Remove fast FD timeouts Signed-off-by: Lalith Suresh <[email protected]>
lalithsuresh · Mar 31, 2021 · a5a9bf2 · a5a9bf2
1 parent d6e228e
commit a5a9bf2
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 6 deletions.
diff --git a/rapid/src/main/java/com/vrg/rapid/FastPaxos.java b/rapid/src/main/java/com/vrg/rapid/FastPaxos.java
@@ -145,7 +145,7 @@ private void handleFastRoundProposal(final FastRoundPhase2bMessage proposalMessa
         final int F = (int) Math.floor((membershipSize - 1) / 4.0); // Fast Paxos resiliency.
         if (votesReceived.size() >= membershipSize - F) {
             if (count >= membershipSize - F) {
-                LOG.trace("Decided on a view change: {}", proposalMessage.getEndpointsList());
+                LOG.trace("Decided on a view change: {}", Utils.loggable(proposalMessage.getEndpointsList()));
                 // We have a successful proposal. Consume it.
                 onDecidedWrapped.accept(proposalMessage.getEndpointsList());
             } else {

diff --git a/rapid/src/main/java/com/vrg/rapid/MembershipService.java b/rapid/src/main/java/com/vrg/rapid/MembershipService.java
@@ -383,6 +383,9 @@ private ListenableFuture<RapidResponse> handleLeaveMessage(final RapidRequest re
      * and any node that is currently in the membership list will be removed from it.
      */
     private void decideViewChange(final List<Endpoint> proposal) {
+        LOG.info("Decide view change called in current configuration {} ({} nodes), for proposal {}",
+                membershipView.getCurrentConfigurationId(), membershipView.getMembershipSize(),
+                Utils.loggable(proposal));
         // The first step is to disable our failure detectors in anticipation of new ones to be created.
         cancelFailureDetectorJobs();
 

diff --git a/rapid/src/test/java/com/vrg/rapid/ClusterTest.java b/rapid/src/test/java/com/vrg/rapid/ClusterTest.java
@@ -341,7 +341,6 @@ public void failTenRandomNodes() throws IOException, InterruptedException {
      */
     @Test
     public void injectAsymmetricDrops() throws IOException, InterruptedException {
-        useFastFailureDetectionTimeouts();
         final int numNodes = 50;
         final int numFailingNodes = 10;
         final Endpoint seedEndpoint = Utils.hostFromParts("127.0.0.1", basePort);
@@ -353,7 +352,7 @@ public void injectAsymmetricDrops() throws IOException, InterruptedException {
         // we may have less than numFailedNodes entries in the set
         failedNodes.forEach(host -> dropFirstNAtServer(host, 100, RapidRequest.ContentCase.PROBEMESSAGE));
         createCluster(numNodes, seedEndpoint);
-        waitAndVerifyAgreement(numNodes - failedNodes.size(), 10, 1000);
+        waitAndVerifyAgreement(numNodes - failedNodes.size(), 20, 1500);
         verifyNumClusterInstances(numNodes);
     }
 

diff --git a/rapid/src/test/resources/log4j.properties b/rapid/src/test/resources/log4j.properties
@@ -11,11 +11,13 @@
 # permissions and limitations under the License.
 #
 
-log4j.rootLogger=ERROR, STDOUT
+log4j.rootLogger=OFF, STDOUT
 log4j.logger.com.vrg.rapid.Cluster=ERROR
-log4j.logger.com.vrg.rapid.MembershipService=ERROR
-log4j.logger.com.vrg.rapid.Paxos=ERROR
+log4j.logger.com.vrg.rapid.MembershipService=INFO
+log4j.logger.com.vrg.rapid.Paxos=INFO
 log4j.logger.com.vrg.rapid.FastPaxos=ERROR
+log4j.logger.com.vrg.rapid.SharedResources=ERROR
+log4j.logger.com.vrg.rapid.messaging.impl.Retries=OFF
 log4j.logger.io.grpc.internal.ManagedChannelOrphanWrapper=OFF
 log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
 log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout