You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by hu...@apache.org on 2020/04/01 22:47:29 UTC
[helix] 34/49: Add rerunFailingTestsCount config to surefire-plugin
(#865)
This is an automated email from the ASF dual-hosted git repository.
hulee pushed a commit to branch zooscalability
in repository https://gitbox.apache.org/repos/asf/helix.git
commit 379682720c71dcbeea15d1d806113f9f045d6581
Author: Hunter Lee <hu...@linkedin.com>
AuthorDate: Thu Mar 5 18:55:27 2020 -0800
Add rerunFailingTestsCount config to surefire-plugin (#865)
It was observed that if build fails (if there is a test failure), then not all of the test goals are executed. There are currently two goals: default-test (single ZK) and multi-zk. If default-test has any test failures, we won't ever see multi-zk get executed, and this is a problem because we don't get to run the test suite in a multi-zk setup.
This config change is a workaround for this - we allow failing tests to be retried up to 3 times to make them pass. If they still fail, we will consider them as flaky tests and have to fix them moving forward.
This PR also address minor comments and fixes a test.
---
.../helix/integration/TestEnableCompression.java | 53 +++++++++++++---------
pom.xml | 6 +++
.../zookeeper/util/HttpRoutingDataReader.java | 6 +--
3 files changed, 40 insertions(+), 25 deletions(-)
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java b/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
index 39e412c..f1da2e6 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestEnableCompression.java
@@ -1,25 +1,5 @@
package org.apache.helix.integration;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.List;
-
-import org.apache.helix.PropertyPathBuilder;
-import org.apache.helix.TestHelper;
-import org.apache.helix.common.ZkTestBase;
-import org.apache.helix.integration.manager.ClusterControllerManager;
-import org.apache.helix.integration.manager.MockParticipantManager;
-import org.apache.helix.zookeeper.api.client.HelixZkClient;
-import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
-import org.apache.helix.model.IdealState;
-import org.apache.helix.model.builder.CustomModeISBuilder;
-import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
-import org.apache.helix.util.GZipCompressionUtil;
-import org.apache.helix.zookeeper.zkclient.serialize.BytesPushThroughSerializer;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -38,6 +18,29 @@ import org.testng.annotations.Test;
* specific language governing permissions and limitations
* under the License.
*/
+
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.helix.PropertyPathBuilder;
+import org.apache.helix.TestHelper;
+import org.apache.helix.common.ZkTestBase;
+import org.apache.helix.integration.manager.ClusterControllerManager;
+import org.apache.helix.integration.manager.MockParticipantManager;
+import org.apache.helix.model.IdealState;
+import org.apache.helix.model.builder.CustomModeISBuilder;
+import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.util.GZipCompressionUtil;
+import org.apache.helix.zookeeper.api.client.HelixZkClient;
+import org.apache.helix.zookeeper.impl.factory.SharedZkClientFactory;
+import org.apache.helix.zookeeper.zkclient.serialize.BytesPushThroughSerializer;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
/**
* Test controller, spectator and participant roles when compression is enabled.
* Compression can be enabled for a specific resource by setting enableCompression=true in the
@@ -66,6 +69,8 @@ public class TestEnableCompression extends ZkTestBase {
List<String> instancesInCluster =
_gSetupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
String resourceName = "TestResource";
+ Set<String> expectedResources = new HashSet<>();
+ expectedResources.add(resourceName);
CustomModeISBuilder customModeISBuilder = new CustomModeISBuilder(resourceName);
int numPartitions = 10000;
@@ -96,15 +101,19 @@ public class TestEnableCompression extends ZkTestBase {
new ClusterControllerManager(ZK_ADDR, clusterName, "controller_0");
controller.syncStart();
+ Set<String> expectedLiveInstances = new HashSet<>();
// start participants
for (int i = 0; i < 5; i++) {
String instanceName = "localhost_" + (12918 + i);
participants[i] = new MockParticipantManager(ZK_ADDR, clusterName, instanceName);
participants[i].syncStart();
+ expectedLiveInstances.add(instanceName);
}
- boolean result = ClusterStateVerifier
- .verifyByPolling(new BestPossAndExtViewZkVerifier(ZK_ADDR, clusterName), 120000L);
+ BestPossibleExternalViewVerifier verifier =
+ new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR)
+ .setExpectLiveInstances(expectedLiveInstances).setResources(expectedResources).build();
+ boolean result = verifier.verify(120000L);
Assert.assertTrue(result);
List<String> compressedPaths = new ArrayList<>();
diff --git a/pom.xml b/pom.xml
index 1681850..e8c9bd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -643,6 +643,10 @@ under the License.
</goals>
<id>default-test</id>
<phase>test</phase>
+ <configuration>
+ <rerunFailingTestsCount>3</rerunFailingTestsCount>
+ <skipAfterFailureCount>10</skipAfterFailureCount>
+ </configuration>
</execution>
<execution>
<goals>
@@ -655,6 +659,8 @@ under the License.
<multiZk>true</multiZk>
<numZk>3</numZk>
</systemPropertyVariables>
+ <rerunFailingTestsCount>3</rerunFailingTestsCount>
+ <skipAfterFailureCount>10</skipAfterFailureCount>
</configuration>
</execution>
</executions>
diff --git a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
index b4c1f9c..f2f907a 100644
--- a/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
+++ b/zookeeper-api/src/main/java/org/apache/helix/zookeeper/util/HttpRoutingDataReader.java
@@ -21,9 +21,9 @@ package org.apache.helix.zookeeper.util;
import java.io.IOException;
import java.util.Collections;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -49,10 +49,10 @@ public class HttpRoutingDataReader {
/** Double-checked locking requires that the following fields be volatile */
// The following map stands for (MSDS endpoint, Raw Routing Data)
private static volatile Map<String, Map<String, List<String>>> _rawRoutingDataMap =
- new HashMap<>();
+ new ConcurrentHashMap<>();
// The following map stands for (MSDS endpoint, MetadataStoreRoutingData)
private static volatile Map<String, MetadataStoreRoutingData> _metadataStoreRoutingDataMap =
- new HashMap<>();
+ new ConcurrentHashMap<>();
/**
* This class is a Singleton.