You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2017/12/20 17:54:40 UTC
[37/50] [abbrv] helix git commit: Fix integration test slowness issue.
Fix integration test slowness issue.
Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/ecdbaa13
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/ecdbaa13
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/ecdbaa13
Branch: refs/heads/master
Commit: ecdbaa13f013aa62bfa5cbc98938d054afccda1f
Parents: b75a841
Author: Lei Xia <lx...@linkedin.com>
Authored: Tue Sep 26 13:27:45 2017 -0700
Committer: Junkai Xue <jx...@linkedin.com>
Committed: Mon Nov 6 17:08:38 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/helix/task/TaskDriver.java | 1 -
.../integration/TestBucketizedResource.java | 61 ++++++++------------
.../apache/helix/integration/TestDisable.java | 2 -
.../messaging/TestBatchMessageWrapper.java | 2 -
.../task/TestTaskRebalancerStopResume.java | 7 ++-
.../integration/task/TestTaskThreadLeak.java | 4 +-
6 files changed, 30 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java b/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
index a34f0ce..40a148a 100644
--- a/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
+++ b/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
@@ -751,7 +751,6 @@ public class TaskDriver {
jobName = jobName.substring(workflowName.length() + 1);
workflowName = ctx.getLastScheduledSingleWorkflow();
}
- jobName = TaskUtil.getNamespacedJobName(workflowName, jobName);
Set<TaskState> allowedStates = new HashSet<>(Arrays.asList(states));
// Wait for state
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java b/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
index 22a28e4..5e678d0 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
@@ -38,8 +38,9 @@ import org.apache.helix.manager.zk.ZKHelixDataAccessor;
import org.apache.helix.model.ExternalView;
import org.apache.helix.model.IdealState;
import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
import org.apache.helix.tools.ClusterStateVerifier.MasterNbInExtViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
import org.apache.helix.tools.DefaultIdealStateCalculator;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -103,10 +104,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
.verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
Assert.assertTrue(result);
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ HelixClusterVerifier _clusterVerifier =
+ new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(_clusterVerifier.verify());
ExternalView ev = accessor.getProperty(evKey);
int v1 = ev.getRecord().getVersion();
@@ -114,10 +114,7 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
_gSetupTool.getClusterManagementTool().enableInstance(clusterName,
participants[0].getInstanceName(), false);
// wait for change in EV
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ Assert.assertTrue(_clusterVerifier.verify());
// read the version in EV
ev = accessor.getProperty(evKey);
@@ -161,20 +158,16 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
participants[i].syncStart();
}
- boolean result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ HelixClusterVerifier _clusterVerifier =
+ new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(_clusterVerifier.verify());
// bounce
participants[0].syncStop();
participants[0] = new MockParticipantManager(ZK_ADDR, clusterName, instanceNames.get(0));
participants[0].syncStart();
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ Assert.assertTrue(_clusterVerifier.verify());
// make sure participants[0]'s current state is bucketzied correctly during carryover
String path =
@@ -186,21 +179,17 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
// disable the bucketize resource
HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
admin.enableResource(clusterName, dbName, false);
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+
+ Assert.assertTrue(_clusterVerifier.verify());
// drop the bucketize resource
_gSetupTool.dropResourceFromCluster(clusterName, dbName);
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+
+ Assert.assertTrue(_clusterVerifier.verify());
// make sure external-view is cleaned up
path = keyBuilder.externalView(dbName).getPath();
- result = _baseAccessor.exists(path, 0);
+ boolean result = _baseAccessor.exists(path, 0);
Assert.assertFalse(result);
// clean up
@@ -252,10 +241,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
participants[i].syncStart();
}
- boolean result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ HelixClusterVerifier _clusterVerifier =
+ new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+ Assert.assertTrue(_clusterVerifier.verify());
// add an external view listener
TestExternalViewListener listener = new TestExternalViewListener();
@@ -263,10 +251,8 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
// remove "TestDB0"
_gSetupTool.dropResourceFromCluster(clusterName, dbName);
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+ Assert.assertTrue(_clusterVerifier.verify());
+
// wait callback to finish
int waitTime =0;
do {
@@ -290,10 +276,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
idealState.setReplicas(Integer.toString(r));
accessor.setProperty(keyBuilder.idealStates(newDbName), idealState);
- result =
- ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
- clusterName));
- Assert.assertTrue(result);
+
+ Assert.assertTrue(_clusterVerifier.verify());
+
Thread.sleep(200);
Assert.assertTrue(listener.cbCnt > 0);
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java b/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
index 3d9a0b0..657dbc9 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
@@ -85,8 +85,6 @@ public class TestDisable extends ZkIntegrationTestBase {
participants[i].syncStart();
}
- Thread.sleep(100000000);
-
boolean result =
ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
clusterName));
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java b/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
index 579acb1..1a9b65a 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
@@ -34,8 +34,6 @@ import org.apache.helix.messaging.handling.BatchMessageWrapper;
import org.apache.helix.mock.participant.MockMSModelFactory;
import org.apache.helix.model.IdealState;
import org.apache.helix.model.Message;
-import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
import org.testng.Assert;
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
index 54a7fe9..8c03eed 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
@@ -256,7 +256,9 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
LOG.info("Flusing job-queue: " + queueName);
_driver.flushQueue(queueName);
- TimeUnit.MILLISECONDS.sleep(1000);
+ // TODO: Use TestHelper.verify() instead of waiting here.
+ TimeUnit.MILLISECONDS.sleep(5000);
+
// verify the cleanup
for (int i = 0; i < currentJobNames.size(); i++) {
String namedSpaceJobName = String.format("%s_%s", queueName, currentJobNames.get(i));
@@ -411,8 +413,7 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
// Create a queue
System.out.println("START " + queueName + " at " + new Date(System.currentTimeMillis()));
- WorkflowConfig wfCfg = new WorkflowConfig.Builder(queueName).setExpiry(2, TimeUnit.MINUTES)
- .setScheduleConfig(ScheduleConfig.recurringFromNow(TimeUnit.MINUTES, 1)).build();
+ WorkflowConfig wfCfg = new WorkflowConfig.Builder(queueName).setExpiry(2, TimeUnit.MINUTES).build();
JobQueue qCfg = new JobQueue.Builder(queueName).fromMap(wfCfg.getResourceConfigMap()).build();
_driver.createQueue(qCfg);
http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
index b5f4d86..188e61c 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
@@ -65,7 +65,9 @@ public class TestTaskThreadLeak extends TaskTestBase {
.setWorkflowConfig(new WorkflowConfig.Builder(queueName).setParallelJobs(10).build());
_driver.start(queueBuilder.build());
- _driver.pollForJobState(queueName, lastJob, TaskState.COMPLETED);
+
+ String nameSpacedJob = TaskUtil.getNamespacedJobName(queueName, lastJob);
+ _driver.pollForJobState(queueName, nameSpacedJob, TaskState.COMPLETED);
Set<Thread> threads = Thread.getAllStackTraces().keySet();
Set<Thread> taskThreads = new HashSet<>();