You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@helix.apache.org by jx...@apache.org on 2017/12/20 17:54:40 UTC

[37/50] [abbrv] helix git commit: Fix integration test slowness issue.

Fix integration test slowness issue.


Project: http://git-wip-us.apache.org/repos/asf/helix/repo
Commit: http://git-wip-us.apache.org/repos/asf/helix/commit/ecdbaa13
Tree: http://git-wip-us.apache.org/repos/asf/helix/tree/ecdbaa13
Diff: http://git-wip-us.apache.org/repos/asf/helix/diff/ecdbaa13

Branch: refs/heads/master
Commit: ecdbaa13f013aa62bfa5cbc98938d054afccda1f
Parents: b75a841
Author: Lei Xia <lx...@linkedin.com>
Authored: Tue Sep 26 13:27:45 2017 -0700
Committer: Junkai Xue <jx...@linkedin.com>
Committed: Mon Nov 6 17:08:38 2017 -0800

----------------------------------------------------------------------
 .../java/org/apache/helix/task/TaskDriver.java  |  1 -
 .../integration/TestBucketizedResource.java     | 61 ++++++++------------
 .../apache/helix/integration/TestDisable.java   |  2 -
 .../messaging/TestBatchMessageWrapper.java      |  2 -
 .../task/TestTaskRebalancerStopResume.java      |  7 ++-
 .../integration/task/TestTaskThreadLeak.java    |  4 +-
 6 files changed, 30 insertions(+), 47 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
----------------------------------------------------------------------
diff --git a/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java b/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
index a34f0ce..40a148a 100644
--- a/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
+++ b/helix-core/src/main/java/org/apache/helix/task/TaskDriver.java
@@ -751,7 +751,6 @@ public class TaskDriver {
       jobName = jobName.substring(workflowName.length() + 1);
       workflowName = ctx.getLastScheduledSingleWorkflow();
     }
-    jobName = TaskUtil.getNamespacedJobName(workflowName, jobName);
 
     Set<TaskState> allowedStates = new HashSet<>(Arrays.asList(states));
     // Wait for state

http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java b/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
index 22a28e4..5e678d0 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestBucketizedResource.java
@@ -38,8 +38,9 @@ import org.apache.helix.manager.zk.ZKHelixDataAccessor;
 import org.apache.helix.model.ExternalView;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
 import org.apache.helix.tools.ClusterStateVerifier.MasterNbInExtViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
+import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
 import org.apache.helix.tools.DefaultIdealStateCalculator;
 import org.testng.Assert;
 import org.testng.annotations.Test;
@@ -103,10 +104,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
             .verifyByZkCallback(new MasterNbInExtViewVerifier(ZK_ADDR, clusterName));
     Assert.assertTrue(result);
 
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    HelixClusterVerifier _clusterVerifier =
+        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+    Assert.assertTrue(_clusterVerifier.verify());
 
     ExternalView ev = accessor.getProperty(evKey);
     int v1 = ev.getRecord().getVersion();
@@ -114,10 +114,7 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
     _gSetupTool.getClusterManagementTool().enableInstance(clusterName,
         participants[0].getInstanceName(), false);
     // wait for change in EV
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // read the version in EV
     ev = accessor.getProperty(evKey);
@@ -161,20 +158,16 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
       participants[i].syncStart();
     }
 
-    boolean result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    HelixClusterVerifier _clusterVerifier =
+        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // bounce
     participants[0].syncStop();
     participants[0] = new MockParticipantManager(ZK_ADDR, clusterName, instanceNames.get(0));
     participants[0].syncStart();
 
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // make sure participants[0]'s current state is bucketzied correctly during carryover
     String path =
@@ -186,21 +179,17 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
     // disable the bucketize resource
     HelixAdmin admin = new ZKHelixAdmin(_gZkClient);
     admin.enableResource(clusterName, dbName, false);
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // drop the bucketize resource
     _gSetupTool.dropResourceFromCluster(clusterName, dbName);
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // make sure external-view is cleaned up
     path = keyBuilder.externalView(dbName).getPath();
-    result = _baseAccessor.exists(path, 0);
+    boolean result = _baseAccessor.exists(path, 0);
     Assert.assertFalse(result);
 
     // clean up
@@ -252,10 +241,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
       participants[i].syncStart();
     }
 
-    boolean result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    HelixClusterVerifier _clusterVerifier =
+        new BestPossibleExternalViewVerifier.Builder(clusterName).setZkAddr(ZK_ADDR).build();
+    Assert.assertTrue(_clusterVerifier.verify());
 
     // add an external view listener
     TestExternalViewListener listener = new TestExternalViewListener();
@@ -263,10 +251,8 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
 
     // remove "TestDB0"
     _gSetupTool.dropResourceFromCluster(clusterName, dbName);
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+    Assert.assertTrue(_clusterVerifier.verify());
+
     // wait callback to finish
     int waitTime =0;
     do {
@@ -290,10 +276,9 @@ public class TestBucketizedResource extends ZkIntegrationTestBase {
     idealState.setRebalanceMode(IdealState.RebalanceMode.CUSTOMIZED);
     idealState.setReplicas(Integer.toString(r));
     accessor.setProperty(keyBuilder.idealStates(newDbName), idealState);
-    result =
-        ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
-            clusterName));
-    Assert.assertTrue(result);
+
+    Assert.assertTrue(_clusterVerifier.verify());
+
     Thread.sleep(200);
     Assert.assertTrue(listener.cbCnt > 0);
 

http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java b/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
index 3d9a0b0..657dbc9 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/TestDisable.java
@@ -85,8 +85,6 @@ public class TestDisable extends ZkIntegrationTestBase {
       participants[i].syncStart();
     }
 
-    Thread.sleep(100000000);
-
     boolean result =
         ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
             clusterName));

http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java b/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
index 579acb1..1a9b65a 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/messaging/TestBatchMessageWrapper.java
@@ -34,8 +34,6 @@ import org.apache.helix.messaging.handling.BatchMessageWrapper;
 import org.apache.helix.mock.participant.MockMSModelFactory;
 import org.apache.helix.model.IdealState;
 import org.apache.helix.model.Message;
-import org.apache.helix.tools.ClusterStateVerifier;
-import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
 import org.apache.helix.tools.ClusterVerifiers.BestPossibleExternalViewVerifier;
 import org.apache.helix.tools.ClusterVerifiers.HelixClusterVerifier;
 import org.testng.Assert;

http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
index 54a7fe9..8c03eed 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskRebalancerStopResume.java
@@ -256,7 +256,9 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
     LOG.info("Flusing job-queue: " + queueName);
     _driver.flushQueue(queueName);
 
-    TimeUnit.MILLISECONDS.sleep(1000);
+    // TODO: Use TestHelper.verify() instead of waiting here.
+    TimeUnit.MILLISECONDS.sleep(5000);
+
     // verify the cleanup
     for (int i = 0; i < currentJobNames.size(); i++) {
       String namedSpaceJobName = String.format("%s_%s", queueName, currentJobNames.get(i));
@@ -411,8 +413,7 @@ public class TestTaskRebalancerStopResume extends TaskTestBase {
 
     // Create a queue
     System.out.println("START " + queueName + " at " + new Date(System.currentTimeMillis()));
-    WorkflowConfig wfCfg = new WorkflowConfig.Builder(queueName).setExpiry(2, TimeUnit.MINUTES)
-        .setScheduleConfig(ScheduleConfig.recurringFromNow(TimeUnit.MINUTES, 1)).build();
+    WorkflowConfig wfCfg = new WorkflowConfig.Builder(queueName).setExpiry(2, TimeUnit.MINUTES).build();
     JobQueue qCfg = new JobQueue.Builder(queueName).fromMap(wfCfg.getResourceConfigMap()).build();
     _driver.createQueue(qCfg);
 

http://git-wip-us.apache.org/repos/asf/helix/blob/ecdbaa13/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
----------------------------------------------------------------------
diff --git a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
index b5f4d86..188e61c 100644
--- a/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
+++ b/helix-core/src/test/java/org/apache/helix/integration/task/TestTaskThreadLeak.java
@@ -65,7 +65,9 @@ public class TestTaskThreadLeak extends TaskTestBase {
         .setWorkflowConfig(new WorkflowConfig.Builder(queueName).setParallelJobs(10).build());
 
     _driver.start(queueBuilder.build());
-    _driver.pollForJobState(queueName, lastJob, TaskState.COMPLETED);
+
+    String nameSpacedJob = TaskUtil.getNamespacedJobName(queueName, lastJob);
+    _driver.pollForJobState(queueName, nameSpacedJob, TaskState.COMPLETED);
 
     Set<Thread> threads = Thread.getAllStackTraces().keySet();
     Set<Thread> taskThreads = new HashSet<>();