You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by hu...@apache.org on 2017/12/01 05:28:49 UTC
incubator-gobblin git commit: [GOBBLIN-328] Fix
GobblinClusterKillTest failure. Path mismatch
Repository: incubator-gobblin
Updated Branches:
refs/heads/master 9fd80690d -> 4a8f7ba8d
[GOBBLIN-328] Fix GobblinClusterKillTest failure. Path mismatch
Issue:
org.apache.gobblin.cluster.GobblinClusterKillTest
failed because it looks at the wrong output path
for the output files.
Cause:
It appears that the paths have changed.
Fix:
Update the path to match.
Also:
* Instead of disabling the tests. Move the tests
to the group
disabledOnTravis to continue to disable it on
Travis until we have
higher confidence with these tests.
This makes it easier to run these tests locally.
* Run the setup and teardown methods per method.
This adds about 10s
overhead per method.
The worker kill test failed without this change
when running with other
tests. I suspect it is related to some state
management issues.
This change makes tests more self contained.
* Some minor code quality improvements.
Closes #2180 from HappyRay/fix-cluster-kill-tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/4a8f7ba8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/4a8f7ba8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/4a8f7ba8
Branch: refs/heads/master
Commit: 4a8f7ba8da986869e7c2426992b9cc4a9f6674ea
Parents: 9fd8069
Author: Ray Yang <ru...@gmail.com>
Authored: Thu Nov 30 21:28:39 2017 -0800
Committer: Hung Tran <hu...@linkedin.com>
Committed: Thu Nov 30 21:28:39 2017 -0800
----------------------------------------------------------------------
.../gobblin/cluster/GobblinClusterKillTest.java | 42 ++++++++------------
1 file changed, 17 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/4a8f7ba8/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java
----------------------------------------------------------------------
diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java
index 2c5d53d..220417b 100644
--- a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java
+++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinClusterKillTest.java
@@ -22,7 +22,6 @@ import com.google.common.base.Predicate;
import com.typesafe.config.Config;
import com.typesafe.config.ConfigFactory;
import com.typesafe.config.ConfigValueFactory;
-import org.apache.gobblin.testing.AssertWithBackoff;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -30,19 +29,18 @@ import java.net.URL;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Iterator;
import java.util.concurrent.TimeoutException;
import org.apache.commons.io.FileUtils;
import org.apache.curator.test.TestingServer;
+import org.apache.gobblin.testing.AssertWithBackoff;
import org.apache.hadoop.fs.Path;
import org.apache.helix.HelixManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -56,9 +54,12 @@ import org.testng.annotations.Test;
* shutdown request message.
* </p>
*/
-// @Test(groups = { "gobblin.cluster" }, singleThreaded = true)
+// The kill tests are unreliable on Travis
+// Disabled GobblinClusterKillTest until reliability improves
+@Test(groups = {"disabledOnTravis"}, singleThreaded = true)
public class GobblinClusterKillTest {
public final static Logger LOG = LoggerFactory.getLogger(GobblinClusterKillTest.class);
+ public static final String CLASS_NAME_BASED_PATH = "org/apache/gobblin/util/test/HelloWorldSource";
private TestingServer _testingZKServer;
@@ -140,7 +141,7 @@ public class GobblinClusterKillTest {
_workerStartThreads[id].start();
}
- // @BeforeClass
+ @BeforeMethod
public void setUp() throws Exception {
// Use a random ZK port
_testingZKServer = new TestingServer(-1);
@@ -176,12 +177,7 @@ public class GobblinClusterKillTest {
}
}
- // The kill tests are unreliable on Travis
- // Disabled GobblinClusterKillTest until reliability improves
- // @Test(groups = { "disabledOnTravis" })
- public void testKillWorker() throws TimeoutException, InterruptedException {
- Collection<File> matches = Collections.EMPTY_LIST;
-
+ public void testKillWorker() throws Exception {
final File testJobFile = new File(_jobDirPath + "/GobblinClusterKillTestJob.conf");
// Job file should exist
@@ -230,11 +226,6 @@ public class GobblinClusterKillTest {
// Disabled GobblinClusterKillTest until reliability improves
// @Test(groups = { "disabledOnTravis" }, dependsOnMethods = "testKillWorker")
public void testKillManager() throws IOException, TimeoutException, InterruptedException {
- Collection<File> matches = Collections.EMPTY_LIST;
-
- // reinitialize test directory
- setupTestDir();
-
// kill a manager to cause leader election. New leader will schedule a new job.
_clusterManagers[0].disconnectHelixManager();
@@ -276,9 +267,7 @@ public class GobblinClusterKillTest {
// Disabled GobblinClusterKillTest until reliability improves
// @Test(groups = { "disabledOnTravis" }, enabled=true, dependsOnMethods = "testKillManager")
public void testRestartManager() throws IOException, TimeoutException, InterruptedException {
- Collection<File> matches = Collections.EMPTY_LIST;
- // reinitialize test directory
- setupTestDir();
+ _clusterManagers[0].disconnectHelixManager();
// At this point there is one connected manager. Disconnect it and reconnect the other one to confirm that a manager
// can continue to function after regaining leadership.
@@ -322,7 +311,7 @@ public class GobblinClusterKillTest {
}, "Waiting for job-completion");
}
- // @AfterClass
+ @AfterMethod
public void tearDown() throws IOException, InterruptedException {
for (int i = 0; i < NUM_MANAGERS; i++) {
@@ -370,11 +359,13 @@ public class GobblinClusterKillTest {
* @return a {@link File} if directory found, otherwise null
*/
private File getWriterOutputDir() {
- File writerOutputJobDir = getFileFromGlob(_testDirPath + "/writer-output", "job*");
+ File writerOutputJobDir = getFileFromGlob(
+ _testDirPath + "/writer-output/GobblinClusterKillTestJob", "job*");
File writerOutputDir = null;
if (writerOutputJobDir != null) {
- writerOutputDir = new File(writerOutputJobDir, "gobblin/util/test/HelloWorldSource");
+ writerOutputDir = new File(writerOutputJobDir,
+ CLASS_NAME_BASED_PATH);
}
return writerOutputDir;
@@ -385,7 +376,8 @@ public class GobblinClusterKillTest {
* @return a {@link File} if directory found, otherwise null
*/
private File getJobOutputDir() {
- return getFileFromGlob(_testDirPath + "/job-output/gobblin/util/test/HelloWorldSource/",
+ return getFileFromGlob(
+ _testDirPath + "/job-output/" + CLASS_NAME_BASED_PATH,
"*_append");
}
}