You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2014/01/06 22:57:56 UTC
git commit: [TWILL-26] Refactor YARN Unit tests so individual tests
can be run directly.
Updated Branches:
refs/heads/master 603ea385a -> fab6d9328
[TWILL-26] Refactor YARN Unit tests so individual tests can be run directly.
It also clean up some minor issues with the tests:
+ The main suite is now empty (as it supposed to be)
+ Imports are organized
+ Test classes are made final
Signed-off-by: Terence Yim <te...@continuuity.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/fab6d932
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/fab6d932
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/fab6d932
Branch: refs/heads/master
Commit: fab6d932825aa969b13ed8f27eb93c529e952763
Parents: 603ea38
Author: Binh Nguyen <bn...@palantir.com>
Authored: Mon Jan 6 13:01:26 2014 -0800
Committer: Terence Yim <te...@continuuity.com>
Committed: Mon Jan 6 13:57:08 2014 -0800
----------------------------------------------------------------------
.../org/apache/twill/yarn/BaseYarnTest.java | 37 +++++
.../twill/yarn/DistributeShellTestRun.java | 22 +--
.../org/apache/twill/yarn/DistributedShell.java | 10 +-
.../apache/twill/yarn/EchoServerTestRun.java | 54 ++++----
.../twill/yarn/FailureRestartTestRun.java | 34 ++---
.../org/apache/twill/yarn/LocalFileTestRun.java | 48 ++++---
.../twill/yarn/ProvisionTimeoutTestRun.java | 26 ++--
.../twill/yarn/ResourceReportTestRun.java | 60 +++++----
.../org/apache/twill/yarn/SocketServer.java | 7 +-
.../apache/twill/yarn/TaskCompletedTestRun.java | 8 +-
.../twill/yarn/TwillSpecificationTest.java | 2 +-
.../org/apache/twill/yarn/YarnTestSuite.java | 94 +------------
.../org/apache/twill/yarn/YarnTestUtils.java | 135 +++++++++++++++++++
13 files changed, 313 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java b/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
new file mode 100644
index 0000000..6943ef9
--- /dev/null
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import java.io.IOException;
+
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Base class for all YARN tests.
+ */
+public abstract class BaseYarnTest {
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @BeforeClass
+ public static final void init() throws IOException {
+ YarnTestUtils.initOnce(tmpFolder.newFolder());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
index 1054ec9..0ed496e 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
@@ -17,29 +17,31 @@
*/
package org.apache.twill.yarn;
+import java.io.PrintWriter;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.util.concurrent.Service;
+
import org.apache.twill.api.TwillController;
import org.apache.twill.api.TwillRunner;
import org.apache.twill.api.logging.PrinterLogHandler;
import org.apache.twill.common.ServiceListenerAdapter;
import org.apache.twill.common.Threads;
-import com.google.common.util.concurrent.Service;
-import org.junit.Assert;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import java.io.PrintWriter;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
/**
* This test is executed by {@link YarnTestSuite}.
*/
-public class DistributeShellTestRun {
+public final class DistributeShellTestRun extends BaseYarnTest {
@Ignore
@Test
public void testDistributedShell() throws InterruptedException {
- TwillRunner twillRunner = YarnTestSuite.getTwillRunner();
+ TwillRunner twillRunner = YarnTestUtils.getTwillRunner();
TwillController controller = twillRunner.prepare(new DistributedShell("pwd", "ls -al"))
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
index c89371c..2f42e31 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
@@ -17,18 +17,20 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.AbstractTwillRunnable;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
+import org.apache.twill.api.AbstractTwillRunnable;
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
index d868eef..a14b326 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -17,21 +17,6 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.ResourceSpecification;
-import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunner;
-import org.apache.twill.api.TwillRunnerService;
-import org.apache.twill.api.logging.PrinterLogHandler;
-import org.apache.twill.common.ServiceListenerAdapter;
-import org.apache.twill.common.Threads;
-import org.apache.twill.discovery.Discoverable;
-import com.google.common.base.Charsets;
-import com.google.common.io.LineReader;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -43,18 +28,35 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.base.Charsets;
+import com.google.common.io.LineReader;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.discovery.Discoverable;
+
/**
* Using echo server to test various behavior of YarnTwillService.
- * This test is executed by {@link YarnTestSuite}.
+ * This test is executed by {@link YarnTestUtils}.
*/
-public class EchoServerTestRun {
+public final class EchoServerTestRun extends BaseYarnTest {
private static final Logger LOG = LoggerFactory.getLogger(EchoServerTestRun.class);
@Test
public void testEchoServer() throws InterruptedException, ExecutionException, IOException,
URISyntaxException, TimeoutException {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new EchoServer(),
ResourceSpecification.Builder.with()
@@ -78,7 +80,7 @@ public class EchoServerTestRun {
Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
Iterable<Discoverable> echoServices = controller.discoverService("echo");
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 2, 60));
for (Discoverable discoverable : echoServices) {
String msg = "Hello: " + discoverable.getSocketAddress();
@@ -98,36 +100,36 @@ public class EchoServerTestRun {
// Increase number of instances
controller.changeInstances("EchoServer", 3);
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 3, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 3, 60));
echoServices = controller.discoverService("echo2");
// Decrease number of instances
controller.changeInstances("EchoServer", 1);
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 1, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 1, 60));
// Increase number of instances again
controller.changeInstances("EchoServer", 2);
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 2, 60));
// Make sure still only one app is running
Iterable<TwillRunner.LiveInfo> apps = runner.lookupLive();
- Assert.assertTrue(YarnTestSuite.waitForSize(apps, 1, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(apps, 1, 60));
// Creates a new runner service to check it can regain control over running app.
- TwillRunnerService runnerService = YarnTestSuite.createTwillRunnerService();
+ TwillRunnerService runnerService = YarnTestUtils.createTwillRunnerService(tmpFolder.newFolder());
runnerService.startAndWait();
try {
Iterable <TwillController> controllers = runnerService.lookup("EchoServer");
- Assert.assertTrue(YarnTestSuite.waitForSize(controllers, 1, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(controllers, 1, 60));
for (TwillController c : controllers) {
LOG.info("Stopping application: " + c.getRunId());
c.stop().get(30, TimeUnit.SECONDS);
}
- Assert.assertTrue(YarnTestSuite.waitForSize(apps, 0, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(apps, 0, 60));
} finally {
runnerService.stopAndWait();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
index b3d3933..06de991 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
@@ -17,18 +17,6 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.Command;
-import org.apache.twill.api.ResourceSpecification;
-import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunner;
-import org.apache.twill.api.logging.PrinterLogHandler;
-import org.apache.twill.discovery.Discoverable;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Sets;
-import com.google.common.io.LineReader;
-import org.junit.Assert;
-import org.junit.Test;
-
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -39,14 +27,28 @@ import java.net.Socket;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Sets;
+import com.google.common.io.LineReader;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.discovery.Discoverable;
+
/**
*
*/
-public class FailureRestartTestRun {
+public final class FailureRestartTestRun extends BaseYarnTest {
@Test
public void testFailureRestart() throws Exception {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
ResourceSpecification resource = ResourceSpecification.Builder.with()
.setVirtualCores(1)
@@ -60,7 +62,7 @@ public class FailureRestartTestRun {
.start();
Iterable<Discoverable> discoverables = controller.discoverService("failure");
- Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 2, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(discoverables, 2, 60));
// Make sure we see the right instance IDs
Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables));
@@ -71,7 +73,7 @@ public class FailureRestartTestRun {
// Do a shot sleep, make sure the runnable is killed.
TimeUnit.SECONDS.sleep(5);
- Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 2, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(discoverables, 2, 60));
// Make sure we see the right instance IDs
Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables));
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
index 54020aa..d7e186f 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
@@ -17,24 +17,6 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.TwillApplication;
-import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunner;
-import org.apache.twill.api.TwillSpecification;
-import org.apache.twill.api.logging.PrinterLogHandler;
-import org.apache.twill.discovery.Discoverable;
-import com.google.common.base.Charsets;
-import com.google.common.base.Preconditions;
-import com.google.common.io.ByteStreams;
-import com.google.common.io.Files;
-import com.google.common.io.LineReader;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
@@ -48,20 +30,36 @@ import java.util.concurrent.TimeUnit;
import java.util.jar.JarEntry;
import java.util.jar.JarOutputStream;
+import org.junit.Test;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import com.google.common.io.LineReader;
+
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.discovery.Discoverable;
+import org.junit.Assert;
+
/**
* Test for local file transfer.
*/
-public class LocalFileTestRun {
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
+public final class LocalFileTestRun extends BaseYarnTest {
@Test
public void testLocalFile() throws Exception {
String header = Files.readFirstLine(new File(getClass().getClassLoader().getResource("header.txt").toURI()),
Charsets.UTF_8);
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
String prevJVMOptions = "";
if (runner instanceof YarnTwillRunnerService) {
YarnTwillRunnerService yarnRunner = (YarnTwillRunnerService) runner;
@@ -80,7 +78,7 @@ public class LocalFileTestRun {
}
Iterable<Discoverable> discoverables = controller.discoverService("local");
- Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 1, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(discoverables, 1, 60));
InetSocketAddress socketAddress = discoverables.iterator().next().getSocketAddress();
Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort());
@@ -98,7 +96,7 @@ public class LocalFileTestRun {
controller.stopAndWait();
- Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 0, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(discoverables, 0, 60));
TimeUnit.SECONDS.sleep(2);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
index 0598ef1..d5e3fc3 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
@@ -17,6 +17,16 @@
*/
package org.apache.twill.yarn;
+import java.io.PrintWriter;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+
import org.apache.twill.api.AbstractTwillRunnable;
import org.apache.twill.api.EventHandler;
import org.apache.twill.api.EventHandlerContext;
@@ -25,27 +35,19 @@ import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
import org.apache.twill.api.TwillRunner;
import org.apache.twill.api.TwillSpecification;
-import org.apache.twill.api.logging.PrinterLogHandler;
-import org.apache.twill.common.Services;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableMap;
import org.junit.Test;
-import java.io.PrintWriter;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.Services;
/**
*
*/
-public class ProvisionTimeoutTestRun {
+public final class ProvisionTimeoutTestRun extends BaseYarnTest {
@Test
public void testProvisionTimeout() throws InterruptedException, ExecutionException, TimeoutException {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new TimeoutApplication())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
index 131f90a..795caed 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
@@ -17,26 +17,6 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.ResourceReport;
-import org.apache.twill.api.ResourceSpecification;
-import org.apache.twill.api.TwillApplication;
-import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunResources;
-import org.apache.twill.api.TwillRunner;
-import org.apache.twill.api.TwillSpecification;
-import org.apache.twill.api.logging.PrinterLogHandler;
-import org.apache.twill.common.ServiceListenerAdapter;
-import org.apache.twill.common.Threads;
-import org.apache.twill.discovery.Discoverable;
-import org.apache.twill.internal.EnvKeys;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Maps;
-import com.google.common.io.LineReader;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -50,11 +30,33 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Maps;
+import com.google.common.io.LineReader;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.internal.EnvKeys;
+
/**
* Using echo server to test resource reports.
- * This test is executed by {@link org.apache.twill.yarn.YarnTestSuite}.
+ * This test is executed by {@link org.apache.twill.yarn.YarnTestUtils}.
*/
-public class ResourceReportTestRun {
+public final class ResourceReportTestRun {
private static final Logger LOG = LoggerFactory.getLogger(ResourceReportTestRun.class);
@@ -80,7 +82,7 @@ public class ResourceReportTestRun {
@Test
public void testRunnablesGetAllowedResourcesInEnv() throws InterruptedException, IOException,
TimeoutException, ExecutionException {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
.setVirtualCores(1)
@@ -104,7 +106,7 @@ public class ResourceReportTestRun {
Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
Iterable<Discoverable> envEchoServices = controller.discoverService("envecho");
- Assert.assertTrue(YarnTestSuite.waitForSize(envEchoServices, 1, 30));
+ Assert.assertTrue(YarnTestUtils.waitForSize(envEchoServices, 1, 30));
// TODO: check virtual cores once yarn adds the ability
Map<String, String> expectedValues = Maps.newHashMap();
@@ -134,7 +136,7 @@ public class ResourceReportTestRun {
@Test
public void testResourceReportWithFailingContainers() throws InterruptedException, IOException,
TimeoutException, ExecutionException {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
.setVirtualCores(1)
@@ -158,7 +160,7 @@ public class ResourceReportTestRun {
Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
Iterable<Discoverable> echoServices = controller.discoverService("echo");
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 2, 60));
// check that we have 2 runnables.
ResourceReport report = controller.getResourceReport();
Assert.assertEquals(2, report.getRunnableResources("BuggyServer").size());
@@ -188,7 +190,7 @@ public class ResourceReportTestRun {
@Test
public void testResourceReport() throws InterruptedException, ExecutionException, IOException,
URISyntaxException, TimeoutException {
- TwillRunner runner = YarnTestSuite.getTwillRunner();
+ TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new ResourceApplication())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
@@ -209,7 +211,7 @@ public class ResourceReportTestRun {
// wait for 3 echo servers to come up
Iterable<Discoverable> echoServices = controller.discoverService("echo");
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 3, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 3, 60));
ResourceReport report = controller.getResourceReport();
// make sure resources for echo1 and echo2 are there
Map<String, Collection<TwillRunResources>> usedResources = report.getResources();
@@ -236,7 +238,7 @@ public class ResourceReportTestRun {
// Decrease number of instances of echo1 from 2 to 1
controller.changeInstances("echo1", 1);
echoServices = controller.discoverService("echo1");
- Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 1, 60));
+ Assert.assertTrue(YarnTestUtils.waitForSize(echoServices, 1, 60));
report = controller.getResourceReport();
// make sure resources for echo1 and echo2 are there
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java b/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
index 5148ed2..dabdc07 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
@@ -17,13 +17,12 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.AbstractTwillRunnable;
-import org.apache.twill.api.TwillContext;
-import org.apache.twill.api.TwillContext;
-import org.apache.twill.common.Cancellable;
import com.google.common.base.Charsets;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.TwillContext;
+import org.apache.twill.common.Cancellable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
index 5a93271..37fc8aa 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
@@ -17,6 +17,8 @@
*/
package org.apache.twill.yarn;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.Service;
import org.apache.twill.api.AbstractTwillRunnable;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillController;
@@ -24,8 +26,6 @@ import org.apache.twill.api.TwillRunner;
import org.apache.twill.api.logging.PrinterLogHandler;
import org.apache.twill.common.ServiceListenerAdapter;
import org.apache.twill.common.Threads;
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.Service;
import org.junit.Assert;
import org.junit.Test;
@@ -38,7 +38,7 @@ import java.util.concurrent.TimeUnit;
* Testing application master will shutdown itself when all tasks are completed.
* This test is executed by {@link YarnTestSuite}.
*/
-public class TaskCompletedTestRun {
+public final class TaskCompletedTestRun extends BaseYarnTest {
public static final class SleepTask extends AbstractTwillRunnable {
@@ -60,7 +60,7 @@ public class TaskCompletedTestRun {
@Test
public void testTaskCompleted() throws InterruptedException {
- TwillRunner twillRunner = YarnTestSuite.getTwillRunner();
+ TwillRunner twillRunner = YarnTestUtils.getTwillRunner();
TwillController controller = twillRunner.prepare(new SleepTask(),
ResourceSpecification.Builder.with()
.setVirtualCores(1)
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
index 8be907b..fca26b7 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
@@ -17,9 +17,9 @@
*/
package org.apache.twill.yarn;
+import com.google.common.collect.ImmutableSet;
import org.apache.twill.api.AbstractTwillRunnable;
import org.apache.twill.api.TwillSpecification;
-import com.google.common.collect.ImmutableSet;
import org.junit.Assert;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
index 6533cdd..912b713 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
@@ -17,26 +17,8 @@
*/
package org.apache.twill.yarn;
-import org.apache.twill.api.TwillRunner;
-import org.apache.twill.api.TwillRunnerService;
-import org.apache.twill.filesystem.LocalLocationFactory;
-import org.apache.twill.internal.zookeeper.InMemoryZKServer;
-import org.apache.twill.internal.yarn.YarnUtils;
-import com.google.common.collect.Iterables;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.MiniYARNCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
/**
* Test suite for all tests with mini yarn cluster.
@@ -51,80 +33,6 @@ import java.util.concurrent.TimeUnit;
FailureRestartTestRun.class,
ProvisionTimeoutTestRun.class
})
-public class YarnTestSuite {
- private static final Logger LOG = LoggerFactory.getLogger(YarnTestSuite.class);
-
- @ClassRule
- public static TemporaryFolder tmpFolder = new TemporaryFolder();
-
- private static InMemoryZKServer zkServer;
- private static MiniYARNCluster cluster;
- private static TwillRunnerService runnerService;
- private static YarnConfiguration config;
-
- @BeforeClass
- public static final void init() throws IOException {
- // Starts Zookeeper
- zkServer = InMemoryZKServer.builder().build();
- zkServer.startAndWait();
-
- // Start YARN mini cluster
- config = new YarnConfiguration(new Configuration());
-
- if (YarnUtils.isHadoop20()) {
- config.set("yarn.resourcemanager.scheduler.class",
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
- } else {
- config.set("yarn.resourcemanager.scheduler.class",
- "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
- config.set("yarn.scheduler.capacity.resource-calculator",
- "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
- }
- config.set("yarn.minicluster.fixed.ports", "true");
- config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
- config.set("yarn.nodemanager.vmem-check-enabled", "false");
- config.set("yarn.scheduler.minimum-allocation-mb", "128");
- config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");
-
- cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
- cluster.init(config);
- cluster.start();
-
- runnerService = createTwillRunnerService();
- runnerService.startAndWait();
- }
-
- @AfterClass
- public static final void finish() {
- runnerService.stopAndWait();
- cluster.stop();
- zkServer.stopAndWait();
- }
-
- public static final TwillRunner getTwillRunner() {
- return runnerService;
- }
-
- /**
- * Creates an unstarted instance of {@link org.apache.twill.api.TwillRunnerService}.
- */
- public static final TwillRunnerService createTwillRunnerService() throws IOException {
- YarnTwillRunnerService runner = new YarnTwillRunnerService(config, zkServer.getConnectionStr() + "/twill",
- new LocalLocationFactory(tmpFolder.newFolder()));
- // disable tests stealing focus
- runner.setJVMOptions("-Djava.awt.headless=true");
- return runner;
- }
+public final class YarnTestSuite {
- public static final <T> boolean waitForSize(Iterable<T> iterable, int count, int limit) throws InterruptedException {
- int trial = 0;
- int size = Iterables.size(iterable);
- while (size != count && trial < limit) {
- LOG.info("Waiting for {} size {} == {}", iterable, size, count);
- TimeUnit.SECONDS.sleep(1);
- trial++;
- size = Iterables.size(iterable);
- }
- return trial < limit;
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/fab6d932/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
new file mode 100644
index 0000000..079db56
--- /dev/null
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestUtils.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.filesystem.LocalLocationFactory;
+import org.apache.twill.internal.yarn.YarnUtils;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Utilities for testing YARN.
+ */
+public final class YarnTestUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTestUtils.class);
+
+ private static InMemoryZKServer zkServer;
+ private static MiniYARNCluster cluster;
+ private static TwillRunnerService runnerService;
+ private static YarnConfiguration config;
+
+ private static final AtomicBoolean once = new AtomicBoolean(false);
+
+ public static final boolean initOnce(File folder) throws IOException {
+ if (once.compareAndSet(false, true)) {
+ init(folder);
+
+ // add shutdown hook because we want to initialized/cleanup once
+ Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
+ @Override
+ public void run() {
+ finish();
+ }
+ }));
+ return true;
+ }
+ return false;
+ }
+
+ private static final void init(File folder) throws IOException {
+ // Starts Zookeeper
+ zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ // Start YARN mini cluster
+ config = new YarnConfiguration(new Configuration());
+
+ if (YarnUtils.isHadoop20()) {
+ config.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
+ } else {
+ config.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
+ config.set("yarn.scheduler.capacity.resource-calculator",
+ "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
+ }
+ config.set("yarn.minicluster.fixed.ports", "true");
+ config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
+ config.set("yarn.nodemanager.vmem-check-enabled", "false");
+ config.set("yarn.scheduler.minimum-allocation-mb", "128");
+ config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");
+
+ cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
+ cluster.init(config);
+ cluster.start();
+
+ runnerService = createTwillRunnerService(folder);
+ runnerService.startAndWait();
+ }
+
+ public static final boolean finish() {
+ if (once.compareAndSet(true, false)) {
+ runnerService.stopAndWait();
+ cluster.stop();
+ zkServer.stopAndWait();
+
+ return true;
+ }
+
+ return false;
+ }
+
+ public static final TwillRunner getTwillRunner() {
+ return runnerService;
+ }
+
+ /**
+ * Creates an unstarted instance of {@link org.apache.twill.api.TwillRunnerService}.
+ */
+ public static final TwillRunnerService createTwillRunnerService(File folder) throws IOException {
+ YarnTwillRunnerService runner = new YarnTwillRunnerService(config, zkServer.getConnectionStr() + "/twill",
+ new LocalLocationFactory(folder));
+ // disable tests stealing focus
+ runner.setJVMOptions("-Djava.awt.headless=true");
+ return runner;
+ }
+
+ public static final <T> boolean waitForSize(Iterable<T> iterable, int count, int limit) throws InterruptedException {
+ int trial = 0;
+ int size = Iterables.size(iterable);
+ while (size != count && trial < limit) {
+ LOG.info("Waiting for {} size {} == {}", iterable, size, count);
+ TimeUnit.SECONDS.sleep(1);
+ trial++;
+ size = Iterables.size(iterable);
+ }
+ return trial < limit;
+ }
+}