You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2014/12/16 20:44:14 UTC
[1/6] incubator-twill git commit: sort twill-yarn test runners,
add missing tests
Repository: incubator-twill
Updated Branches:
refs/heads/site ff3b99b58 -> 2c673aeb7
sort twill-yarn test runners, add missing tests
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/f1c1e6bc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/f1c1e6bc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/f1c1e6bc
Branch: refs/heads/site
Commit: f1c1e6bc2b8ad7c298a8da6662b4d8146905558d
Parents: 925b85f
Author: Erick Tryzelaar <er...@lab41.org>
Authored: Fri Jul 25 15:56:36 2014 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Nov 18 11:07:11 2014 -0800
----------------------------------------------------------------------
.../src/test/java/org/apache/twill/yarn/YarnTestSuite.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/f1c1e6bc/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
index 87c380f..8427041 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
@@ -25,6 +25,7 @@ import org.junit.runners.Suite;
*/
@RunWith(Suite.class)
@Suite.SuiteClasses({
+ PlacementPolicyTestRun.class,
EchoServerTestRun.class,
ResourceReportTestRun.class,
TaskCompletedTestRun.class,
@@ -36,7 +37,8 @@ import org.junit.runners.Suite;
SessionExpireTestRun.class,
ServiceDiscoveryTestRun.class,
DebugTestRun.class,
- PlacementPolicyTestRun.class
+ ContainerSizeTestRun.class,
+ InitializeFailTestRun.class
})
public final class YarnTestSuite {
[3/6] incubator-twill git commit: (TWILL-110) added state guard to
prevent possible race-condition when shutting down zk client,
which causes deadlock.
Posted by ch...@apache.org.
(TWILL-110) added state guard to prevent possible race-condition when shutting down zk client, which causes deadlock.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/73ecf8e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/73ecf8e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/73ecf8e3
Branch: refs/heads/site
Commit: 73ecf8e3a1ba9869ba759f0d9d2ea9dabc0cddbf
Parents: 8db2d47
Author: Terence Yim <ch...@apache.org>
Authored: Mon Dec 1 14:12:11 2014 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Dec 2 14:11:20 2014 -0800
----------------------------------------------------------------------
.../zookeeper/DefaultZKClientService.java | 20 ++++++++++++--
.../apache/twill/zookeeper/ZKClientTest.java | 28 ++++++++++++++++++++
2 files changed, 46 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/73ecf8e3/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
index 7c9bd08..73ca274 100644
--- a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
+++ b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
@@ -57,6 +57,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
@@ -371,6 +372,9 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
private final class ServiceDelegate extends AbstractService implements Watcher {
+ private final AtomicBoolean stopNotified = new AtomicBoolean(false);
+ private volatile boolean executorStopped;
+
@Override
protected void doStart() {
// A single thread executor
@@ -379,7 +383,12 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
@Override
protected void terminated() {
super.terminated();
- notifyStopped();
+
+ // Only call notifyStopped if the executor.shutdown() returned, otherwise deadlock (TWILL-110) can occur.
+ // Also, notifyStopped() should only be called once.
+ if (executorStopped && stopNotified.compareAndSet(false, true)) {
+ notifyStopped();
+ }
}
};
@@ -400,6 +409,13 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
notifyFailed(e);
} finally {
eventExecutor.shutdown();
+ executorStopped = true;
+
+ // If the executor state is terminated, meaning the terminate() method is triggered,
+ // call notifyStopped() if it hasn't been called yet.
+ if (eventExecutor.isTerminated() && stopNotified.compareAndSet(false, true)) {
+ notifyStopped();
+ }
}
}
}
@@ -408,7 +424,7 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
public void process(WatchedEvent event) {
try {
if (event.getState() == Event.KeeperState.SyncConnected && state() == State.STARTING) {
- LOG.info("Connected to ZooKeeper: " + zkStr);
+ LOG.debug("Connected to ZooKeeper: " + zkStr);
notifyStarted();
return;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/73ecf8e3/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
index b0c7507..dc1b9eb 100644
--- a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
+++ b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
@@ -35,6 +35,8 @@ import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -54,6 +56,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
*/
public class ZKClientTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ZKClientTest.class);
+
@ClassRule
public static TemporaryFolder tmpFolder = new TemporaryFolder();
@@ -320,4 +324,28 @@ public class ZKClientTest {
zkServer.stopAndWait();
}
}
+
+ @Test (timeout = 120000L)
+ public void testDeadlock() throws IOException, InterruptedException {
+ // This is to test deadlock bug as described in (TWILL-110)
+ // This test has very high chance to get deadlock before the bug fix, hence failed with timeout.
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(tmpFolder.newFolder()).build();
+ zkServer.startAndWait();
+ try {
+ for (int i = 0; i < 5000; i++) {
+ final ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClient.addConnectionWatcher(new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ LOG.debug("Connection event: {}", event);
+ }
+ });
+ zkClient.startAndWait();
+ zkClient.stopAndWait();
+ }
+
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
}
[2/6] incubator-twill git commit: Make PlacementPolicyTestRun more
robust.
Posted by ch...@apache.org.
Make PlacementPolicyTestRun more robust.
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/8db2d473
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/8db2d473
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/8db2d473
Branch: refs/heads/site
Commit: 8db2d473235d80b7a3abe4d92587e4c20f90ed24
Parents: f1c1e6b
Author: Terence Yim <ch...@apache.org>
Authored: Tue Nov 18 11:48:29 2014 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Nov 18 11:48:29 2014 -0800
----------------------------------------------------------------------
.../twill/yarn/PlacementPolicyTestRun.java | 48 +++++++++++---------
1 file changed, 26 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/8db2d473/twill-yarn/src/test/java/org/apache/twill/yarn/PlacementPolicyTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/PlacementPolicyTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/PlacementPolicyTestRun.java
index 0da9dd2..a886491 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/PlacementPolicyTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/PlacementPolicyTestRun.java
@@ -17,16 +17,13 @@
*/
package org.apache.twill.yarn;
-import com.google.common.collect.Sets;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.twill.api.Hosts;
import org.apache.twill.api.Racks;
-import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
-import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.TwillRunner;
import org.apache.twill.api.TwillSpecification;
import org.apache.twill.api.logging.PrinterLogHandler;
@@ -36,17 +33,20 @@ import org.junit.Assert;
import org.junit.Assume;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.PrintWriter;
-import java.util.Collection;
import java.util.List;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
/**
* Tests for placement Policies.
*/
public class PlacementPolicyTestRun extends BaseYarnTest {
+ private static final Logger LOG = LoggerFactory.getLogger(PlacementPolicyTestRun.class);
+
private static final int RUNNABLE_MEMORY = 512;
private static final int RUNNABLE_CORES = 1;
@@ -103,12 +103,8 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
// Ignore test if it is running against older Hadoop versions which does not support blacklists.
Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22));
- ServiceDiscovered serviceDiscovered;
- ResourceReport resourceReport;
- Set<Integer> nmPorts = Sets.newHashSet();
- Collection<TwillRunResources> distributedResource;
+ waitNodeManagerCount(0, 10, TimeUnit.SECONDS);
- Assert.assertEquals(0, getProvisionedNodeManagerCount());
TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new PlacementPolicyApplication())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
@@ -120,7 +116,7 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
try {
// All runnables should get started.
- serviceDiscovered = controller.discoverService("PlacementPolicyTest");
+ ServiceDiscovered serviceDiscovered = controller.discoverService("PlacementPolicyTest");
Assert.assertTrue(YarnTestUtils.waitForSize(serviceDiscovered, 4, 80));
// DISTRIBUTED runnables should be provisioned on different nodes.
@@ -164,13 +160,8 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
// Ignore test if it is running against older Hadoop versions which does not support blacklists.
Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22));
- ServiceDiscovered serviceDiscovered;
- ResourceReport resourceReport;
- Set<Integer> nmPorts = Sets.newHashSet();
- Collection<TwillRunResources> aliceResources;
- Collection<TwillRunResources> bobResources;
+ waitNodeManagerCount(0, 10, TimeUnit.SECONDS);
- Assert.assertEquals(0, getProvisionedNodeManagerCount());
TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new DistributedApplication())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
@@ -182,7 +173,7 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
try {
// All runnables should get started with DISTRIBUTED ones being on different nodes.
- serviceDiscovered = controller.discoverService("DistributedTest");
+ ServiceDiscovered serviceDiscovered = controller.discoverService("DistributedTest");
Assert.assertTrue(YarnTestUtils.waitForSize(serviceDiscovered, 3, 60));
Assert.assertTrue(getProvisionedNodeManagerCount() >= 2);
@@ -212,6 +203,23 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
TimeUnit.SECONDS.sleep(2);
}
+ private void waitNodeManagerCount(int expected, long timeout, TimeUnit unit) throws Exception {
+ int count = getProvisionedNodeManagerCount();
+ long startTime = System.currentTimeMillis();
+ long elapse = 0L;
+
+ while (count != expected && elapse < unit.toMillis(timeout)) {
+ LOG.info("Waiting for expected number of node managers. Expected: {}. Actual: {}", expected, count);
+ TimeUnit.SECONDS.sleep(1);
+ count = getProvisionedNodeManagerCount();
+ elapse = System.currentTimeMillis() - startTime;
+ }
+ if (count != expected) {
+ throw new TimeoutException("Failed to get expected number of node managers. " +
+ "Expected: " + expected + ". Actual: " + count);
+ }
+ }
+
/**
* An application that runs three runnables, with a DISTRIBUTED placement policy for two of them.
*/
@@ -241,10 +249,6 @@ public class PlacementPolicyTestRun extends BaseYarnTest {
Assume.assumeTrue(YarnUtils.getHadoopVersion().equals(YarnUtils.HadoopVersions.HADOOP_22));
ServiceDiscovered serviceDiscovered;
- ResourceReport resourceReport;
- Set<Integer> nmPorts = Sets.newHashSet();
- Collection<TwillRunResources> aliceResources;
- Collection<TwillRunResources> bobResources;
TwillRunner runner = YarnTestUtils.getTwillRunner();
TwillController controller = runner.prepare(new ChangeInstanceApplication())
[5/6] incubator-twill git commit: Temporarily disable Travis-CI
hadoop-2.3 profile as it always fail due to hostname issue. Looks like it is
a known issue (https://github.com/travis-ci/travis-ci/issues/2970)
Posted by ch...@apache.org.
Temporarily disable Travis-CI hadoop-2.3 profile as it always fail due to hostname issue. Looks like it is a known issue (https://github.com/travis-ci/travis-ci/issues/2970)
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/b1adf69d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/b1adf69d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/b1adf69d
Branch: refs/heads/site
Commit: b1adf69d69b1922bc9af900ecdb8d12319d233af
Parents: b9e668c
Author: Terence Yim <ch...@apache.org>
Authored: Tue Dec 16 11:13:20 2014 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Dec 16 11:13:20 2014 -0800
----------------------------------------------------------------------
.travis.yml | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b1adf69d/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index e051942..b8a16bf 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,7 +35,6 @@ env:
- PROFILE='hadoop-2.0'
- PROFILE='hadoop-2.1'
- PROFILE='hadoop-2.2'
- - PROFILE='hadoop-2.3'
- PROFILE='hadoop-2.4'
- PROFILE='hadoop-2.5'
- PROFILE='cdh-4.4.0'
[6/6] incubator-twill git commit: Merge branch 'master' into site
Posted by ch...@apache.org.
Merge branch 'master' into site
Conflicts:
src/site/markdown/GettingStarted.md
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/2c673aeb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/2c673aeb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/2c673aeb
Branch: refs/heads/site
Commit: 2c673aeb7a1e322c69875856f2658cb9b0291528
Parents: ff3b99b b1adf69
Author: Terence Yim <ch...@apache.org>
Authored: Tue Dec 16 11:35:55 2014 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Dec 16 11:35:55 2014 -0800
----------------------------------------------------------------------
.gitignore | 6 ++
.travis.yml | 1 -
src/site/markdown/GettingStarted.md | 65 ++++++++++++++++++++
twill-examples/echo/pom.xml | 6 ++
twill-examples/yarn/pom.xml | 8 ++-
.../twill/example/yarn/BundledJarExample.java | 4 +-
.../twill/yarn/PlacementPolicyTestRun.java | 48 ++++++++-------
.../org/apache/twill/yarn/YarnTestSuite.java | 4 +-
.../zookeeper/DefaultZKClientService.java | 20 +++++-
.../apache/twill/zookeeper/ZKClientTest.java | 28 +++++++++
10 files changed, 159 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2c673aeb/src/site/markdown/GettingStarted.md
----------------------------------------------------------------------
diff --cc src/site/markdown/GettingStarted.md
index 9847ac1,85345ab..9343d92
--- a/src/site/markdown/GettingStarted.md
+++ b/src/site/markdown/GettingStarted.md
@@@ -147,3 -124,68 +147,68 @@@ controller.stop().get()
```
This will shut down the application master and all the containers.
+
+ ### Hello World
+
+ To see Twill in action, you can run the "hello world" example applications located in the twill-examples module.
+
+ #### Prerequisites
+
+ * Single Node or Cluster installation of Hadoop with YARN (Hadoop >= 2.2.0) set-up and running.
+ * Single Node or Cluster installation of ZooKeeper set-up and running.
+ * Build of Twill Library Code (minimum, build of twill-examples module)
+
+ #### Running the Examples
+
+ There are two example applications you can run: HelloWorld and BundledJarExample.
+
+ ##### HelloWorld Application
+
+ The HelloWorld application creates a simple YARN application that prints a line to the log.
+
+ You can run the HelloWorld application from any node of the Hadoop cluster using the below command
+ (be sure to add your ZooKeeper Host and Port):
+
+ ```sh
+ $ CP=twill-examples-yarn-0.5.0-incubating-SNAPSHOT.jar:`hadoop classpath`; java -cp $CP org.apache.twill.example.yarn.HelloWorld {zookeeper_host:port}
+ ```
+
+ If successful, you should see logs output to the terminal with details of the running application. Once the application
+ is finished running, check the YARN logs and you should see output like the following:
+
+ ```
+ 14:49:45.944 [TwillContainerService] INFO o.a.twill.example.yarn.HelloWorld - Hello World. My first distributed application.
+ ```
+
+ ##### BundledJarExample Application
+
+ The BundledJarExample application demonstrates the Twill functionality that allows you to run any Java application
+ in Twill without worrying about library version conflicts between your application and Hadoop. The example
+ calls the main class in a sample application `Echo`, which simply logs the command line argument(s) passed to it.
+ The `Echo` application uses a different version of Guava from Twill and Hadoop distributions. BundledJarExample
+ looks for the dependency in a `lib` folder packaged at the root of the `Echo` jar.
+
+ You can run the BundleJarExample application from any node of the Hadoop cluster using the below command
+ (be sure to add your ZooKeeper Host and Port):
+
+ ```sh
+ $ CP=twill-examples-yarn-0.5.0-incubating-SNAPSHOT.jar:`hadoop classpath`;
+ java -cp $CP org.apache.twill.example.yarn.BundledJarExample {zookeeper_host:port} twill-examples-echo-0.5.0-incubating-SNAPSHOT.jar echo.EchoMain arg1
+ ```
+
+ Like with the HelloWorld example, you should see logs output to the terminal. Once the application is complete, check
+ the YARN logs as before and you should see output like the following:
+
+ ```
+ [TwillContainerService] INFO echo.EchoMain - Hello from EchoMain: 6
+ err HELLO from scatch
+ [TwillContainerService] INFO echo.EchoMain - Got args: [arg1]
+
+ ...
+
+ out HELLO from scatch
+ Got args: [arg1]
+
+ ```
+
-Congratulations! You have run your first Twill applications.
++Congratulations! You have run your first Twill applications.
[4/6] incubator-twill git commit: (TWILL-100) Fix examples POMs and
update documentations - Updates to examples - Updates to BundleJarExample
application and documentation.
Posted by ch...@apache.org.
(TWILL-100) Fix examples POMs and update documentations
- Updates to examples
- Updates to BundleJarExample application and documentation.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/b9e668c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/b9e668c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/b9e668c4
Branch: refs/heads/site
Commit: b9e668c41c9b89a8984d46a65bf6dfbfcb5e119e
Parents: 73ecf8e
Author: Matt Anderson <ma...@liaison.com>
Authored: Mon Dec 15 16:48:25 2014 -0500
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Dec 16 10:41:33 2014 -0800
----------------------------------------------------------------------
.gitignore | 6 ++
src/site/markdown/GettingStarted.md | 67 +++++++++++++++++++-
twill-examples/echo/pom.xml | 6 ++
twill-examples/yarn/pom.xml | 8 ++-
.../twill/example/yarn/BundledJarExample.java | 4 +-
5 files changed, 85 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b9e668c4/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index bcb6216..87061cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,11 @@ out/
lib/
.idea
+# Eclipse Files and Dir #
+.project
+.classpath
+.settings
+
# Gradle Files & Dir #
build/
.stickyStorage
@@ -26,3 +31,4 @@ target/
logs/
zookeeper.out
+/bin
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b9e668c4/src/site/markdown/GettingStarted.md
----------------------------------------------------------------------
diff --git a/src/site/markdown/GettingStarted.md b/src/site/markdown/GettingStarted.md
index c8384a3..85345ab 100644
--- a/src/site/markdown/GettingStarted.md
+++ b/src/site/markdown/GettingStarted.md
@@ -123,4 +123,69 @@ You can also stop the application through the `TwillController`:
controller.stop().get();
```
-This will shut down the application master and all the containers.
\ No newline at end of file
+This will shut down the application master and all the containers.
+
+### Hello World
+
+To see Twill in action, you can run the "hello world" example applications located in the twill-examples module.
+
+#### Prerequisites
+
+* Single Node or Cluster installation of Hadoop with YARN (Hadoop >= 2.2.0) set-up and running.
+* Single Node or Cluster installation of ZooKeeper set-up and running.
+* Build of Twill Library Code (minimum, build of twill-examples module)
+
+#### Running the Examples
+
+There are two example applications you can run: HelloWorld and BundledJarExample.
+
+##### HelloWorld Application
+
+The HelloWorld application creates a simple YARN application that prints a line to the log.
+
+You can run the HelloWorld application from any node of the Hadoop cluster using the below command
+(be sure to add your ZooKeeper Host and Port):
+
+```sh
+$ CP=twill-examples-yarn-0.5.0-incubating-SNAPSHOT.jar:`hadoop classpath`; java -cp $CP org.apache.twill.example.yarn.HelloWorld {zookeeper_host:port}
+```
+
+If successful, you should see logs output to the terminal with details of the running application. Once the application
+is finished running, check the YARN logs and you should see output like the following:
+
+```
+14:49:45.944 [TwillContainerService] INFO o.a.twill.example.yarn.HelloWorld - Hello World. My first distributed application.
+```
+
+##### BundledJarExample Application
+
+The BundledJarExample application demonstrates the Twill functionality that allows you to run any Java application
+in Twill without worrying about library version conflicts between your application and Hadoop. The example
+calls the main class in a sample application `Echo`, which simply logs the command line argument(s) passed to it.
+The `Echo` application uses a different version of Guava from Twill and Hadoop distributions. BundledJarExample
+looks for the dependency in a `lib` folder packaged at the root of the `Echo` jar.
+
+You can run the BundleJarExample application from any node of the Hadoop cluster using the below command
+(be sure to add your ZooKeeper Host and Port):
+
+```sh
+$ CP=twill-examples-yarn-0.5.0-incubating-SNAPSHOT.jar:`hadoop classpath`;
+ java -cp $CP org.apache.twill.example.yarn.BundledJarExample {zookeeper_host:port} twill-examples-echo-0.5.0-incubating-SNAPSHOT.jar echo.EchoMain arg1
+```
+
+Like with the HelloWorld example, you should see logs output to the terminal. Once the application is complete, check
+the YARN logs as before and you should see output like the following:
+
+```
+[TwillContainerService] INFO echo.EchoMain - Hello from EchoMain: 6
+err HELLO from scatch
+[TwillContainerService] INFO echo.EchoMain - Got args: [arg1]
+
+...
+
+out HELLO from scatch
+Got args: [arg1]
+
+```
+
+Congratulations! You have run your first Twill applications.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b9e668c4/twill-examples/echo/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/echo/pom.xml b/twill-examples/echo/pom.xml
index f05cef7..cd6376a 100644
--- a/twill-examples/echo/pom.xml
+++ b/twill-examples/echo/pom.xml
@@ -46,6 +46,12 @@ limitations under the License.
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
+ <version>1.7.7</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-simple</artifactId>
+ <version>1.7.7</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b9e668c4/twill-examples/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/pom.xml b/twill-examples/yarn/pom.xml
index 19c04a7..7c3e344 100644
--- a/twill-examples/yarn/pom.xml
+++ b/twill-examples/yarn/pom.xml
@@ -30,24 +30,26 @@ limitations under the License.
<name>Apache Twill examples: YARN</name>
<description>Examples demonstrating usage of twill-yarn</description>
<artifactId>twill-examples-yarn</artifactId>
+
+ <properties>
+ <guava.version>13.0.1</guava.version>
+ </properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>twill-yarn</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>twill-ext</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <scope>provided</scope>
+ <version>${guava.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/b9e668c4/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/BundledJarExample.java
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/BundledJarExample.java b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/BundledJarExample.java
index 4a5cbf4..cec5bdf 100644
--- a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/BundledJarExample.java
+++ b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/BundledJarExample.java
@@ -79,8 +79,8 @@ public class BundledJarExample {
}
String zkStr = args[0];
- BundledJarRunner.Arguments arguments = BundledJarRunner.Arguments.fromArray(
- Arrays.copyOfRange(args, 1, args.length));
+ BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(
+ args[1], "/lib", args[2], Arrays.copyOfRange(args, 3, args.length));
File jarFile = new File(arguments.getJarFileName());
Preconditions.checkState(jarFile.exists());