You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by mw...@apache.org on 2018/11/19 16:52:10 UTC
[accumulo-testing] branch master updated: Fixed running continuous
ingest in YARN (#41)
This is an automated email from the ASF dual-hosted git repository.
mwalch pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/master by this push:
new 0085330 Fixed running continuous ingest in YARN (#41)
0085330 is described below
commit 0085330b5a03134fa227da83b66e319a053c6d28
Author: Mike Walch <mw...@apache.org>
AuthorDate: Mon Nov 19 11:52:06 2018 -0500
Fixed running continuous ingest in YARN (#41)
* Excluded license directory from shaded jar to prevent
Twill from failing
* Upgraded Twill to 0.13 and made minor improvement to code.
---
core/pom.xml | 2 ++
pom.xml | 2 +-
.../testing/yarn/YarnAccumuloTestRunner.java | 22 ++++++++++++++++------
3 files changed, 19 insertions(+), 7 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 0d03f95..d982022 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -116,6 +116,8 @@
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
+ <!-- Twill 0.13 fails if shaded jar includes this directory !-->
+ <exclude>META-INF/license/*</exclude>
</excludes>
</filter>
</filters>
diff --git a/pom.xml b/pom.xml
index 0de1eed..7ea2a47 100644
--- a/pom.xml
+++ b/pom.xml
@@ -42,7 +42,7 @@
<hadoop.version>3.0.3</hadoop.version>
<zookeeper.version>3.4.9</zookeeper.version>
<slf4j.version>1.7.21</slf4j.version>
- <twill.version>0.12.1</twill.version>
+ <twill.version>0.13.0</twill.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<formatter.config>${project.parent.basedir}/contrib/Eclipse-Accumulo-Codestyle.xml</formatter.config>
diff --git a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
index 56fe6f3..5ac22dc 100644
--- a/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
+++ b/yarn/src/main/java/org/apache/accumulo/testing/yarn/YarnAccumuloTestRunner.java
@@ -37,13 +37,14 @@ import java.io.File;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import java.util.Properties;
public class YarnAccumuloTestRunner {
private static final Logger LOG = LoggerFactory.getLogger(YarnAccumuloTestRunner.class);
- private static final String RUNNABLE_ID = "BundledJarRunnable";
+ private static final String RUNNABLE_ID = "AccumuloTest";
private static class YarnTestApp implements TwillApplication {
@@ -111,7 +112,9 @@ public class YarnAccumuloTestRunner {
public static void main(String[] args) throws Exception {
TestRunnerOpts opts = new TestRunnerOpts();
- new JCommander(opts, args);
+ JCommander commander = new JCommander();
+ commander.addObject(opts);
+ commander.parse(args);
verifyPath(opts.jarPath);
verifyPath(opts.testProps);
@@ -119,10 +122,17 @@ public class YarnAccumuloTestRunner {
verifyPath(opts.logProps);
String jarFileName = Paths.get(opts.jarPath).getFileName().toString();
-
String[] mainArgs = opts.mainArgs.stream().toArray(String[]::new);
- BundledJarRunner.Arguments arguments = new BundledJarRunner.Arguments(jarFileName, "/lib",
- opts.mainClass, mainArgs);
+
+ Objects.requireNonNull(jarFileName);
+ Objects.requireNonNull(opts.mainClass);
+ Objects.requireNonNull(mainArgs);
+
+ BundledJarRunner.Arguments arguments =
+ new BundledJarRunner.Arguments.Builder().setJarFileName(jarFileName)
+ .setLibFolder("lib").setMainClassName(opts.mainClass)
+ .setMainArgs(mainArgs).createArguments();
+
TestEnv env = new TestEnv(opts.testProps, opts.clientProps);
YarnConfiguration yarnConfig = new YarnConfiguration(env.getHadoopConfiguration());
@@ -131,7 +141,7 @@ public class YarnAccumuloTestRunner {
twillRunner.prepare(new YarnTestApp(opts, env.getTestProperties()))
.addJVMOptions("-Dlog4j.configuration=file:$PWD/" + new File(opts.logProps).getName())
- .withArguments("BundledJarRunnable", arguments.toArray()).start();
+ .withArguments(RUNNABLE_ID, arguments.toArray()).start();
LOG.info("{} containers will start in YARN.", opts.numContainers);
LOG.info("Press Ctrl-C when these containers have started.");