You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@apex.apache.org by pr...@apache.org on 2017/08/29 23:30:48 UTC
[apex-core] branch master updated: APEXCORE-779 In unit tests Yarn
containers must use the same JVM as the test itself.
This is an automated email from the ASF dual-hosted git repository.
pramod pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/apex-core.git
The following commit(s) were added to refs/heads/master by this push:
new 821184e APEXCORE-779 In unit tests Yarn containers must use the same JVM as the test itself.
821184e is described below
commit 821184e296d2ecc982c3b3275d28c7167d0768e8
Author: Vlad Rozov <vr...@apache.org>
AuthorDate: Sat Aug 26 10:03:45 2017 -0700
APEXCORE-779 In unit tests Yarn containers must use the same JVM as the test itself.
---
.../java/com/datatorrent/stram/StramClient.java | 3 ++-
.../datatorrent/stram/StramMiniClusterTest.java | 30 +++-------------------
2 files changed, 6 insertions(+), 27 deletions(-)
diff --git a/engine/src/main/java/com/datatorrent/stram/StramClient.java b/engine/src/main/java/com/datatorrent/stram/StramClient.java
index 06abdae..3a3c135 100644
--- a/engine/src/main/java/com/datatorrent/stram/StramClient.java
+++ b/engine/src/main/java/com/datatorrent/stram/StramClient.java
@@ -103,6 +103,8 @@ public class StramClient
@Deprecated
public static final String YARN_APPLICATION_TYPE_DEPRECATED = "DataTorrent";
+ private static final String javaCmd = "${JAVA_HOME}" + "/bin/java";
+
public static final String LIB_JARS_SEP = ",";
// Configuration
@@ -114,7 +116,6 @@ public class StramClient
private final int amPriority = 0;
private ApplicationId appId;
private final LogicalPlan dag;
- public String javaCmd = "${JAVA_HOME}" + "/bin/java";
// log4j.properties file
// if available, add to local resources and set into classpath
private final String log4jPropFile = "";
diff --git a/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java b/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
index 455604b..0e3c0ab 100644
--- a/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
+++ b/engine/src/test/java/com/datatorrent/stram/StramMiniClusterTest.java
@@ -44,7 +44,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.apex.common.util.JarHelper;
-import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -125,19 +124,10 @@ public class StramMiniClusterTest
conf = StramClientUtils.addDTDefaultResources(conf);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.setInt("yarn.nodemanager.vmem-pmem-ratio", 20); // workaround to avoid containers being killed because java allocated too much vmem
- conf.setStrings("yarn.scheduler.capacity.root.queues", "default");
- conf.setStrings("yarn.scheduler.capacity.root.default.capacity", "100");
-
- StringBuilder adminEnv = new StringBuilder(1024);
- if (System.getenv("JAVA_HOME") == null) {
- adminEnv.append("JAVA_HOME=").append(System.getProperty("java.home"));
- adminEnv.append(",");
- }
- adminEnv.append("MALLOC_ARENA_MAX=4"); // see MAPREDUCE-3068, MAPREDUCE-3065
- adminEnv.append(",");
- adminEnv.append("CLASSPATH=").append(getTestRuntimeClasspath());
-
- conf.set(YarnConfiguration.NM_ADMIN_USER_ENV, adminEnv.toString());
+ conf.set("yarn.scheduler.capacity.root.queues", "default");
+ conf.set("yarn.scheduler.capacity.root.default.capacity", "100");
+ conf.set(YarnConfiguration.NM_ADMIN_USER_ENV, String.format("JAVA_HOME=%s,CLASSPATH=%s", System.getProperty("java.home"), getTestRuntimeClasspath()));
+ conf.set(YarnConfiguration.NM_ENV_WHITELIST, YarnConfiguration.DEFAULT_NM_ENV_WHITELIST.replaceAll("JAVA_HOME,*", ""));
if (yarnCluster == null) {
yarnCluster = new MiniYARNCluster(StramMiniClusterTest.class.getName(),
@@ -229,9 +219,6 @@ public class StramMiniClusterTest
StramClient client = new StramClient(yarnConf, dag);
try {
client.start();
- if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
- client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
- }
LOG.info("Running client");
client.startApplication();
boolean result = client.monitorApplication();
@@ -275,9 +262,6 @@ public class StramMiniClusterTest
tb.addFromProperties(props, null);
StramClient client = new StramClient(new Configuration(yarnCluster.getConfig()), createDAG(tb));
- if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
- client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
- }
try {
client.start();
client.startApplication();
@@ -382,9 +366,6 @@ public class StramMiniClusterTest
LOG.info("Initializing Client");
StramClient client = new StramClient(conf, dag);
- if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
- client.javaCmd = "java"; // JAVA_HOME not set in the yarn mini cluster
- }
try {
client.start();
client.startApplication();
@@ -614,9 +595,6 @@ public class StramMiniClusterTest
dag.getContextAttributes(operator).put(OperatorContext.RECOVERY_ATTEMPTS, 0);
StramClient client = new StramClient(conf, dag);
- if (StringUtils.isBlank(System.getenv("JAVA_HOME"))) {
- client.javaCmd = "java";
- }
try {
client.start();
client.startApplication();
--
To stop receiving notification emails like this one, please contact
['"commits@apex.apache.org" <co...@apex.apache.org>'].