You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2018/05/16 21:23:49 UTC
hadoop git commit: YARN-8071. Add ability to specify nodemanager
environment variables individually. Contributed by Jim Brennan
Repository: hadoop
Updated Branches:
refs/heads/trunk e3b7d7ac1 -> be5396904
YARN-8071. Add ability to specify nodemanager environment variables individually. Contributed by Jim Brennan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be539690
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be539690
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be539690
Branch: refs/heads/trunk
Commit: be539690477f7fee8f836bf3612cbe7ff6a3506e
Parents: e3b7d7a
Author: Jason Lowe <jl...@apache.org>
Authored: Wed May 16 16:17:28 2018 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed May 16 16:17:28 2018 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/util/Apps.java | 33 +++++--
.../src/main/resources/yarn-default.xml | 11 ++-
.../launcher/ContainerLaunch.java | 15 ++--
.../launcher/TestContainerLaunch.java | 94 +++++++++++++++++++-
4 files changed, 134 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be539690/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 7a02874..1ea7646 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -23,10 +23,11 @@ import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import java.io.File;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -191,20 +192,36 @@ public class Apps {
/**
*
* @param envString String containing env variable definitions
- * @param classPathSeparator String that separates the definitions
- * @return ArrayList of environment variable names
+ * @return Set of environment variable names
*/
- public static ArrayList<String> getEnvVarsFromInputString(String envString,
- String classPathSeparator) {
- ArrayList<String> envList = new ArrayList<>();
+ private static Set<String> getEnvVarsFromInputString(String envString) {
+ Set<String> envSet = new HashSet<>();
if (envString != null && envString.length() > 0) {
Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
while (varValMatcher.find()) {
String envVar = varValMatcher.group(1);
- envList.add(envVar);
+ envSet.add(envVar);
}
}
- return envList;
+ return envSet;
+ }
+
+ /**
+ * Return the list of environment variable names specified in the
+ * given property or default string and those specified individually
+ * with the propname.VARNAME syntax (e.g., mapreduce.map.env.VARNAME=value).
+ * @param propName the name of the property
+ * @param defaultPropValue the default value for propName
+ * @param conf configuration containing properties
+ * @return Set of environment variable names
+ */
+ public static Set<String> getEnvVarsFromInputProperty(
+ String propName, String defaultPropValue, Configuration conf) {
+ String envString = conf.get(propName, defaultPropValue);
+ Set<String> varSet = getEnvVarsFromInputString(envString);
+ Map<String, String> propMap = conf.getPropsWithPrefix(propName + ".");
+ varSet.addAll(propMap.keySet());
+ return varSet;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be539690/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 4eb509f..e078206 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1059,7 +1059,16 @@
</property>
<property>
- <description>Environment variables that should be forwarded from the NodeManager's environment to the container's.</description>
+ <description>
+ Environment variables that should be forwarded from the NodeManager's
+ environment to the container's, specified as a comma separated list of
+ VARNAME=value pairs.
+
+ To define environment variables individually, you can specify
+ multiple properties of the form yarn.nodemanager.admin-env.VARNAME,
+ where VARNAME is the name of the environment variable. This is the only
+ way to add a variable when its value contains commas.
+ </description>
<name>yarn.nodemanager.admin-env</name>
<value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be539690/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index d43c069..57abfc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1735,13 +1735,14 @@ public class ContainerLaunch implements Callable<Integer> {
addToEnvMap(environment, nmVars, "JVM_PID", "$$");
}
- // variables here will be forced in, even if the container has specified them.
- String nmAdminUserEnv = conf.get(
- YarnConfiguration.NM_ADMIN_USER_ENV,
- YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV);
- Apps.setEnvFromInputString(environment, nmAdminUserEnv, File.pathSeparator);
- nmVars.addAll(Apps.getEnvVarsFromInputString(nmAdminUserEnv,
- File.pathSeparator));
+ // variables here will be forced in, even if the container has
+ // specified them.
+ String defEnvStr = conf.get(YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV);
+ Apps.setEnvFromInputProperty(environment,
+ YarnConfiguration.NM_ADMIN_USER_ENV, defEnvStr, conf,
+ File.pathSeparator);
+ nmVars.addAll(Apps.getEnvVarsFromInputProperty(
+ YarnConfiguration.NM_ADMIN_USER_ENV, defEnvStr, conf));
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be539690/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index fb17c8c..f8cf92d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -491,7 +491,6 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
}
-
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException {
@@ -688,9 +687,10 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
resources.put(userjar, lpaths);
Path nmp = new Path(testDir);
+ Set<String> nmEnvTrack = new LinkedHashSet<>();
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
- resources, nmp, Collections.emptySet());
+ resources, nmp, nmEnvTrack);
List<String> result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -709,7 +709,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
dispatcher, exec, null, container, dirsHandler, containerManager);
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
- resources, nmp, Collections.emptySet());
+ resources, nmp, nmEnvTrack);
result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -721,6 +721,94 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
@Test
+ public void testSanitizeNMEnvVars() throws Exception {
+ // Valid only for unix
+ assumeNotWindows();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
+ Map<String, String> userSetEnv = new HashMap<String, String>();
+ Set<String> nmEnvTrack = new LinkedHashSet<>();
+ userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
+ userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST");
+ userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
+ userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
+ userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
+ userSetEnv.put(Environment.USER.key(), "user_set_" +
+ Environment.USER.key());
+ userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
+ userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
+ userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
+ userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
+ // This one should be appended to.
+ String userMallocArenaMaxVal = "test_user_max_val";
+ userSetEnv.put("MALLOC_ARENA_MAX", userMallocArenaMaxVal);
+ containerLaunchContext.setEnvironment(userSetEnv);
+ Container container = mock(Container.class);
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(containerLaunchContext);
+ when(container.getLocalizedResources()).thenReturn(null);
+ Dispatcher dispatcher = mock(Dispatcher.class);
+ EventHandler<Event> eventHandler = new EventHandler<Event>() {
+ public void handle(Event event) {
+ Assert.assertTrue(event instanceof ContainerExitEvent);
+ ContainerExitEvent exitEvent = (ContainerExitEvent) event;
+ Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ exitEvent.getType());
+ }
+ };
+ when(dispatcher.getEventHandler()).thenReturn(eventHandler);
+
+ // these should eclipse anything in the user environment
+ YarnConfiguration conf = new YarnConfiguration();
+ String mallocArenaMaxVal = "test_nm_max_val";
+ conf.set("yarn.nodemanager.admin-env",
+ "MALLOC_ARENA_MAX=" + mallocArenaMaxVal);
+ String testKey1 = "TEST_KEY1";
+ String testVal1 = "testVal1";
+ conf.set("yarn.nodemanager.admin-env." + testKey1, testVal1);
+ String testKey2 = "TEST_KEY2";
+ String testVal2 = "testVal2";
+ conf.set("yarn.nodemanager.admin-env." + testKey2, testVal2);
+ String testKey3 = "MOUNT_LIST";
+ String testVal3 = "/home/a/b/c,/home/d/e/f,/home/g/e/h";
+ conf.set("yarn.nodemanager.admin-env." + testKey3, testVal3);
+ Map<String, String> environment = new HashMap<>();
+ LinkedHashSet<String> nmVars = new LinkedHashSet<>();
+ ContainerLaunch launch = new ContainerLaunch(distContext, conf,
+ dispatcher, exec, null, container, dirsHandler, containerManager);
+ String testDir = System.getProperty("test.build.data",
+ "target/test-dir");
+ Path pwd = new Path(testDir);
+ List<Path> appDirs = new ArrayList<Path>();
+ List<String> userLocalDirs = new ArrayList<>();
+ List<String> containerLogs = new ArrayList<String>();
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ Path userjar = new Path("user.jar");
+ List<String> lpaths = new ArrayList<String>();
+ lpaths.add("userjarlink.jar");
+ resources.put(userjar, lpaths);
+ Path nmp = new Path(testDir);
+
+ launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+ resources, nmp, nmEnvTrack);
+ Assert.assertTrue(userSetEnv.containsKey("MALLOC_ARENA_MAX"));
+ Assert.assertTrue(userSetEnv.containsKey(testKey1));
+ Assert.assertTrue(userSetEnv.containsKey(testKey2));
+ Assert.assertTrue(userSetEnv.containsKey(testKey3));
+ Assert.assertTrue(nmEnvTrack.contains("MALLOC_ARENA_MAX"));
+ Assert.assertTrue(nmEnvTrack.contains("MOUNT_LIST"));
+ Assert.assertEquals(userMallocArenaMaxVal + File.pathSeparator
+ + mallocArenaMaxVal, userSetEnv.get("MALLOC_ARENA_MAX"));
+ Assert.assertEquals(testVal1, userSetEnv.get(testKey1));
+ Assert.assertEquals(testVal2, userSetEnv.get(testKey2));
+ Assert.assertEquals(testVal3, userSetEnv.get(testKey3));
+ }
+
+ @Test
public void testErrorLogOnContainerExit() throws Exception {
verifyTailErrorLogOnContainerExit(new Configuration(), "/stderr", false);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org