You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jb...@apache.org on 2021/02/05 18:02:13 UTC
[hadoop] branch branch-3.2 updated: [YARN-10607] User environment
is unable to prepend PATH when mapreduce.admin.user.env also sets PATH.
Contributed by Eric Badger.
This is an automated email from the ASF dual-hosted git repository.
jbrennan pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 6cc0eb3 [YARN-10607] User environment is unable to prepend PATH when mapreduce.admin.user.env also sets PATH. Contributed by Eric Badger.
6cc0eb3 is described below
commit 6cc0eb3e304386eb1993ddbfed2a2a93582a5393
Author: Jim Brennan <jb...@apache.org>
AuthorDate: Fri Feb 5 17:33:01 2021 +0000
[YARN-10607] User environment is unable to prepend PATH when mapreduce.admin.user.env also sets PATH. Contributed by Eric Badger.
(cherry picked from commit c22c77af4368cd4fb1d630f84d0d0c5cc11b224f)
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 9 ++++
.../src/main/resources/yarn-default.xml | 11 ++++
.../containermanager/launcher/ContainerLaunch.java | 21 ++++++++
.../launcher/TestContainerLaunch.java | 63 ++++++++++++++++++++++
4 files changed, 104 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ad4d87d..90804d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1202,6 +1202,15 @@ public class YarnConfiguration extends Configuration {
public static final String NM_ADMIN_USER_ENV = NM_PREFIX + "admin-env";
public static final String DEFAULT_NM_ADMIN_USER_ENV = "MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX";
+ /**
+ * PATH components that will be prepended to the user's path.
+ * If this is defined and the user does not define PATH, NM will also
+ * append ":$PATH" to prevent this from eclipsing the PATH defined in
+ * the container. This feature is only available for Linux.
+ * */
+ public static final String NM_ADMIN_FORCE_PATH = NM_PREFIX + "force.path";
+ public static final String DEFAULT_NM_ADMIN_FORCE_PATH = "";
+
/** Environment variables that containers may override rather than use NodeManager's default.*/
public static final String NM_ENV_WHITELIST = NM_PREFIX + "env-whitelist";
public static final String DEFAULT_NM_ENV_WHITELIST = StringUtils.join(",",
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d096e8c..dff7e31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1146,6 +1146,17 @@
</property>
<property>
+ <description>
+ * PATH components that will be prepended to the user's path.
+ * If this is defined and the user does not define PATH, NM will also
+ * append ":$PATH" to prevent this from eclipsing the PATH defined in
+ * the container. This feature is only available for Linux.
+ </description>
+ <name>yarn.nodemanager.force.path</name>
+ <value></value>
+ </property>
+
+ <property>
<description>Environment variables that containers may override rather than use NodeManager's default.</description>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ</value>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index a8f4602..84cc79e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1572,6 +1572,27 @@ public class ContainerLaunch implements Callable<Integer> {
nmVars.addAll(Apps.getEnvVarsFromInputProperty(
YarnConfiguration.NM_ADMIN_USER_ENV, defEnvStr, conf));
+ if (!Shell.WINDOWS) {
+ // maybe force path components
+ String forcePath = conf.get(YarnConfiguration.NM_ADMIN_FORCE_PATH,
+ YarnConfiguration.DEFAULT_NM_ADMIN_FORCE_PATH);
+ if (!forcePath.isEmpty()) {
+ String userPath = environment.get(Environment.PATH.name());
+ environment.remove(Environment.PATH.name());
+ if (userPath == null || userPath.isEmpty()) {
+ Apps.addToEnvironment(environment, Environment.PATH.name(),
+ forcePath, File.pathSeparator);
+ Apps.addToEnvironment(environment, Environment.PATH.name(),
+ "$PATH", File.pathSeparator);
+ } else {
+ Apps.addToEnvironment(environment, Environment.PATH.name(),
+ forcePath, File.pathSeparator);
+ Apps.addToEnvironment(environment, Environment.PATH.name(),
+ userPath, File.pathSeparator);
+ }
+ }
+ }
+
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
if (Shell.WINDOWS) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 0f2f76c..d700850 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -815,6 +815,69 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
@Test
+ public void testNmForcePath() throws Exception {
+ // Valid only for unix
+ assumeNotWindows();
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
+ Map<String, String> userSetEnv = new HashMap<>();
+ Set<String> nmEnvTrack = new LinkedHashSet<>();
+ containerLaunchContext.setEnvironment(userSetEnv);
+ Container container = mock(Container.class);
+ when(container.getContainerId()).thenReturn(cId);
+ when(container.getLaunchContext()).thenReturn(containerLaunchContext);
+ when(container.getLocalizedResources()).thenReturn(null);
+ Dispatcher dispatcher = mock(Dispatcher.class);
+ EventHandler<Event> eventHandler = new EventHandler<Event>() {
+ public void handle(Event event) {
+ Assert.assertTrue(event instanceof ContainerExitEvent);
+ ContainerExitEvent exitEvent = (ContainerExitEvent) event;
+ Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ exitEvent.getType());
+ }
+ };
+ when(dispatcher.getEventHandler()).thenReturn(eventHandler);
+
+ String testDir = System.getProperty("test.build.data",
+ "target/test-dir");
+ Path pwd = new Path(testDir);
+ List<Path> appDirs = new ArrayList<>();
+ List<String> userLocalDirs = new ArrayList<>();
+ List<String> containerLogs = new ArrayList<>();
+ Map<Path, List<String>> resources = new HashMap<>();
+ Path nmp = new Path(testDir);
+
+ YarnConfiguration conf = new YarnConfiguration();
+ String forcePath = "./force-path";
+ conf.set("yarn.nodemanager.force.path", forcePath);
+
+ ContainerLaunch launch = new ContainerLaunch(distContext, conf,
+ dispatcher, exec, null, container, dirsHandler, containerManager);
+ launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+ resources, nmp, nmEnvTrack);
+
+ Assert.assertTrue(userSetEnv.containsKey(Environment.PATH.name()));
+ Assert.assertEquals(forcePath + ":$PATH",
+ userSetEnv.get(Environment.PATH.name()));
+
+ String userPath = "/usr/bin:/usr/local/bin";
+ userSetEnv.put(Environment.PATH.name(), userPath);
+ containerLaunchContext.setEnvironment(userSetEnv);
+ when(container.getLaunchContext()).thenReturn(containerLaunchContext);
+
+ launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
+ resources, nmp, nmEnvTrack);
+
+ Assert.assertTrue(userSetEnv.containsKey(Environment.PATH.name()));
+ Assert.assertEquals(forcePath + ":" + userPath,
+ userSetEnv.get(Environment.PATH.name()));
+ }
+
+ @Test
public void testErrorLogOnContainerExit() throws Exception {
verifyTailErrorLogOnContainerExit(new Configuration(), "/stderr", false);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org