You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/12/20 06:17:20 UTC
hadoop git commit: YARN-5877. Allow all env's from
yarn.nodemanager.env-whitelist to get overridden during launch. Contributed
by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/trunk d2026ae9e -> 575773a35
YARN-5877. Allow all env's from yarn.nodemanager.env-whitelist to get overridden during launch. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/575773a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/575773a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/575773a3
Branch: refs/heads/trunk
Commit: 575773a3570b85293fdf7b8aeb8467580ec7f896
Parents: d2026ae
Author: Sunil G <su...@apache.org>
Authored: Tue Dec 20 11:39:06 2016 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Dec 20 11:39:06 2016 +0530
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 11 ++--
.../launcher/TestContainerLaunch.java | 63 ++++++++++++++++----
2 files changed, 58 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/575773a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index f880506..158585e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -320,11 +319,11 @@ public abstract class ContainerExecutor implements Configurable {
ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<>();
- whitelist.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name());
- whitelist.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name());
- whitelist.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name());
- whitelist.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name());
- whitelist.add(ApplicationConstants.Environment.JAVA_HOME.name());
+ String[] nmWhiteList = conf.get(YarnConfiguration.NM_ENV_WHITELIST,
+ YarnConfiguration.DEFAULT_NM_ENV_WHITELIST).split(",");
+ for (String param : nmWhiteList) {
+ whitelist.add(param);
+ }
if (environment != null) {
for (Map.Entry<String, String> env : environment.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/575773a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 4ce816a..23b99d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.apache.hadoop.test.PlatformAssumptions.assumeWindows;
+import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
@@ -33,6 +34,9 @@ import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -172,10 +176,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add("/bin/sh ./\\\"" + badSymlink + "\\\"");
}
- new DefaultContainerExecutor()
- .writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user",
- tempFile.getName());
+ DefaultContainerExecutor defaultContainerExecutor =
+ new DefaultContainerExecutor();
+ defaultContainerExecutor.setConf(new YarnConfiguration());
+ defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -242,9 +247,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
} else {
commands.add("/bin/sh ./\\\"" + symLink + "\\\"");
}
- new DefaultContainerExecutor()
- .writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user");
+ DefaultContainerExecutor defaultContainerExecutor =
+ new DefaultContainerExecutor();
+ defaultContainerExecutor.setConf(new YarnConfiguration());
+ defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -279,6 +286,39 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
}
}
+ @Test(timeout = 20000)
+ public void testWriteEnvExport() throws Exception {
+ // Valid only for unix
+ assumeNotWindows();
+ File shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ Map<String, String> env = new HashMap<String, String>();
+ env.put("HADOOP_COMMON_HOME", "/opt/hadoopcommon");
+ env.put("HADOOP_MAPRED_HOME", "/opt/hadoopbuild");
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ FileOutputStream fos = new FileOutputStream(shellFile);
+ List<String> commands = new ArrayList<String>();
+ DefaultContainerExecutor defaultContainerExecutor =
+ new DefaultContainerExecutor();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.NM_ENV_WHITELIST,
+ "HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
+ defaultContainerExecutor.setConf(conf);
+ defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user");
+ String shellContent =
+ new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
+ StandardCharsets.UTF_8);
+ Assert.assertTrue(shellContent
+ .contains("export HADOOP_COMMON_HOME=\"/opt/hadoopcommon\""));
+ // Not available in env and whitelist
+ Assert.assertTrue(shellContent.contains("export HADOOP_MAPRED_HOME="
+ + "${HADOOP_MAPRED_HOME:-\"/opt/hadoopbuild\"}"));
+ // Not available in env but in whitelist
+ Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
+ fos.flush();
+ fos.close();
+ }
+
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException {
@@ -297,9 +337,11 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"\"workflowName\":\"\n\ninsert table " +
"\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
List<String> commands = new ArrayList<String>();
- new DefaultContainerExecutor()
- .writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user");
+ DefaultContainerExecutor defaultContainerExecutor =
+ new DefaultContainerExecutor();
+ defaultContainerExecutor.setConf(new YarnConfiguration());
+ defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -377,6 +419,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
List<String> commands = new ArrayList<String>();
commands.add(command);
ContainerExecutor exec = new DefaultContainerExecutor();
+ exec.setConf(new YarnConfiguration());
exec.writeLaunchEnv(fos, env, resources, commands,
new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org