You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/02/23 19:45:04 UTC
[23/50] [abbrv] hadoop git commit: Revert "YARN-7677. Docker image
cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan"
Revert "YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan"
This reverts commit 8013475d447a8377b5aed858208bf8b91dd32366.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9a429bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9a429bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9a429bb
Branch: refs/heads/HDFS-12996
Commit: b9a429bb2854910add8d4cf787e6ee65ebdfc9cf
Parents: 83e2bb9
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Feb 19 08:16:25 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Feb 19 08:16:25 2018 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/yarn/util/Apps.java | 22 +--
.../yarn/util/AuxiliaryServiceHelper.java | 2 +-
.../server/nodemanager/ContainerExecutor.java | 62 +++------
.../nodemanager/LinuxContainerExecutor.java | 8 ++
.../launcher/ContainerLaunch.java | 88 ++++--------
.../runtime/DefaultLinuxContainerRuntime.java | 6 +
.../DelegatingLinuxContainerRuntime.java | 11 ++
.../runtime/DockerLinuxContainerRuntime.java | 7 +
.../runtime/ContainerRuntime.java | 11 ++
.../launcher/TestContainerLaunch.java | 133 ++-----------------
10 files changed, 110 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index 1c90d55..685c6d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.util.StringHelper.sjoin;
import java.io.File;
-import java.util.ArrayList;
import java.util.Iterator;
import java.util.Map;
import java.util.regex.Matcher;
@@ -106,26 +105,7 @@ public class Apps {
}
}
}
-
- /**
- *
- * @param envString String containing env variable definitions
- * @param classPathSeparator String that separates the definitions
- * @return ArrayList of environment variable names
- */
- public static ArrayList<String> getEnvVarsFromInputString(String envString,
- String classPathSeparator) {
- ArrayList<String> envList = new ArrayList<>();
- if (envString != null && envString.length() > 0) {
- Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
- while (varValMatcher.find()) {
- String envVar = varValMatcher.group(1);
- envList.add(envVar);
- }
- }
- return envList;
- }
-
+
/**
* This older version of this method is kept around for compatibility
* because downstream frameworks like Spark and Tez have been using it.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java
index 1374d96..cb118f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AuxiliaryServiceHelper.java
@@ -45,7 +45,7 @@ public class AuxiliaryServiceHelper {
Base64.encodeBase64String(byteData));
}
- public static String getPrefixServiceName(String serviceName) {
+ private static String getPrefixServiceName(String serviceName) {
return NM_AUX_SERVICE + serviceName;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 01cd992..f4279a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -27,7 +27,6 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.LinkedHashSet;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -317,15 +316,14 @@ public abstract class ContainerExecutor implements Configurable {
* @param command the command that will be run
* @param logDir the log dir to which to copy debugging information
* @param user the username of the job owner
- * @param nmVars the set of environment vars that are explicitly set by NM
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
- String user, LinkedHashSet<String> nmVars) throws IOException {
+ String user) throws IOException {
this.writeLaunchEnv(out, environment, resources, command, logDir, user,
- ContainerLaunch.CONTAINER_SCRIPT, nmVars);
+ ContainerLaunch.CONTAINER_SCRIPT);
}
/**
@@ -341,15 +339,14 @@ public abstract class ContainerExecutor implements Configurable {
* @param logDir the log dir to which to copy debugging information
* @param user the username of the job owner
* @param outFilename the path to which to write the launch environment
- * @param nmVars the set of environment vars that are explicitly set by NM
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
@VisibleForTesting
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
- String user, String outFilename, LinkedHashSet<String> nmVars)
- throws IOException {
+ String user, String outFilename) throws IOException {
+ updateEnvForWhitelistVars(environment);
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -364,40 +361,8 @@ public abstract class ContainerExecutor implements Configurable {
if (environment != null) {
sb.echo("Setting up env variables");
- // Whitelist environment variables are treated specially.
- // Only add them if they are not already defined in the environment.
- // Add them using special syntax to prevent them from eclipsing
- // variables that may be set explicitly in the container image (e.g,
- // in a docker image). Put these before the others to ensure the
- // correct expansion is used.
- for(String var : whitelistVars) {
- if (!environment.containsKey(var)) {
- String val = getNMEnvVar(var);
- if (val != null) {
- sb.whitelistedEnv(var, val);
- }
- }
- }
- // Now write vars that were set explicitly by nodemanager, preserving
- // the order they were written in.
- for (String nmEnvVar : nmVars) {
- sb.env(nmEnvVar, environment.get(nmEnvVar));
- }
- // Now write the remaining environment variables.
for (Map.Entry<String, String> env : environment.entrySet()) {
- if (!nmVars.contains(env.getKey())) {
- sb.env(env.getKey(), env.getValue());
- }
- }
- // Add the whitelist vars to the environment. Do this after writing
- // environment variables so they are not written twice.
- for(String var : whitelistVars) {
- if (!environment.containsKey(var)) {
- String val = getNMEnvVar(var);
- if (val != null) {
- environment.put(var, val);
- }
- }
+ sb.env(env.getKey(), env.getValue());
}
}
@@ -698,6 +663,23 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ /**
+ * Propagate variables from the nodemanager's environment into the
+ * container's environment if unspecified by the container.
+ * @param env the environment to update
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ for(String var : whitelistVars) {
+ if (!env.containsKey(var)) {
+ String val = getNMEnvVar(var);
+ if (val != null) {
+ env.put(var, val);
+ }
+ }
+ }
+ }
+
@VisibleForTesting
protected String getNMEnvVar(String varname) {
return System.getenv(varname);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 44edc21..fe54e2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -66,6 +66,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -472,6 +473,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ if (linuxContainerRuntime.useWhitelistEnv(env)) {
+ super.updateEnvForWhitelistVars(env);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index ca62a5c..112f54a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -33,9 +33,7 @@ import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
-import java.util.LinkedHashSet;
import java.util.Map;
-import java.util.Set;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -219,9 +217,6 @@ public class ContainerLaunch implements Callable<Integer> {
launchContext, containerLogDir);
// /////////////////////////// End of variable expansion
- // Use this to track variables that are added to the environment by nm.
- LinkedHashSet<String> nmEnvVars = new LinkedHashSet<String>();
-
FileContext lfs = FileContext.getLocalFSFileContext();
Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(
@@ -266,7 +261,7 @@ public class ContainerLaunch implements Callable<Integer> {
}
// Set the token location too.
- addToEnvMap(environment, nmEnvVars,
+ environment.put(
ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME,
new Path(containerWorkDir,
FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
@@ -277,15 +272,14 @@ public class ContainerLaunch implements Callable<Integer> {
EnumSet.of(CREATE, OVERWRITE))) {
// Sanitize the container's environment
sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs,
- containerLogDirs, localResources, nmPrivateClasspathJarDir,
- nmEnvVars);
+ containerLogDirs, localResources, nmPrivateClasspathJarDir);
prepareContainer(localResources, containerLocalDirs);
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
localResources, launchContext.getCommands(),
- containerLogDir, user, nmEnvVars);
+ containerLogDir, user);
}
// /////////// End of writing out container-script
@@ -1177,9 +1171,6 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void env(String key, String value) throws IOException;
- public abstract void whitelistedEnv(String key, String value)
- throws IOException;
-
public abstract void echo(String echoStr) throws IOException;
public final void symlink(Path src, Path dst) throws IOException {
@@ -1300,11 +1291,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- line("export ", key, "=${", key, ":-", "\"", value, "\"}");
- }
-
- @Override
public void echo(final String echoStr) throws IOException {
line("echo \"" + echoStr + "\"");
}
@@ -1395,11 +1381,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- env(key, value);
- }
-
- @Override
public void echo(final String echoStr) throws IOException {
lineWithLenCheck("@echo \"", echoStr, "\"");
}
@@ -1454,70 +1435,60 @@ public class ContainerLaunch implements Callable<Integer> {
putEnvIfNotNull(environment, variable, System.getenv(variable));
}
}
-
- private static void addToEnvMap(
- Map<String, String> envMap, Set<String> envSet,
- String envName, String envValue) {
- envMap.put(envName, envValue);
- envSet.add(envName);
- }
-
+
public void sanitizeEnv(Map<String, String> environment, Path pwd,
List<Path> appDirs, List<String> userLocalDirs, List<String>
- containerLogDirs, Map<Path, List<String>> resources,
- Path nmPrivateClasspathJarDir,
- Set<String> nmVars) throws IOException {
+ containerLogDirs,
+ Map<Path, List<String>> resources,
+ Path nmPrivateClasspathJarDir) throws IOException {
/**
* Non-modifiable environment variables
*/
- addToEnvMap(environment, nmVars, Environment.CONTAINER_ID.name(),
- container.getContainerId().toString());
+ environment.put(Environment.CONTAINER_ID.name(), container
+ .getContainerId().toString());
- addToEnvMap(environment, nmVars, Environment.NM_PORT.name(),
+ environment.put(Environment.NM_PORT.name(),
String.valueOf(this.context.getNodeId().getPort()));
- addToEnvMap(environment, nmVars, Environment.NM_HOST.name(),
- this.context.getNodeId().getHost());
+ environment.put(Environment.NM_HOST.name(), this.context.getNodeId()
+ .getHost());
- addToEnvMap(environment, nmVars, Environment.NM_HTTP_PORT.name(),
+ environment.put(Environment.NM_HTTP_PORT.name(),
String.valueOf(this.context.getHttpPort()));
- addToEnvMap(environment, nmVars, Environment.LOCAL_DIRS.name(),
+ environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
- addToEnvMap(environment, nmVars, Environment.LOCAL_USER_DIRS.name(),
- StringUtils.join(",", userLocalDirs));
+ environment.put(Environment.LOCAL_USER_DIRS.name(), StringUtils.join(",",
+ userLocalDirs));
- addToEnvMap(environment, nmVars, Environment.LOG_DIRS.name(),
+ environment.put(Environment.LOG_DIRS.name(),
StringUtils.join(",", containerLogDirs));
- addToEnvMap(environment, nmVars, Environment.USER.name(),
- container.getUser());
-
- addToEnvMap(environment, nmVars, Environment.LOGNAME.name(),
- container.getUser());
+ environment.put(Environment.USER.name(), container.getUser());
+
+ environment.put(Environment.LOGNAME.name(), container.getUser());
- addToEnvMap(environment, nmVars, Environment.HOME.name(),
+ environment.put(Environment.HOME.name(),
conf.get(
YarnConfiguration.NM_USER_HOME_DIR,
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR
)
);
-
- addToEnvMap(environment, nmVars, Environment.PWD.name(), pwd.toString());
+
+ environment.put(Environment.PWD.name(), pwd.toString());
+
+ putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
if (!Shell.WINDOWS) {
- addToEnvMap(environment, nmVars, "JVM_PID", "$$");
+ environment.put("JVM_PID", "$$");
}
// variables here will be forced in, even if the container has specified them.
- String nmAdminUserEnv = conf.get(
- YarnConfiguration.NM_ADMIN_USER_ENV,
- YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV);
- Apps.setEnvFromInputString(environment, nmAdminUserEnv, File.pathSeparator);
- nmVars.addAll(Apps.getEnvVarsFromInputString(nmAdminUserEnv,
- File.pathSeparator));
+ Apps.setEnvFromInputString(environment, conf.get(
+ YarnConfiguration.NM_ADMIN_USER_ENV,
+ YarnConfiguration.DEFAULT_NM_ADMIN_USER_ENV), File.pathSeparator);
// TODO: Remove Windows check and use this approach on all platforms after
// additional testing. See YARN-358.
@@ -1531,7 +1502,6 @@ public class ContainerLaunch implements Callable<Integer> {
.getAuxServiceMetaData().entrySet()) {
AuxiliaryServiceHelper.setServiceDataIntoEnv(
meta.getKey(), meta.getValue(), environment);
- nmVars.add(AuxiliaryServiceHelper.getPrefixServiceName(meta.getKey()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 83380ee..b50d56c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -73,6 +74,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ return true;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
//nothing to do here at the moment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 675bffb..dd10617 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -94,6 +94,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ try {
+ LinuxContainerRuntime runtime = pickContainerRuntime(env);
+ return runtime.useWhitelistEnv(env);
+ } catch (ContainerExecutionException e) {
+ LOG.debug("Unable to determine runtime");
+ return false;
+ }
+ }
+
@VisibleForTesting
LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment) throws ContainerExecutionException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index de225e6..401fc4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -371,6 +371,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return capabilities;
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ // Avoid propagating nodemanager environment variables into the container
+ // so those variables can be picked up from the Docker image instead.
+ return false;
+ }
+
private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand,
Container container) throws ContainerExecutionException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 7caa0ed..aa294fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import java.util.Map;
+
/**
* An abstraction for various container runtime implementations. Examples
* include Process Tree, Docker, Appc runtimes etc. These implementations
@@ -83,4 +85,13 @@ public interface ContainerRuntime {
* and hostname
*/
String[] getIpAndHost(Container container) throws ContainerExecutionException;
+
+ /**
+ * Whether to propagate the whitelist of environment variables from the
+ * nodemanager environment into the container environment.
+ * @param env the container's environment variables
+ * @return true if whitelist variables should be propagated, false otherwise
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ boolean useWhitelistEnv(Map<String, String> env);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9a429bb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 47e268c..5923f8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -41,7 +41,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
@@ -186,10 +185,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
DefaultContainerExecutor defaultContainerExecutor =
new DefaultContainerExecutor();
defaultContainerExecutor.setConf(new YarnConfiguration());
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName(),
- nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user", tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -263,9 +260,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
DefaultContainerExecutor defaultContainerExecutor =
new DefaultContainerExecutor();
defaultContainerExecutor.setConf(new YarnConfiguration());
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -327,9 +323,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
conf.set(YarnConfiguration.NM_ENV_WHITELIST,
"HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
defaultContainerExecutor.setConf(conf);
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user");
String shellContent =
new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
StandardCharsets.UTF_8);
@@ -342,8 +337,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
// Available in env and in whitelist
Assert.assertTrue(shellContent.contains(
- "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
- ));
+ "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\""));
fos.flush();
fos.close();
}
@@ -378,9 +372,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
conf.set(YarnConfiguration.NM_ENV_WHITELIST,
"HADOOP_MAPRED_HOME,HADOOP_YARN_HOME");
lce.setConf(conf);
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
lce.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user");
String shellContent =
new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
StandardCharsets.UTF_8);
@@ -389,106 +382,13 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
// Whitelisted variable overridden by container
Assert.assertTrue(shellContent.contains(
"export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
- // Available in env but not in whitelist
+ // Verify no whitelisted variables inherited from NM env
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
- // Available in env and in whitelist
- Assert.assertTrue(shellContent.contains(
- "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
- ));
- fos.flush();
- fos.close();
- }
-
- @Test(timeout = 20000)
- public void testWriteEnvOrder() throws Exception {
- // Valid only for unix
- assumeNotWindows();
- List<String> commands = new ArrayList<String>();
-
- // Setup user-defined environment
- Map<String, String> env = new HashMap<String, String>();
- env.put("USER_VAR_1", "1");
- env.put("USER_VAR_2", "2");
- env.put("NM_MODIFIED_VAR_1", "nm 1");
- env.put("NM_MODIFIED_VAR_2", "nm 2");
-
- // These represent vars explicitly set by NM
- LinkedHashSet<String> trackedNmVars = new LinkedHashSet<>();
- trackedNmVars.add("NM_MODIFIED_VAR_1");
- trackedNmVars.add("NM_MODIFIED_VAR_2");
-
- // Setup Nodemanager environment
- final Map<String, String> nmEnv = new HashMap<>();
- nmEnv.put("WHITELIST_VAR_1", "wl 1");
- nmEnv.put("WHITELIST_VAR_2", "wl 2");
- nmEnv.put("NON_WHITELIST_VAR_1", "nwl 1");
- nmEnv.put("NON_WHITELIST_VAR_2", "nwl 2");
- DefaultContainerExecutor defaultContainerExecutor =
- new DefaultContainerExecutor() {
- @Override
- protected String getNMEnvVar(String varname) {
- return nmEnv.get(varname);
- }
- };
-
- // Setup conf with whitelisted variables
- ArrayList<String> whitelistVars = new ArrayList<>();
- whitelistVars.add("WHITELIST_VAR_1");
- whitelistVars.add("WHITELIST_VAR_2");
- YarnConfiguration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.NM_ENV_WHITELIST,
- whitelistVars.get(0) + "," + whitelistVars.get(1));
-
- // These are in the NM env, but not in the whitelist.
- ArrayList<String> nonWhiteListEnv = new ArrayList<>();
- nonWhiteListEnv.add("NON_WHITELIST_VAR_1");
- nonWhiteListEnv.add("NON_WHITELIST_VAR_2");
-
- // Write the launch script
- File shellFile = Shell.appendScriptExtension(tmpDir, "hello");
- Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
- FileOutputStream fos = new FileOutputStream(shellFile);
- defaultContainerExecutor.setConf(conf);
- defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", trackedNmVars);
+ Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
fos.flush();
fos.close();
-
- // Examine the script
- String shellContent =
- new String(Files.readAllBytes(Paths.get(shellFile.getAbsolutePath())),
- StandardCharsets.UTF_8);
- // First make sure everything is there that's supposed to be
- for (String envVar : env.keySet()) {
- Assert.assertTrue(shellContent.contains(envVar + "="));
- }
- for (String wlVar : whitelistVars) {
- Assert.assertTrue(shellContent.contains(wlVar + "="));
- }
- for (String nwlVar : nonWhiteListEnv) {
- Assert.assertFalse(shellContent.contains(nwlVar + "="));
- }
- // Explicitly Set NM vars should be before user vars
- for (String nmVar : trackedNmVars) {
- for (String userVar : env.keySet()) {
- // Need to skip nm vars and whitelist vars
- if (!trackedNmVars.contains(userVar) &&
- !whitelistVars.contains(userVar)) {
- Assert.assertTrue(shellContent.indexOf(nmVar + "=") <
- shellContent.indexOf(userVar + "="));
- }
- }
- }
- // Whitelisted vars should be before explicitly set NM vars
- for (String wlVar : whitelistVars) {
- for (String nmVar : trackedNmVars) {
- Assert.assertTrue(shellContent.indexOf(wlVar + "=") <
- shellContent.indexOf(nmVar + "="));
- }
- }
}
-
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException {
@@ -510,9 +410,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
DefaultContainerExecutor defaultContainerExecutor =
new DefaultContainerExecutor();
defaultContainerExecutor.setConf(new YarnConfiguration());
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
defaultContainerExecutor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -594,9 +493,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add(command);
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(new YarnConfiguration());
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
exec.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), "user", nmVars);
+ new Path(localLogDir.getAbsolutePath()), "user");
fos.flush();
fos.close();
@@ -687,7 +585,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Path nmp = new Path(testDir);
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
- resources, nmp, Collections.emptySet());
+ resources, nmp);
List<String> result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -706,7 +604,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
dispatcher, exec, null, container, dirsHandler, containerManager);
launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs,
- resources, nmp, Collections.emptySet());
+ resources, nmp);
result =
getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
@@ -1630,10 +1528,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
FileOutputStream fos = new FileOutputStream(tempFile);
ContainerExecutor exec = new DefaultContainerExecutor();
exec.setConf(conf);
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
exec.writeLaunchEnv(fos, env, resources, commands,
new Path(localLogDir.getAbsolutePath()), "user",
- tempFile.getName(), nmVars);
+ tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -1856,9 +1753,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
List<String> commands = new ArrayList<String>();
DefaultContainerExecutor executor = new DefaultContainerExecutor();
executor.setConf(new Configuration());
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
executor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), user, nmVars);
+ new Path(localLogDir.getAbsolutePath()), user);
fos.flush();
fos.close();
@@ -1902,9 +1798,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Configuration execConf = new Configuration();
execConf.setBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO, false);
executor.setConf(execConf);
- LinkedHashSet<String> nmVars = new LinkedHashSet<>();
executor.writeLaunchEnv(fos, env, resources, commands,
- new Path(localLogDir.getAbsolutePath()), user, nmVars);
+ new Path(localLogDir.getAbsolutePath()), user);
fos.flush();
fos.close();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org