You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2018/02/01 18:17:01 UTC
hadoop git commit: Revert "YARN-7677. Docker image cannot set
HADOOP_CONF_DIR. Contributed by Jim Brennan"
Repository: hadoop
Updated Branches:
refs/heads/trunk ae2177d29 -> 682ea21f2
Revert "YARN-7677. Docker image cannot set HADOOP_CONF_DIR. Contributed by Jim Brennan"
This reverts commit 12eaae383ad06de8f9959241b2451dec82cf9ceb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/682ea21f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/682ea21f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/682ea21f
Branch: refs/heads/trunk
Commit: 682ea21f2bbc587e1b727b3c895c2f513a908432
Parents: ae2177d
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Feb 1 12:14:09 2018 -0600
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Feb 1 12:14:09 2018 -0600
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 31 ++++++++++++--------
.../nodemanager/LinuxContainerExecutor.java | 8 +++++
.../launcher/ContainerLaunch.java | 15 ++--------
.../runtime/DefaultLinuxContainerRuntime.java | 6 ++++
.../DelegatingLinuxContainerRuntime.java | 11 +++++++
.../runtime/DockerLinuxContainerRuntime.java | 7 +++++
.../runtime/ContainerRuntime.java | 11 +++++++
.../launcher/TestContainerLaunch.java | 10 ++-----
8 files changed, 66 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 6241733..f4279a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -346,6 +346,7 @@ public abstract class ContainerExecutor implements Configurable {
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
Map<Path, List<String>> resources, List<String> command, Path logDir,
String user, String outFilename) throws IOException {
+ updateEnvForWhitelistVars(environment);
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -363,19 +364,6 @@ public abstract class ContainerExecutor implements Configurable {
for (Map.Entry<String, String> env : environment.entrySet()) {
sb.env(env.getKey(), env.getValue());
}
- // Whitelist environment variables are treated specially.
- // Only add them if they are not already defined in the environment.
- // Add them using special syntax to prevent them from eclipsing
- // variables that may be set explicitly in the container image (e.g,
- // in a docker image)
- for(String var : whitelistVars) {
- if (!environment.containsKey(var)) {
- String val = getNMEnvVar(var);
- if (val != null) {
- sb.whitelistedEnv(var, val);
- }
- }
- }
}
if (resources != null) {
@@ -675,6 +663,23 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ /**
+ * Propagate variables from the nodemanager's environment into the
+ * container's environment if unspecified by the container.
+ * @param env the environment to update
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ for(String var : whitelistVars) {
+ if (!env.containsKey(var)) {
+ String val = getNMEnvVar(var);
+ if (val != null) {
+ env.put(var, val);
+ }
+ }
+ }
+ }
+
@VisibleForTesting
protected String getNMEnvVar(String varname) {
return System.getenv(varname);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index f0b9797..d359f31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -66,6 +66,7 @@ import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -472,6 +473,13 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ protected void updateEnvForWhitelistVars(Map<String, String> env) {
+ if (linuxContainerRuntime.useWhitelistEnv(env)) {
+ super.updateEnvForWhitelistVars(env);
+ }
+ }
+
+ @Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
Container container = ctx.getContainer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 309a1d8..7f43458 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1135,9 +1135,6 @@ public class ContainerLaunch implements Callable<Integer> {
public abstract void env(String key, String value) throws IOException;
- public abstract void whitelistedEnv(String key, String value)
- throws IOException;
-
public abstract void echo(String echoStr) throws IOException;
public final void symlink(Path src, Path dst) throws IOException {
@@ -1258,11 +1255,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- line("export ", key, "=${", key, ":-", "\"", value, "\"}");
- }
-
- @Override
public void echo(final String echoStr) throws IOException {
line("echo \"" + echoStr + "\"");
}
@@ -1353,11 +1345,6 @@ public class ContainerLaunch implements Callable<Integer> {
}
@Override
- public void whitelistedEnv(String key, String value) throws IOException {
- env(key, value);
- }
-
- @Override
public void echo(final String echoStr) throws IOException {
lineWithLenCheck("@echo \"", echoStr, "\"");
}
@@ -1456,6 +1443,8 @@ public class ContainerLaunch implements Callable<Integer> {
environment.put(Environment.PWD.name(), pwd.toString());
+ putEnvIfAbsent(environment, Environment.HADOOP_CONF_DIR.name());
+
if (!Shell.WINDOWS) {
environment.put("JVM_PID", "$$");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 83380ee..b50d56c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -37,6 +37,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -73,6 +74,11 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
}
@Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ return true;
+ }
+
+ @Override
public void prepareContainer(ContainerRuntimeContext ctx)
throws ContainerExecutionException {
//nothing to do here at the moment.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 675bffb..dd10617 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -94,6 +94,17 @@ public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ try {
+ LinuxContainerRuntime runtime = pickContainerRuntime(env);
+ return runtime.useWhitelistEnv(env);
+ } catch (ContainerExecutionException e) {
+ LOG.debug("Unable to determine runtime");
+ return false;
+ }
+ }
+
@VisibleForTesting
LinuxContainerRuntime pickContainerRuntime(
Map<String, String> environment) throws ContainerExecutionException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 26af9a0..601c32c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -366,6 +366,13 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
return capabilities;
}
+ @Override
+ public boolean useWhitelistEnv(Map<String, String> env) {
+ // Avoid propagating nodemanager environment variables into the container
+ // so those variables can be picked up from the Docker image instead.
+ return false;
+ }
+
private String runDockerVolumeCommand(DockerVolumeCommand dockerVolumeCommand,
Container container) throws ContainerExecutionException {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
index 7caa0ed..aa294fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerRuntime.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import java.util.Map;
+
/**
* An abstraction for various container runtime implementations. Examples
* include Process Tree, Docker, Appc runtimes etc. These implementations
@@ -83,4 +85,13 @@ public interface ContainerRuntime {
* and hostname
*/
String[] getIpAndHost(Container container) throws ContainerExecutionException;
+
+ /**
+ * Whether to propagate the whitelist of environment variables from the
+ * nodemanager environment into the container environment.
+ * @param env the container's environment variables
+ * @return true if whitelist variables should be propagated, false otherwise
+ * @see org.apache.hadoop.yarn.conf.YarnConfiguration#NM_ENV_WHITELIST
+ */
+ boolean useWhitelistEnv(Map<String, String> env);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/682ea21f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 5e57bb5..5923f8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -337,8 +337,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
// Available in env and in whitelist
Assert.assertTrue(shellContent.contains(
- "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
- ));
+ "export HADOOP_YARN_HOME=\"nodemanager_yarn_home\""));
fos.flush();
fos.close();
}
@@ -383,12 +382,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
// Whitelisted variable overridden by container
Assert.assertTrue(shellContent.contains(
"export HADOOP_MAPRED_HOME=\"/opt/hadoopbuild\""));
- // Available in env but not in whitelist
+ // Verify no whitelisted variables inherited from NM env
Assert.assertFalse(shellContent.contains("HADOOP_HDFS_HOME"));
- // Available in env and in whitelist
- Assert.assertTrue(shellContent.contains(
- "export HADOOP_YARN_HOME=${HADOOP_YARN_HOME:-\"nodemanager_yarn_home\"}"
- ));
+ Assert.assertFalse(shellContent.contains("HADOOP_YARN_HOME"));
fos.flush();
fos.close();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org