You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2016/04/29 22:20:43 UTC
[47/50] [abbrv] hadoop git commit: Remove parent's env vars from
child processes
Remove parent's env vars from child processes
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d4d3024
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d4d3024
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d4d3024
Branch: refs/heads/HDFS-7240
Commit: 9d4d30243b0fc9630da51a2c17b543ef671d035c
Parents: af9b000
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Apr 28 19:24:38 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Fri Apr 29 09:25:51 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/util/Shell.java | 23 ++++++++++--
.../java/org/apache/hadoop/util/TestShell.java | 37 ++++++++++++++++++++
.../nodemanager/DefaultContainerExecutor.java | 4 ++-
.../nodemanager/DockerContainerExecutor.java | 4 ++-
.../nodemanager/LinuxContainerExecutor.java | 2 +-
.../privileged/PrivilegedOperationExecutor.java | 8 +++--
.../runtime/DefaultLinuxContainerRuntime.java | 4 +--
.../runtime/DockerLinuxContainerRuntime.java | 4 +--
.../runtime/TestDockerContainerRuntime.java | 2 +-
9 files changed, 75 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 0af3752..ea8db07 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -362,6 +363,9 @@ public abstract class Shell {
/** If or not script timed out*/
private final AtomicBoolean timedOut = new AtomicBoolean(false);
+ /** Indicates if the parent env vars should be inherited or not*/
+ protected boolean inheritParentEnv = true;
+
/**
* Centralized logic to discover and validate the sanity of the Hadoop
* home directory.
@@ -854,9 +858,16 @@ public abstract class Shell {
timedOut.set(false);
completed.set(false);
+ // Remove all env vars from the Builder to prevent leaking of env vars from
+ // the parent process.
+ if (!inheritParentEnv) {
+ builder.environment().clear();
+ }
+
if (environment != null) {
builder.environment().putAll(this.environment);
}
+
if (dir != null) {
builder.directory(this.dir);
}
@@ -1084,6 +1095,11 @@ public abstract class Shell {
this(execString, dir, env , 0L);
}
+ public ShellCommandExecutor(String[] execString, File dir,
+ Map<String, String> env, long timeout) {
+ this(execString, dir, env , timeout, true);
+ }
+
/**
* Create a new instance of the ShellCommandExecutor to execute a command.
*
@@ -1096,10 +1112,12 @@ public abstract class Shell {
* environment is not modified.
* @param timeout Specifies the time in milliseconds, after which the
* command will be killed and the status marked as timed-out.
- * If 0, the command will not be timed out.
+ * If 0, the command will not be timed out.
+ * @param inheritParentEnv Indicates if the process should inherit the env
+ * vars from the parent process or not.
*/
public ShellCommandExecutor(String[] execString, File dir,
- Map<String, String> env, long timeout) {
+ Map<String, String> env, long timeout, boolean inheritParentEnv) {
command = execString.clone();
if (dir != null) {
setWorkingDirectory(dir);
@@ -1108,6 +1126,7 @@ public abstract class Shell {
setEnvironment(env);
}
timeOutInterval = timeout;
+ this.inheritParentEnv = inheritParentEnv;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
index f20c140..16ec867 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestShell.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.util;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.security.alias.AbstractJavaKeyStoreProvider;
import org.junit.Assert;
import java.io.BufferedReader;
@@ -29,6 +30,8 @@ import java.io.PrintWriter;
import java.lang.management.ManagementFactory;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.test.GenericTestUtils;
@@ -145,6 +148,40 @@ public class TestShell extends Assert {
shellFile.delete();
assertTrue("Script did not timeout" , shexc.isTimedOut());
}
+
+ @Test
+ public void testEnvVarsWithInheritance() throws Exception {
+ Assume.assumeFalse(WINDOWS);
+ testEnvHelper(true);
+ }
+
+ @Test
+ public void testEnvVarsWithoutInheritance() throws Exception {
+ Assume.assumeFalse(WINDOWS);
+ testEnvHelper(false);
+ }
+
+ private void testEnvHelper(boolean inheritParentEnv) throws Exception {
+ Map<String, String> customEnv = new HashMap<>();
+ customEnv.put("AAA" + System.currentTimeMillis(), "AAA");
+ customEnv.put("BBB" + System.currentTimeMillis(), "BBB");
+ customEnv.put("CCC" + System.currentTimeMillis(), "CCC");
+ Shell.ShellCommandExecutor command = new ShellCommandExecutor(
+ new String[]{"env"}, null, customEnv, 0L, inheritParentEnv);
+ command.execute();
+ String[] varsArr = command.getOutput().split("\n");
+ Map<String, String> vars = new HashMap<>();
+ for (String var : varsArr) {
+ int eqIndex = var.indexOf('=');
+ vars.put(var.substring(0, eqIndex), var.substring(eqIndex + 1));
+ }
+ Map<String, String> expectedEnv = new HashMap<>();
+ expectedEnv.putAll(customEnv);
+ if (inheritParentEnv) {
+ expectedEnv.putAll(System.getenv());
+ }
+ assertEquals(expectedEnv, vars);
+ }
private static int countTimerThreads() {
ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 49398e4..c8048e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -284,7 +284,9 @@ public class DefaultContainerExecutor extends ContainerExecutor {
return new ShellCommandExecutor(
command,
wordDir,
- environment);
+ environment,
+ 0L,
+ false);
}
protected LocalWrapperScriptBuilder getLocalWrapperScriptBuilder(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index b089947..72da236 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -284,7 +284,9 @@ public class DockerContainerExecutor extends ContainerExecutor {
shExec = new ShellCommandExecutor(
command,
new File(containerWorkDir.toUri().getPath()),
- container.getLaunchContext().getEnvironment()); // sanitized env
+ container.getLaunchContext().getEnvironment(), // sanitized env
+ 0L,
+ false);
if (isContainerActive(containerId)) {
shExec.execute();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 5a48e09..e46ce56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -282,7 +282,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
PrivilegedOperationExecutor.getInstance(conf);
privilegedOperationExecutor.executePrivilegedOperation(prefixCommands,
- initializeContainerOp, null, null, false);
+ initializeContainerOp, null, null, false, true);
} catch (PrivilegedOperationException e) {
int exitCode = e.getExitCode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
index 7370daa..f865c14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
@@ -133,18 +133,19 @@ public class PrivilegedOperationExecutor {
* @param workingDir (optional) working directory for execution
* @param env (optional) env of the command will include specified vars
* @param grabOutput return (possibly large) shell command output
+ * @param inheritParentEnv inherit the env vars from the parent process
* @return stdout contents from shell executor - useful for some privileged
* operations - e.g --tc_read
* @throws org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException
*/
public String executePrivilegedOperation(List<String> prefixCommands,
PrivilegedOperation operation, File workingDir,
- Map<String, String> env, boolean grabOutput)
+ Map<String, String> env, boolean grabOutput, boolean inheritParentEnv)
throws PrivilegedOperationException {
String[] fullCommandArray = getPrivilegedOperationExecutionCommand
(prefixCommands, operation);
ShellCommandExecutor exec = new ShellCommandExecutor(fullCommandArray,
- workingDir, env);
+ workingDir, env, 0L, inheritParentEnv);
try {
exec.execute();
@@ -199,7 +200,8 @@ public class PrivilegedOperationExecutor {
*/
public String executePrivilegedOperation(PrivilegedOperation operation,
boolean grabOutput) throws PrivilegedOperationException {
- return executePrivilegedOperation(null, operation, null, null, grabOutput);
+ return executePrivilegedOperation(null, operation, null, null, grabOutput,
+ true);
}
//Utility functions for squashing together operations in supported ways
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index 3862b92..e78f460 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -102,7 +102,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
try {
privilegedOperationExecutor.executePrivilegedOperation(prefixCommands,
launchOp, null, container.getLaunchContext().getEnvironment(),
- false);
+ false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
@@ -134,7 +134,7 @@ public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
executor.executePrivilegedOperation(null,
signalOp, null, container.getLaunchContext().getEnvironment(),
- false);
+ false, true);
} catch (PrivilegedOperationException e) {
//Don't log the failure here. Some kinds of signaling failures are
// acceptable. Let the calling executor decide what to do.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index c66189d..681cae2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -331,7 +331,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
try {
privilegedOperationExecutor.executePrivilegedOperation(null,
launchOp, null, container.getLaunchContext().getEnvironment(),
- false);
+ false, false);
} catch (PrivilegedOperationException e) {
LOG.warn("Launch container failed. Exception: ", e);
@@ -360,7 +360,7 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
executor.executePrivilegedOperation(null,
signalOp, null, container.getLaunchContext().getEnvironment(),
- false);
+ false, true);
} catch (PrivilegedOperationException e) {
LOG.warn("Signal container failed. Exception: ", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d4d3024/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index e05719c..d1bdabe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -179,7 +179,7 @@ public class TestDockerContainerRuntime {
// warning annotation on the entire method
verify(mockExecutor, times(1))
.executePrivilegedOperation(anyList(), opCaptor.capture(), any(
- File.class), any(Map.class), eq(false));
+ File.class), any(Map.class), eq(false), eq(false));
PrivilegedOperation op = opCaptor.getValue();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org