You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/08/29 22:29:35 UTC
hadoop git commit: YARN-5219. When an export var command fails in
launch_container.sh,
the full container launch should fail. (Sunil G via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk ad45d1999 -> f59332b97
YARN-5219. When an export var command fails in launch_container.sh, the full container launch should fail. (Sunil G via wangda)
Change-Id: Iaa6b978bb89482e9d1d77ba57f4adfdc48e39a3c
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59332b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59332b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59332b9
Branch: refs/heads/trunk
Commit: f59332b97b9a57e3cf1dcdeb47d7838d287100eb
Parents: ad45d19
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Aug 29 15:29:22 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Aug 29 15:29:22 2017 -0700
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 3 +
.../launcher/ContainerLaunch.java | 9 ++
.../launcher/TestContainerLaunch.java | 105 +++++++++++++++++++
3 files changed, 117 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 9767fb9..b6fb4ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -337,6 +337,9 @@ public abstract class ContainerExecutor implements Configurable {
whitelist.add(param);
}
+ // Add "set -o pipefail -e" to validate launch_container script.
+ sb.setExitOnFailure();
+
if (environment != null) {
for (Map.Entry<String, String> env : environment.entrySet()) {
if (!whitelist.contains(env.getKey())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 3f50c3f..50443f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -931,6 +931,10 @@ public class ContainerLaunch implements Callable<Integer> {
sb.append(LINE_SEPARATOR);
}
+ public void setExitOnFailure() {
+ // Dummy implementation
+ }
+
protected abstract void link(Path src, Path dst) throws IOException;
protected abstract void mkdir(Path path) throws IOException;
@@ -1008,6 +1012,11 @@ public class ContainerLaunch implements Callable<Integer> {
output.toString(), "\"");
line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\"");
}
+
+ @Override
+ public void setExitOnFailure() {
+ line("set -o pipefail -e");
+ }
}
private static final class WindowsShellScriptBuilder
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59332b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index e71ce75..085b60f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -39,6 +39,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
@@ -1529,4 +1530,108 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
verify(updaterNoCall, never()).reportException(any());
}
+
+ /*
+ * ${foo.version} is substituted to suffix a specific version number
+ */
+ @Test
+ public void testInvalidEnvVariableSubstitutionType1() throws IOException {
+ Map<String, String> env = new HashMap<String, String>();
+ // invalid env
+ env.put("testVar", "version${foo.version}");
+ validateShellExecutorForDifferentEnvs(env);
+ }
+
+ /*
+ * Multiple paths are substituted in a path variable
+ */
+ @Test
+ public void testInvalidEnvVariableSubstitutionType2() throws IOException {
+ Map<String, String> env = new HashMap<String, String>();
+ // invalid env
+ env.put("testPath", "/abc:/${foo.path}:/$bar");
+ validateShellExecutorForDifferentEnvs(env);
+ }
+
+ private void validateShellExecutorForDifferentEnvs(Map<String, String> env)
+ throws IOException {
+ File shellFile = null;
+ try {
+ shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ FileOutputStream fos = new FileOutputStream(shellFile);
+ FileUtil.setExecutable(shellFile, true);
+
+ List<String> commands = new ArrayList<String>();
+ DefaultContainerExecutor executor = new DefaultContainerExecutor();
+ executor.setConf(new Configuration());
+ executor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), user);
+ fos.flush();
+ fos.close();
+
+ // It is supposed that LANG is set as C.
+ Map<String, String> cmdEnv = new HashMap<String, String>();
+ cmdEnv.put("LANG", "C");
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[] { shellFile.getAbsolutePath() }, tmpDir, cmdEnv);
+ try {
+ shexc.execute();
+ Assert.fail("Should catch exception");
+ } catch (ExitCodeException e) {
+ Assert.assertTrue(shexc.getExitCode() != 0);
+ }
+ } finally {
+ // cleanup
+ if (shellFile != null && shellFile.exists()) {
+ shellFile.delete();
+ }
+ }
+ }
+
+ @Test
+ public void testValidEnvVariableSubstitution() throws IOException {
+ File shellFile = null;
+ try {
+ shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ Map<Path, List<String>> resources =
+ new HashMap<Path, List<String>>();
+ FileOutputStream fos = new FileOutputStream(shellFile);
+ FileUtil.setExecutable(shellFile, true);
+
+ Map<String, String> env = new LinkedHashMap<String, String>();
+ // valid env
+ env.put(
+ "foo", "2.4.6" );
+ env.put(
+ "testVar", "version${foo}" );
+ List<String> commands = new ArrayList<String>();
+ DefaultContainerExecutor executor = new DefaultContainerExecutor();
+ executor.setConf(new Configuration());
+ executor.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), user);
+ fos.flush();
+ fos.close();
+
+ // It is supposed that LANG is set as C.
+ Map<String, String> cmdEnv = new HashMap<String, String>();
+ cmdEnv.put("LANG", "C");
+ Shell.ShellCommandExecutor shexc
+ = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
+ tmpDir, cmdEnv);
+ try {
+ shexc.execute();
+ } catch(ExitCodeException e){
+ Assert.fail("Should not catch exception");
+ }
+ Assert.assertTrue(shexc.getExitCode() == 0);
+ }
+ finally {
+ // cleanup
+ if (shellFile != null
+ && shellFile.exists()) {
+ shellFile.delete();
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org