You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/01/27 19:40:59 UTC
[41/50] [abbrv] hadoop git commit: YARN-3088.
LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns
an error. Contributed by Eric Payne
YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native executor returns an error. Contributed by Eric Payne
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/902c6ea7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/902c6ea7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/902c6ea7
Branch: refs/heads/YARN-2928
Commit: 902c6ea7e4d3b49e49d9ce51ae9d12694ecfcf89
Parents: 2b0fa20
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Jan 26 15:40:21 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Jan 26 15:40:21 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../nodemanager/LinuxContainerExecutor.java | 9 +++-
.../TestLinuxContainerExecutorWithMocks.java | 49 +++++++++++++++++++-
3 files changed, 58 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0808678..872f16e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -409,6 +409,9 @@ Release 2.7.0 - UNRELEASED
YARN-3082. Non thread safe access to systemCredentials in NodeHeartbeatResponse
processing. (Anubhav Dhoot via ozawa)
+ YARN-3088. LinuxContainerExecutor.deleteAsUser can throw NPE if native
+ executor returns an error (Eric Payne via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 4606f0c..d6e6894 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -392,18 +392,23 @@ public class LinuxContainerExecutor extends ContainerExecutor {
verifyUsernamePattern(user);
String runAsUser = getRunAsUser(user);
+ String dirString = dir == null ? "" : dir.toUri().getPath();
+
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
runAsUser,
user,
Integer.toString(Commands.DELETE_AS_USER.getValue()),
- dir == null ? "" : dir.toUri().getPath()));
+ dirString));
+ List<String> pathsToDelete = new ArrayList<String>();
if (baseDirs == null || baseDirs.length == 0) {
LOG.info("Deleting absolute path : " + dir);
+ pathsToDelete.add(dirString);
} else {
for (Path baseDir : baseDirs) {
Path del = dir == null ? baseDir : new Path(baseDir, dir);
LOG.info("Deleting path : " + del);
+ pathsToDelete.add(del.toString());
command.add(baseDir.toUri().getPath());
}
}
@@ -419,7 +424,7 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
} catch (IOException e) {
int exitCode = shExec.getExitCode();
- LOG.error("DeleteAsUser for " + dir.toUri().getPath()
+ LOG.error("DeleteAsUser for " + StringUtils.join(" ", pathsToDelete)
+ " returned with exit code: " + exitCode, e);
LOG.error("Output from LinuxContainerExecutor's deleteAsUser follows:");
logOutput(shExec.getOutput());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/902c6ea7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index d54367a..98ab8e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -319,10 +319,57 @@ public class TestLinuxContainerExecutorWithMocks {
String cmd = String.valueOf(
LinuxContainerExecutor.Commands.DELETE_AS_USER.getValue());
Path dir = new Path("/tmp/testdir");
-
+ Path testFile = new Path("testfile");
+ Path baseDir0 = new Path("/grid/0/BaseDir");
+ Path baseDir1 = new Path("/grid/1/BaseDir");
+
+ mockExec.deleteAsUser(appSubmitter, dir);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, "/tmp/testdir"),
+ readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, null);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, ""),
+ readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+ readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+ readMockParams());
+
+ File f = new File("./src/test/resources/mock-container-executer-with-error");
+ if (!FileUtil.canExecute(f)) {
+ FileUtil.setExecutable(f, true);
+ }
+ String executorPath = f.getAbsolutePath();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, executorPath);
+ mockExec.setConf(conf);
+
mockExec.deleteAsUser(appSubmitter, dir);
assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
appSubmitter, cmd, "/tmp/testdir"),
readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, null);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, ""),
+ readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, testFile, baseDir0, baseDir1);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, testFile.toString(), baseDir0.toString(), baseDir1.toString()),
+ readMockParams());
+
+ mockExec.deleteAsUser(appSubmitter, null, baseDir0, baseDir1);
+ assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER,
+ appSubmitter, cmd, "", baseDir0.toString(), baseDir1.toString()),
+ readMockParams());
}
}