You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/07/27 23:58:32 UTC
[21/37] hadoop git commit: YARN-3925.
ContainerLogsUtils#getContainerLogFile fails to read container log files from
full disks. Contributed by zhihai xu
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. Contributed by zhihai xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff9c13e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff9c13e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff9c13e0
Branch: refs/heads/HDFS-7240
Commit: ff9c13e0a739bb13115167dc661b6a16b2ed2c04
Parents: 83fe34a
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Jul 24 22:14:39 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Jul 24 22:14:39 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodemanager/LocalDirsHandlerService.java | 35 +++++++++++++-
.../webapp/TestContainerLogsPage.java | 48 ++++++++++++++++++++
3 files changed, 83 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf00fe5..c295784 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -716,6 +716,8 @@ Release 2.7.2 - UNRELEASED
YARN-3969. Allow jobs to be submitted to reservation that is active
but does not have any allocations. (subru via curino)
+ YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
+ log files from full disks. (zhihai xu via jlowe)
Release 2.7.1 - 2015-07-06
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 0a61035..6709c90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -31,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -467,6 +469,35 @@ public class LocalDirsHandlerService extends AbstractService {
return disksTurnedGood;
}
+ private Path getPathToRead(String pathStr, List<String> dirs)
+ throws IOException {
+ // remove the leading slash from the path (to make sure that the uri
+ // resolution results in a valid path on the dir being checked)
+ if (pathStr.startsWith("/")) {
+ pathStr = pathStr.substring(1);
+ }
+
+ FileSystem localFS = FileSystem.getLocal(getConfig());
+ for (String dir : dirs) {
+ try {
+ Path tmpDir = new Path(dir);
+ File tmpFile = tmpDir.isAbsolute()
+ ? new File(localFS.makeQualified(tmpDir).toUri())
+ : new File(dir);
+ Path file = new Path(tmpFile.getPath(), pathStr);
+ if (localFS.exists(file)) {
+ return file;
+ }
+ } catch (IOException ie) {
+ // ignore
+ LOG.warn("Failed to find " + pathStr + " at " + dir, ie);
+ }
+ }
+
+ throw new IOException("Could not find " + pathStr + " in any of" +
+ " the directories");
+ }
+
public Path getLocalPathForWrite(String pathStr) throws IOException {
return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
}
@@ -484,9 +515,9 @@ public class LocalDirsHandlerService extends AbstractService {
}
public Path getLogPathToRead(String pathStr) throws IOException {
- return logDirsAllocator.getLocalPathToRead(pathStr, getConfig());
+ return getPathToRead(pathStr, getLogDirsForRead());
}
-
+
public static String[] validatePaths(String[] paths) {
ArrayList<String> validPaths = new ArrayList<String>();
for (int i = 0; i < paths.length; ++i) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index e63f681..84e42fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.security.UserGroupInformation;
@@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -144,6 +146,52 @@ public class TestContainerLogsPage {
Assert.assertTrue(dirs.contains(containerLogDir));
}
+ @Test(timeout=30000)
+ public void testContainerLogFile() throws IOException, YarnException {
+ File absLogDir = new File("target",
+ TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
+ String logdirwithFile = absLogDir.toURI().toString();
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
+ conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
+ 0.0f);
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ dirsHandler.init(conf);
+ NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
+ new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ // Add an application and the corresponding containers
+ String user = "nobody";
+ long clusterTimeStamp = 1234;
+ ApplicationId appId = BuilderUtils.newApplicationId(
+ clusterTimeStamp, 1);
+ Application app = mock(Application.class);
+ when(app.getUser()).thenReturn(user);
+ when(app.getAppId()).thenReturn(appId);
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ ContainerId containerId = BuilderUtils.newContainerId(
+ appAttemptId, 1);
+ nmContext.getApplications().put(appId, app);
+
+ MockContainer container =
+ new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
+ appId, 1);
+ container.setState(ContainerState.RUNNING);
+ nmContext.getContainers().put(containerId, container);
+ File containerLogDir = new File(absLogDir,
+ ContainerLaunch.getRelativeContainerLogDir(appId.toString(),
+ containerId.toString()));
+ containerLogDir.mkdirs();
+ String fileName = "fileName";
+ File containerLogFile = new File(containerLogDir, fileName);
+ containerLogFile.createNewFile();
+ File file = ContainerLogsUtils.getContainerLogFile(containerId,
+ fileName, user, nmContext);
+ Assert.assertEquals(containerLogFile.toURI().toString(),
+ file.toURI().toString());
+ FileUtil.fullyDelete(absLogDir);
+ }
+
@Test(timeout = 10000)
public void testContainerLogPageAccess() throws IOException {
// SecureIOUtils require Native IO to be enabled. This test will run