You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2014/10/28 22:11:30 UTC
git commit: YARN-2741. Made NM web UI serve logs on the drive other
than C: on Windows. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk c6f04f391 -> 8984e9b17
YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8984e9b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8984e9b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8984e9b1
Branch: refs/heads/trunk
Commit: 8984e9b1774033e379b57da1bd30a5c81888c7a3
Parents: c6f04f3
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Oct 28 14:10:16 2014 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Oct 28 14:11:19 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/webapp/ContainerLogsUtils.java | 10 +--
.../webapp/TestContainerLogsPage.java | 82 ++++++++++++++++++++
3 files changed, 87 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0bff6d4..01fcc8f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED
YARN-2758. Update TestApplicationHistoryClientService to use the new generic
history store. (Zhijie Shen via xgong)
+ YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. (Craig
+ Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
index 3d0f19d..c588a89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
@@ -77,11 +77,7 @@ public class ContainerLogsUtils {
List<String> logDirs = dirsHandler.getLogDirs();
List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
for (String logDir : logDirs) {
- try {
- logDir = new URI(logDir).getPath();
- } catch (URISyntaxException e) {
- throw new YarnException("Internal error", e);
- }
+ logDir = new File(logDir).toURI().getPath();
String appIdStr = ConverterUtils.toString(containerId
.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
@@ -109,11 +105,9 @@ public class ContainerLogsUtils {
application.getAppId().toString(), containerId.toString());
Path logPath = dirsHandler.getLogPathToRead(
relativeContainerLogDir + Path.SEPARATOR + fileName);
- URI logPathURI = new URI(logPath.toString());
+ URI logPathURI = new File(logPath.toString()).toURI();
File logFile = new File(logPathURI.getPath());
return logFile;
- } catch (URISyntaxException e) {
- throw new YarnException("Internal error", e);
} catch (IOException e) {
LOG.warn("Failed to find log file", e);
throw new NotFoundException("Cannot find this log on the local disk.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 9305b84..b1d4397 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -28,6 +28,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -36,11 +37,16 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -210,4 +216,80 @@ public class TestContainerLogsPage {
}
}
}
+
+ @Test
+ public void testLogDirWithDriveLetter() throws Exception {
+ //To verify that logs paths which include drive letters (Windows)
+ //do not lose their drive letter specification
+ LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
+ List<String> logDirs = new ArrayList<String>();
+ logDirs.add("F:/nmlogs");
+ when(localDirs.getLogDirs()).thenReturn(logDirs);
+
+ ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
+ when(appId.toString()).thenReturn("app_id_1");
+
+ ApplicationAttemptIdPBImpl appAttemptId =
+ mock(ApplicationAttemptIdPBImpl.class);
+ when(appAttemptId.getApplicationId()).thenReturn(appId);
+
+ ContainerId containerId = mock(ContainerIdPBImpl.class);
+ when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
+
+ List<File> logDirFiles = ContainerLogsUtils.getContainerLogDirs(
+ containerId, localDirs);
+
+ Assert.assertTrue("logDir lost drive letter " +
+ logDirFiles.get(0),
+ logDirFiles.get(0).toString().indexOf("F:" + File.separator +
+ "nmlogs") > -1);
+ }
+
+ @Test
+ public void testLogFileWithDriveLetter() throws Exception {
+
+ ContainerImpl container = mock(ContainerImpl.class);
+
+ ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
+ when(appId.toString()).thenReturn("appId");
+
+ Application app = mock(Application.class);
+ when(app.getAppId()).thenReturn(appId);
+
+ ApplicationAttemptIdPBImpl appAttemptId =
+ mock(ApplicationAttemptIdPBImpl.class);
+ when(appAttemptId.getApplicationId()).thenReturn(appId);
+
+ ConcurrentMap<ApplicationId, Application> applications =
+ new ConcurrentHashMap<ApplicationId, Application>();
+ applications.put(appId, app);
+
+ ContainerId containerId = mock(ContainerIdPBImpl.class);
+ when(containerId.toString()).thenReturn("containerId");
+ when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
+
+ ConcurrentMap<ContainerId, Container> containers =
+ new ConcurrentHashMap<ContainerId, Container>();
+
+ containers.put(containerId, container);
+
+ LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
+ when(localDirs.getLogPathToRead("appId" + Path.SEPARATOR + "containerId" +
+ Path.SEPARATOR + "fileName"))
+ .thenReturn(new Path("F:/nmlogs/appId/containerId/fileName"));
+
+ NMContext context = mock(NMContext.class);
+ when(context.getLocalDirsHandler()).thenReturn(localDirs);
+ when(context.getApplications()).thenReturn(applications);
+ when(context.getContainers()).thenReturn(containers);
+
+ File logFile = ContainerLogsUtils.getContainerLogFile(containerId,
+ "fileName", null, context);
+
+ Assert.assertTrue("logFile lost drive letter " +
+ logFile,
+ logFile.toString().indexOf("F:" + File.separator + "nmlogs") > -1);
+
+ }
+
}