You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2014/10/24 23:11:51 UTC
git commit: YARN-2703. Added logUploadedTime into LogValue for better
display. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk a52eb4bc5 -> f81dc3f99
YARN-2703. Added logUploadedTime into LogValue for better display. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f81dc3f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f81dc3f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f81dc3f9
Branch: refs/heads/trunk
Commit: f81dc3f995579c1b94b11d60e9fc6da56c8a9496
Parents: a52eb4b
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri Oct 24 14:10:46 2014 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri Oct 24 14:10:46 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 1 +
.../yarn/logaggregation/AggregatedLogFormat.java | 19 +++++++++++++++++++
.../yarn/webapp/log/AggregatedLogsBlock.java | 4 +++-
.../logaggregation/TestAggregatedLogFormat.java | 4 +++-
.../TestLogAggregationService.java | 15 +++++++++------
6 files changed, 38 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cfa0727..c30499f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -391,6 +391,9 @@ Release 2.6.0 - UNRELEASED
YARN-2209. Replaced AM resync/shutdown command with corresponding exceptions and
made related MR changes. (Jian He via zjshen)
+ YARN-2703. Added logUploadedTime into LogValue for better display. (Xuan Gong
+ via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 132dca2..2e9e92d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -227,6 +227,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
"Hello container_0_0001_01_000001!"));
+ assertTrue(sysOutStream.toString().contains("LogUploadTime"));
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 46e39cc..035d61d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
@@ -226,6 +227,9 @@ public class AggregatedLogFormat {
// Write the logFile Type
out.writeUTF(logFile.getName());
+ // Write the uploaded TimeStamp
+ out.writeLong(System.currentTimeMillis());
+
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(fileLength));
@@ -636,6 +640,7 @@ public class AggregatedLogFormat {
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
String fileType;
+ long uploadTime;
String fileLengthStr;
long fileLength;
@@ -646,10 +651,13 @@ public class AggregatedLogFormat {
// EndOfFile
return;
}
+ uploadTime = valueStream.readLong();
fileLengthStr = valueStream.readUTF();
fileLength = Long.parseLong(fileLengthStr);
writer.write("\n\nLogType:");
writer.write(fileType);
+ writer.write("\nLogUploadTime:");
+ writer.write(String.valueOf(uploadTime));
writer.write("\nLogLength:");
writer.write(fileLengthStr);
writer.write("\nLog Contents:\n");
@@ -681,10 +689,13 @@ public class AggregatedLogFormat {
byte[] buf = new byte[65535];
String fileType = valueStream.readUTF();
+ long uploadTime = valueStream.readLong();
String fileLengthStr = valueStream.readUTF();
long fileLength = Long.parseLong(fileLengthStr);
out.print("LogType: ");
out.println(fileType);
+ out.print("LogUploadTime: ");
+ out.println(Times.format(uploadTime));
out.print("LogLength: ");
out.println(fileLengthStr);
out.println("Log Contents:");
@@ -715,6 +726,7 @@ public class AggregatedLogFormat {
public static class ContainerLogsReader {
private DataInputStream valueStream;
private String currentLogType = null;
+ private long currentLogUpLoadTime = 0;
private long currentLogLength = 0;
private BoundedInputStream currentLogData = null;
private InputStreamReader currentLogISR;
@@ -735,12 +747,14 @@ public class AggregatedLogFormat {
}
currentLogType = null;
+ currentLogUpLoadTime = 0;
currentLogLength = 0;
currentLogData = null;
currentLogISR = null;
try {
String logType = valueStream.readUTF();
+ long logUpLoadTime = valueStream.readLong();
String logLengthStr = valueStream.readUTF();
currentLogLength = Long.parseLong(logLengthStr);
currentLogData =
@@ -748,6 +762,7 @@ public class AggregatedLogFormat {
currentLogData.setPropagateClose(false);
currentLogISR = new InputStreamReader(currentLogData);
currentLogType = logType;
+ currentLogUpLoadTime = logUpLoadTime;
} catch (EOFException e) {
}
@@ -758,6 +773,10 @@ public class AggregatedLogFormat {
return currentLogType;
}
+ public long getCurrentLogUpLoadTime() {
+ return currentLogUpLoadTime;
+ }
+
public long getCurrentLogLength() {
return currentLogLength;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
index 16e6359..bba3258 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/log/AggregatedLogsBlock.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -198,12 +199,13 @@ public class AggregatedLogsBlock extends HtmlBlock {
if (desiredLogType == null || desiredLogType.isEmpty()
|| desiredLogType.equals(logType)) {
long logLength = logReader.getCurrentLogLength();
-
+ long logUpLoadTime = logReader.getCurrentLogUpLoadTime();
if (foundLog) {
html.pre()._("\n\n")._();
}
html.p()._("Log Type: " + logType)._();
+ html.p()._("Log UpLoadTime: " + Times.format(logUpLoadTime))._();
html.p()._("Log Length: " + Long.toString(logLength))._();
long start = logLimits.start < 0
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index 0fae77d..fa9de74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -232,7 +232,9 @@ public class TestAggregatedLogFormat {
// aggregated.
String s = writer.toString();
int expectedLength =
- "\n\nLogType:stdout".length() + ("\nLogLength:" + numChars).length()
+ "\n\nLogType:stdout".length()
+ + ("\nLogUploadTime:" + System.currentTimeMillis()).length()
+ + ("\nLogLength:" + numChars).length()
+ "\nLog Contents:\n".length() + numChars;
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 8a5441a..cea71fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -61,7 +61,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -768,19 +767,23 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
Assert.assertEquals("LogType:", writtenLines[0].substring(0, 8));
String fileType = writtenLines[0].substring(9);
- Assert.assertEquals("LogLength:", writtenLines[1].substring(0, 10));
- String fileLengthStr = writtenLines[1].substring(11);
+ Assert.assertEquals("LogUploadTime:", writtenLines[1].substring(0, 14));
+ String fileUploadedTimeStr = writtenLines[1].substring(15);
+
+ Assert.assertEquals("LogLength:", writtenLines[2].substring(0, 10));
+ String fileLengthStr = writtenLines[2].substring(11);
long fileLength = Long.parseLong(fileLengthStr);
Assert.assertEquals("Log Contents:",
- writtenLines[2].substring(0, 13));
+ writtenLines[3].substring(0, 13));
String logContents = StringUtils.join(
- Arrays.copyOfRange(writtenLines, 3, writtenLines.length), "\n");
+ Arrays.copyOfRange(writtenLines, 4, writtenLines.length), "\n");
perContainerMap.put(fileType, logContents);
LOG.info("LogType:" + fileType);
- LOG.info("LogType:" + fileLength);
+ LOG.info("LogUploadTime:" + fileUploadedTimeStr);
+ LOG.info("LogLength:" + fileLength);
LOG.info("Log Contents:\n" + perContainerMap.get(fileType));
} catch (EOFException eof) {
break;