You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/06/22 00:26:25 UTC
[27/51] [abbrv] hadoop git commit: YARN-5251. Yarn CLI to obtain App
logs for last 'n' bytes fails. Contributed by Xuan Gong.
YARN-5251. Yarn CLI to obtain App logs for last 'n' bytes fails. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c35fa4a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c35fa4a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c35fa4a0
Branch: refs/heads/YARN-2915
Commit: c35fa4a0e57b99415e284c7c2f3b1a0bebc610c2
Parents: 09e82ac
Author: Junping Du <ju...@apache.org>
Authored: Fri Jun 17 08:24:24 2016 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Fri Jun 17 08:24:24 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/cli/TestLogsCLI.java | 2 +-
.../logaggregation/AggregatedLogFormat.java | 26 +++++++++++---------
.../logaggregation/TestAggregatedLogFormat.java | 2 +-
.../webapp/AHSWebServices.java | 11 +++++----
.../nodemanager/webapp/NMWebServices.java | 9 +++----
.../TestLogAggregationService.java | 2 +-
6 files changed, 27 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index df1c6b7..6683fc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -400,7 +400,7 @@ public class TestLogsCLI {
String logMessage = "Hello container_0_0001_01_000003 in stdout!";
int fileContentSize = logMessage.getBytes().length;
- int tailContentSize = "End of LogType:syslog\n\n".getBytes().length;
+ int tailContentSize = "\nEnd of LogType:syslog\n\n".getBytes().length;
// specify how many bytes we should get from logs
// specify a position number, it would get the first n bytes from
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 2fb18ed..7508dd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -787,20 +787,19 @@ public class AggregatedLogFormat {
long toSkip = 0;
long totalBytesToRead = fileLength;
+ long skipAfterRead = 0;
if (bytes < 0) {
long absBytes = Math.abs(bytes);
if (absBytes < fileLength) {
toSkip = fileLength - absBytes;
totalBytesToRead = absBytes;
}
- long skippedBytes = valueStream.skip(toSkip);
- if (skippedBytes != toSkip) {
- throw new IOException("The bytes were skipped are "
- + "different from the caller requested");
- }
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, toSkip);
} else {
if (bytes < fileLength) {
totalBytesToRead = bytes;
+ skipAfterRead = fileLength - bytes;
}
}
@@ -818,7 +817,9 @@ public class AggregatedLogFormat {
pendingRead > buf.length ? buf.length : (int) pendingRead;
len = valueStream.read(buf, 0, toRead);
}
- out.println("End of LogType:" + fileType);
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, skipAfterRead);
+ out.println("\nEnd of LogType:" + fileType);
out.println("");
}
@@ -913,20 +914,19 @@ public class AggregatedLogFormat {
long toSkip = 0;
long totalBytesToRead = fileLength;
+ long skipAfterRead = 0;
if (bytes < 0) {
long absBytes = Math.abs(bytes);
if (absBytes < fileLength) {
toSkip = fileLength - absBytes;
totalBytesToRead = absBytes;
}
- long skippedBytes = valueStream.skip(toSkip);
- if (skippedBytes != toSkip) {
- throw new IOException("The bytes were skipped are "
- + "different from the caller requested");
- }
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, toSkip);
} else {
if (bytes < fileLength) {
totalBytesToRead = bytes;
+ skipAfterRead = fileLength - bytes;
}
}
@@ -942,7 +942,9 @@ public class AggregatedLogFormat {
toRead = pendingRead > buf.length ? buf.length : (int) pendingRead;
len = valueStream.read(buf, 0, toRead);
}
- out.println("End of LogType:" + fileType);
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, skipAfterRead);
+ out.println("\nEnd of LogType:" + fileType);
out.println("");
return 0;
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index 21e1655..45dd8ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -258,7 +258,7 @@ public class TestAggregatedLogFormat {
.currentTimeMillis())).length() : 0)
+ ("\nLogLength:" + numChars).length()
+ "\nLog Contents:\n".length() + numChars + "\n".length()
- + "End of LogType:stdout\n".length();
+ + "\nEnd of LogType:stdout\n".length();
Assert.assertTrue("LogType not matched", s.contains("LogType:stdout"));
Assert.assertTrue("log file:stderr should not be aggregated.", !s.contains("LogType:stderr"));
Assert.assertTrue("log file:logs should not be aggregated.", !s.contains("LogType:logs"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
index 692b172..b56fae9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebServices.java
@@ -404,20 +404,19 @@ public class AHSWebServices extends WebServices {
long toSkip = 0;
long totalBytesToRead = fileLength;
+ long skipAfterRead = 0;
if (bytes < 0) {
long absBytes = Math.abs(bytes);
if (absBytes < fileLength) {
toSkip = fileLength - absBytes;
totalBytesToRead = absBytes;
}
- long skippedBytes = valueStream.skip(toSkip);
- if (skippedBytes != toSkip) {
- throw new IOException("The bytes were skipped are "
- + "different from the caller requested");
- }
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, toSkip);
} else {
if (bytes < fileLength) {
totalBytesToRead = bytes;
+ skipAfterRead = fileLength - bytes;
}
}
@@ -435,6 +434,8 @@ public class AHSWebServices extends WebServices {
: (int) pendingRead;
len = valueStream.read(buf, 0, toRead);
}
+ org.apache.hadoop.io.IOUtils.skipFully(
+ valueStream, skipAfterRead);
sb = new StringBuilder();
sb.append("\nEnd of LogType:" + fileType + "\n");
b = sb.toString().getBytes(Charset.forName("UTF-8"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 3a30392..4e38e90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -264,20 +264,18 @@ public class NMWebServices {
byte[] buf = new byte[bufferSize];
long toSkip = 0;
long totalBytesToRead = fileLength;
+ long skipAfterRead = 0;
if (bytes < 0) {
long absBytes = Math.abs(bytes);
if (absBytes < fileLength) {
toSkip = fileLength - absBytes;
totalBytesToRead = absBytes;
}
- long skippedBytes = fis.skip(toSkip);
- if (skippedBytes != toSkip) {
- throw new IOException("The bytes were skipped are different "
- + "from the caller requested");
- }
+ org.apache.hadoop.io.IOUtils.skipFully(fis, toSkip);
} else {
if (bytes < fileLength) {
totalBytesToRead = bytes;
+ skipAfterRead = fileLength - bytes;
}
}
@@ -295,6 +293,7 @@ public class NMWebServices {
: (int) pendingRead;
len = fis.read(buf, 0, toRead);
}
+ org.apache.hadoop.io.IOUtils.skipFully(fis, skipAfterRead);
os.flush();
} finally {
IOUtils.closeQuietly(fis);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c35fa4a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index c98e366..1edb841 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -948,7 +948,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
Assert.assertEquals(numOfLogsPerContainer, thisContainerMap.size());
for (String fileType : logFiles) {
String expectedValue =
- containerStr + " Hello " + fileType + "!End of LogType:"
+ containerStr + " Hello " + fileType + "!\nEnd of LogType:"
+ fileType;
LOG.info("Expected log-content : " + new String(expectedValue));
String foundValue = thisContainerMap.remove(fileType);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org