You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ju...@apache.org on 2017/08/29 22:14:23 UTC
hadoop git commit: YARN-7037. Optimize data transfer with zero-copy
approach for containerlogs REST API in NMWebServices. Contributed by Tao
Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk cc8893edc -> ad45d1999
YARN-7037. Optimize data transfer with zero-copy approach for containerlogs REST API in NMWebServices. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad45d199
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad45d199
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad45d199
Branch: refs/heads/trunk
Commit: ad45d19998c1b0da25754d0016854046731fa623
Parents: cc8893e
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 29 15:16:23 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Tue Aug 29 15:16:23 2017 -0700
----------------------------------------------------------------------
.../yarn/logaggregation/LogToolUtils.java | 52 ++++++++++++++++++++
.../nodemanager/webapp/NMWebServices.java | 10 ++--
2 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad45d199/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
index ae2517a..74f694e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogToolUtils.java
@@ -19,9 +19,13 @@ package org.apache.hadoop.yarn.logaggregation;
import java.io.DataInputStream;
import java.io.EOFException;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.WritableByteChannel;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
@@ -195,6 +199,54 @@ public final class LogToolUtils {
os.flush();
}
+ public static void outputContainerLogThroughZeroCopy(String containerId,
+ String nodeId, String fileName, long fileLength, long outputSize,
+ String lastModifiedTime, FileInputStream fis, OutputStream os,
+ ContainerLogAggregationType logType) throws IOException {
+ long toSkip = 0;
+ long totalBytesToRead = fileLength;
+ if (outputSize < 0) {
+ long absBytes = Math.abs(outputSize);
+ if (absBytes < fileLength) {
+ toSkip = fileLength - absBytes;
+ totalBytesToRead = absBytes;
+ }
+ } else {
+ if (outputSize < fileLength) {
+ totalBytesToRead = outputSize;
+ }
+ }
+
+ if (totalBytesToRead > 0) {
+ // output log summary
+ StringBuilder sb = new StringBuilder();
+ String containerStr = String.format(
+ LogToolUtils.CONTAINER_ON_NODE_PATTERN,
+ containerId, nodeId);
+ sb.append(containerStr + "\n");
+ sb.append("LogAggregationType: " + logType + "\n");
+ sb.append(StringUtils.repeat("=", containerStr.length()) + "\n");
+ sb.append("LogType:" + fileName + "\n");
+ sb.append("LogLastModifiedTime:" + lastModifiedTime + "\n");
+ sb.append("LogLength:" + Long.toString(fileLength) + "\n");
+ sb.append("LogContents:\n");
+ byte[] b = sb.toString().getBytes(
+ Charset.forName("UTF-8"));
+ os.write(b, 0, b.length);
+ // output log content
+ FileChannel inputChannel = fis.getChannel();
+ WritableByteChannel outputChannel = Channels.newChannel(os);
+ long position = toSkip;
+ while (totalBytesToRead > 0) {
+ long transferred =
+ inputChannel.transferTo(position, totalBytesToRead, outputChannel);
+ totalBytesToRead -= transferred;
+ position += transferred;
+ }
+ os.flush();
+ }
+ }
+
public static boolean outputAggregatedContainerLog(Configuration conf,
ApplicationId appId, String appOwner,
String containerId, String nodeId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad45d199/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 7a6aa0f..3ba7d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -426,11 +426,9 @@ public class NMWebServices {
public void write(OutputStream os) throws IOException,
WebApplicationException {
try {
- int bufferSize = 65536;
- byte[] buf = new byte[bufferSize];
- LogToolUtils.outputContainerLog(containerId.toString(),
- nmContext.getNodeId().toString(), outputFileName, fileLength,
- bytes, lastModifiedTime, fis, os, buf,
+ LogToolUtils.outputContainerLogThroughZeroCopy(
+ containerId.toString(), nmContext.getNodeId().toString(),
+ outputFileName, fileLength, bytes, lastModifiedTime, fis, os,
ContainerLogAggregationType.LOCAL);
StringBuilder sb = new StringBuilder();
String endOfFile = "End of LogType:" + outputFileName;
@@ -451,6 +449,8 @@ public class NMWebServices {
Application app = nmContext.getApplications().get(appId);
String appOwner = app == null ? null : app.getUser();
try {
+ int bufferSize = 65536;
+ byte[] buf = new byte[bufferSize];
LogToolUtils.outputAggregatedContainerLog(nmContext.getConf(),
appId, appOwner, containerId.toString(),
nmContext.getNodeId().toString(), outputFileName, bytes,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org