You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by da...@apache.org on 2014/04/16 18:06:28 UTC
svn commit: r1587955 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/
Author: daryn
Date: Wed Apr 16 16:06:28 2014
New Revision: 1587955
URL: http://svn.apache.org/r1587955
Log:
svn merge -c 1587954 FIXES: HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Apr 16 16:06:28 2014
@@ -51,6 +51,8 @@ Release 2.5.0 - UNRELEASED
OPTIMIZATIONS
+ HDFS-6214. Webhdfs has poor throughput for files >2GB (daryn)
+
BUG FIXES
HDFS-6112. NFS Gateway docs are incorrect for allowed hosts configuration.
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/DatanodeWebHdfsMethods.java Wed Apr 16 16:06:28 2014
@@ -430,10 +430,18 @@ public class DatanodeWebHdfsMethods {
Math.min(length.getValue(), in.getVisibleLength() - offset.getValue()) :
in.getVisibleLength() - offset.getValue();
+ // jetty 6 reserves 12 bytes in the out buffer for chunked responses
+ // (file length > 2GB) which causes extremely poor performance when
+ // 12 bytes of the output spill into another buffer which results
+ // in a big and little write
+ int outBufferSize = response.getBufferSize();
+ if (n > Integer.MAX_VALUE) {
+ outBufferSize -= 12;
+ }
/**
* Allow the Web UI to perform an AJAX request to get the data.
*/
- return Response.ok(new OpenEntity(in, n, dfsclient))
+ return Response.ok(new OpenEntity(in, n, outBufferSize, dfsclient))
.type(MediaType.APPLICATION_OCTET_STREAM)
.header("Access-Control-Allow-Methods", "GET")
.header("Access-Control-Allow-Origin", "*")
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java?rev=1587955&r1=1587954&r2=1587955&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/resources/OpenEntity.java Wed Apr 16 16:06:28 2014
@@ -37,12 +37,14 @@ import org.apache.hadoop.io.IOUtils;
public class OpenEntity {
private final HdfsDataInputStream in;
private final long length;
+ private final int outBufferSize;
private final DFSClient dfsclient;
OpenEntity(final HdfsDataInputStream in, final long length,
- final DFSClient dfsclient) {
+ final int outBufferSize, final DFSClient dfsclient) {
this.in = in;
this.length = length;
+ this.outBufferSize = outBufferSize;
this.dfsclient = dfsclient;
}
@@ -71,7 +73,17 @@ public class OpenEntity {
MultivaluedMap<String, Object> httpHeaders, OutputStream out
) throws IOException {
try {
- IOUtils.copyBytes(e.in, out, e.length, false);
+ byte[] buf = new byte[e.outBufferSize];
+ long remaining = e.length;
+ while (remaining > 0) {
+ int read = e.in.read(buf, 0, (int)Math.min(buf.length, remaining));
+ if (read == -1) { // EOF
+ break;
+ }
+ out.write(buf, 0, read);
+ out.flush();
+ remaining -= read;
+ }
} finally {
IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.in);
IOUtils.cleanup(DatanodeWebHdfsMethods.LOG, e.dfsclient);