You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2017/02/20 16:22:43 UTC
[1/3] hadoop git commit: HADOOP-14081. S3A: Consider avoiding array
copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh
Balamohan
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8a88e8ea5 -> 274c02d2f
refs/heads/branch-2.8 b1c1f05b1 -> 8a05ea4ab
refs/heads/trunk 172b23af3 -> 8035749c2
HADOOP-14081. S3A: Consider avoiding array copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/274c02d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/274c02d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/274c02d2
Branch: refs/heads/branch-2
Commit: 274c02d2fa06581891ba9bb9041fa20baa83582a
Parents: 8a88e8e
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 20 16:21:00 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 20 16:21:00 2017 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/274c02d2/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 0fe2af7..05f8efe 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -298,6 +298,25 @@ final class S3ADataBlocks {
}
+ static class S3AByteArrayOutputStream extends ByteArrayOutputStream {
+
+ S3AByteArrayOutputStream(int size) {
+ super(size);
+ }
+
+ /**
+ * InputStream backed by the internal byte array
+ *
+ * @return
+ */
+ ByteArrayInputStream getInputStream() {
+ ByteArrayInputStream bin = new ByteArrayInputStream(this.buf, 0, count);
+ this.reset();
+ this.buf = null;
+ return bin;
+ }
+ }
+
/**
* Stream to memory via a {@code ByteArrayOutputStream}.
*
@@ -310,14 +329,14 @@ final class S3ADataBlocks {
*/
static class ByteArrayBlock extends DataBlock {
- private ByteArrayOutputStream buffer;
+ private S3AByteArrayOutputStream buffer;
private final int limit;
// cache data size so that it is consistent after the buffer is reset.
private Integer dataSize;
ByteArrayBlock(int limit) {
this.limit = limit;
- buffer = new ByteArrayOutputStream();
+ buffer = new S3AByteArrayOutputStream(limit);
}
/**
@@ -333,8 +352,7 @@ final class S3ADataBlocks {
InputStream startUpload() throws IOException {
super.startUpload();
dataSize = buffer.size();
- ByteArrayInputStream bufferData = new ByteArrayInputStream(
- buffer.toByteArray());
+ ByteArrayInputStream bufferData = buffer.getInputStream();
buffer = null;
return bufferData;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[3/3] hadoop git commit: HADOOP-14081. S3A: Consider avoiding array
copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh
Balamohan
Posted by st...@apache.org.
HADOOP-14081. S3A: Consider avoiding array copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8035749c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8035749c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8035749c
Branch: refs/heads/trunk
Commit: 8035749c26947dc641ef87dac041050d439a16d1
Parents: 172b23a
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 20 16:21:00 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 20 16:21:46 2017 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8035749c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 0fe2af7..05f8efe 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -298,6 +298,25 @@ final class S3ADataBlocks {
}
+ static class S3AByteArrayOutputStream extends ByteArrayOutputStream {
+
+ S3AByteArrayOutputStream(int size) {
+ super(size);
+ }
+
+ /**
+ * InputStream backed by the internal byte array
+ *
+ * @return
+ */
+ ByteArrayInputStream getInputStream() {
+ ByteArrayInputStream bin = new ByteArrayInputStream(this.buf, 0, count);
+ this.reset();
+ this.buf = null;
+ return bin;
+ }
+ }
+
/**
* Stream to memory via a {@code ByteArrayOutputStream}.
*
@@ -310,14 +329,14 @@ final class S3ADataBlocks {
*/
static class ByteArrayBlock extends DataBlock {
- private ByteArrayOutputStream buffer;
+ private S3AByteArrayOutputStream buffer;
private final int limit;
// cache data size so that it is consistent after the buffer is reset.
private Integer dataSize;
ByteArrayBlock(int limit) {
this.limit = limit;
- buffer = new ByteArrayOutputStream();
+ buffer = new S3AByteArrayOutputStream(limit);
}
/**
@@ -333,8 +352,7 @@ final class S3ADataBlocks {
InputStream startUpload() throws IOException {
super.startUpload();
dataSize = buffer.size();
- ByteArrayInputStream bufferData = new ByteArrayInputStream(
- buffer.toByteArray());
+ ByteArrayInputStream bufferData = buffer.getInputStream();
buffer = null;
return bufferData;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/3] hadoop git commit: HADOOP-14081. S3A: Consider avoiding array
copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh
Balamohan
Posted by st...@apache.org.
HADOOP-14081. S3A: Consider avoiding array copy in S3ABlockOutputStream (ByteArrayBlock). Contributed by Rajesh Balamohan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a05ea4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a05ea4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a05ea4a
Branch: refs/heads/branch-2.8
Commit: 8a05ea4abad52ea30c227f57239dea09db665e14
Parents: b1c1f05
Author: Steve Loughran <st...@apache.org>
Authored: Mon Feb 20 16:21:00 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Feb 20 16:21:12 2017 +0000
----------------------------------------------------------------------
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 26 +++++++++++++++++---
1 file changed, 22 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a05ea4a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 0fe2af7..05f8efe 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -298,6 +298,25 @@ final class S3ADataBlocks {
}
+ static class S3AByteArrayOutputStream extends ByteArrayOutputStream {
+
+ S3AByteArrayOutputStream(int size) {
+ super(size);
+ }
+
+ /**
+ * InputStream backed by the internal byte array
+ *
+ * @return
+ */
+ ByteArrayInputStream getInputStream() {
+ ByteArrayInputStream bin = new ByteArrayInputStream(this.buf, 0, count);
+ this.reset();
+ this.buf = null;
+ return bin;
+ }
+ }
+
/**
* Stream to memory via a {@code ByteArrayOutputStream}.
*
@@ -310,14 +329,14 @@ final class S3ADataBlocks {
*/
static class ByteArrayBlock extends DataBlock {
- private ByteArrayOutputStream buffer;
+ private S3AByteArrayOutputStream buffer;
private final int limit;
// cache data size so that it is consistent after the buffer is reset.
private Integer dataSize;
ByteArrayBlock(int limit) {
this.limit = limit;
- buffer = new ByteArrayOutputStream();
+ buffer = new S3AByteArrayOutputStream(limit);
}
/**
@@ -333,8 +352,7 @@ final class S3ADataBlocks {
InputStream startUpload() throws IOException {
super.startUpload();
dataSize = buffer.size();
- ByteArrayInputStream bufferData = new ByteArrayInputStream(
- buffer.toByteArray());
+ ByteArrayInputStream bufferData = buffer.getInputStream();
buffer = null;
return bufferData;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org