You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2015/08/14 12:55:00 UTC
[46/50] hadoop git commit: HDFS-8804. Erasure Coding: use
DirectBufferPool in DFSStripedInputStream for buffer allocation. Contributed
by Jing Zhao.
HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for buffer allocation. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675a00e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675a00e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675a00e8
Branch: refs/heads/HDFS-7285-REBASE
Commit: 675a00e8e6958f8e074a73b3d3ac279b731de68a
Parents: 3d53690
Author: Jing Zhao <ji...@apache.org>
Authored: Mon Aug 3 17:03:15 2015 -0700
Committer: Vinayakumar B <vi...@apache.org>
Committed: Thu Aug 13 17:32:18 2015 +0530
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 ++
.../hadoop/hdfs/DFSStripedInputStream.java | 33 ++++++++++++++++++--
2 files changed, 33 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675a00e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 673fbab..f087bb4 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -382,3 +382,6 @@
HDFS-8202. Improve end to end stirpping file test to add erasure recovering
test. (Xinwei Qin via zhz)
+
+ HDFS-8804. Erasure Coding: use DirectBufferPool in DFSStripedInputStream for
+ buffer allocation. (jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675a00e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index 1f64d4e..3612063 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.io.erasurecode.ECSchema;
import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.util.DirectBufferPool;
import java.io.EOFException;
import java.io.IOException;
@@ -136,6 +137,8 @@ public class DFSStripedInputStream extends DFSInputStream {
}
}
+ private static final DirectBufferPool bufferPool = new DirectBufferPool();
+
private final BlockReaderInfo[] blockReaders;
private final int cellSize;
private final short dataBlkNum;
@@ -143,6 +146,7 @@ public class DFSStripedInputStream extends DFSInputStream {
private final int groupSize;
/** the buffer for a complete stripe */
private ByteBuffer curStripeBuf;
+ private ByteBuffer parityBuf;
private final ECSchema schema;
private final RawErasureDecoder decoder;
@@ -177,12 +181,20 @@ public class DFSStripedInputStream extends DFSInputStream {
private void resetCurStripeBuffer() {
if (curStripeBuf == null) {
- curStripeBuf = ByteBuffer.allocateDirect(cellSize * dataBlkNum);
+ curStripeBuf = bufferPool.getBuffer(cellSize * dataBlkNum);
}
curStripeBuf.clear();
curStripeRange = new StripeRange(0, 0);
}
+ private ByteBuffer getParityBuffer() {
+ if (parityBuf == null) {
+ parityBuf = bufferPool.getBuffer(cellSize * parityBlkNum);
+ }
+ parityBuf.clear();
+ return parityBuf;
+ }
+
/**
* When seeking into a new block group, create blockReader for each internal
* block in the group.
@@ -204,6 +216,19 @@ public class DFSStripedInputStream extends DFSInputStream {
currentLocatedBlock = targetBlockGroup;
}
+ @Override
+ public synchronized void close() throws IOException {
+ super.close();
+ if (curStripeBuf != null) {
+ bufferPool.returnBuffer(curStripeBuf);
+ curStripeBuf = null;
+ }
+ if (parityBuf != null) {
+ bufferPool.returnBuffer(parityBuf);
+ parityBuf = null;
+ }
+ }
+
/**
* Extend the super method with the logic of switching between cells.
* When reaching the end of a cell, proceed to the next cell and read it
@@ -830,8 +855,10 @@ public class DFSStripedInputStream extends DFSInputStream {
}
final int decodeIndex = StripedBlockUtil.convertIndex4Decode(index,
dataBlkNum, parityBlkNum);
- decodeInputs[decodeIndex] = ByteBuffer.allocateDirect(
- (int) alignedStripe.range.spanInBlock);
+ ByteBuffer buf = getParityBuffer().duplicate();
+ buf.position(cellSize * decodeIndex);
+ buf.limit(cellSize * decodeIndex + (int) alignedStripe.range.spanInBlock);
+ decodeInputs[decodeIndex] = buf.slice();
alignedStripe.chunks[index] = new StripingChunk(decodeInputs[decodeIndex]);
return true;
}