You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2011/02/24 01:59:12 UTC
svn commit: r1074001 - in /hbase/trunk: CHANGES.txt
src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
Author: rawson
Date: Thu Feb 24 00:59:11 2011
New Revision: 1074001
URL: http://svn.apache.org/viewvc?rev=1074001&view=rev
Log:
HBASE-3514 Speedup HFile.Writer append
Modified:
hbase/trunk/CHANGES.txt
hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
Modified: hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hbase/trunk/CHANGES.txt?rev=1074001&r1=1074000&r2=1074001&view=diff
==============================================================================
--- hbase/trunk/CHANGES.txt (original)
+++ hbase/trunk/CHANGES.txt Thu Feb 24 00:59:11 2011
@@ -51,6 +51,7 @@ Release 0.91.0 - Unreleased
unflushable regions.
HBASE-3550 FilterList reports false positives (Bill Graham via Andrew
Purtell)
+ HBASE-3514 Speedup HFile.Writer append
IMPROVEMENTS
HBASE-3290 Max Compaction Size (Nicolas Spiegelberg via Stack)
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java?rev=1074001&r1=1074000&r2=1074001&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java Thu Feb 24 00:59:11 2011
@@ -221,9 +221,6 @@ public class HFile {
// Used to ensure we write in order.
private final RawComparator<byte []> comparator;
- // A stream made per block written.
- private DataOutputStream out;
-
// Number of uncompressed bytes per block. Reinitialized when we start
// new block.
private int blocksize;
@@ -264,9 +261,9 @@ public class HFile {
// Block cache to optionally fill on write
private BlockCache blockCache;
- // Additional byte array output stream used to fill block cache
- private ByteArrayOutputStream baos;
- private DataOutputStream baosDos;
+ // Byte array output stream made per block written.
+ private ByteArrayOutputStream baos = null;
+ private DataOutputStream baosDos = null;
private int blockNumber = 0;
/**
@@ -360,7 +357,7 @@ public class HFile {
* @throws IOException
*/
private void checkBlockBoundary() throws IOException {
- if (this.out != null && this.out.size() < blocksize) return;
+ if (baosDos != null && baosDos.size() < blocksize) return;
finishBlock();
newBlock();
}
@@ -370,11 +367,18 @@ public class HFile {
* @throws IOException
*/
private void finishBlock() throws IOException {
- if (this.out == null) return;
+ if (baosDos == null) return;
+
+ // Flush Data Output Stream
+ baosDos.flush();
+
+ // Compress Data and write to output stream
+ DataOutputStream compressStream = getCompressingStream();
+ baos.writeTo(compressStream);
+ int size = releaseCompressingStream(compressStream);
+
long now = System.currentTimeMillis();
- int size = releaseCompressingStream(this.out);
- this.out = null;
blockKeys.add(firstKey);
blockOffsets.add(Long.valueOf(blockBegin));
blockDataSizes.add(Integer.valueOf(size));
@@ -384,14 +388,17 @@ public class HFile {
writeOps++;
if (blockCache != null) {
- baosDos.flush();
byte [] bytes = baos.toByteArray();
ByteBuffer blockToCache = ByteBuffer.wrap(bytes, DATABLOCKMAGIC.length,
bytes.length - DATABLOCKMAGIC.length);
String blockName = path.toString() + blockNumber;
blockCache.cacheBlock(blockName, blockToCache);
- baosDos.close();
}
+
+ baosDos.close();
+ baosDos = null;
+ baos = null;
+
blockNumber++;
}
@@ -402,14 +409,14 @@ public class HFile {
private void newBlock() throws IOException {
// This is where the next block begins.
blockBegin = outputStream.getPos();
- this.out = getCompressingStream();
- this.out.write(DATABLOCKMAGIC);
+
firstKey = null;
- if (blockCache != null) {
- this.baos = new ByteArrayOutputStream();
- this.baosDos = new DataOutputStream(baos);
- this.baosDos.write(DATABLOCKMAGIC);
- }
+
+ // to avoid too many calls to realloc(),
+ // pre-allocates the byte stream to the block size + 25%
+ baos = new ByteArrayOutputStream(blocksize + (int)(blocksize * 0.25));
+ baosDos = new DataOutputStream(baos);
+ baosDos.write(DATABLOCKMAGIC);
}
/*
@@ -467,7 +474,7 @@ public class HFile {
for (i = 0; i < metaNames.size(); ++i) {
// stop when the current key is greater than our own
byte[] cur = metaNames.get(i);
- if (Bytes.BYTES_RAWCOMPARATOR.compare(cur, 0, cur.length,
+ if (Bytes.BYTES_RAWCOMPARATOR.compare(cur, 0, cur.length,
key, 0, key.length) > 0) {
break;
}
@@ -563,12 +570,12 @@ public class HFile {
checkBlockBoundary();
}
// Write length of key and value and then actual key and value bytes.
- this.out.writeInt(klength);
+ this.baosDos.writeInt(klength);
this.keylength += klength;
- this.out.writeInt(vlength);
+ this.baosDos.writeInt(vlength);
this.valuelength += vlength;
- this.out.write(key, koffset, klength);
- this.out.write(value, voffset, vlength);
+ this.baosDos.write(key, koffset, klength);
+ this.baosDos.write(value, voffset, vlength);
// Are we the first key in this block?
if (this.firstKey == null) {
// Copy the key.
@@ -579,13 +586,6 @@ public class HFile {
this.lastKeyOffset = koffset;
this.lastKeyLength = klength;
this.entryCount ++;
- // If we are pre-caching blocks on write, fill byte array stream
- if (blockCache != null) {
- this.baosDos.writeInt(klength);
- this.baosDos.writeInt(vlength);
- this.baosDos.write(key, koffset, klength);
- this.baosDos.write(value, voffset, vlength);
- }
}
/*