You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2016/09/01 01:25:11 UTC
[1/2] hbase git commit: HBASE-16502 Reduce garbage in
BufferedDataBlockEncoder (binlijin)
Repository: hbase
Updated Branches:
refs/heads/0.98 5a1b3a548 -> e792d570a
HBASE-16502 Reduce garbage in BufferedDataBlockEncoder (binlijin)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b861c331
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b861c331
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b861c331
Branch: refs/heads/0.98
Commit: b861c331f05df8a585d03959dc471e9e8b045047
Parents: 5a1b3a5
Author: tedyu <yu...@gmail.com>
Authored: Thu Aug 25 20:39:42 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Aug 31 17:58:46 2016 -0700
----------------------------------------------------------------------
.../io/encoding/BufferedDataBlockEncoder.java | 21 ++++++++++++++++----
.../hbase/io/encoding/DiffKeyDeltaEncoder.java | 6 +++++-
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 6 +++++-
.../encoding/TestBufferedDataBlockEncoder.java | 4 ++--
4 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b861c331/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 93f43b3..32be3c8 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -84,11 +84,19 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
/** We need to store a copy of the key. */
protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
- protected byte[] tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
+ protected byte[] tagsBuffer = null;
protected long memstoreTS;
protected int nextKvOffset;
+ public SeekerState(boolean tagsCompressed) {
+ if (tagsCompressed) {
+ tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
+ } else {
+ tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
+ }
+ }
+
protected boolean isValid() {
return valueOffset != -1;
}
@@ -164,8 +172,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected final KVComparator comparator;
protected final SamePrefixComparator<byte[]> samePrefixComparator;
protected ByteBuffer currentBuffer;
- protected STATE current = createSeekerState(); // always valid
- protected STATE previous = createSeekerState(); // may not be valid
+ protected STATE current, previous;
protected TagCompressionContext tagCompressionContext = null;
public BufferedEncodedSeeker(KVComparator comparator,
@@ -180,6 +187,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
throw new RuntimeException("Failed to initialize TagCompressionContext", e);
}
}
+ current = createSeekerState(); // always valid
+ previous = createSeekerState(); // may not be valid
}
protected boolean includesMvcc() {
@@ -190,6 +199,10 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return this.decodingCtx.getHFileContext().isIncludesTags();
}
+ protected boolean tagsCompressed() {
+ return this.decodingCtx.getHFileContext().isCompressTags();
+ }
+
@Override
public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
return comparator.compareFlatKey(key, offset, length,
@@ -390,7 +403,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected STATE createSeekerState() {
// This will fail for non-default seeker state if the subclass does not
// override this method.
- return (STATE) new SeekerState();
+ return (STATE) new SeekerState(this.tagsCompressed());
}
abstract protected void decodeFirst();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b861c331/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
index c63acb1..d2b93be 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -392,6 +392,10 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize;
private long timestamp;
+ public DiffSeekerState(boolean tagsCompressed) {
+ super(tagsCompressed);
+ }
+
@Override
protected void copyFromNext(SeekerState that) {
super.copyFromNext(that);
@@ -527,7 +531,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
@Override
protected DiffSeekerState createSeekerState() {
- return new DiffSeekerState();
+ return new DiffSeekerState(this.tagsCompressed());
}
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b861c331/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index ef31a77..d09d8d7 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -404,6 +404,10 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize;
private int familyLengthWithSize;
+ public FastDiffSeekerState(boolean tagsCompressed) {
+ super(tagsCompressed);
+ }
+
@Override
protected void copyFromNext(SeekerState that) {
super.copyFromNext(that);
@@ -543,7 +547,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
@Override
protected FastDiffSeekerState createSeekerState() {
- return new FastDiffSeekerState();
+ return new FastDiffSeekerState(this.tagsCompressed());
}
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b861c331/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
index 5d9e56a..44ae89d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
@@ -27,8 +27,8 @@ public class TestBufferedDataBlockEncoder {
@Test
public void testEnsureSpaceForKey() {
- BufferedDataBlockEncoder.SeekerState state =
- new BufferedDataBlockEncoder.SeekerState();
+ BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState(
+ false);
for (int i = 1; i <= 65536; ++i) {
state.keyLength = i;
state.ensureSpaceForKey();
[2/2] hbase git commit: HBASE-16502 Reduce garbage in
BufferedDataBlockEncoder - addendum adopts Hiroshi's suggestion (binlijin)
Posted by ap...@apache.org.
HBASE-16502 Reduce garbage in BufferedDataBlockEncoder - addendum adopts Hiroshi's suggestion (binlijin)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e792d570
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e792d570
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e792d570
Branch: refs/heads/0.98
Commit: e792d570a98374a9829522b22d63b90e25c89104
Parents: b861c33
Author: tedyu <yu...@gmail.com>
Authored: Tue Aug 30 18:06:24 2016 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Wed Aug 31 18:12:08 2016 -0700
----------------------------------------------------------------------
.../io/encoding/BufferedDataBlockEncoder.java | 31 +++++---------------
.../hbase/io/encoding/DiffKeyDeltaEncoder.java | 6 ++--
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 6 ++--
.../encoding/TestBufferedDataBlockEncoder.java | 3 +-
4 files changed, 15 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e792d570/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
index 32be3c8..8d103af 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
@@ -83,18 +83,13 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected boolean uncompressTags = true;
/** We need to store a copy of the key. */
- protected byte[] keyBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
- protected byte[] tagsBuffer = null;
+ protected byte[] keyBuffer = HConstants.EMPTY_BYTE_ARRAY;
+ protected byte[] tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
protected long memstoreTS;
protected int nextKvOffset;
- public SeekerState(boolean tagsCompressed) {
- if (tagsCompressed) {
- tagsBuffer = new byte[INITIAL_KEY_BUFFER_SIZE];
- } else {
- tagsBuffer = HConstants.EMPTY_BYTE_ARRAY;
- }
+ public SeekerState() {
}
protected boolean isValid() {
@@ -109,11 +104,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected void ensureSpaceForKey() {
if (keyLength > keyBuffer.length) {
- // rare case, but we need to handle arbitrary length of key
- int newKeyBufferLength = Math.max(keyBuffer.length, 1) * 2;
- while (keyLength > newKeyBufferLength) {
- newKeyBufferLength *= 2;
- }
+ int newKeyBufferLength = Integer.highestOneBit(Math.max(
+ INITIAL_KEY_BUFFER_SIZE, keyLength) - 1) << 1;
byte[] newKeyBuffer = new byte[newKeyBufferLength];
System.arraycopy(keyBuffer, 0, newKeyBuffer, 0, keyBuffer.length);
keyBuffer = newKeyBuffer;
@@ -122,11 +114,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected void ensureSpaceForTags() {
if (tagsLength > tagsBuffer.length) {
- // rare case, but we need to handle arbitrary length of tags
- int newTagsBufferLength = Math.max(tagsBuffer.length, 1) * 2;
- while (tagsLength > newTagsBufferLength) {
- newTagsBufferLength *= 2;
- }
+ int newTagsBufferLength = Integer.highestOneBit(Math.max(
+ INITIAL_KEY_BUFFER_SIZE, tagsLength) - 1) << 1;
byte[] newTagsBuffer = new byte[newTagsBufferLength];
System.arraycopy(tagsBuffer, 0, newTagsBuffer, 0, tagsBuffer.length);
tagsBuffer = newTagsBuffer;
@@ -199,10 +188,6 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
return this.decodingCtx.getHFileContext().isIncludesTags();
}
- protected boolean tagsCompressed() {
- return this.decodingCtx.getHFileContext().isCompressTags();
- }
-
@Override
public int compareKey(KVComparator comparator, byte[] key, int offset, int length) {
return comparator.compareFlatKey(key, offset, length,
@@ -403,7 +388,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder {
protected STATE createSeekerState() {
// This will fail for non-default seeker state if the subclass does not
// override this method.
- return (STATE) new SeekerState(this.tagsCompressed());
+ return (STATE) new SeekerState();
}
abstract protected void decodeFirst();
http://git-wip-us.apache.org/repos/asf/hbase/blob/e792d570/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
index d2b93be..832d4f6 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -392,8 +392,8 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize;
private long timestamp;
- public DiffSeekerState(boolean tagsCompressed) {
- super(tagsCompressed);
+ public DiffSeekerState() {
+ super();
}
@Override
@@ -531,7 +531,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
@Override
protected DiffSeekerState createSeekerState() {
- return new DiffSeekerState(this.tagsCompressed());
+ return new DiffSeekerState();
}
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e792d570/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index d09d8d7..6f57f0f 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -404,8 +404,8 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
private int rowLengthWithSize;
private int familyLengthWithSize;
- public FastDiffSeekerState(boolean tagsCompressed) {
- super(tagsCompressed);
+ public FastDiffSeekerState() {
+ super();
}
@Override
@@ -547,7 +547,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
@Override
protected FastDiffSeekerState createSeekerState() {
- return new FastDiffSeekerState(this.tagsCompressed());
+ return new FastDiffSeekerState();
}
};
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e792d570/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
index 44ae89d..fb9253e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestBufferedDataBlockEncoder.java
@@ -27,8 +27,7 @@ public class TestBufferedDataBlockEncoder {
@Test
public void testEnsureSpaceForKey() {
- BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState(
- false);
+ BufferedDataBlockEncoder.SeekerState state = new BufferedDataBlockEncoder.SeekerState();
for (int i = 1; i <= 65536; ++i) {
state.keyLength = i;
state.ensureSpaceForKey();