You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2015/06/26 17:18:11 UTC
hbase git commit: HBASE-13939 - Make
HFileReaderImpl.getFirstKeyInBlock() to return a Cell (Ram)
Repository: hbase
Updated Branches:
refs/heads/master 4167a3f69 -> 130817b55
HBASE-13939 - Make HFileReaderImpl.getFirstKeyInBlock() to return a Cell
(Ram)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/130817b5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/130817b5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/130817b5
Branch: refs/heads/master
Commit: 130817b55904155a300423cd20004dd0531d8b83
Parents: 4167a3f
Author: ramkrishna <ra...@gmail.com>
Authored: Fri Jun 26 20:47:30 2015 +0530
Committer: ramkrishna <ra...@gmail.com>
Committed: Fri Jun 26 20:47:30 2015 +0530
----------------------------------------------------------------------
.../io/encoding/CopyKeyDataBlockEncoder.java | 4 ++--
.../hbase/io/encoding/DataBlockEncoder.java | 6 ++---
.../hbase/io/encoding/DiffKeyDeltaEncoder.java | 4 ++--
.../hbase/io/encoding/FastDiffDeltaEncoder.java | 4 ++--
.../io/encoding/PrefixKeyDeltaEncoder.java | 4 ++--
.../org/apache/hadoop/hbase/util/Bytes.java | 2 --
.../hbase/codec/prefixtree/PrefixTreeCodec.java | 4 ++--
.../codec/prefixtree/PrefixTreeSeeker.java | 5 +---
.../hadoop/hbase/io/hfile/HFileReaderImpl.java | 21 ++++++++---------
.../io/encoding/TestDataBlockEncoders.java | 24 ++++++++++++++------
10 files changed, 41 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
index 6fa01c0..4eea272 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java
@@ -66,13 +66,13 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder {
}
@Override
- public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+ public Cell getFirstKeyCellInBlock(ByteBuffer block) {
int keyLength = block.getInt(Bytes.SIZEOF_INT);
ByteBuffer dup = block.duplicate();
int pos = 3 * Bytes.SIZEOF_INT;
dup.position(pos);
dup.limit(pos + keyLength);
- return dup.slice();
+ return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
index 9019376..eddc689 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java
@@ -83,14 +83,14 @@ public interface DataBlockEncoder {
throws IOException;
/**
- * Return first key in block. Useful for indexing. Typically does not make
+ * Return first key in block as a cell. Useful for indexing. Typically does not make
* a deep copy but returns a buffer wrapping a segment of the actual block's
* byte array. This is because the first key in block is usually stored
* unencoded.
* @param block encoded block we want index, the position will not change
- * @return First key in block.
+ * @return First key in block as a cell.
*/
- ByteBuffer getFirstKeyInBlock(ByteBuffer block);
+ Cell getFirstKeyCellInBlock(ByteBuffer block);
/**
* Create a HFileBlock seeker which find KeyValues within a block.
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
index c55400b..f2d4751 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
@@ -305,7 +305,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
- public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+ public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT);
byte familyLength = block.get();
@@ -350,7 +350,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder {
block.get(result.array(), pos, Bytes.SIZEOF_BYTE);
block.reset();
- return result;
+ return new KeyValue.KeyOnlyKeyValue(result.array(), 0, keyLength);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
index 3000993..f750e09 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
@@ -354,7 +354,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
- public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+ public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE);
int keyLength = ByteBufferUtils.readCompressedInt(block);
@@ -365,7 +365,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder {
ByteBuffer dup = block.duplicate();
dup.position(pos);
dup.limit(pos + keyLength);
- return dup.slice();
+ return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
index 1350366..15608cc 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
@@ -172,7 +172,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
}
@Override
- public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+ public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.mark();
block.position(Bytes.SIZEOF_INT);
int keyLength = ByteBufferUtils.readCompressedInt(block);
@@ -187,7 +187,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder {
ByteBuffer dup = block.duplicate();
dup.position(pos);
dup.limit(pos + keyLength);
- return dup.slice();
+ return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
index bc245a8..685e402 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java
@@ -2056,8 +2056,6 @@ public class Bytes implements Comparable<Bytes> {
int low = 0;
int high = arr.length - 1;
- KeyValue.KeyOnlyKeyValue r = new KeyValue.KeyOnlyKeyValue();
- r.setKey(key, offset, length);
while (low <= high) {
int mid = (low + high) >>> 1;
// we have to compare in this order, because the comparator order
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java
index f703eef..7fceaa5 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java
@@ -114,7 +114,7 @@ public class PrefixTreeCodec implements DataBlockEncoder {
@Override
- public ByteBuffer getFirstKeyInBlock(ByteBuffer block) {
+ public Cell getFirstKeyCellInBlock(ByteBuffer block) {
block.rewind();
PrefixTreeArraySearcher searcher = null;
try {
@@ -123,7 +123,7 @@ public class PrefixTreeCodec implements DataBlockEncoder {
if (!searcher.positionAtFirstCell()) {
return null;
}
- return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current());
+ return searcher.current();
} finally {
DecoderFactory.checkIn(searcher);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
index d812f67..d64bce3 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
@@ -23,7 +23,6 @@ import java.nio.ByteBuffer;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.SettableSequenceId;
@@ -204,10 +203,8 @@ public class PrefixTreeSeeker implements EncodedSeeker {
@Override
public int compareKey(CellComparator comparator, Cell key) {
- // can't optimize this, make a copy of the key
- ByteBuffer bb = getKeyDeepCopy();
return comparator.compare(key,
- new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit()));
+ ptSearcher.current());
}
/**
* Cloned version of the PrefixTreeCell where except the value part, the rest
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
index 617d82f..86544c9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java
@@ -758,12 +758,9 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
if (seekToBlock == null) {
return false;
}
- ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock);
- // Will be changed as part of HBASE-13939
+ Cell firstKey = getFirstKeyCellInBlock(seekToBlock);
if (reader.getComparator()
- .compareKeyIgnoresMvcc(
- new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(),
- firstKey.limit()), key) >= 0) {
+ .compareKeyIgnoresMvcc(firstKey, key) >= 0) {
long previousBlockOffset = seekToBlock.getPrevBlockOffset();
// The key we are interested in
if (previousBlockOffset == -1) {
@@ -780,8 +777,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
// TODO shortcut: seek forward in this block to the last key of the
// block.
}
- Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey));
- loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true);
+ loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true);
return true;
}
@@ -1033,7 +1029,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
this.nextIndexedKey = null;
}
- protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
+ protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
ByteBuffer buffer = curBlock.getBufferWithoutHeader();
// It is safe to manipulate this buffer because we own the buffer object.
buffer.rewind();
@@ -1042,7 +1038,10 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
ByteBuffer keyBuff = buffer.slice();
keyBuff.limit(klen);
keyBuff.rewind();
- return keyBuff;
+ // Create a KeyOnlyKv now.
+ // TODO : Will change when Buffer backed cells come
+ return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset()
+ + keyBuff.position(), klen);
}
@Override
@@ -1556,8 +1555,8 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
}
}
- protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) {
- return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock));
+ protected Cell getFirstKeyCellInBlock(HFileBlock curBlock) {
+ return dataBlockEncoder.getFirstKeyCellInBlock(getEncodedBuffer(curBlock));
}
protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey,
http://git-wip-us.apache.org/repos/asf/hbase/blob/130817b5/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
index fbe8164..ede0aef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
@@ -30,7 +30,9 @@ import java.util.Collection;
import java.util.List;
import java.util.Random;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
+import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
@@ -300,23 +302,31 @@ public class TestDataBlockEncoders {
DataBlockEncoder encoder = encoding.getEncoder();
ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv,
getEncodingContext(Compression.Algorithm.NONE, encoding));
- ByteBuffer keyBuffer = encoder.getFirstKeyInBlock(encodedBuffer);
+ Cell key = encoder.getFirstKeyCellInBlock(encodedBuffer);
+ KeyValue keyBuffer = null;
+ if(encoding == DataBlockEncoding.PREFIX_TREE) {
+ // This is not an actual case. So the Prefix tree block is not loaded in case of Prefix_tree
+ // Just copy only the key part to form a keyBuffer
+ byte[] serializedKey = CellUtil.getCellKeySerializedAsKeyValueKey(key);
+ keyBuffer = KeyValueUtil.createKeyValueFromKey(serializedKey);
+ } else {
+ keyBuffer = KeyValueUtil.ensureKeyValue(key);
+ }
KeyValue firstKv = sampleKv.get(0);
- if (0 != Bytes.compareTo(keyBuffer.array(), keyBuffer.arrayOffset(), keyBuffer.limit(),
- firstKv.getBuffer(), firstKv.getKeyOffset(), firstKv.getKeyLength())) {
+ if (0 != CellComparator.COMPARATOR.compareKeyIgnoresMvcc(keyBuffer, firstKv)) {
int commonPrefix = 0;
- int length = Math.min(keyBuffer.limit(), firstKv.getKeyLength());
+ int length = Math.min(keyBuffer.getKeyLength(), firstKv.getKeyLength());
while (commonPrefix < length
- && keyBuffer.array()[keyBuffer.arrayOffset() + commonPrefix] == firstKv.getBuffer()[firstKv
- .getKeyOffset() + commonPrefix]) {
+ && keyBuffer.getBuffer()[keyBuffer.getKeyOffset() + commonPrefix] == firstKv
+ .getBuffer()[firstKv.getKeyOffset() + commonPrefix]) {
commonPrefix++;
}
fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix));
}
}
}
-
+
private void checkSeekingConsistency(List<DataBlockEncoder.EncodedSeeker> encodedSeekers,
boolean seekBefore, KeyValue keyValue) {
ByteBuffer expectedKeyValue = null;