You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ra...@apache.org on 2014/10/10 19:18:58 UTC
git commit: HBASE-12210 Avoid KeyValue in Prefix Tree
Repository: hbase
Updated Branches:
refs/heads/branch-1 052a6f070 -> a5b386412
HBASE-12210 Avoid KeyValue in Prefix Tree
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a5b38641
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a5b38641
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a5b38641
Branch: refs/heads/branch-1
Commit: a5b386412141fe0891157f20809575f906a10c2e
Parents: 052a6f0
Author: Ramkrishna <ra...@intel.com>
Authored: Fri Oct 10 22:47:37 2014 +0530
Committer: Ramkrishna <ra...@intel.com>
Committed: Fri Oct 10 22:47:37 2014 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/CellComparator.java | 26 ++-
.../codec/prefixtree/PrefixTreeSeeker.java | 192 ++++++++++++++++++-
.../codec/prefixtree/decode/PrefixTreeCell.java | 18 +-
.../hbase/io/encoding/TestPrefixTree.java | 7 +
4 files changed, 228 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5b38641/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
index 16cfef8..741718d 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java
@@ -21,9 +21,9 @@ package org.apache.hadoop.hbase;
import java.io.Serializable;
import java.util.Comparator;
+import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.util.Bytes;
import com.google.common.primitives.Longs;
@@ -255,6 +255,29 @@ public class CellComparator implements Comparator<Cell>, Serializable{
return 0;
}
+ int hash = calculateHashForKeyValue(cell);
+ hash = 31 * hash + (int)cell.getMvccVersion();
+ return hash;
+ }
+
+ /**
+ * Returns a hash code that is always the same for two Cells having a matching
+ * equals(..) result. Currently does not guard against nulls, but it could if
+ * necessary. Note : Ignore mvcc while calculating the hashcode
+ *
+ * @param cell
+ * @return hashCode
+ */
+ public static int hashCodeIgnoreMvcc(Cell cell) {
+ if (cell == null) {// return 0 for empty Cell
+ return 0;
+ }
+
+ int hash = calculateHashForKeyValue(cell);
+ return hash;
+ }
+
+ private static int calculateHashForKeyValue(Cell cell) {
//pre-calculate the 3 hashes made of byte ranges
int rowHash = Bytes.hashCode(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
int familyHash =
@@ -267,7 +290,6 @@ public class CellComparator implements Comparator<Cell>, Serializable{
hash = 31 * hash + qualifierHash;
hash = 31 * hash + (int)cell.getTimestamp();
hash = 31 * hash + cell.getTypeByte();
- hash = 31 * hash + (int)cell.getMvccVersion();
return hash;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5b38641/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
index 012b3e5..fb73443 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java
@@ -20,12 +20,13 @@ package org.apache.hadoop.hbase.codec.prefixtree;
import java.nio.ByteBuffer;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.KVComparator;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.SettableSequenceId;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory;
import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher;
import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition;
@@ -91,11 +92,17 @@ public class PrefixTreeSeeker implements EncodedSeeker {
* currently must do deep copy into new array
*/
@Override
- public KeyValue getKeyValue() {
- if (ptSearcher.current() == null) {
+ public Cell getKeyValue() {
+ Cell cell = ptSearcher.current();
+ if (cell == null) {
return null;
}
- return KeyValueUtil.copyToNewKeyValue(ptSearcher.current());
+ return new ClonedPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
+ cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(),
+ cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(),
+ cell.getValueArray(), cell.getValueOffset(), cell.getValueLength(), cell.getTagsArray(),
+ cell.getTagsOffset(), cell.getTagsLength(), cell.getTimestamp(), cell.getTypeByte(),
+ cell.getSequenceId());
}
/**
@@ -248,7 +255,7 @@ public class PrefixTreeSeeker implements EncodedSeeker {
public int seekToKeyInBlock(Cell key, boolean forceBeforeOnExactMatch) {
if (USE_POSITION_BEFORE) {
return seekToOrBeforeUsingPositionAtOrBefore(key, forceBeforeOnExactMatch);
- }else{
+ } else {
return seekToOrBeforeUsingPositionAtOrAfter(key, forceBeforeOnExactMatch);
}
}
@@ -259,4 +266,179 @@ public class PrefixTreeSeeker implements EncodedSeeker {
return comparator.compare(key,
new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit()));
}
+ /**
+ * Cloned version of the PrefixTreeCell where except the value part, the rest
+ * of the key part is deep copied
+ *
+ */
+ private static class ClonedPrefixTreeCell implements Cell, SettableSequenceId {
+ private byte[] row;
+ private short rowLength;
+ private byte[] fam;
+ private byte famLength;
+ private byte[] qual;
+ private int qualLength;
+ private byte[] val;
+ private int valOffset;
+ private int valLength;
+ private byte[] tag;
+ private int tagLength;
+ private long ts;
+ private long seqId;
+ private byte type;
+
+ public ClonedPrefixTreeCell(byte[] row, int rowOffset, short rowLength, byte[] fam,
+ int famOffset, byte famLength, byte[] qual, int qualOffset, int qualLength, byte[] val,
+ int valOffset, int valLength, byte[] tag, int tagOffset, int tagLength, long ts, byte type,
+ long seqId) {
+ this.row = new byte[rowLength];
+ System.arraycopy(row, rowOffset, this.row, 0, rowLength);
+ this.rowLength = rowLength;
+ this.fam = new byte[famLength];
+ System.arraycopy(fam, famOffset, this.fam, 0, famLength);
+ this.famLength = famLength;
+ this.qual = new byte[qualLength];
+ System.arraycopy(qual, qualOffset, this.qual, 0, qualLength);
+ this.qualLength = qualLength;
+ this.tag = new byte[tagLength];
+ System.arraycopy(tag, tagOffset, this.tag, 0, tagLength);
+ this.tagLength = tagLength;
+ this.val = val;
+ this.valLength = valLength;
+ this.valOffset = valOffset;
+ this.ts = ts;
+ this.seqId = seqId;
+ this.type = type;
+ }
+
+ @Override
+ public void setSequenceId(long seqId) {
+ this.seqId = seqId;
+ }
+
+ @Override
+ public byte[] getRowArray() {
+ return this.row;
+ }
+
+ @Override
+ public int getRowOffset() {
+ return 0;
+ }
+
+ @Override
+ public short getRowLength() {
+ return this.rowLength;
+ }
+
+ @Override
+ public byte[] getFamilyArray() {
+ return this.fam;
+ }
+
+ @Override
+ public int getFamilyOffset() {
+ return 0;
+ }
+
+ @Override
+ public byte getFamilyLength() {
+ return this.famLength;
+ }
+
+ @Override
+ public byte[] getQualifierArray() {
+ return this.qual;
+ }
+
+ @Override
+ public int getQualifierOffset() {
+ return 0;
+ }
+
+ @Override
+ public int getQualifierLength() {
+ return this.qualLength;
+ }
+
+ @Override
+ public long getTimestamp() {
+ return ts;
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return type;
+ }
+
+ @Override
+ @Deprecated
+ public long getMvccVersion() {
+ return getSequenceId();
+ }
+
+ @Override
+ public long getSequenceId() {
+ return seqId;
+ }
+
+ @Override
+ public byte[] getValueArray() {
+ return val;
+ }
+
+ @Override
+ public int getValueOffset() {
+ return this.valOffset;
+ }
+
+ @Override
+ public int getValueLength() {
+ return this.valLength;
+ }
+
+ @Override
+ public byte[] getTagsArray() {
+ return this.tag;
+ }
+
+ @Override
+ public int getTagsOffset() {
+ return 0;
+ }
+
+ @Override
+ public int getTagsLength() {
+ return this.tagLength;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getValue() {
+ return this.val;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getFamily() {
+ return this.fam;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getQualifier() {
+ return this.qual;
+ }
+
+ @Override
+ @Deprecated
+ public byte[] getRow() {
+ return this.row;
+ }
+
+ @Override
+ public String toString() {
+ return KeyValueUtil.copyToNewKeyValue(this).toString();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5b38641/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java
----------------------------------------------------------------------
diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java
index 5bc6196..c29a704 100644
--- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java
+++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java
@@ -18,12 +18,13 @@
package org.apache.hadoop.hbase.codec.prefixtree.decode;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.SettableSequenceId;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* As the PrefixTreeArrayScanner moves through the tree bytes, it changes the values in the fields
@@ -31,7 +32,7 @@ import org.apache.hadoop.hbase.KeyValueUtil;
* iterated through.
*/
@InterfaceAudience.Private
-public class PrefixTreeCell implements Cell, Comparable<Cell> {
+public class PrefixTreeCell implements Cell, SettableSequenceId, Comparable<Cell> {
/********************** static **********************/
@@ -96,12 +97,8 @@ public class PrefixTreeCell implements Cell, Comparable<Cell> {
}
@Override
- public int hashCode(){
- //Temporary hack to maintain backwards compatibility with KeyValue.hashCode
- //I don't think this is used in any hot code paths
- return KeyValueUtil.copyToNewKeyValue(this).hashCode();
-
- //TODO return CellComparator.hashCode(this);//see HBASE-6907
+ public int hashCode() {
+ return CellComparator.hashCodeIgnoreMvcc(this);
}
@Override
@@ -237,4 +234,9 @@ public class PrefixTreeCell implements Cell, Comparable<Cell> {
public byte[] getTagsArray() {
return this.tagsBuffer;
}
+
+ @Override
+ public void setSequenceId(long seqId) {
+ mvccVersion = seqId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a5b38641/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
index 710aed9..8541bdc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTree.java
@@ -115,6 +115,10 @@ public class TestPrefixTree {
rows[0] = row1;
rows[1] = row2;
rows[2] = row3;
+ byte[][] val = new byte[3][];
+ val[0] = Bytes.toBytes("c1-value");
+ val[1] = Bytes.toBytes("c2-value");
+ val[2] = Bytes.toBytes("c2-value-2");
Scan scan = new Scan();
scan.setStartRow(row1_bytes);
scan.setStopRow(Bytes.toBytes("a-b-A-1:"));
@@ -127,6 +131,9 @@ public class TestPrefixTree {
while (cellScanner.advance()) {
assertEquals(rows[i], Bytes.toString(cellScanner.current().getRowArray(), cellScanner
.current().getRowOffset(), cellScanner.current().getRowLength()));
+ assertEquals(Bytes.toString(val[i]), Bytes.toString(
+ cellScanner.current().getValueArray(), cellScanner.current().getValueOffset(),
+ cellScanner.current().getValueLength()));
}
i++;
}