You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/05/02 21:19:02 UTC
svn commit: r1333159 - in /hbase/trunk/src:
main/java/org/apache/hadoop/hbase/KeyValue.java
main/java/org/apache/hadoop/hbase/client/Result.java
test/java/org/apache/hadoop/hbase/TestKeyValue.java
test/java/org/apache/hadoop/hbase/client/TestResult.java
Author: tedyu
Date: Wed May 2 19:19:01 2012
New Revision: 1333159
URL: http://svn.apache.org/viewvc?rev=1333159&view=rev
Log:
HBASE-2625 Avoid byte buffer allocations when reading a value from a Result object (Tudor Scurtu)
Modified:
hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1333159&r1=1333158&r2=1333159&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/KeyValue.java Wed May 2 19:19:01 2012
@@ -22,6 +22,7 @@ package org.apache.hadoop.hbase;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
import java.util.Comparator;
import java.util.HashMap;
@@ -167,6 +168,37 @@ public class KeyValue implements Writabl
public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET;
/**
+ * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
+ * characteristics would take up for its underlying data structure.
+ *
+ * @param rlength row length
+ * @param flength family length
+ * @param qlength qualifier length
+ * @param vlength value length
+ *
+ * @return the <code>KeyValue</code> data structure length
+ */
+ public static long getKeyValueDataStructureSize(int rlength,
+ int flength, int qlength, int vlength) {
+ return KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE +
+ getKeyDataStructureSize(rlength, flength, qlength) + vlength;
+ }
+
+ /**
+ * Computes the number of bytes that a <code>KeyValue</code> instance with the provided
+ * characteristics would take up in its underlying data structure for the key.
+ *
+ * @param rlength row length
+ * @param flength family length
+ * @param qlength qualifier length
+ *
+ * @return the key data structure length
+ */
+ public static long getKeyDataStructureSize(int rlength, int flength, int qlength) {
+ return KeyValue.KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
+ }
+
+ /**
* Key type.
* Has space for other key types to be added later. Cannot rely on
* enum ordinals . They change if item is removed or moved. Do our own codes.
@@ -478,7 +510,7 @@ public class KeyValue implements Writabl
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
// Key length
- long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
+ long longkeylength = getKeyDataStructureSize(rlength, flength, qlength);
if (longkeylength > Integer.MAX_VALUE) {
throw new IllegalArgumentException("keylength " + longkeylength + " > " +
Integer.MAX_VALUE);
@@ -491,7 +523,8 @@ public class KeyValue implements Writabl
}
// Allocate right-sized byte array.
- byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength];
+ byte [] bytes =
+ new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)];
// Write the correct size markers
int pos = 0;
pos = Bytes.putInt(bytes, pos, keylength);
@@ -506,8 +539,12 @@ public class KeyValue implements Writabl
}
/**
- * Write KeyValue format into a byte array.
+ * Constructs KeyValue structure filled with specified values. Uses the provided buffer as its
+ * backing data buffer.
+ * <p>
+ * Column is split into two fields, family and qualifier.
*
+ * @param buffer the bytes buffer to use
* @param row row key
* @param roffset row offset
* @param rlength row length
@@ -522,13 +559,84 @@ public class KeyValue implements Writabl
* @param value column value
* @param voffset value offset
* @param vlength value length
- * @return The newly created byte array.
+ * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
+ * remaining in the buffer
*/
- static byte [] createByteArray(final byte [] row, final int roffset,
- final int rlength, final byte [] family, final int foffset, int flength,
- final byte [] qualifier, final int qoffset, int qlength,
+ public KeyValue(byte [] buffer,
+ final byte [] row, final int roffset, final int rlength,
+ final byte [] family, final int foffset, final int flength,
+ final byte [] qualifier, final int qoffset, final int qlength,
final long timestamp, final Type type,
- final byte [] value, final int voffset, int vlength) {
+ final byte [] value, final int voffset, final int vlength) {
+
+ this(buffer, 0,
+ row, roffset, rlength,
+ family, foffset, flength,
+ qualifier, qoffset, qlength,
+ timestamp, type,
+ value, voffset, vlength);
+ }
+
+ /**
+ * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the
+ * data buffer.
+ * <p>
+ * Column is split into two fields, family and qualifier.
+ *
+ * @param buffer the bytes buffer to use
+ * @param boffset buffer offset
+ * @param row row key
+ * @param roffset row offset
+ * @param rlength row length
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ * @param timestamp version timestamp
+ * @param type key type
+ * @param value column value
+ * @param voffset value offset
+ * @param vlength value length
+ * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
+ * remaining in the buffer
+ */
+ public KeyValue(byte [] buffer, final int boffset,
+ final byte [] row, final int roffset, final int rlength,
+ final byte [] family, final int foffset, final int flength,
+ final byte [] qualifier, final int qoffset, final int qlength,
+ final long timestamp, final Type type,
+ final byte [] value, final int voffset, final int vlength) {
+
+ this.bytes = buffer;
+ this.length = writeByteArray(buffer, boffset,
+ row, roffset, rlength,
+ family, foffset, flength, qualifier, qoffset, qlength,
+ timestamp, type, value, voffset, vlength);
+ this.offset = boffset;
+ }
+
+ /**
+ * Checks the parameters passed to a constructor.
+ *
+ * @param row row key
+ * @param rlength row length
+ * @param family family name
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qlength qualifier length
+ * @param value column value
+ * @param vlength value length
+ *
+ * @throws IllegalArgumentException an illegal value was passed
+ */
+ private static void checkParameters(final byte [] row, final int rlength,
+ final byte [] family, int flength,
+ final byte [] qualifier, int qlength,
+ final byte [] value, int vlength)
+ throws IllegalArgumentException {
+
if (rlength > Short.MAX_VALUE) {
throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
}
@@ -546,24 +654,116 @@ public class KeyValue implements Writabl
throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
}
// Key length
- long longkeylength = KEY_INFRASTRUCTURE_SIZE + rlength + flength + qlength;
- if (longkeylength > Integer.MAX_VALUE) {
- throw new IllegalArgumentException("keylength " + longkeylength + " > " +
- Integer.MAX_VALUE);
+ long longKeyLength = getKeyDataStructureSize(rlength, flength, qlength);
+ if (longKeyLength > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("keylength " + longKeyLength + " > " +
+ Integer.MAX_VALUE);
}
- int keylength = (int)longkeylength;
// Value length
vlength = value == null? 0 : vlength;
if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
- throw new IllegalArgumentException("Valuer > " +
+ throw new IllegalArgumentException("Value length " + vlength + " > " +
HConstants.MAXIMUM_VALUE_LENGTH);
}
+ }
+
+ /**
+ * Write KeyValue format into the provided byte array.
+ *
+ * @param buffer the bytes buffer to use
+ * @param boffset buffer offset
+ * @param row row key
+ * @param roffset row offset
+ * @param rlength row length
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ * @param timestamp version timestamp
+ * @param type key type
+ * @param value column value
+ * @param voffset value offset
+ * @param vlength value length
+ *
+ * @return The number of useful bytes in the buffer.
+ *
+ * @throws IllegalArgumentException an illegal value was passed or there is insufficient space
+ * remaining in the buffer
+ */
+ static int writeByteArray(byte [] buffer, final int boffset,
+ final byte [] row, final int roffset, final int rlength,
+ final byte [] family, final int foffset, int flength,
+ final byte [] qualifier, final int qoffset, int qlength,
+ final long timestamp, final Type type,
+ final byte [] value, final int voffset, int vlength) {
+
+ checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
+
+ int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
+ int keyValueLength = (int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength);
+ if (keyValueLength > buffer.length - boffset) {
+ throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
+ keyValueLength);
+ }
+
+ // Write key, value and key row length.
+ int pos = boffset;
+ pos = Bytes.putInt(buffer, pos, keyLength);
+ pos = Bytes.putInt(buffer, pos, vlength);
+ pos = Bytes.putShort(buffer, pos, (short)(rlength & 0x0000ffff));
+ pos = Bytes.putBytes(buffer, pos, row, roffset, rlength);
+ pos = Bytes.putByte(buffer, pos, (byte) (flength & 0x0000ff));
+ if (flength != 0) {
+ pos = Bytes.putBytes(buffer, pos, family, foffset, flength);
+ }
+ if (qlength != 0) {
+ pos = Bytes.putBytes(buffer, pos, qualifier, qoffset, qlength);
+ }
+ pos = Bytes.putLong(buffer, pos, timestamp);
+ pos = Bytes.putByte(buffer, pos, type.getCode());
+ if (value != null && value.length > 0) {
+ pos = Bytes.putBytes(buffer, pos, value, voffset, vlength);
+ }
+
+ return keyValueLength;
+ }
+
+ /**
+ * Write KeyValue format into a byte array.
+ *
+ * @param row row key
+ * @param roffset row offset
+ * @param rlength row length
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ * @param timestamp version timestamp
+ * @param type key type
+ * @param value column value
+ * @param voffset value offset
+ * @param vlength value length
+ * @return The newly created byte array.
+ */
+ static byte [] createByteArray(final byte [] row, final int roffset,
+ final int rlength, final byte [] family, final int foffset, int flength,
+ final byte [] qualifier, final int qoffset, int qlength,
+ final long timestamp, final Type type,
+ final byte [] value, final int voffset, int vlength) {
+
+ checkParameters(row, rlength, family, flength, qualifier, qlength, value, vlength);
// Allocate right-sized byte array.
- byte [] bytes = new byte[KEYVALUE_INFRASTRUCTURE_SIZE + keylength + vlength];
+ int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
+ byte [] bytes =
+ new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength)];
// Write key, value and key row length.
int pos = 0;
- pos = Bytes.putInt(bytes, pos, keylength);
+ pos = Bytes.putInt(bytes, pos, keyLength);
pos = Bytes.putInt(bytes, pos, vlength);
pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
@@ -913,8 +1113,7 @@ public class KeyValue implements Writabl
* @return Qualifier length
*/
public int getQualifierLength(int rlength, int flength) {
- return getKeyLength() -
- (KEY_INFRASTRUCTURE_SIZE + rlength + flength);
+ return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0);
}
/**
@@ -1008,6 +1207,28 @@ public class KeyValue implements Writabl
}
/**
+ * Returns the value wrapped in a new <code>ByteBuffer</code>.
+ *
+ * @return the value
+ */
+ public ByteBuffer getValueAsByteBuffer() {
+ return ByteBuffer.wrap(getBuffer(), getValueOffset(), getValueLength());
+ }
+
+ /**
+ * Loads this object's value into the provided <code>ByteBuffer</code>.
+ * <p>
+ * Does not clear or flip the buffer.
+ *
+ * @param dst the buffer where to write the value
+ *
+ * @throws BufferOverflowException if there is insufficient space remaining in the buffer
+ */
+ public void loadValue(ByteBuffer dst) throws BufferOverflowException {
+ dst.put(getBuffer(), getValueOffset(), getValueLength());
+ }
+
+ /**
* Primarily for use client-side. Returns the row of this KeyValue in a new
* byte array.<p>
*
@@ -1278,21 +1499,36 @@ public class KeyValue implements Writabl
* @return True if column matches
*/
public boolean matchingColumn(final byte[] family, final byte[] qualifier) {
+ return matchingColumn(family, 0, family == null ? 0 : family.length,
+ qualifier, 0, qualifier == null ? 0 : qualifier.length);
+ }
+
+ /**
+ * Checks if column matches.
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return True if column matches
+ */
+ public boolean matchingColumn(final byte [] family, final int foffset, final int flength,
+ final byte [] qualifier, final int qoffset, final int qlength) {
int rl = getRowLength();
int o = getFamilyOffset(rl);
int fl = getFamilyLength(o);
- int ql = getQualifierLength(rl,fl);
- if (!Bytes.equals(family, 0, family.length, this.bytes, o, fl)) {
+ if (!Bytes.equals(family, foffset, flength, this.bytes, o, fl)) {
return false;
}
- if (qualifier == null || qualifier.length == 0) {
- if (ql == 0) {
- return true;
- }
- return false;
+
+ int ql = getQualifierLength(rl, fl);
+ if (qualifier == null || qlength == 0) {
+ return (ql == 0);
}
- return Bytes.equals(qualifier, 0, qualifier.length,
- this.bytes, o + fl, ql);
+ return Bytes.equals(qualifier, qoffset, qlength, this.bytes, o + fl, ql);
}
/**
@@ -1822,6 +2058,78 @@ public class KeyValue implements Writabl
/**
* Create a KeyValue for the specified row, family and qualifier that would be
+ * smaller than all other possible KeyValues that have the same row,
+ * family, qualifier.
+ * Used for seeking.
+ *
+ * @param buffer the buffer to use for the new <code>KeyValue</code> object
+ * @param row the value key
+ * @param family family name
+ * @param qualifier column qualifier
+ *
+ * @return First possible key on passed Row, Family, Qualifier.
+ *
+ * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
+ * than the provided buffer or than <code>Integer.MAX_VALUE</code>
+ */
+ public static KeyValue createFirstOnRow(byte [] buffer, final byte [] row,
+ final byte [] family, final byte [] qualifier)
+ throws IllegalArgumentException {
+
+ return createFirstOnRow(buffer, 0, row, 0, row.length,
+ family, 0, family.length,
+ qualifier, 0, qualifier.length);
+ }
+
+ /**
+ * Create a KeyValue for the specified row, family and qualifier that would be
+ * smaller than all other possible KeyValues that have the same row,
+ * family, qualifier.
+ * Used for seeking.
+ *
+ * @param buffer the buffer to use for the new <code>KeyValue</code> object
+ * @param boffset buffer offset
+ * @param row the value key
+ * @param roffset row offset
+ * @param rlength row length
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return First possible key on passed Row, Family, Qualifier.
+ *
+ * @throws IllegalArgumentException The resulting <code>KeyValue</code> object would be larger
+ * than the provided buffer or than <code>Integer.MAX_VALUE</code>
+ */
+ public static KeyValue createFirstOnRow(byte [] buffer, final int boffset,
+ final byte [] row, final int roffset, final int rlength,
+ final byte [] family, final int foffset, final int flength,
+ final byte [] qualifier, final int qoffset, final int qlength)
+ throws IllegalArgumentException {
+
+ long lLength = getKeyValueDataStructureSize(rlength, flength, qlength, 0);
+
+ if (lLength > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("KeyValue length " + lLength + " > " + Integer.MAX_VALUE);
+ }
+ int iLength = (int) lLength;
+ if (buffer.length - boffset < iLength) {
+ throw new IllegalArgumentException("Buffer size " + (buffer.length - boffset) + " < " +
+ iLength);
+ }
+ return new KeyValue(buffer, boffset,
+ row, roffset, rlength,
+ family, foffset, flength,
+ qualifier, qoffset, qlength,
+ HConstants.LATEST_TIMESTAMP, KeyValue.Type.Maximum,
+ null, 0, 0);
+ }
+
+ /**
+ * Create a KeyValue for the specified row, family and qualifier that would be
* larger than or equal to all other possible KeyValues that have the same
* row, family, qualifier.
* Used for reseeking.
Modified: hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java?rev=1333159&r1=1333158&r2=1333159&view=diff
==============================================================================
--- hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hbase/trunk/src/main/java/org/apache/hadoop/hbase/client/Result.java Wed May 2 19:19:01 2012
@@ -23,6 +23,8 @@ package org.apache.hadoop.hbase.client;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.nio.BufferOverflowException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
@@ -71,6 +73,7 @@ import org.apache.hadoop.io.Writable;
@InterfaceStability.Stable
public class Result implements Writable, WritableWithSize {
private static final byte RESULT_VERSION = (byte)1;
+ private static final int DEFAULT_BUFFER_SIZE = 1024;
private KeyValue [] kvs = null;
private NavigableMap<byte[],
@@ -80,6 +83,10 @@ public class Result implements Writable,
private transient byte [] row = null;
private ImmutableBytesWritable bytes = null;
+ // never use directly
+ private static byte [] buffer = null;
+ private static final int PAD_WIDTH = 128;
+
/**
* Constructor used for Writable.
*/
@@ -228,13 +235,56 @@ public class Result implements Writable,
}
/**
- * The KeyValue for the most recent for a given column. If the column does
- * not exist in the result set - if it wasn't selected in the query (Get/Scan)
- * or just does not exist in the row the return value is null.
+ * Searches for the latest value for the specified column.
+ *
+ * @param kvs the array to search
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return the index where the value was found, or -1 otherwise
+ */
+ protected int binarySearch(final KeyValue [] kvs,
+ final byte [] family, final int foffset, final int flength,
+ final byte [] qualifier, final int qoffset, final int qlength) {
+
+ double keyValueSize = (double)
+ KeyValue.getKeyValueDataStructureSize(kvs[0].getRowLength(), flength, qlength, 0);
+
+ if (buffer == null || keyValueSize > buffer.length) {
+ // pad to the smallest multiple of the pad width
+ buffer = new byte[(int) Math.ceil(keyValueSize / PAD_WIDTH) * PAD_WIDTH];
+ }
+
+ KeyValue searchTerm = KeyValue.createFirstOnRow(buffer, 0,
+ kvs[0].getBuffer(), kvs[0].getRowOffset(), kvs[0].getRowLength(),
+ family, foffset, flength,
+ qualifier, qoffset, qlength);
+
+ // pos === ( -(insertion point) - 1)
+ int pos = Arrays.binarySearch(kvs, searchTerm, KeyValue.COMPARATOR);
+ // never will exact match
+ if (pos < 0) {
+ pos = (pos+1) * -1;
+ // pos is now insertion point
+ }
+ if (pos == kvs.length) {
+ return -1; // doesn't exist
+ }
+ return pos;
+ }
+
+ /**
+ * The KeyValue for the most recent timestamp for a given column.
*
* @param family
* @param qualifier
- * @return KeyValue for the column or null
+ *
+ * @return the KeyValue for the column, or null if no value exists in the row or none have been
+ * selected in the query (Get/Scan)
*/
public KeyValue getColumnLatest(byte [] family, byte [] qualifier) {
KeyValue [] kvs = raw(); // side effect possibly.
@@ -253,6 +303,37 @@ public class Result implements Writable,
}
/**
+ * The KeyValue for the most recent timestamp for a given column.
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return the KeyValue for the column, or null if no value exists in the row or none have been
+ * selected in the query (Get/Scan)
+ */
+ public KeyValue getColumnLatest(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength) {
+
+ KeyValue [] kvs = raw(); // side effect possibly.
+ if (kvs == null || kvs.length == 0) {
+ return null;
+ }
+ int pos = binarySearch(kvs, family, foffset, flength, qualifier, qoffset, qlength);
+ if (pos == -1) {
+ return null;
+ }
+ KeyValue kv = kvs[pos];
+ if (kv.matchingColumn(family, foffset, flength, qualifier, qoffset, qlength)) {
+ return kv;
+ }
+ return null;
+ }
+
+ /**
* Get the latest version of the specified column.
* @param family family name
* @param qualifier column qualifier
@@ -267,9 +348,164 @@ public class Result implements Writable,
}
/**
- * Checks for existence of the specified column.
+ * Returns the value wrapped in a new <code>ByteBuffer</code>.
+ *
+ * @param family family name
+ * @param qualifier column qualifier
+ *
+ * @return the latest version of the column, or <code>null</code> if none found
+ */
+ public ByteBuffer getValueAsByteBuffer(byte [] family, byte [] qualifier) {
+
+ KeyValue kv = getColumnLatest(family, 0, family.length, qualifier, 0, qualifier.length);
+
+ if (kv == null) {
+ return null;
+ }
+ return kv.getValueAsByteBuffer();
+ }
+
+ /**
+ * Returns the value wrapped in a new <code>ByteBuffer</code>.
+ *
* @param family family name
+ * @param foffset family offset
+ * @param flength family length
* @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return the latest version of the column, or <code>null</code> if none found
+ */
+ public ByteBuffer getValueAsByteBuffer(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength) {
+
+ KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
+
+ if (kv == null) {
+ return null;
+ }
+ return kv.getValueAsByteBuffer();
+ }
+
+ /**
+ * Loads the latest version of the specified column into the provided <code>ByteBuffer</code>.
+ * <p>
+ * Does not clear or flip the buffer.
+ *
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param dst the buffer where to write the value
+ *
+ * @return <code>true</code> if a value was found, <code>false</code> otherwise
+ *
+ * @throws BufferOverflowException there is insufficient space remaining in the buffer
+ */
+ public boolean loadValue(byte [] family, byte [] qualifier, ByteBuffer dst)
+ throws BufferOverflowException {
+ return loadValue(family, 0, family.length, qualifier, 0, qualifier.length, dst);
+ }
+
+ /**
+ * Loads the latest version of the specified column into the provided <code>ByteBuffer</code>.
+ * <p>
+ * Does not clear or flip the buffer.
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ * @param dst the buffer where to write the value
+ *
+ * @return <code>true</code> if a value was found, <code>false</code> otherwise
+ *
+ * @throws BufferOverflowException there is insufficient space remaining in the buffer
+ */
+ public boolean loadValue(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength, ByteBuffer dst)
+ throws BufferOverflowException {
+ KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
+
+ if (kv == null) {
+ return false;
+ }
+ kv.loadValue(dst);
+ return true;
+ }
+
+ /**
+ * Checks if the specified column contains a non-empty value (not a zero-length byte array).
+ *
+ * @param family family name
+ * @param qualifier column qualifier
+ *
+ * @return whether or not a latest value exists and is not empty
+ */
+ public boolean containsNonEmptyColumn(byte [] family, byte [] qualifier) {
+
+ return containsNonEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length);
+ }
+
+ /**
+ * Checks if the specified column contains a non-empty value (not a zero-length byte array).
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return whether or not a latest value exists and is not empty
+ */
+ public boolean containsNonEmptyColumn(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength) {
+
+ KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
+
+ return (kv != null) && (kv.getValueLength() > 0);
+ }
+
+ /**
+ * Checks if the specified column contains an empty value (a zero-length byte array).
+ *
+ * @param family family name
+ * @param qualifier column qualifier
+ *
+ * @return whether or not a latest value exists and is empty
+ */
+ public boolean containsEmptyColumn(byte [] family, byte [] qualifier) {
+
+ return containsEmptyColumn(family, 0, family.length, qualifier, 0, qualifier.length);
+ }
+
+ /**
+ * Checks if the specified column contains an empty value (a zero-length byte array).
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return whether or not a latest value exists and is empty
+ */
+ public boolean containsEmptyColumn(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength) {
+ KeyValue kv = getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength);
+
+ return (kv != null) && (kv.getValueLength() == 0);
+ }
+
+ /**
+ * Checks for existence of a value for the specified column (empty or not).
+ *
+ * @param family family name
+ * @param qualifier column qualifier
+ *
* @return true if at least one value exists in the result, false if not
*/
public boolean containsColumn(byte [] family, byte [] qualifier) {
@@ -278,6 +514,24 @@ public class Result implements Writable,
}
/**
+ * Checks for existence of a value for the specified column (empty or not).
+ *
+ * @param family family name
+ * @param foffset family offset
+ * @param flength family length
+ * @param qualifier column qualifier
+ * @param qoffset qualifier offset
+ * @param qlength qualifier length
+ *
+ * @return true if at least one value exists in the result, false if not
+ */
+ public boolean containsColumn(byte [] family, int foffset, int flength,
+ byte [] qualifier, int qoffset, int qlength) {
+
+ return getColumnLatest(family, foffset, flength, qualifier, qoffset, qlength) != null;
+ }
+
+ /**
* Map of families to all versions of its qualifiers and values.
* <p>
* Returns a three level Map of the form:
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java?rev=1333159&r1=1333158&r2=1333159&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java Wed May 2 19:19:01 2012
@@ -344,31 +344,47 @@ public class TestKeyValue extends TestCa
public void testFirstLastOnRow() {
final KVComparator c = KeyValue.COMPARATOR;
long ts = 1;
+ byte[] bufferA = new byte[128];
+ int offsetA = 0;
+ byte[] bufferB = new byte[128];
+ int offsetB = 7;
// These are listed in sort order (ie: every one should be less
// than the one on the next line).
final KeyValue firstOnRowA = KeyValue.createFirstOnRow(rowA);
+ final KeyValue firstOnRowABufferFamQual = KeyValue.createFirstOnRow(bufferA, offsetA,
+ rowA, 0, rowA.length, family, 0, family.length, qualA, 0, qualA.length);
final KeyValue kvA_1 = new KeyValue(rowA, null, null, ts, Type.Put);
final KeyValue kvA_2 = new KeyValue(rowA, family, qualA, ts, Type.Put);
-
+
final KeyValue lastOnRowA = KeyValue.createLastOnRow(rowA);
final KeyValue firstOnRowB = KeyValue.createFirstOnRow(rowB);
+ final KeyValue firstOnRowBBufferFam = KeyValue.createFirstOnRow(bufferB, offsetB,
+ rowB, 0, rowB.length, family, 0, family.length, null, 0, 0);
final KeyValue kvB = new KeyValue(rowB, family, qualA, ts, Type.Put);
assertKVLess(c, firstOnRowA, firstOnRowB);
+ assertKVLess(c, firstOnRowA, firstOnRowBBufferFam);
+ assertKVLess(c, firstOnRowABufferFamQual, firstOnRowB);
assertKVLess(c, firstOnRowA, kvA_1);
assertKVLess(c, firstOnRowA, kvA_2);
+ assertKVLess(c, firstOnRowABufferFamQual, kvA_2);
assertKVLess(c, kvA_1, kvA_2);
assertKVLess(c, kvA_2, firstOnRowB);
assertKVLess(c, kvA_1, firstOnRowB);
+ assertKVLess(c, kvA_2, firstOnRowBBufferFam);
+ assertKVLess(c, kvA_1, firstOnRowBBufferFam);
assertKVLess(c, lastOnRowA, firstOnRowB);
+ assertKVLess(c, lastOnRowA, firstOnRowBBufferFam);
assertKVLess(c, firstOnRowB, kvB);
+ assertKVLess(c, firstOnRowBBufferFam, kvB);
assertKVLess(c, lastOnRowA, kvB);
assertKVLess(c, kvA_2, lastOnRowA);
assertKVLess(c, kvA_1, lastOnRowA);
assertKVLess(c, firstOnRowA, lastOnRowA);
+ assertKVLess(c, firstOnRowABufferFamQual, lastOnRowA);
}
public void testCreateKeyOnly() throws Exception {
Modified: hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestResult.java
URL: http://svn.apache.org/viewvc/hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestResult.java?rev=1333159&r1=1333158&r2=1333159&view=diff
==============================================================================
--- hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestResult.java (original)
+++ hbase/trunk/src/test/java/org/apache/hadoop/hbase/client/TestResult.java Wed May 2 19:19:01 2012
@@ -21,6 +21,8 @@
package org.apache.hadoop.hbase.client;
import junit.framework.TestCase;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -28,6 +30,7 @@ import org.junit.experimental.categories
import static org.apache.hadoop.hbase.HBaseTestCase.assertByteEquals;
+import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
@@ -36,6 +39,8 @@ import java.util.NavigableMap;
@Category(SmallTests.class)
public class TestResult extends TestCase {
+ private static final Log LOG = LogFactory.getLog(TestResult.class.getName());
+
static KeyValue[] genKVs(final byte[] row, final byte[] family,
final byte[] value,
final long timestamp,
@@ -55,7 +60,7 @@ public class TestResult extends TestCase
static final byte [] family = Bytes.toBytes("family");
static final byte [] value = Bytes.toBytes("value");
- public void testBasic() throws Exception {
+ public void testBasicGetColumn() throws Exception {
KeyValue [] kvs = genKVs(row, family, value, 1, 100);
Arrays.sort(kvs, KeyValue.COMPARATOR);
@@ -68,13 +73,11 @@ public class TestResult extends TestCase
List<KeyValue> ks = r.getColumn(family, qf);
assertEquals(1, ks.size());
assertByteEquals(qf, ks.get(0).getQualifier());
-
assertEquals(ks.get(0), r.getColumnLatest(family, qf));
- assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
- assertTrue(r.containsColumn(family, qf));
}
}
- public void testMultiVersion() throws Exception {
+
+ public void testMultiVersionGetColumn() throws Exception {
KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
@@ -92,13 +95,89 @@ public class TestResult extends TestCase
assertEquals(2, ks.size());
assertByteEquals(qf, ks.get(0).getQualifier());
assertEquals(200, ks.get(0).getTimestamp());
-
assertEquals(ks.get(0), r.getColumnLatest(family, qf));
+ }
+ }
+
+ public void testBasicGetValue() throws Exception {
+ KeyValue [] kvs = genKVs(row, family, value, 1, 100);
+
+ Arrays.sort(kvs, KeyValue.COMPARATOR);
+
+ Result r = new Result(kvs);
+
+ for (int i = 0; i < 100; ++i) {
+ final byte[] qf = Bytes.toBytes(i);
+
+ assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
+ assertTrue(r.containsColumn(family, qf));
+ }
+ }
+
+ public void testMultiVersionGetValue() throws Exception {
+ KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
+ KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
+
+ KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length];
+ System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
+ System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
+
+ Arrays.sort(kvs, KeyValue.COMPARATOR);
+
+ Result r = new Result(kvs);
+ for (int i = 0; i < 100; ++i) {
+ final byte[] qf = Bytes.toBytes(i);
+
assertByteEquals(Bytes.add(value, Bytes.toBytes(i)), r.getValue(family, qf));
assertTrue(r.containsColumn(family, qf));
}
}
+ public void testBasicLoadValue() throws Exception {
+ KeyValue [] kvs = genKVs(row, family, value, 1, 100);
+
+ Arrays.sort(kvs, KeyValue.COMPARATOR);
+
+ Result r = new Result(kvs);
+ ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
+
+ for (int i = 0; i < 100; ++i) {
+ final byte[] qf = Bytes.toBytes(i);
+
+ loadValueBuffer.clear();
+ r.loadValue(family, qf, loadValueBuffer);
+ loadValueBuffer.flip();
+ assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
+ assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
+ r.getValueAsByteBuffer(family, qf));
+ }
+ }
+
+ public void testMultiVersionLoadValue() throws Exception {
+ KeyValue [] kvs1 = genKVs(row, family, value, 1, 100);
+ KeyValue [] kvs2 = genKVs(row, family, value, 200, 100);
+
+ KeyValue [] kvs = new KeyValue[kvs1.length+kvs2.length];
+ System.arraycopy(kvs1, 0, kvs, 0, kvs1.length);
+ System.arraycopy(kvs2, 0, kvs, kvs1.length, kvs2.length);
+
+ Arrays.sort(kvs, KeyValue.COMPARATOR);
+
+ ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
+
+ Result r = new Result(kvs);
+ for (int i = 0; i < 100; ++i) {
+ final byte[] qf = Bytes.toBytes(i);
+
+ loadValueBuffer.clear();
+ r.loadValue(family, qf, loadValueBuffer);
+ loadValueBuffer.flip();
+ assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))), loadValueBuffer);
+ assertEquals(ByteBuffer.wrap(Bytes.add(value, Bytes.toBytes(i))),
+ r.getValueAsByteBuffer(family, qf));
+ }
+ }
+
/**
* Verify that Result.compareResults(...) behaves correctly.
*/
@@ -125,5 +204,82 @@ public class TestResult extends TestCase
@org.junit.Rule
public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
-}
+ /**
+ * Microbenchmark that compares {@link Result#getValue} and {@link Result#loadValue} performance.
+ *
+ * @throws Exception
+ */
+ public void doReadBenchmark() throws Exception {
+
+ final int n = 5;
+ final int m = 100000000;
+
+ StringBuilder valueSB = new StringBuilder();
+ for (int i = 0; i < 100; i++) {
+ valueSB.append((byte)(Math.random() * 10));
+ }
+
+ StringBuilder rowSB = new StringBuilder();
+ for (int i = 0; i < 50; i++) {
+ rowSB.append((byte)(Math.random() * 10));
+ }
+
+ KeyValue [] kvs = genKVs(Bytes.toBytes(rowSB.toString()), family,
+ Bytes.toBytes(valueSB.toString()), 1, n);
+ Arrays.sort(kvs, KeyValue.COMPARATOR);
+ ByteBuffer loadValueBuffer = ByteBuffer.allocate(1024);
+ Result r = new Result(kvs);
+
+ byte[][] qfs = new byte[n][Bytes.SIZEOF_INT];
+ for (int i = 0; i < n; ++i) {
+ System.arraycopy(qfs[i], 0, Bytes.toBytes(i), 0, Bytes.SIZEOF_INT);
+ }
+
+ // warm up
+ for (int k = 0; k < 100000; k++) {
+ for (int i = 0; i < n; ++i) {
+ r.getValue(family, qfs[i]);
+ loadValueBuffer.clear();
+ r.loadValue(family, qfs[i], loadValueBuffer);
+ loadValueBuffer.flip();
+ }
+ }
+
+ System.gc();
+ long start = System.nanoTime();
+ for (int k = 0; k < m; k++) {
+ for (int i = 0; i < n; ++i) {
+ loadValueBuffer.clear();
+ r.loadValue(family, qfs[i], loadValueBuffer);
+ loadValueBuffer.flip();
+ }
+ }
+ long stop = System.nanoTime();
+ System.out.println("loadValue(): " + (stop - start));
+
+ System.gc();
+ start = System.nanoTime();
+ for (int k = 0; k < m; k++) {
+ for (int i = 0; i < n; i++) {
+ r.getValue(family, qfs[i]);
+ }
+ }
+ stop = System.nanoTime();
+ System.out.println("getValue(): " + (stop - start));
+ }
+
+ /**
+ * Calls non-functional test methods.
+ *
+ * @param args
+ */
+ public static void main(String[] args) {
+ TestResult testResult = new TestResult();
+ try {
+ testResult.doReadBenchmark();
+ } catch (Exception e) {
+ LOG.error("Unexpected exception", e);
+ }
+ }
+}