You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2012/11/16 07:53:28 UTC
svn commit: r1410213 [2/2] - in /hbase/trunk:
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/
hbase-common/src/main/java/org/apache/hadoop/hbase/util/
hbase-common/src/main/java/org/ap...
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hadoop.hbase.util.test;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.ByteBufferUtils;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Generate list of key values which are very useful to test data block encoding
+ * and compression.
+ */
+public class RedundantKVGenerator {
+ // row settings
+ static int DEFAULT_NUMBER_OF_ROW_PREFIXES = 10;
+ static int DEFAULT_AVERAGE_PREFIX_LENGTH = 6;
+ static int DEFAULT_PREFIX_LENGTH_VARIANCE = 3;
+ static int DEFAULT_AVERAGE_SUFFIX_LENGTH = 3;
+ static int DEFAULT_SUFFIX_LENGTH_VARIANCE = 3;
+ static int DEFAULT_NUMBER_OF_ROW = 500;
+
+ // qualifier
+ static float DEFAULT_CHANCE_FOR_SAME_QUALIFIER = 0.5f;
+ static float DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER = 0.4f;
+ static int DEFAULT_AVERAGE_QUALIFIER_LENGTH = 9;
+ static int DEFAULT_QUALIFIER_LENGTH_VARIANCE = 3;
+
+ static int DEFAULT_COLUMN_FAMILY_LENGTH = 9;
+ static int DEFAULT_VALUE_LENGTH = 8;
+ static float DEFAULT_CHANCE_FOR_ZERO_VALUE = 0.5f;
+
+ static int DEFAULT_BASE_TIMESTAMP_DIVIDE = 1000000;
+ static int DEFAULT_TIMESTAMP_DIFF_SIZE = 100000000;
+
+ /**
+ * Default constructor, assumes all parameters from class constants.
+ */
+ public RedundantKVGenerator() {
+ this(new Random(42L),
+ DEFAULT_NUMBER_OF_ROW_PREFIXES,
+ DEFAULT_AVERAGE_PREFIX_LENGTH,
+ DEFAULT_PREFIX_LENGTH_VARIANCE,
+ DEFAULT_AVERAGE_SUFFIX_LENGTH,
+ DEFAULT_SUFFIX_LENGTH_VARIANCE,
+ DEFAULT_NUMBER_OF_ROW,
+
+ DEFAULT_CHANCE_FOR_SAME_QUALIFIER,
+ DEFAULT_CHANCE_FOR_SIMILIAR_QUALIFIER,
+ DEFAULT_AVERAGE_QUALIFIER_LENGTH,
+ DEFAULT_QUALIFIER_LENGTH_VARIANCE,
+
+ DEFAULT_COLUMN_FAMILY_LENGTH,
+ DEFAULT_VALUE_LENGTH,
+ DEFAULT_CHANCE_FOR_ZERO_VALUE,
+
+ DEFAULT_BASE_TIMESTAMP_DIVIDE,
+ DEFAULT_TIMESTAMP_DIFF_SIZE
+ );
+ }
+
+
+ /**
+ * Various configuration options for generating key values
+ * @param randomizer pick things by random
+ */
+ public RedundantKVGenerator(Random randomizer,
+ int numberOfRowPrefixes,
+ int averagePrefixLength,
+ int prefixLengthVariance,
+ int averageSuffixLength,
+ int suffixLengthVariance,
+ int numberOfRows,
+
+ float chanceForSameQualifier,
+ float chanceForSimiliarQualifier,
+ int averageQualifierLength,
+ int qualifierLengthVariance,
+
+ int columnFamilyLength,
+ int valueLength,
+ float chanceForZeroValue,
+
+ int baseTimestampDivide,
+ int timestampDiffSize
+ ) {
+ this.randomizer = randomizer;
+
+ this.numberOfRowPrefixes = numberOfRowPrefixes;
+ this.averagePrefixLength = averagePrefixLength;
+ this.prefixLengthVariance = prefixLengthVariance;
+ this.averageSuffixLength = averageSuffixLength;
+ this.suffixLengthVariance = suffixLengthVariance;
+ this.numberOfRows = numberOfRows;
+
+ this.chanceForSameQualifier = chanceForSameQualifier;
+ this.chanceForSimiliarQualifier = chanceForSimiliarQualifier;
+ this.averageQualifierLength = averageQualifierLength;
+ this.qualifierLengthVariance = qualifierLengthVariance;
+
+ this.columnFamilyLength = columnFamilyLength;
+ this.valueLength = valueLength;
+ this.chanceForZeroValue = chanceForZeroValue;
+
+ this.baseTimestampDivide = baseTimestampDivide;
+ this.timestampDiffSize = timestampDiffSize;
+ }
+
+ /** Used to generate dataset */
+ private Random randomizer;
+
+ // row settings
+ private int numberOfRowPrefixes;
+ private int averagePrefixLength = 6;
+ private int prefixLengthVariance = 3;
+ private int averageSuffixLength = 3;
+ private int suffixLengthVariance = 3;
+ private int numberOfRows = 500;
+
+ // qualifier
+ private float chanceForSameQualifier = 0.5f;
+ private float chanceForSimiliarQualifier = 0.4f;
+ private int averageQualifierLength = 9;
+ private int qualifierLengthVariance = 3;
+
+ private int columnFamilyLength = 9;
+ private int valueLength = 8;
+ private float chanceForZeroValue = 0.5f;
+
+ private int baseTimestampDivide = 1000000;
+ private int timestampDiffSize = 100000000;
+
+ private List<byte[]> generateRows() {
+ // generate prefixes
+ List<byte[]> prefixes = new ArrayList<byte[]>();
+ prefixes.add(new byte[0]);
+ for (int i = 1; i < numberOfRowPrefixes; ++i) {
+ int prefixLength = averagePrefixLength;
+ prefixLength += randomizer.nextInt(2 * prefixLengthVariance + 1) -
+ prefixLengthVariance;
+ byte[] newPrefix = new byte[prefixLength];
+ randomizer.nextBytes(newPrefix);
+ prefixes.add(newPrefix);
+ }
+
+ // generate rest of the row
+ List<byte[]> rows = new ArrayList<byte[]>();
+ for (int i = 0; i < numberOfRows; ++i) {
+ int suffixLength = averageSuffixLength;
+ suffixLength += randomizer.nextInt(2 * suffixLengthVariance + 1) -
+ suffixLengthVariance;
+ int randomPrefix = randomizer.nextInt(prefixes.size());
+ byte[] row = new byte[prefixes.get(randomPrefix).length +
+ suffixLength];
+ rows.add(row);
+ }
+
+ return rows;
+ }
+
+ /**
+ * Generate test data useful to test encoders.
+ * @param howMany How many Key values should be generated.
+ * @return sorted list of key values
+ */
+ public List<KeyValue> generateTestKeyValues(int howMany) {
+ List<KeyValue> result = new ArrayList<KeyValue>();
+
+ List<byte[]> rows = generateRows();
+ Map<Integer, List<byte[]>> rowsToQualifier =
+ new HashMap<Integer, List<byte[]>>();
+
+ byte[] family = new byte[columnFamilyLength];
+ randomizer.nextBytes(family);
+
+ long baseTimestamp = Math.abs(randomizer.nextLong()) /
+ baseTimestampDivide;
+
+ byte[] value = new byte[valueLength];
+
+ for (int i = 0; i < howMany; ++i) {
+ long timestamp = baseTimestamp + randomizer.nextInt(
+ timestampDiffSize);
+ Integer rowId = randomizer.nextInt(rows.size());
+ byte[] row = rows.get(rowId);
+
+ // generate qualifier, sometimes it is same, sometimes similar,
+ // occasionally completely different
+ byte[] qualifier;
+ float qualifierChance = randomizer.nextFloat();
+ if (!rowsToQualifier.containsKey(rowId) ||
+ qualifierChance > chanceForSameQualifier +
+ chanceForSimiliarQualifier) {
+ int qualifierLength = averageQualifierLength;
+ qualifierLength +=
+ randomizer.nextInt(2 * qualifierLengthVariance + 1) -
+ qualifierLengthVariance;
+ qualifier = new byte[qualifierLength];
+ randomizer.nextBytes(qualifier);
+
+ // add it to map
+ if (!rowsToQualifier.containsKey(rowId)) {
+ rowsToQualifier.put(rowId, new ArrayList<byte[]>());
+ }
+ rowsToQualifier.get(rowId).add(qualifier);
+ } else if (qualifierChance > chanceForSameQualifier) {
+ // similar qualifier
+ List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
+ byte[] originalQualifier = previousQualifiers.get(
+ randomizer.nextInt(previousQualifiers.size()));
+
+ qualifier = new byte[originalQualifier.length];
+ int commonPrefix = randomizer.nextInt(qualifier.length);
+ System.arraycopy(originalQualifier, 0, qualifier, 0, commonPrefix);
+ for (int j = commonPrefix; j < qualifier.length; ++j) {
+ qualifier[j] = (byte) (randomizer.nextInt() & 0xff);
+ }
+
+ rowsToQualifier.get(rowId).add(qualifier);
+ } else {
+ // same qualifier
+ List<byte[]> previousQualifiers = rowsToQualifier.get(rowId);
+ qualifier = previousQualifiers.get(
+ randomizer.nextInt(previousQualifiers.size()));
+ }
+
+ if (randomizer.nextFloat() < chanceForZeroValue) {
+ for (int j = 0; j < value.length; ++j) {
+ value[j] = (byte) 0;
+ }
+ } else {
+ randomizer.nextBytes(value);
+ }
+
+ result.add(new KeyValue(row, family, qualifier, timestamp, value));
+ }
+
+ Collections.sort(result, KeyValue.COMPARATOR);
+
+ return result;
+ }
+
+ /**
+ * Convert list of KeyValues to byte buffer.
+ * @param keyValues list of KeyValues to be converted.
+ * @return buffer with content from key values
+ */
+ public static ByteBuffer convertKvToByteBuffer(List<KeyValue> keyValues,
+ boolean includesMemstoreTS) {
+ int totalSize = 0;
+ for (KeyValue kv : keyValues) {
+ totalSize += kv.getLength();
+ if (includesMemstoreTS) {
+ totalSize += WritableUtils.getVIntSize(kv.getMemstoreTS());
+ }
+ }
+
+ ByteBuffer result = ByteBuffer.allocate(totalSize);
+ for (KeyValue kv : keyValues) {
+ result.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
+ if (includesMemstoreTS) {
+ ByteBufferUtils.writeVLong(result, kv.getMemstoreTS());
+ }
+ }
+
+ return result;
+ }
+
+}
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/Cell.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/Cell.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/Cell.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/Cell.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hbase;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hbase.cell.CellTool;
+
+
+/**
+ * The unit of storage in HBase consisting of the following fields:<br/>
+ * <pre>
+ * 1) row
+ * 2) column family
+ * 3) column qualifier
+ * 4) timestamp
+ * 5) type
+ * 6) MVCC version
+ * 7) value
+ * </pre>
+ * <p/>
+ * Uniqueness is determined by the combination of row, column family, column qualifier,
+ * timestamp, and type.
+ * <p/>
+ * The natural comparator will perform a bitwise comparison on row, column family, and column
+ * qualifier. Less intuitively, it will then treat the greater timestamp as the lesser value with
+ * the goal of sorting newer cells first.
+ * <p/>
+ * This interface does not include methods that allocate new byte[]'s such as those used in client
+ * or debugging code. These should be placed in a sub-interface or the {@link CellTool} class.
+ * <p/>
+ * Cell implements Comparable<Cell> which is only meaningful when comparing to other keys in the
+ * same table. It uses {@link #CellComparator} which does not work on the -ROOT- and .META. tables.
+ * <p/>
+ * In the future, we may consider adding a boolean isOnHeap() method and a getValueBuffer() method
+ * that can be used to pass a value directly from an off-heap ByteBuffer to the network without
+ * copying into an on-heap byte[].
+ * <p/>
+ * Historic note: the original Cell implementation (KeyValue) requires that all fields be encoded as
+ * consecutive bytes in the same byte[], whereas this interface allows fields to reside in separate
+ * byte[]'s.
+ * <p/>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface Cell {
+
+ //1) Row
+
+ /**
+ * Contiguous raw bytes that may start at any index in the containing array. Max length is
+ * Short.MAX_VALUE which is 32,767 bytes.
+ * @return The array containing the row bytes.
+ */
+ byte[] getRowArray();
+
+ /**
+ * @return Array index of first row byte
+ */
+ int getRowOffset();
+
+ /**
+ * @return Number of row bytes. Must be < rowArray.length - offset.
+ */
+ short getRowLength();
+
+
+ //2) Family
+
+ /**
+ * Contiguous bytes composed of legal HDFS filename characters which may start at any index in the
+ * containing array. Max length is Byte.MAX_VALUE, which is 127 bytes.
+ * @return the array containing the family bytes.
+ */
+ byte[] getFamilyArray();
+
+ /**
+ * @return Array index of first row byte
+ */
+ int getFamilyOffset();
+
+ /**
+ * @return Number of family bytes. Must be < familyArray.length - offset.
+ */
+ byte getFamilyLength();
+
+
+ //3) Qualifier
+
+ /**
+ * Contiguous raw bytes that may start at any index in the containing array. Max length is
+ * Short.MAX_VALUE which is 32,767 bytes.
+ * @return The array containing the qualifier bytes.
+ */
+ byte[] getQualifierArray();
+
+ /**
+ * @return Array index of first qualifier byte
+ */
+ int getQualifierOffset();
+
+ /**
+ * @return Number of qualifier bytes. Must be < qualifierArray.length - offset.
+ */
+ int getQualifierLength();
+
+
+ //4) Timestamp
+
+ /**
+ * @return Long value representing time at which this cell was "Put" into the row. Typically
+ * represents the time of insertion, but can be any value from Long.MIN_VALUE to Long.MAX_VALUE.
+ */
+ long getTimestamp();
+
+
+ //5) Type
+
+ /**
+ * see {@link #KeyValue.TYPE}
+ * @return The byte representation of the KeyValue.TYPE of this cell: one of Put, Delete, etc
+ */
+ byte getTypeByte();
+
+
+ //6) MvccVersion
+
+ /**
+ * Internal use only. A region-specific sequence ID given to each operation. It always exists for
+ * cells in the memstore but is not retained forever. It may survive several flushes, but
+ * generally becomes irrelevant after the cell's row is no longer involved in any operations that
+ * require strict consistency.
+ * @return mvccVersion (always >= 0 if exists), or 0 if it no longer exists
+ */
+ long getMvccVersion();
+
+
+ //7) Value
+
+ /**
+ * Contiguous raw bytes that may start at any index in the containing array. Max length is
+ * Integer.MAX_VALUE which is 2,147,483,648 bytes.
+ * @return The array containing the value bytes.
+ */
+ byte[] getValueArray();
+
+ /**
+ * @return Array index of first value byte
+ */
+ int getValueOffset();
+
+ /**
+ * @return Number of value bytes. Must be < valueArray.length - offset.
+ */
+ int getValueLength();
+
+}
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellComparator.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellComparator.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellComparator.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellComparator.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hbase.cell;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.Cell;
+
+import com.google.common.primitives.Longs;
+
+/**
+ * Compare two traditional HBase cells.
+ *
+ * Note: This comparator is not valid for -ROOT- and .META. tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class CellComparator implements Comparator<Cell>, Serializable{
+ private static final long serialVersionUID = -8760041766259623329L;
+
+ @Override
+ public int compare(Cell a, Cell b) {
+ return compareStatic(a, b);
+ }
+
+
+ public static int compareStatic(Cell a, Cell b) {
+ //row
+ int c = Bytes.compareTo(
+ a.getRowArray(), a.getRowOffset(), a.getRowLength(),
+ b.getRowArray(), b.getRowOffset(), b.getRowLength());
+ if (c != 0) return c;
+
+ //family
+ c = Bytes.compareTo(
+ a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(),
+ b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength());
+ if (c != 0) return c;
+
+ //qualifier
+ c = Bytes.compareTo(
+ a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(),
+ b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength());
+ if (c != 0) return c;
+
+ //timestamp: later sorts first
+ c = -Longs.compare(a.getTimestamp(), b.getTimestamp());
+ if (c != 0) return c;
+
+ //type
+ c = (0xff & a.getTypeByte()) - (0xff & b.getTypeByte());
+ if (c != 0) return c;
+
+ //mvccVersion: later sorts first
+ return -Longs.compare(a.getMvccVersion(), b.getMvccVersion());
+ }
+
+
+ /**************** equals ****************************/
+
+ public static boolean equals(Cell a, Cell b){
+ if (!areKeyLengthsEqual(a, b)) {
+ return false;
+ }
+ //TODO compare byte[]'s in reverse since later bytes more likely to differ
+ return 0 == compareStatic(a, b);
+ }
+
+ public static boolean equalsRow(Cell a, Cell b){
+ if(!areRowLengthsEqual(a, b)){
+ return false;
+ }
+ return 0 == Bytes.compareTo(
+ a.getRowArray(), a.getRowOffset(), a.getRowLength(),
+ b.getRowArray(), b.getRowOffset(), b.getRowLength());
+ }
+
+
+ /********************* hashCode ************************/
+
+ /**
+ * Returns a hash code that is always the same for two Cells having a matching equals(..) result.
+ * Currently does not guard against nulls, but it could if necessary.
+ */
+ public static int hashCode(Cell cell){
+ if (cell == null) {// return 0 for empty Cell
+ return 0;
+ }
+
+ //pre-calculate the 3 hashes made of byte ranges
+ int rowHash = Bytes.hashCode(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+ int familyHash = Bytes.hashCode(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+ int qualifierHash = Bytes.hashCode(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
+
+ //combine the 6 sub-hashes
+ int hash = 31 * rowHash + familyHash;
+ hash = 31 * hash + qualifierHash;
+ hash = 31 * hash + (int)cell.getTimestamp();
+ hash = 31 * hash + cell.getTypeByte();
+ hash = 31 * hash + (int)cell.getMvccVersion();
+ return hash;
+ }
+
+
+ /******************** lengths *************************/
+
+ public static boolean areKeyLengthsEqual(Cell a, Cell b) {
+ return a.getRowLength() == b.getRowLength()
+ && a.getFamilyLength() == b.getFamilyLength()
+ && a.getQualifierLength() == b.getQualifierLength();
+ }
+
+ public static boolean areRowLengthsEqual(Cell a, Cell b) {
+ return a.getRowLength() == b.getRowLength();
+ }
+
+
+ /***************** special cases ****************************/
+
+ /**
+ * special case for KeyValue.equals
+ */
+ private static int compareStaticIgnoreMvccVersion(Cell a, Cell b) {
+ //row
+ int c = Bytes.compareTo(
+ a.getRowArray(), a.getRowOffset(), a.getRowLength(),
+ b.getRowArray(), b.getRowOffset(), b.getRowLength());
+ if (c != 0) return c;
+
+ //family
+ c = Bytes.compareTo(
+ a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(),
+ b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength());
+ if (c != 0) return c;
+
+ //qualifier
+ c = Bytes.compareTo(
+ a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(),
+ b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength());
+ if (c != 0) return c;
+
+ //timestamp: later sorts first
+ c = -Longs.compare(a.getTimestamp(), b.getTimestamp());
+ if (c != 0) return c;
+
+ //type
+ c = (0xff & a.getTypeByte()) - (0xff & b.getTypeByte());
+ return c;
+ }
+
+ /**
+ * special case for KeyValue.equals
+ */
+ public static boolean equalsIgnoreMvccVersion(Cell a, Cell b){
+ return 0 == compareStaticIgnoreMvccVersion(a, b);
+ }
+
+}
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellOutputStream.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellOutputStream.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellOutputStream.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellOutputStream.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hbase.cell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hbase.Cell;
+
+/**
+ * Accepts a stream of Cells and adds them to its internal data structure. This can be used to build
+ * a block of cells during compactions and flushes, or to build a byte[] to send to the client. This
+ * could be backed by a List<KeyValue>, but more efficient implementations will append results to a
+ * byte[] to eliminate overhead, and possibly encode the cells further.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public interface CellOutputStream {
+
+ /**
+ * Implementation must copy the entire state of the Cell. If the appended Cell is modified
+ * immediately after the append method returns, the modifications must have absolutely no effect
+ * on the copy of the Cell that was added to the appender. For example, calling someList.add(cell)
+ * is not correct.
+ */
+ void write(Cell cell);
+
+ /**
+ * Let the implementation decide what to do. Usually means writing accumulated data into a byte[]
+ * that can then be read from the implementation to be sent to disk, put in the block cache, or
+ * sent over the network.
+ */
+ void flush();
+
+}
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellScannerPosition.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellScannerPosition.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellScannerPosition.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellScannerPosition.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hbase.cell;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An indicator of the state of the scanner after an operation such as nextCell() or positionAt(..).
+ * For example:
+ * <ul>
+ * <li>In a DataBlockScanner, the AFTER_LAST position indicates to the parent StoreFileScanner that
+ * it should load the next block.</li>
+ * <li>In a StoreFileScanner, the AFTER_LAST position indicates that the file has been exhausted.</li>
+ * <li>In a RegionScanner, the AFTER_LAST position indicates that the scanner should move to the
+ * next region.</li>
+ * </ul>
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public enum CellScannerPosition {
+
+ /**
+ * getCurrentCell() will NOT return a valid cell. Calling nextCell() will advance to the first
+ * cell.
+ */
+ BEFORE_FIRST,
+
+ /**
+ * getCurrentCell() will return a valid cell, but it is not the cell requested by positionAt(..),
+ * rather it is the nearest cell before the requested cell.
+ */
+ BEFORE,
+
+ /**
+ * getCurrentCell() will return a valid cell, and it is exactly the cell that was requested by
+ * positionAt(..).
+ */
+ AT,
+
+ /**
+ * getCurrentCell() will return a valid cell, but it is not the cell requested by positionAt(..),
+ * rather it is the nearest cell after the requested cell.
+ */
+ AFTER,
+
+ /**
+ * getCurrentCell() will NOT return a valid cell. Calling nextCell() will have no effect.
+ */
+ AFTER_LAST
+
+}
Added: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellTool.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellTool.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellTool.java (added)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/cell/CellTool.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hbase.cell;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.ByteRange;
+import org.apache.hbase.Cell;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class CellTool {
+
+ /******************* ByteRange *******************************/
+
+ public static ByteRange fillRowRange(Cell cell, ByteRange range) {
+ return range.set(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
+ }
+
+ public static ByteRange fillFamilyRange(Cell cell, ByteRange range) {
+ return range.set(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
+ }
+
+ public static ByteRange fillQualifierRange(Cell cell, ByteRange range) {
+ return range.set(cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ }
+
+
+ /***************** get individual arrays for tests ************/
+
+ public static byte[] getRowArray(Cell cell){
+ byte[] output = new byte[cell.getRowLength()];
+ copyRowTo(cell, output, 0);
+ return output;
+ }
+
+ public static byte[] getFamilyArray(Cell cell){
+ byte[] output = new byte[cell.getFamilyLength()];
+ copyFamilyTo(cell, output, 0);
+ return output;
+ }
+
+ public static byte[] getQualifierArray(Cell cell){
+ byte[] output = new byte[cell.getQualifierLength()];
+ copyQualifierTo(cell, output, 0);
+ return output;
+ }
+
+ public static byte[] getValueArray(Cell cell){
+ byte[] output = new byte[cell.getValueLength()];
+ copyValueTo(cell, output, 0);
+ return output;
+ }
+
+
+ /******************** copyTo **********************************/
+
+ public static int copyRowTo(Cell cell, byte[] destination, int destinationOffset) {
+ System.arraycopy(cell.getRowArray(), cell.getRowOffset(), destination, destinationOffset,
+ cell.getRowLength());
+ return destinationOffset + cell.getRowLength();
+ }
+
+ public static int copyFamilyTo(Cell cell, byte[] destination, int destinationOffset) {
+ System.arraycopy(cell.getFamilyArray(), cell.getFamilyOffset(), destination, destinationOffset,
+ cell.getFamilyLength());
+ return destinationOffset + cell.getFamilyLength();
+ }
+
+ public static int copyQualifierTo(Cell cell, byte[] destination, int destinationOffset) {
+ System.arraycopy(cell.getQualifierArray(), cell.getQualifierOffset(), destination,
+ destinationOffset, cell.getQualifierLength());
+ return destinationOffset + cell.getQualifierLength();
+ }
+
+ public static int copyValueTo(Cell cell, byte[] destination, int destinationOffset) {
+ System.arraycopy(cell.getValueArray(), cell.getValueOffset(), destination, destinationOffset,
+ cell.getValueLength());
+ return destinationOffset + cell.getValueLength();
+ }
+
+
+ /********************* misc *************************************/
+
+ public static byte getRowByte(Cell cell, int index) {
+ return cell.getRowArray()[cell.getRowOffset() + index];
+ }
+
+
+ /********************** KeyValue (move to KeyValueUtils) *********************/
+
+ public static ByteBuffer getValueBufferShallowCopy(Cell cell) {
+ ByteBuffer buffer = ByteBuffer.wrap(cell.getValueArray(), cell.getValueOffset(),
+ cell.getValueLength());
+// buffer.position(buffer.limit());//make it look as if value was appended
+ return buffer;
+ }
+
+}
Added: hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRange.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRange.java?rev=1410213&view=auto
==============================================================================
--- hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRange.java (added)
+++ hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteRange.java Fri Nov 16 06:53:23 2012
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.util;
+
+import junit.framework.Assert;
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hbase.SmallTests;
+import org.junit.experimental.categories.Category;
+
+
+@Category(SmallTests.class)
+public class TestByteRange extends TestCase {
+
+ public void testEmpty(){
+ Assert.assertTrue(ByteRange.isEmpty(null));
+ ByteRange r = new ByteRange();
+ Assert.assertTrue(ByteRange.isEmpty(r));
+ Assert.assertFalse(ByteRange.notEmpty(r));
+ Assert.assertTrue(r.isEmpty());
+ Assert.assertFalse(r.notEmpty());
+ Assert.assertNotNull(r.getBytes());//should be empty byte[], but could change this behavior
+ Assert.assertEquals(0, r.getBytes().length);
+ Assert.assertEquals(0, r.getOffset());
+ Assert.assertEquals(0, r.getLength());
+ Assert.assertTrue(Bytes.equals(new byte[0], r.deepCopyToNewArray()));
+ Assert.assertEquals(0, r.compareTo(new ByteRange(new byte[0], 0, 0)));
+ Assert.assertEquals(0, r.hashCode());
+ }
+
+ public void testBasics(){
+ ByteRange r = new ByteRange(new byte[]{1, 3, 2});
+ Assert.assertFalse(ByteRange.isEmpty(r));
+ Assert.assertNotNull(r.getBytes());//should be empty byte[], but could change this behavior
+ Assert.assertEquals(3, r.getBytes().length);
+ Assert.assertEquals(0, r.getOffset());
+ Assert.assertEquals(3, r.getLength());
+
+ //cloning (deep copying)
+ Assert.assertTrue(Bytes.equals(new byte[]{1, 3, 2}, r.deepCopyToNewArray()));
+ Assert.assertNotSame(r.getBytes(), r.deepCopyToNewArray());
+
+ //hash code
+ Assert.assertTrue(r.hashCode() > 0);
+ Assert.assertEquals(r.hashCode(), r.deepCopy().hashCode());
+
+ //copying to arrays
+ byte[] destination = new byte[]{-59};//junk
+ r.deepCopySubRangeTo(2, 1, destination, 0);
+ Assert.assertTrue(Bytes.equals(new byte[]{2}, destination));
+
+ //set length
+ r.setLength(1);
+ Assert.assertTrue(Bytes.equals(new byte[]{1}, r.deepCopyToNewArray()));
+ r.setLength(2);//verify we retained the 2nd byte, but dangerous in real code
+ Assert.assertTrue(Bytes.equals(new byte[]{1, 3}, r.deepCopyToNewArray()));
+ }
+
+}
+
Modified: hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java (original)
+++ hbase/trunk/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestBytes.java Fri Nov 16 06:53:23 2012
@@ -28,8 +28,10 @@ import java.util.Arrays;
import java.util.Random;
import junit.framework.TestCase;
-import org.junit.experimental.categories.Category;
+
import org.apache.hadoop.hbase.SmallTests;
+import org.junit.Assert;
+import org.junit.experimental.categories.Category;
@Category(SmallTests.class)
@@ -376,7 +378,7 @@ public class TestBytes extends TestCase
assertEquals("World", Bytes.readStringFixedSize(dis, 18));
assertEquals("", Bytes.readStringFixedSize(dis, 9));
}
-
+
public void testCopy() throws Exception {
byte [] bytes = Bytes.toBytes("ABCDEFGHIJKLMNOPQRSTUVWXYZ");
byte [] copy = Bytes.copy(bytes);
@@ -396,5 +398,32 @@ public class TestBytes extends TestCase
String bytes = Bytes.toStringBinary(Bytes.toBytes(2.17));
assertEquals(2.17, Bytes.toDouble(Bytes.toBytesBinary(bytes)), 0);
}
+
+ public void testUnsignedBinarySearch(){
+ byte[] bytes = new byte[]{0,5,123,127,-128,-100,-1};
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)5), 1);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)127), 3);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-128), 4);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-100), 5);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-1), 6);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)2), -1-1);
+ Assert.assertEquals(Bytes.unsignedBinarySearch(bytes, 0, bytes.length, (byte)-5), -6-1);
+ }
+
+ public void testUnsignedIncrement(){
+ byte[] a = Bytes.toBytes(0);
+ int a2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(a), 0);
+ Assert.assertTrue(a2==1);
+
+ byte[] b = Bytes.toBytes(-1);
+ byte[] actual = Bytes.unsignedCopyAndIncrement(b);
+ Assert.assertNotSame(b, actual);
+ byte[] expected = new byte[]{1,0,0,0,0};
+ Assert.assertArrayEquals(expected, actual);
+
+ byte[] c = Bytes.toBytes(255);//should wrap to the next significant byte
+ int c2 = Bytes.toInt(Bytes.unsignedCopyAndIncrement(c), 0);
+ Assert.assertTrue(c2==256);
+ }
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java Fri Nov 16 06:53:23 2012
@@ -352,4 +352,21 @@ abstract class BufferedDataBlockEncoder
}
}
+ /**
+ * Asserts that there is at least the given amount of unfilled space
+ * remaining in the given buffer.
+ * @param out typically, the buffer we are writing to
+ * @param length the required space in the buffer
+ * @throws EncoderBufferTooSmallException If there are no enough bytes.
+ */
+ protected static void ensureSpace(ByteBuffer out, int length)
+ throws EncoderBufferTooSmallException {
+ if (out.position() + length > out.limit()) {
+ throw new EncoderBufferTooSmallException(
+ "Buffer position=" + out.position() +
+ ", buffer limit=" + out.limit() +
+ ", length to be written=" + length);
+ }
+ }
+
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java Fri Nov 16 06:53:23 2012
@@ -233,8 +233,7 @@ public class DiffKeyDeltaEncoder extends
// create KeyValue buffer and fill it prefix
int keyOffset = buffer.position();
- ByteBufferUtils.ensureSpace(buffer, keyLength + valueLength
- + KeyValue.ROW_OFFSET);
+ ensureSpace(buffer, keyLength + valueLength + KeyValue.ROW_OFFSET);
buffer.putInt(keyLength);
buffer.putInt(valueLength);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java Fri Nov 16 06:53:23 2012
@@ -230,8 +230,7 @@ public class FastDiffDeltaEncoder extend
}
int commonLength = ByteBufferUtils.readCompressedInt(source);
- ByteBufferUtils.ensureSpace(out, state.keyLength + state.valueLength +
- KeyValue.ROW_OFFSET);
+ ensureSpace(out, state.keyLength + state.valueLength + KeyValue.ROW_OFFSET);
int kvPos = out.position();
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java Fri Nov 16 06:53:23 2012
@@ -126,8 +126,7 @@ public class PrefixKeyDeltaEncoder exten
int keyOffset;
keyLength += commonLength;
- ByteBufferUtils.ensureSpace(buffer, keyLength + valueLength
- + KeyValue.ROW_OFFSET);
+ ensureSpace(buffer, keyLength + valueLength + KeyValue.ROW_OFFSET);
buffer.putInt(keyLength);
buffer.putInt(valueLength);
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlock.java Fri Nov 16 06:53:23 2012
@@ -725,7 +725,7 @@ public class HFileBlock implements Cache
this.dataBlockEncoder = dataBlockEncoder != null
? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
defaultBlockEncodingCtx =
- new HFileBlockDefaultEncodingContext(compressionAlgorithm, null);
+ new HFileBlockDefaultEncodingContext(compressionAlgorithm, null, DUMMY_HEADER);
dataBlockEncodingCtx =
this.dataBlockEncoder.newOnDiskDataBlockEncodingContext(
compressionAlgorithm, DUMMY_HEADER);
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java Fri Nov 16 06:53:23 2012
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.LargeTest
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.io.hfile.HFileBlock;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
@@ -75,7 +76,7 @@ public class TestDataBlockEncoders {
return encoder.newDataBlockEncodingContext(algo, encoding,
HFileBlock.DUMMY_HEADER);
} else {
- return new HFileBlockDefaultEncodingContext(algo, encoding);
+ return new HFileBlockDefaultEncodingContext(algo, encoding, HFileBlock.DUMMY_HEADER);
}
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestEncodedSeekers.java Fri Nov 16 06:53:23 2012
@@ -27,19 +27,17 @@ import java.util.Map;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.MediumTests;
-import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
-import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
import org.apache.hadoop.hbase.regionserver.HRegion;
-import org.apache.hadoop.hbase.regionserver.StoreFile.BloomType;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.MultiThreadedWriter;
+import org.apache.hadoop.hbase.util.Strings;
import org.apache.hadoop.hbase.util.test.LoadTestKVGenerator;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -59,9 +57,10 @@ public class TestEncodedSeekers {
private static final byte[] CF_BYTES = Bytes.toBytes(CF_NAME);
private static final int MAX_VERSIONS = 5;
+ private static final int BLOCK_SIZE = 64 * 1024;
private static final int MIN_VALUE_SIZE = 30;
private static final int MAX_VALUE_SIZE = 60;
- private static final int NUM_ROWS = 1000;
+ private static final int NUM_ROWS = 1003;
private static final int NUM_COLS_PER_ROW = 20;
private static final int NUM_HFILES = 4;
private static final int NUM_ROWS_PER_FLUSH = NUM_ROWS / NUM_HFILES;
@@ -101,61 +100,73 @@ public class TestEncodedSeekers {
.setMaxVersions(MAX_VERSIONS)
.setDataBlockEncoding(encoding)
.setEncodeOnDisk(encodeOnDisk)
+ .setBlocksize(BLOCK_SIZE)
);
- LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(
- MIN_VALUE_SIZE, MAX_VALUE_SIZE);
- // Write
- for (int i = 0; i < NUM_ROWS; ++i) {
+ //write the data, but leave some in the memstore
+ doPuts(region);
+
+ //verify correctness when memstore contains data
+ doGets(region);
+
+ //verify correctness again after compacting
+ region.compactStores();
+ doGets(region);
+
+
+ Map<DataBlockEncoding, Integer> encodingCounts = cache.getEncodingCountsForTest();
+
+ // Ensure that compactions don't pollute the cache with unencoded blocks
+ // in case of in-cache-only encoding.
+ System.err.println("encodingCounts=" + encodingCounts);
+ assertEquals(1, encodingCounts.size());
+ DataBlockEncoding encodingInCache = encodingCounts.keySet().iterator().next();
+ assertEquals(encoding, encodingInCache);
+ assertTrue(encodingCounts.get(encodingInCache) > 0);
+ }
+
+
+ private void doPuts(HRegion region) throws IOException{
+ LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
+ for (int i = 0; i < NUM_ROWS; ++i) {
byte[] key = MultiThreadedWriter.longToByteArrayKey(i);
for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
Put put = new Put(key);
String colAsStr = String.valueOf(j);
+ byte[] col = Bytes.toBytes(colAsStr);
byte[] value = dataGenerator.generateRandomSizeValue(i, colAsStr);
put.add(CF_BYTES, Bytes.toBytes(colAsStr), value);
+ if(VERBOSE){
+ KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
+ System.err.println(Strings.padFront(i+"", ' ', 4)+" "+kvPut);
+ }
region.put(put);
}
if (i % NUM_ROWS_PER_FLUSH == 0) {
region.flushcache();
}
}
-
- for (int doneCompaction = 0; doneCompaction <= 1; ++doneCompaction) {
- // Read
- for (int i = 0; i < NUM_ROWS; ++i) {
- final byte[] rowKey = MultiThreadedWriter.longToByteArrayKey(i);
- for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
- if (VERBOSE) {
- System.err.println("Reading row " + i + ", column " + j);
- }
- final String qualStr = String.valueOf(j);
- final byte[] qualBytes = Bytes.toBytes(qualStr);
- Get get = new Get(rowKey);
- get.addColumn(CF_BYTES, qualBytes);
- Result result = region.get(get, null);
- assertEquals(1, result.size());
- assertTrue(LoadTestKVGenerator.verify(Bytes.toString(rowKey), qualStr,
- result.getValue(CF_BYTES, qualBytes)));
+ }
+
+
+ private void doGets(HRegion region) throws IOException{
+ for (int i = 0; i < NUM_ROWS; ++i) {
+ final byte[] rowKey = MultiThreadedWriter.longToByteArrayKey(i);
+ for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
+ final String qualStr = String.valueOf(j);
+ if (VERBOSE) {
+ System.err.println("Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey)+"/"
+ +qualStr);
}
- }
-
- if (doneCompaction == 0) {
- // Compact, then read again at the next loop iteration.
- region.compactStores();
+ final byte[] qualBytes = Bytes.toBytes(qualStr);
+ Get get = new Get(rowKey);
+ get.addColumn(CF_BYTES, qualBytes);
+ Result result = region.get(get, null);
+ assertEquals(1, result.size());
+ assertTrue(LoadTestKVGenerator.verify(Bytes.toString(rowKey), qualStr,
+ result.getValue(CF_BYTES, qualBytes)));
}
}
-
- Map<DataBlockEncoding, Integer> encodingCounts =
- cache.getEncodingCountsForTest();
-
- // Ensure that compactions don't pollute the cache with unencoded blocks
- // in case of in-cache-only encoding.
- System.err.println("encodingCounts=" + encodingCounts);
- assertEquals(1, encodingCounts.size());
- DataBlockEncoding encodingInCache =
- encodingCounts.keySet().iterator().next();
- assertEquals(encoding, encodingInCache);
- assertTrue(encodingCounts.get(encodingInCache) > 0);
}
}
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileDataBlockEncoder.java Fri Nov 16 06:53:23 2012
@@ -33,9 +33,8 @@ import org.apache.hadoop.hbase.io.compre
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext;
import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext;
-import org.apache.hadoop.hbase.io.encoding.RedundantKVGenerator;
import org.apache.hadoop.hbase.util.ChecksumType;
-import org.junit.After;
+import org.apache.hadoop.hbase.util.test.RedundantKVGenerator;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -112,9 +111,8 @@ public class TestHFileDataBlockEncoder {
public void testEncodingWritePath() throws IOException {
// usually we have just block without headers, but don't complicate that
HFileBlock block = getSampleHFileBlock();
- HFileBlockEncodingContext context =
- new HFileBlockDefaultEncodingContext(
- Compression.Algorithm.NONE, blockEncoder.getEncodingOnDisk());
+ HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
+ Compression.Algorithm.NONE, blockEncoder.getEncodingOnDisk(), HFileBlock.DUMMY_HEADER);
blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
includesMemstoreTS, context, block.getBlockType());
Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java?rev=1410213&r1=1410212&r2=1410213&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java Fri Nov 16 06:53:23 2012
@@ -310,4 +310,16 @@ public class TestByteBufferUtils {
throw new RuntimeException("Bug in test!", e);
}
}
+
+ @Test
+ public void testToBytes(){
+ ByteBuffer buffer = ByteBuffer.allocate(5);
+ buffer.put(new byte[]{0,1,2,3,4});
+ assertEquals(5, buffer.position());
+ assertEquals(5, buffer.limit());
+ byte[] copy = ByteBufferUtils.toBytes(buffer, 2);
+ assertArrayEquals(new byte[]{2,3,4}, copy);
+ assertEquals(5, buffer.position());
+ assertEquals(5, buffer.limit());
+ }
}