You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2013/02/26 05:33:37 UTC
svn commit: r1450017 [1/2] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/
hbase-common/src/main/java/org/apache/hadoop/hbase/
hbase-common/src/main/java/org/apa...
Author: stack
Date: Tue Feb 26 04:33:36 2013
New Revision: 1450017
URL: http://svn.apache.org/r1450017
Log:
HBASE-7900 Have client Mutations (Put/Delete/etc.) and Result implement CellScanner Interface
Modified:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Operation.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Row.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
hbase/trunk/hbase-common/src/main/java/org/apache/hbase/CellUtil.java
hbase/trunk/hbase-common/src/test/java/org/apache/hbase/TestCellUtil.java
hbase/trunk/hbase-server/pom.xml
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MultiRowMutationProcessor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/rest/client/RemoteHTable.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/security/access/AccessController.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/thrift2/ThriftUtilities.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/SimpleRegionObserver.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionObserverBypass.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestWALObserver.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHRegion.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStore.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/HLogPerformanceEvaluation.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorScanPolicy.java
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Action.java Tue Feb 26 04:33:36 2013
@@ -29,33 +29,16 @@ import org.apache.hadoop.classification.
@InterfaceAudience.Public
@InterfaceStability.Stable
public class Action<R> implements Comparable<R> {
-
+ // TODO: This class should not be visible outside of the client package.
private Row action;
private int originalIndex;
private R result;
- /**
- * This constructor is replaced by {@link #Action(Row, int)}
- */
- @Deprecated
- public Action(byte[] regionName, Row action, int originalIndex) {
- this(action, originalIndex);
- }
-
public Action(Row action, int originalIndex) {
super();
this.action = action;
this.originalIndex = originalIndex;
}
-
- @Deprecated
- public byte[] getRegionName() {
- return null;
- }
-
- @Deprecated
- public void setRegionName(byte[] regionName) {
- }
public R getResult() {
return result;
@@ -73,6 +56,7 @@ public class Action<R> implements Compar
return originalIndex;
}
+ @SuppressWarnings("rawtypes")
@Override
public int compareTo(Object o) {
return action.compareTo(((Action) o).getAction());
@@ -85,4 +69,4 @@ public class Action<R> implements Compar
Action<?> other = (Action<?>) obj;
return compareTo(other) == 0;
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Append.java Tue Feb 26 04:33:36 2013
@@ -20,7 +20,9 @@ package org.apache.hadoop.hbase.client;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.Cell;
import java.util.ArrayList;
import java.util.Arrays;
@@ -78,13 +80,28 @@ public class Append extends Mutation {
* @return this
*/
public Append add(byte [] family, byte [] qualifier, byte [] value) {
- List<KeyValue> list = familyMap.get(family);
- if(list == null) {
- list = new ArrayList<KeyValue>();
+ KeyValue kv = new KeyValue(this.row, family, qualifier, this.ts, KeyValue.Type.Put, value);
+ return add(kv);
+ }
+
+ /**
+ * Add column and value to this Append operation.
+ * @param cell
+ * @return This instance
+ */
+ @SuppressWarnings("unchecked")
+ public Append add(final Cell cell) {
+ // Presume it is KeyValue for now.
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ byte [] family = kv.getFamily();
+ List<? extends Cell> list = this.familyMap.get(family);
+ if (list == null) {
+ list = new ArrayList<Cell>();
}
- list.add(new KeyValue(
- this.row, family, qualifier, this.ts, KeyValue.Type.Put, value));
- familyMap.put(family, list);
+ // Cast so explicit list type rather than ? extends Cell. Help the compiler out. See
+ // http://stackoverflow.com/questions/6474784/java-using-generics-with-lists-and-interfaces
+ ((List<KeyValue>)list).add(kv);
+ this.familyMap.put(family, list);
return this;
}
}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Delete.java Tue Feb 26 04:33:36 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.Cell;
import java.io.IOException;
import java.util.ArrayList;
@@ -111,7 +112,9 @@ public class Delete extends Mutation imp
* @return this for invocation chaining
* @throws IOException
*/
+ @SuppressWarnings("unchecked")
public Delete addDeleteMarker(KeyValue kv) throws IOException {
+ // TODO: Deprecate and rename 'add' so it matches how we add KVs to Puts.
if (!kv.isDelete()) {
throw new IOException("The recently added KeyValue is not of type "
+ "delete. Rowkey: " + Bytes.toStringBinary(this.row));
@@ -124,11 +127,13 @@ public class Delete extends Mutation imp
+ Bytes.toStringBinary(this.row));
}
byte [] family = kv.getFamily();
- List<KeyValue> list = familyMap.get(family);
+ List<? extends Cell> list = familyMap.get(family);
if (list == null) {
- list = new ArrayList<KeyValue>();
+ list = new ArrayList<Cell>();
}
- list.add(kv);
+ // Cast so explicit list type rather than ? extends Cell. Help the compiler out. See
+ // http://stackoverflow.com/questions/6474784/java-using-generics-with-lists-and-interfaces
+ ((List<KeyValue>)list).add(kv);
familyMap.put(family, list);
return this;
}
@@ -156,14 +161,18 @@ public class Delete extends Mutation imp
* @param timestamp maximum version timestamp
* @return this for invocation chaining
*/
+ @SuppressWarnings("unchecked")
public Delete deleteFamily(byte [] family, long timestamp) {
- List<KeyValue> list = familyMap.get(family);
+ List<? extends Cell> list = familyMap.get(family);
if(list == null) {
- list = new ArrayList<KeyValue>();
+ list = new ArrayList<Cell>();
} else if(!list.isEmpty()) {
list.clear();
}
- list.add(new KeyValue(row, family, null, timestamp, KeyValue.Type.DeleteFamily));
+ KeyValue kv = new KeyValue(row, family, null, timestamp, KeyValue.Type.DeleteFamily);
+ // Cast so explicit list type rather than ? extends Cell. Help the compiler out. See
+ // http://stackoverflow.com/questions/6474784/java-using-generics-with-lists-and-interfaces
+ ((List<KeyValue>)list).add(kv);
familyMap.put(family, list);
return this;
}
@@ -187,13 +196,16 @@ public class Delete extends Mutation imp
* @param timestamp maximum version timestamp
* @return this for invocation chaining
*/
+ @SuppressWarnings("unchecked")
public Delete deleteColumns(byte [] family, byte [] qualifier, long timestamp) {
- List<KeyValue> list = familyMap.get(family);
+ List<? extends Cell> list = familyMap.get(family);
if (list == null) {
- list = new ArrayList<KeyValue>();
+ list = new ArrayList<Cell>();
}
- list.add(new KeyValue(this.row, family, qualifier, timestamp,
- KeyValue.Type.DeleteColumn));
+ // Cast so explicit list type rather than ? extends Cell. Help the compiler out. See
+ // http://stackoverflow.com/questions/6474784/java-using-generics-with-lists-and-interfaces
+ ((List<KeyValue>)list).add(new KeyValue(this.row, family, qualifier, timestamp,
+ KeyValue.Type.DeleteColumn));
familyMap.put(family, list);
return this;
}
@@ -219,13 +231,16 @@ public class Delete extends Mutation imp
* @param timestamp version timestamp
* @return this for invocation chaining
*/
+ @SuppressWarnings("unchecked")
public Delete deleteColumn(byte [] family, byte [] qualifier, long timestamp) {
- List<KeyValue> list = familyMap.get(family);
+ List<? extends Cell> list = familyMap.get(family);
if(list == null) {
- list = new ArrayList<KeyValue>();
+ list = new ArrayList<Cell>();
}
- list.add(new KeyValue(
- this.row, family, qualifier, timestamp, KeyValue.Type.Delete));
+ // Cast so explicit list type rather than ? extends Cell. Help the compiler out. See
+ // http://stackoverflow.com/questions/6474784/java-using-generics-with-lists-and-interfaces
+ KeyValue kv = new KeyValue(this.row, family, qualifier, timestamp, KeyValue.Type.Delete);
+ ((List<KeyValue>)list).add(kv);
familyMap.put(family, list);
return this;
}
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Get.java Tue Feb 26 04:33:36 2013
@@ -41,8 +41,7 @@ import java.util.TreeSet;
* Used to perform Get operations on a single row.
* <p>
* To get everything for a row, instantiate a Get object with the row to get.
- * To further define the scope of what to get, perform additional methods as
- * outlined below.
+ * To further narrow the scope of what to Get, use the methods below.
* <p>
* To get all columns from specific families, execute {@link #addFamily(byte[]) addFamily}
* for each family to retrieve.
@@ -59,7 +58,7 @@ import java.util.TreeSet;
* To limit the number of versions of each column to be returned, execute
* {@link #setMaxVersions(int) setMaxVersions}.
* <p>
- * To add a filter, execute {@link #setFilter(Filter) setFilter}.
+ * To add a filter, call {@link #setFilter(Filter) setFilter}.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HTable.java Tue Feb 26 04:33:36 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.HRegionIn
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.HConnectionManager.HConnectable;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
@@ -50,6 +51,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hbase.Cell;
import java.io.Closeable;
import java.io.IOException;
@@ -1098,8 +1100,10 @@ public class HTable implements HTableInt
throw new IllegalArgumentException("No columns to insert");
}
if (maxKeyValueSize > 0) {
- for (List<KeyValue> list : put.getFamilyMap().values()) {
- for (KeyValue kv : list) {
+ for (List<? extends Cell> list : put.getFamilyMap().values()) {
+ for (Cell cell : list) {
+ // KeyValue v1 expectation. Cast for now.
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (kv.getLength() > maxKeyValueSize) {
throw new IllegalArgumentException("KeyValue size too large");
}
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Increment.java Tue Feb 26 04:33:36 2013
@@ -18,16 +18,19 @@
*/
package org.apache.hadoop.hbase.client;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.util.Bytes;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
+import org.apache.hbase.Cell;
/**
* Used to perform Increment operations on a single row.
@@ -43,15 +46,8 @@ import java.util.TreeMap;
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class Increment implements Row {
- private byte [] row = null;
- private boolean writeToWAL = true;
+public class Increment extends Mutation implements Comparable<Row> {
private TimeRange tr = new TimeRange();
- private Map<byte [], NavigableMap<byte [], Long>> familyMap =
- new TreeMap<byte [], NavigableMap<byte [], Long>>(Bytes.BYTES_COMPARATOR);
-
- /** Constructor for Writable. DO NOT USE */
- public Increment() {}
/**
* Create a Increment operation for the specified row, using an existing row
@@ -61,10 +57,10 @@ public class Increment implements Row {
* @param row row key
*/
public Increment(byte [] row) {
- if (row == null) {
- throw new IllegalArgumentException("Cannot increment a null row");
+ if (row == null || row.length > HConstants.MAX_ROW_LENGTH) {
+ throw new IllegalArgumentException("Row key is invalid");
}
- this.row = row;
+ this.row = Arrays.copyOf(row, row.length);
}
/**
@@ -77,6 +73,7 @@ public class Increment implements Row {
* @param amount amount to increment by
* @return the Increment object
*/
+ @SuppressWarnings("unchecked")
public Increment addColumn(byte [] family, byte [] qualifier, long amount) {
if (family == null) {
throw new IllegalArgumentException("family cannot be null");
@@ -84,40 +81,10 @@ public class Increment implements Row {
if (qualifier == null) {
throw new IllegalArgumentException("qualifier cannot be null");
}
- NavigableMap<byte [], Long> set = familyMap.get(family);
- if(set == null) {
- set = new TreeMap<byte [], Long>(Bytes.BYTES_COMPARATOR);
- }
- set.put(qualifier, amount);
- familyMap.put(family, set);
- return this;
- }
-
- /* Accessors */
-
- /**
- * Method for retrieving the increment's row
- * @return row
- */
- public byte [] getRow() {
- return this.row;
- }
-
- /**
- * Method for retrieving whether WAL will be written to or not
- * @return true if WAL should be used, false if not
- */
- public boolean getWriteToWAL() {
- return this.writeToWAL;
- }
-
- /**
- * Sets whether this operation should write to the WAL or not.
- * @param writeToWAL true if WAL should be used, false if not
- * @return this increment operation
- */
- public Increment setWriteToWAL(boolean writeToWAL) {
- this.writeToWAL = writeToWAL;
+ List<? extends Cell> list = getCellList(family);
+ KeyValue kv = createPutKeyValue(family, qualifier, ts, Bytes.toBytes(amount));
+ ((List<KeyValue>)list).add(kv);
+ familyMap.put(kv.getFamily(), list);
return this;
}
@@ -150,14 +117,6 @@ public class Increment implements Row {
}
/**
- * Method for retrieving the keys in the familyMap
- * @return keys in the current familyMap
- */
- public Set<byte[]> familySet() {
- return this.familyMap.keySet();
- }
-
- /**
* Method for retrieving the number of families to increment from
* @return number of families
*/
@@ -166,19 +125,6 @@ public class Increment implements Row {
}
/**
- * Method for retrieving the number of columns to increment
- * @return number of columns across all families
- */
- public int numColumns() {
- if (!hasFamilies()) return 0;
- int num = 0;
- for (NavigableMap<byte [], Long> family : familyMap.values()) {
- num += family.size();
- }
- return num;
- }
-
- /**
* Method for checking if any families have been inserted into this Increment
* @return true if familyMap is non empty false otherwise
*/
@@ -187,14 +133,6 @@ public class Increment implements Row {
}
/**
- * Method for retrieving the increment's familyMap
- * @return familyMap
- */
- public Map<byte[],NavigableMap<byte[], Long>> getFamilyMap() {
- return this.familyMap;
- }
-
- /**
* @return String
*/
@Override
@@ -208,8 +146,7 @@ public class Increment implements Row {
}
sb.append(", families=");
boolean moreThanOne = false;
- for(Map.Entry<byte [], NavigableMap<byte[], Long>> entry :
- this.familyMap.entrySet()) {
+ for(Map.Entry<byte [], List<? extends Cell>> entry: this.familyMap.entrySet()) {
if(moreThanOne) {
sb.append("), ");
} else {
@@ -224,13 +161,14 @@ public class Increment implements Row {
} else {
sb.append("{");
boolean moreThanOneB = false;
- for(Map.Entry<byte [], Long> column : entry.getValue().entrySet()) {
+ for(Cell cell : entry.getValue()) {
if(moreThanOneB) {
sb.append(", ");
} else {
moreThanOneB = true;
}
- sb.append(Bytes.toStringBinary(column.getKey()) + "+=" + column.getValue());
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ sb.append(Bytes.toStringBinary(kv.getKey()) + "+=" + Bytes.toLong(kv.getValue()));
}
sb.append("}");
}
@@ -255,4 +193,4 @@ public class Increment implements Row {
Row other = (Row) obj;
return compareTo(other) == 0;
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/MultiAction.java Tue Feb 26 04:33:36 2013
@@ -35,9 +35,11 @@ import java.util.TreeMap;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public final class MultiAction<R> {
+ // TODO: This class should not be visible outside of the client package.
// map of regions to lists of puts/gets/deletes for that region.
- public Map<byte[], List<Action<R>>> actions = new TreeMap<byte[], List<Action<R>>>(Bytes.BYTES_COMPARATOR);
+ public Map<byte[], List<Action<R>>> actions =
+ new TreeMap<byte[], List<Action<R>>>(Bytes.BYTES_COMPARATOR);
public MultiAction() {
super();
@@ -87,4 +89,4 @@ public final class MultiAction<R> {
}
return res;
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Mutation.java Tue Feb 26 04:33:36 2013
@@ -23,26 +23,78 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hbase.Cell;
+import org.apache.hbase.CellScannable;
+import org.apache.hbase.CellScanner;
+import org.apache.hbase.CellUtil;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.NavigableMap;
import java.util.TreeMap;
import java.util.UUID;
@InterfaceAudience.Public
@InterfaceStability.Evolving
-public abstract class Mutation extends OperationWithAttributes implements Row {
+public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable {
+ static final long MUTATION_OVERHEAD = ClassSize.align(
+ // This
+ ClassSize.OBJECT +
+ // OperationWithAttributes map reference? I don't know what the other reference is and if I
+ // remove it it breaks TestHeapSize so just leaving it.
+ 2 * ClassSize.REFERENCE +
+ // Timestamp
+ 1 * Bytes.SIZEOF_LONG +
+ // writeToWAL
+ Bytes.SIZEOF_BOOLEAN +
+ // familyMap
+ ClassSize.REFERENCE +
+ // familyMap
+ ClassSize.TREEMAP);
+
// Attribute used in Mutations to indicate the originating cluster.
private static final String CLUSTER_ID_ATTR = "_c.id_";
protected byte [] row = null;
protected long ts = HConstants.LATEST_TIMESTAMP;
protected boolean writeToWAL = true;
- protected Map<byte [], List<KeyValue>> familyMap =
- new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
+ // A Map sorted by column family.
+ protected NavigableMap<byte [], List<? extends Cell>> familyMap =
+ new TreeMap<byte [], List<? extends Cell>>(Bytes.BYTES_COMPARATOR);
+
+ @Override
+ public CellScanner cellScanner() {
+ return CellUtil.createCellScanner(getFamilyMap());
+ }
+
+ /**
+ * Creates an empty list if one doesn't exist for the given column family
+ * or else it returns the associated list of Cell objects.
+ *
+ * @param family column family
+ * @return a list of Cell objects, returns an empty list if one doesn't exist.
+ */
+ List<? extends Cell> getCellList(byte[] family) {
+ List<? extends Cell> list = this.familyMap.get(family);
+ if (list == null) {
+ list = new ArrayList<Cell>();
+ }
+ return list;
+ }
+
+ /*
+ * Create a nnnnnnnn with this objects row key and the Put identifier.
+ *
+ * @return a KeyValue with this objects row key and the Put identifier.
+ */
+ KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) {
+ return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value);
+ }
/**
* Compile the column family (i.e. schema) information
@@ -57,9 +109,9 @@ public abstract class Mutation extends O
// ideally, we would also include table information, but that information
// is not stored in each Operation instance.
map.put("families", families);
- for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
+ for (Map.Entry<byte [], List<? extends Cell>> entry : this.familyMap.entrySet()) {
families.add(Bytes.toStringBinary(entry.getKey()));
- }
+ }
return map;
}
@@ -74,7 +126,7 @@ public abstract class Mutation extends O
public Map<String, Object> toMap(int maxCols) {
// we start with the fingerprint map and build on top of it.
Map<String, Object> map = getFingerprint();
- // replace the fingerprint's simple list of families with a
+ // replace the fingerprint's simple list of families with a
// map from column families to lists of qualifiers and kv details
Map<String, List<Map<String, Object>>> columns =
new HashMap<String, List<Map<String, Object>>>();
@@ -82,20 +134,21 @@ public abstract class Mutation extends O
map.put("row", Bytes.toStringBinary(this.row));
int colCount = 0;
// iterate through all column families affected
- for (Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
- // map from this family to details for each kv affected within the family
- List<Map<String, Object>> qualifierDetails =
- new ArrayList<Map<String, Object>>();
+ for (Map.Entry<byte [], List<? extends Cell>> entry : this.familyMap.entrySet()) {
+ // map from this family to details for each cell affected within the family
+ List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>();
columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails);
colCount += entry.getValue().size();
if (maxCols <= 0) {
continue;
}
- // add details for each kv
- for (KeyValue kv : entry.getValue()) {
+ // add details for each cell
+ for (Cell cell: entry.getValue()) {
if (--maxCols <= 0 ) {
continue;
}
+ // KeyValue v1 expectation. Cast for now until we go all Cell all the time.
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
Map<String, Object> kvMap = kv.toStringMap();
// row and family information are already available in the bigger map
kvMap.remove("row");
@@ -131,14 +184,16 @@ public abstract class Mutation extends O
* Method for retrieving the put's familyMap
* @return familyMap
*/
- public Map<byte [], List<KeyValue>> getFamilyMap() {
+ public NavigableMap<byte [], List<? extends Cell>> getFamilyMap() {
return this.familyMap;
}
/**
* Method for setting the put's familyMap
*/
- public void setFamilyMap(Map<byte [], List<KeyValue>> map) {
+ public void setFamilyMap(NavigableMap<byte [], List<? extends Cell>> map) {
+ // TODO: Shut this down or move it up to be a Constructor. Get new object rather than change
+ // this internal data member.
this.familyMap = map;
}
@@ -199,8 +254,8 @@ public abstract class Mutation extends O
*/
public int size() {
int size = 0;
- for(List<KeyValue> kvList : this.familyMap.values()) {
- size += kvList.size();
+ for (List<? extends Cell> cells : this.familyMap.values()) {
+ size += cells.size();
}
return size;
}
@@ -211,4 +266,37 @@ public abstract class Mutation extends O
public int numFamilies() {
return familyMap.size();
}
+
+ /**
+ * @return Calculate what Mutation adds to class heap size.
+ */
+ long heapSize() {
+ long heapsize = MUTATION_OVERHEAD;
+ // Adding row
+ heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
+
+ // Adding map overhead
+ heapsize +=
+ ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
+ for(Map.Entry<byte [], List<? extends Cell>> entry : this.familyMap.entrySet()) {
+ //Adding key overhead
+ heapsize +=
+ ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
+
+ //This part is kinds tricky since the JVM can reuse references if you
+ //store the same value, but have a good match with SizeOf at the moment
+ //Adding value overhead
+ heapsize += ClassSize.align(ClassSize.ARRAYLIST);
+ int size = entry.getValue().size();
+ heapsize += ClassSize.align(ClassSize.ARRAY +
+ size * ClassSize.REFERENCE);
+
+ for(Cell cell : entry.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ heapsize += kv.heapSize();
+ }
+ }
+ heapsize += getAttributeSize();
+ return heapsize;
+ }
}
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Operation.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Operation.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Operation.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Operation.java Tue Feb 26 04:33:36 2013
@@ -34,6 +34,7 @@ import java.util.Map;
@InterfaceStability.Evolving
public abstract class Operation {
// TODO make this configurable
+ // TODO Do we need this anymore now we have protobuffed it all?
private static final int DEFAULT_MAX_COLS = 5;
/**
@@ -109,5 +110,4 @@ public abstract class Operation {
public String toString() {
return toString(DEFAULT_MAX_COLS);
}
-}
-
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/OperationWithAttributes.java Tue Feb 26 04:33:36 2013
@@ -31,7 +31,7 @@ import java.util.Map;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public abstract class OperationWithAttributes extends Operation implements Attributes {
- // a opaque blob of attributes
+ // An opaque blob of attributes
private Map<String, byte[]> attributes;
// used for uniquely identifying an operation
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Put.java Tue Feb 26 04:33:36 2013
@@ -19,20 +19,22 @@
package org.apache.hadoop.hbase.client;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
+import org.apache.hbase.Cell;
/**
* Used to perform Put operations for a single row.
@@ -44,11 +46,6 @@ import java.util.TreeMap;
@InterfaceAudience.Public
@InterfaceStability.Stable
public class Put extends Mutation implements HeapSize, Comparable<Row> {
- private static final long OVERHEAD = ClassSize.align(
- ClassSize.OBJECT + 2 * ClassSize.REFERENCE +
- 1 * Bytes.SIZEOF_LONG + Bytes.SIZEOF_BOOLEAN +
- ClassSize.REFERENCE + ClassSize.TREEMAP);
-
/**
* Create a Put operation for the specified row.
* @param row row key
@@ -77,10 +74,8 @@ public class Put extends Mutation implem
*/
public Put(Put putToCopy) {
this(putToCopy.getRow(), putToCopy.ts);
- this.familyMap =
- new TreeMap<byte [], List<KeyValue>>(Bytes.BYTES_COMPARATOR);
- for(Map.Entry<byte [], List<KeyValue>> entry :
- putToCopy.getFamilyMap().entrySet()) {
+ this.familyMap = new TreeMap<byte [], List<? extends Cell>>(Bytes.BYTES_COMPARATOR);
+ for(Map.Entry<byte [], List<? extends Cell>> entry: putToCopy.getFamilyMap().entrySet()) {
this.familyMap.put(entry.getKey(), entry.getValue());
}
this.writeToWAL = putToCopy.writeToWAL;
@@ -106,10 +101,11 @@ public class Put extends Mutation implem
* @param value column value
* @return this
*/
+ @SuppressWarnings("unchecked")
public Put add(byte [] family, byte [] qualifier, long ts, byte [] value) {
- List<KeyValue> list = getKeyValueList(family);
+ List<? extends Cell> list = getCellList(family);
KeyValue kv = createPutKeyValue(family, qualifier, ts, value);
- list.add(kv);
+ ((List<KeyValue>)list).add(kv);
familyMap.put(kv.getFamily(), list);
return this;
}
@@ -122,9 +118,10 @@ public class Put extends Mutation implem
* @return this
* @throws java.io.IOException e
*/
+ @SuppressWarnings("unchecked")
public Put add(KeyValue kv) throws IOException{
byte [] family = kv.getFamily();
- List<KeyValue> list = getKeyValueList(family);
+ List<? extends Cell> list = getCellList(family);
//Checking that the row of the kv is the same as the put
int res = Bytes.compareTo(this.row, 0, row.length,
kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
@@ -134,22 +131,11 @@ public class Put extends Mutation implem
kv.getRowLength()) + " doesn't match the original one " +
Bytes.toStringBinary(this.row));
}
- list.add(kv);
+ ((List<KeyValue>)list).add(kv);
familyMap.put(family, list);
return this;
}
- /*
- * Create a KeyValue with this objects row key and the Put identifier.
- *
- * @return a KeyValue with this objects row key and the Put identifier.
- */
- private KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts,
- byte[] value) {
- return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put,
- value);
- }
-
/**
* A convenience method to determine if this object's familyMap contains
* a value assigned to the given family & qualifier.
@@ -226,7 +212,7 @@ public class Put extends Mutation implem
*/
private boolean has(byte[] family, byte[] qualifier, long ts, byte[] value,
boolean ignoreTS, boolean ignoreValue) {
- List<KeyValue> list = getKeyValueList(family);
+ List<? extends Cell> list = getCellList(family);
if (list.size() == 0) {
return false;
}
@@ -236,7 +222,8 @@ public class Put extends Mutation implem
// F T => 2
// F F => 1
if (!ignoreTS && !ignoreValue) {
- for (KeyValue kv : list) {
+ for (Cell cell : list) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (Arrays.equals(kv.getFamily(), family) &&
Arrays.equals(kv.getQualifier(), qualifier) &&
Arrays.equals(kv.getValue(), value) &&
@@ -245,21 +232,24 @@ public class Put extends Mutation implem
}
}
} else if (ignoreValue && !ignoreTS) {
- for (KeyValue kv : list) {
+ for (Cell cell : list) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
&& kv.getTimestamp() == ts) {
return true;
}
}
} else if (!ignoreValue && ignoreTS) {
- for (KeyValue kv : list) {
+ for (Cell cell : list) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (Arrays.equals(kv.getFamily(), family) && Arrays.equals(kv.getQualifier(), qualifier)
&& Arrays.equals(kv.getValue(), value)) {
return true;
}
}
} else {
- for (KeyValue kv : list) {
+ for (Cell cell : list) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (Arrays.equals(kv.getFamily(), family) &&
Arrays.equals(kv.getQualifier(), qualifier)) {
return true;
@@ -279,7 +269,8 @@ public class Put extends Mutation implem
*/
public List<KeyValue> get(byte[] family, byte[] qualifier) {
List<KeyValue> filteredList = new ArrayList<KeyValue>();
- for (KeyValue kv: getKeyValueList(family)) {
+ for (Cell cell: getCellList(family)) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (Arrays.equals(kv.getQualifier(), qualifier)) {
filteredList.add(kv);
}
@@ -287,49 +278,8 @@ public class Put extends Mutation implem
return filteredList;
}
- /**
- * Creates an empty list if one doesnt exist for the given column family
- * or else it returns the associated list of KeyValue objects.
- *
- * @param family column family
- * @return a list of KeyValue objects, returns an empty list if one doesnt exist.
- */
- private List<KeyValue> getKeyValueList(byte[] family) {
- List<KeyValue> list = familyMap.get(family);
- if(list == null) {
- list = new ArrayList<KeyValue>(0);
- }
- return list;
- }
-
- //HeapSize
+ @Override
public long heapSize() {
- long heapsize = OVERHEAD;
- //Adding row
- heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length);
-
- //Adding map overhead
- heapsize +=
- ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY);
- for(Map.Entry<byte [], List<KeyValue>> entry : this.familyMap.entrySet()) {
- //Adding key overhead
- heapsize +=
- ClassSize.align(ClassSize.ARRAY + entry.getKey().length);
-
- //This part is kinds tricky since the JVM can reuse references if you
- //store the same value, but have a good match with SizeOf at the moment
- //Adding value overhead
- heapsize += ClassSize.align(ClassSize.ARRAYLIST);
- int size = entry.getValue().size();
- heapsize += ClassSize.align(ClassSize.ARRAY +
- size * ClassSize.REFERENCE);
-
- for(KeyValue kv : entry.getValue()) {
- heapsize += kv.heapSize();
- }
- }
- heapsize += getAttributeSize();
-
- return ClassSize.align((int)heapsize);
+ return ClassSize.align((int)super.heapSize());
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Result.java Tue Feb 26 04:33:36 2013
@@ -24,6 +24,10 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValue.SplitKeyValue;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hbase.Cell;
+import org.apache.hbase.CellScannable;
+import org.apache.hbase.CellScanner;
+import org.apache.hbase.CellUtil;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
@@ -38,7 +42,7 @@ import java.util.TreeMap;
/**
* Single row result of a {@link Get} or {@link Scan} query.<p>
*
- * This class is NOT THREAD SAFE.<p>
+ * This class is <b>NOT THREAD SAFE</b>.<p>
*
* Convenience methods are available that return various {@link Map}
* structures and values directly.<p>
@@ -61,13 +65,13 @@ import java.util.TreeMap;
* Each KeyValue can then be accessed through
* {@link KeyValue#getRow()}, {@link KeyValue#getFamily()}, {@link KeyValue#getQualifier()},
* {@link KeyValue#getTimestamp()}, and {@link KeyValue#getValue()}.<p>
- *
+ *
* If you need to overwrite a Result with another Result instance -- as in the old 'mapred' RecordReader next
* invocations -- then create an empty Result with the null constructor and in then use {@link #copyFrom(Result)}
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
-public class Result {
+public class Result implements CellScannable {
private KeyValue [] kvs;
// We're not using java serialization. Transient here is just a marker to say
// that this is where we cache row if we're ever asked for it.
@@ -78,6 +82,7 @@ public class Result {
// never use directly
private static byte [] buffer = null;
private static final int PAD_WIDTH = 128;
+ public static final Result EMPTY_RESULT = new Result();
/**
* Creates an empty Result w/ no KeyValue payload; returns null if you call {@link #raw()}.
@@ -105,7 +110,8 @@ public class Result {
* are already sorted
* @param kvs List of KeyValues
*/
- public Result(List<KeyValue> kvs) {
+ public Result(List<? extends Cell> kvs) {
+ // TODO: Here we presume the passed in Cells are KVs. One day this won't always be so.
this(kvs.toArray(new KeyValue[kvs.size()]));
}
@@ -678,7 +684,7 @@ public class Result {
*/
public static void compareResults(Result res1, Result res2)
throws Exception {
- if (res2 == null) {
+ if (res2 == null) {
throw new Exception("There wasn't enough rows, we stopped at "
+ Bytes.toStringBinary(res1.getRow()));
}
@@ -706,4 +712,9 @@ public class Result {
this.familyMap = null;
this.kvs = other.kvs;
}
-}
\ No newline at end of file
+
+ @Override
+ public CellScanner cellScanner() {
+ return CellUtil.createCellScanner(this.kvs);
+ }
+}
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Row.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Row.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Row.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Row.java Tue Feb 26 04:33:36 2013
@@ -31,4 +31,4 @@ public interface Row extends Comparable<
* @return The row.
*/
public byte [] getRow();
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RowMutations.java Tue Feb 26 04:33:36 2013
@@ -38,9 +38,8 @@ import java.util.List;
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class RowMutations implements Row {
- private List<Mutation> mutations = new ArrayList<Mutation>();
+ private final List<Mutation> mutations = new ArrayList<Mutation>();
private byte [] row;
- private static final byte VERSION = (byte)0;
/** Constructor for Writable. DO NOT USE */
public RowMutations() {}
@@ -100,4 +99,4 @@ public class RowMutations implements Row
public List<Mutation> getMutations() {
return Collections.unmodifiableList(mutations);
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ScannerCallable.java Tue Feb 26 04:33:36 2013
@@ -19,6 +19,8 @@
package org.apache.hadoop.hbase.client;
import com.google.protobuf.ServiceException;
+import com.google.protobuf.TextFormat;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -54,7 +56,7 @@ public class ScannerCallable extends Ser
= "hbase.client.log.scanner.latency.cutoff";
public static final String LOG_SCANNER_ACTIVITY = "hbase.client.log.scanner.activity";
- private static final Log LOG = LogFactory.getLog(ScannerCallable.class);
+ public static final Log LOG = LogFactory.getLog(ScannerCallable.class);
private long scannerId = -1L;
private boolean instantiated = false;
private boolean closed = false;
@@ -135,9 +137,10 @@ public class ScannerCallable extends Ser
this.scannerId = openScanner();
} else {
Result [] rrs = null;
+ ScanRequest request = null;
try {
incRPCcallsMetrics();
- ScanRequest request =
+ request =
RequestConverter.buildScanRequest(scannerId, caching, false, nextCallSeq);
ScanResponse response = null;
try {
@@ -174,8 +177,7 @@ public class ScannerCallable extends Ser
updateResultsMetrics(response);
} catch (IOException e) {
if (logScannerActivity) {
- LOG.info("Got exception in fetching from scanner="
- + scannerId, e);
+ LOG.info("Got exception making request " + TextFormat.shortDebugString(request), e);
}
IOException ioe = e;
if (e instanceof RemoteException) {
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ServerCallable.java Tue Feb 26 04:33:36 2013
@@ -245,4 +245,4 @@ public abstract class ServerCallable<T>
}
return t;
}
-}
+}
\ No newline at end of file
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java Tue Feb 26 04:33:36 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.HConstant
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.MasterAdminProtocol;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.Action;
@@ -118,7 +119,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.NavigableMap;
import java.util.NavigableSet;
import static org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.RegionSpecifier.RegionSpecifierType.REGION_NAME;
@@ -747,16 +747,15 @@ public final class ProtobufUtil {
}
ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
- for (Map.Entry<byte[],NavigableMap<byte[], Long>>
- family: increment.getFamilyMap().entrySet()) {
+ for (Map.Entry<byte[], List<? extends Cell>> family: increment.getFamilyMap().entrySet()) {
columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
columnBuilder.clearQualifierValue();
- NavigableMap<byte[], Long> values = family.getValue();
+ List<? extends Cell> values = family.getValue();
if (values != null && values.size() > 0) {
- for (Map.Entry<byte[], Long> value: values.entrySet()) {
- valueBuilder.setQualifier(ByteString.copyFrom(value.getKey()));
- valueBuilder.setValue(ByteString.copyFrom(
- Bytes.toBytes(value.getValue().longValue())));
+ for (Cell cell: values) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier()));
+ valueBuilder.setValue(ByteString.copyFrom(kv.getValue()));
columnBuilder.addQualifierValue(valueBuilder.build());
}
}
@@ -791,16 +790,16 @@ public final class ProtobufUtil {
}
ColumnValue.Builder columnBuilder = ColumnValue.newBuilder();
QualifierValue.Builder valueBuilder = QualifierValue.newBuilder();
- for (Map.Entry<byte[],List<KeyValue>>
- family: mutation.getFamilyMap().entrySet()) {
+ for (Map.Entry<byte[],List<? extends Cell>> family: mutation.getFamilyMap().entrySet()) {
columnBuilder.setFamily(ByteString.copyFrom(family.getKey()));
columnBuilder.clearQualifierValue();
- for (KeyValue value: family.getValue()) {
- valueBuilder.setQualifier(ByteString.copyFrom(value.getQualifier()));
- valueBuilder.setValue(ByteString.copyFrom(value.getValue()));
- valueBuilder.setTimestamp(value.getTimestamp());
+ for (Cell cell: family.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ valueBuilder.setQualifier(ByteString.copyFrom(kv.getQualifier()));
+ valueBuilder.setValue(ByteString.copyFrom(kv.getValue()));
+ valueBuilder.setTimestamp(kv.getTimestamp());
if (mutateType == MutateType.DELETE) {
- KeyValue.Type keyValueType = KeyValue.Type.codeToType(value.getType());
+ KeyValue.Type keyValueType = KeyValue.Type.codeToType(kv.getType());
valueBuilder.setDeleteType(toDeleteType(keyValueType));
}
columnBuilder.addQualifierValue(valueBuilder.build());
@@ -1765,8 +1764,8 @@ public final class ProtobufUtil {
* @throws IOException
*/
@SuppressWarnings("unchecked")
- public static <T extends Message>
- T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
+ public static <T extends Message>
+ T getParsedGenericInstance(Class<?> runtimeClass, int position, ByteString b)
throws IOException {
Type type = runtimeClass.getGenericSuperclass();
Type argType = ((ParameterizedType)type).getActualTypeArguments()[position];
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java Tue Feb 26 04:33:36 2013
@@ -201,7 +201,8 @@ public class KeyValueUtil {
* @return <code>cell<code> if it is an instance of {@link KeyValue} else we will return a
* new {@link KeyValue} instance made from <code>cell</code>
*/
- public static Cell ensureKeyValue(final Cell cell) {
- return cell instanceof KeyValue? cell: copyToNewKeyValue(cell);
+ public static KeyValue ensureKeyValue(final Cell cell) {
+ if (cell == null) return null;
+ return cell instanceof KeyValue? (KeyValue)cell: copyToNewKeyValue(cell);
}
}
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hbase/CellUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hbase/CellUtil.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hbase/CellUtil.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hbase/CellUtil.java Tue Feb 26 04:33:36 2013
@@ -213,10 +213,12 @@ public final class CellUtil {
* inside Put, etc., keeping Cells organized by family.
* @return CellScanner interface over <code>cellIterable</code>
*/
- public static CellScanner createCellScanner(final NavigableMap<byte [], List<Cell>> map) {
+ public static CellScanner createCellScanner(final NavigableMap<byte [],
+ List<? extends Cell>> map) {
return new CellScanner() {
- private final Iterator<Entry<byte[], List<Cell>>> entries = map.entrySet().iterator();
- private Iterator<Cell> currentIterator = null;
+ private final Iterator<Entry<byte[], List<? extends Cell>>> entries =
+ map.entrySet().iterator();
+ private Iterator<? extends Cell> currentIterator = null;
private Cell currentCell;
@Override
Modified: hbase/trunk/hbase-common/src/test/java/org/apache/hbase/TestCellUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/test/java/org/apache/hbase/TestCellUtil.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/test/java/org/apache/hbase/TestCellUtil.java (original)
+++ hbase/trunk/hbase-common/src/test/java/org/apache/hbase/TestCellUtil.java Tue Feb 26 04:33:36 2013
@@ -45,11 +45,11 @@ public class TestCellUtil {
@Test
public void testCreateCellScannerFamilyMap() {
final int count = 3;
- final NavigableMap<byte [], List<Cell>> map =
- new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR);
+ final NavigableMap<byte [], List<? extends Cell>> map =
+ new TreeMap<byte [], List<? extends Cell>>(Bytes.BYTES_COMPARATOR);
for (int i = 0; i < count; i++) {
byte [] key = Bytes.toBytes(i);
- Cell [] cs = getCells(count, key);
+ KeyValue [] cs = getCells(count, key);
map.put(key, Arrays.asList(cs));
}
CellScanner scanner = CellUtil.createCellScanner(map);
@@ -60,8 +60,8 @@ public class TestCellUtil {
assertEquals(count * count, i);
}
- static Cell [] getCells(final int howMany, final byte [] family) {
- Cell [] cells = new Cell[howMany];
+ static KeyValue [] getCells(final int howMany, final byte [] family) {
+ KeyValue [] cells = new KeyValue[howMany];
for (int i = 0; i < howMany; i++) {
byte [] index = Bytes.toBytes(i);
KeyValue kv = new KeyValue(index, family, index, index);
Modified: hbase/trunk/hbase-server/pom.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/pom.xml?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-server/pom.xml (original)
+++ hbase/trunk/hbase-server/pom.xml Tue Feb 26 04:33:36 2013
@@ -50,8 +50,8 @@
</testResource>
</testResources>
<plugins>
- <!-- Run with -Dmaven.test.skip.exec=true to build -tests.jar without running
- tests (this is needed for upstream projects whose tests need this jar simply for
+ <!-- Run with -Dmaven.test.skip.exec=true to build -tests.jar without running
+ tests (this is needed for upstream projects whose tests need this jar simply for
compilation) -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -62,8 +62,8 @@
<mainClass>org/apache/hadoop/hbase/mapreduce/Driver</mainClass>
</manifest>
</archive>
- <!-- Exclude these 2 packages, because their dependency _binary_ files
- include the sources, and Maven 2.2 appears to add them to the sources to compile,
+ <!-- Exclude these 2 packages, because their dependency _binary_ files
+ include the sources, and Maven 2.2 appears to add them to the sources to compile,
weird -->
<excludes>
<exclude>org/apache/jute/**</exclude>
@@ -200,9 +200,10 @@
</plugins>
<!-- General Resources -->
<pluginManagement>
- <plugins>
- <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on
- the Maven build itself and needs to be kept in plugin management, not in the actual plugins. -->
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no
+ influence on the Maven build itself and needs to be kept in plugin management, not in
+ the actual plugins. -->
<plugin>
<groupId>org.eclipse.m2e</groupId>
<artifactId>lifecycle-mapping</artifactId>
@@ -532,11 +533,13 @@
<configuration>
<target>
<mkdir dir="${project.build.directory}/native"/>
- <exec executable="cmake" dir="${project.build.directory}/native"
+ <exec executable="cmake" dir="${project.build.directory}/native"
failonerror="true">
- <arg line="${basedir}/src/main/native -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
+ <arg
+ line="${basedir}/src/main/native -DJVM_ARCH_DATA_MODEL=${sun.arch.data.model}"/>
</exec>
- <exec executable="make" dir="${project.build.directory}/native" failonerror="true">
+ <exec executable="make" dir="${project.build.directory}/native"
+ failonerror="true">
<arg line="VERBOSE=1"/>
</exec>
</target>
@@ -550,8 +553,8 @@
<!-- Profiles for building against different hadoop versions -->
<!-- There are a lot of common dependencies used here, should investigate
if we can combine these profiles somehow -->
- <!-- profile against Hadoop 1.0.x: This is the default. It has to have the same
- activation property as the parent Hadoop 1.0.x profile to make sure it gets run at
+ <!-- profile against Hadoop 1.0.x: This is the default. It has to have the same
+ activation property as the parent Hadoop 1.0.x profile to make sure it gets run at
the same time. -->
<profile>
<id>hadoop-1.0</id>
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/PutSortReducer.java Tue Feb 26 04:33:36 2013
@@ -25,10 +25,12 @@ import java.util.TreeSet;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hbase.Cell;
/**
* Emits sorted Puts.
@@ -61,8 +63,9 @@ public class PutSortReducer extends
// stop at the end or the RAM threshold
while (iter.hasNext() && curSize < threshold) {
Put p = iter.next();
- for (List<KeyValue> kvs : p.getFamilyMap().values()) {
- for (KeyValue kv : kvs) {
+ for (List<? extends Cell> cells: p.getFamilyMap().values()) {
+ for (Cell cell: cells) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
map.add(kv);
curSize += kv.getLength();
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java Tue Feb 26 04:33:36 2013
@@ -76,6 +76,7 @@ import org.apache.hadoop.hbase.exception
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
import org.apache.hadoop.hbase.HDFSBlocksDistribution;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -135,6 +136,7 @@ import org.apache.hadoop.hbase.util.Pair
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hbase.Cell;
import org.cliffc.high_scale_lib.Counter;
import com.google.common.base.Preconditions;
@@ -1826,7 +1828,7 @@ public class HRegion implements HeapSize
* @param writeToWAL
* @throws IOException
*/
- void delete(Map<byte[], List<KeyValue>> familyMap, UUID clusterId,
+ void delete(NavigableMap<byte[], List<? extends Cell>> familyMap, UUID clusterId,
boolean writeToWAL) throws IOException {
Delete delete = new Delete(HConstants.EMPTY_BYTE_ARRAY);
delete.setFamilyMap(familyMap);
@@ -1842,15 +1844,16 @@ public class HRegion implements HeapSize
* @param byteNow
* @throws IOException
*/
- void prepareDeleteTimestamps(Map<byte[], List<KeyValue>> familyMap, byte[] byteNow)
+ void prepareDeleteTimestamps(Map<byte[], List<? extends Cell>> familyMap, byte[] byteNow)
throws IOException {
- for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
+ for (Map.Entry<byte[], List<? extends Cell>> e : familyMap.entrySet()) {
byte[] family = e.getKey();
- List<KeyValue> kvs = e.getValue();
+ List<? extends Cell> cells = e.getValue();
Map<byte[], Integer> kvCount = new TreeMap<byte[], Integer>(Bytes.BYTES_COMPARATOR);
- for (KeyValue kv: kvs) {
+ for (Cell cell: cells) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
// Check if time is LATEST, change to time of most recent addition if so
// This is expensive.
if (kv.isLatestTimestamp() && kv.isDeleteType()) {
@@ -2064,7 +2067,7 @@ public class HRegion implements HeapSize
/** Keep track of the locks we hold so we can release them in finally clause */
List<Integer> acquiredLocks = Lists.newArrayListWithCapacity(batchOp.operations.length);
// reference family maps directly so coprocessors can mutate them if desired
- Map<byte[],List<KeyValue>>[] familyMaps = new Map[batchOp.operations.length];
+ Map<byte[], List<? extends Cell>>[] familyMaps = new Map[batchOp.operations.length];
// We try to set up a batch in the range [firstIndex,lastIndexExclusive)
int firstIndex = batchOp.nextIndexToProcess;
int lastIndexExclusive = firstIndex;
@@ -2083,7 +2086,7 @@ public class HRegion implements HeapSize
boolean isPutMutation = mutation instanceof Put;
Integer providedLockId = nextPair.getSecond();
- Map<byte[], List<KeyValue>> familyMap = mutation.getFamilyMap();
+ Map<byte[], List<? extends Cell>> familyMap = mutation.getFamilyMap();
// store the family map reference to allow for mutations
familyMaps[lastIndexExclusive] = familyMap;
@@ -2520,15 +2523,15 @@ public class HRegion implements HeapSize
}
/**
- * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the provided current
- * timestamp.
+ * Replaces any KV timestamps set to {@link HConstants#LATEST_TIMESTAMP} with the
+ * provided current timestamp.
*/
- void updateKVTimestamps(
- final Iterable<List<KeyValue>> keyLists, final byte[] now) {
- for (List<KeyValue> keys: keyLists) {
- if (keys == null) continue;
- for (KeyValue key : keys) {
- key.updateLatestStamp(now);
+ void updateKVTimestamps(final Iterable<List<? extends Cell>> keyLists, final byte[] now) {
+ for (List<? extends Cell> cells: keyLists) {
+ if (cells == null) continue;
+ for (Cell cell : cells) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ kv.updateLatestStamp(now);
}
}
}
@@ -2616,10 +2619,10 @@ public class HRegion implements HeapSize
* @praram now
* @throws IOException
*/
- private void put(final byte [] row, byte [] family, List<KeyValue> edits)
+ private void put(final byte [] row, byte [] family, List<? extends Cell> edits)
throws IOException {
- Map<byte[], List<KeyValue>> familyMap;
- familyMap = new HashMap<byte[], List<KeyValue>>();
+ NavigableMap<byte[], List<? extends Cell>> familyMap;
+ familyMap = new TreeMap<byte[], List<? extends Cell>>(Bytes.BYTES_COMPARATOR);
familyMap.put(family, edits);
Put p = new Put(row);
@@ -2641,7 +2644,7 @@ public class HRegion implements HeapSize
* @return the additional memory usage of the memstore caused by the
* new entries.
*/
- private long applyFamilyMapToMemstore(Map<byte[], List<KeyValue>> familyMap,
+ private long applyFamilyMapToMemstore(Map<byte[], List<? extends Cell>> familyMap,
MultiVersionConsistencyControl.WriteEntry localizedWriteEntry) {
long size = 0;
boolean freemvcc = false;
@@ -2652,12 +2655,13 @@ public class HRegion implements HeapSize
freemvcc = true;
}
- for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
+ for (Map.Entry<byte[], List<? extends Cell>> e : familyMap.entrySet()) {
byte[] family = e.getKey();
- List<KeyValue> edits = e.getValue();
+ List<? extends Cell> cells = e.getValue();
Store store = getStore(family);
- for (KeyValue kv: edits) {
+ for (Cell cell: cells) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
kv.setMemstoreTS(localizedWriteEntry.getWriteNumber());
size += store.add(kv);
}
@@ -2677,7 +2681,7 @@ public class HRegion implements HeapSize
* the wal. This method is then invoked to rollback the memstore.
*/
private void rollbackMemstore(BatchOperationInProgress<Pair<Mutation, Integer>> batchOp,
- Map<byte[], List<KeyValue>>[] familyMaps,
+ Map<byte[], List<? extends Cell>>[] familyMaps,
int start, int end) {
int kvsRolledback = 0;
for (int i = start; i < end; i++) {
@@ -2688,17 +2692,17 @@ public class HRegion implements HeapSize
}
// Rollback all the kvs for this row.
- Map<byte[], List<KeyValue>> familyMap = familyMaps[i];
- for (Map.Entry<byte[], List<KeyValue>> e : familyMap.entrySet()) {
+ Map<byte[], List<? extends Cell>> familyMap = familyMaps[i];
+ for (Map.Entry<byte[], List<? extends Cell>> e : familyMap.entrySet()) {
byte[] family = e.getKey();
- List<KeyValue> edits = e.getValue();
+ List<? extends Cell> cells = e.getValue();
// Remove those keys from the memstore that matches our
// key's (row, cf, cq, timestamp, memstoreTS). The interesting part is
// that even the memstoreTS has to match for keys that will be rolleded-back.
Store store = getStore(family);
- for (KeyValue kv: edits) {
- store.rollback(kv);
+ for (Cell cell: cells) {
+ store.rollback(KeyValueUtil.ensureKeyValue(cell));
kvsRolledback++;
}
}
@@ -2718,18 +2722,19 @@ public class HRegion implements HeapSize
}
}
- void checkTimestamps(final Map<byte[], List<KeyValue>> familyMap,
+ void checkTimestamps(final Map<byte[], List<? extends Cell>> familyMap,
long now) throws FailedSanityCheckException {
if (timestampSlop == HConstants.LATEST_TIMESTAMP) {
return;
}
long maxTs = now + timestampSlop;
- for (List<KeyValue> kvs : familyMap.values()) {
- for (KeyValue kv : kvs) {
+ for (List<? extends Cell> kvs : familyMap.values()) {
+ for (Cell cell : kvs) {
// see if the user-side TS is out of range. latest = server-side
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (!kv.isLatestTimestamp() && kv.getTimestamp() > maxTs) {
throw new FailedSanityCheckException("Timestamp for KV out of range "
- + kv + " (too.new=" + timestampSlop + ")");
+ + cell + " (too.new=" + timestampSlop + ")");
}
}
}
@@ -2741,11 +2746,11 @@ public class HRegion implements HeapSize
* @param familyMap map of family->edits
* @param walEdit the destination entry to append into
*/
- private void addFamilyMapToWALEdit(Map<byte[], List<KeyValue>> familyMap,
+ private void addFamilyMapToWALEdit(Map<byte[], List<? extends Cell>> familyMap,
WALEdit walEdit) {
- for (List<KeyValue> edits : familyMap.values()) {
- for (KeyValue kv : edits) {
- walEdit.add(kv);
+ for (List<? extends Cell> edits : familyMap.values()) {
+ for (Cell cell : edits) {
+ walEdit.add(KeyValueUtil.ensureKeyValue(cell));
}
}
}
@@ -3450,7 +3455,7 @@ public class HRegion implements HeapSize
public HRegionInfo getRegionInfo() {
return regionInfo;
}
-
+
RegionScannerImpl(Scan scan, List<KeyValueScanner> additionalScanners, HRegion region)
throws IOException {
// DebugPrint.println("HRegionScanner.<init>");
@@ -3600,11 +3605,11 @@ public class HRegion implements HeapSize
return next(outResults, batch, metric);
}
- private void populateFromJoinedHeap(List<KeyValue> results, int limit, String metric)
+ private void populateFromJoinedHeap(List<KeyValue> results, int limit, String metric)
throws IOException {
assert joinedContinuationRow != null;
- KeyValue kv = populateResult(results, this.joinedHeap, limit,
- joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
+ KeyValue kv = populateResult(results, this.joinedHeap, limit,
+ joinedContinuationRow.getBuffer(), joinedContinuationRow.getRowOffset(),
joinedContinuationRow.getRowLength(), metric);
if (kv != KV_LIMIT) {
// We are done with this row, reset the continuation.
@@ -3626,7 +3631,7 @@ public class HRegion implements HeapSize
* @param metric Metric key to be passed into KeyValueHeap::next().
* @return KV_LIMIT if limit reached, next KeyValue otherwise.
*/
- private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
+ private KeyValue populateResult(List<KeyValue> results, KeyValueHeap heap, int limit,
byte[] currentRow, int offset, short length, String metric) throws IOException {
KeyValue nextKv;
do {
@@ -4214,15 +4219,15 @@ public class HRegion implements HeapSize
// The row key is the region name
byte[] row = r.getRegionName();
final long now = EnvironmentEdgeManager.currentTimeMillis();
- final List<KeyValue> edits = new ArrayList<KeyValue>(2);
- edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
+ final List<KeyValue> cells = new ArrayList<KeyValue>(2);
+ cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
HConstants.REGIONINFO_QUALIFIER, now,
r.getRegionInfo().toByteArray()));
// Set into the root table the version of the meta table.
- edits.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
+ cells.add(new KeyValue(row, HConstants.CATALOG_FAMILY,
HConstants.META_VERSION_QUALIFIER, now,
Bytes.toBytes(HConstants.META_VERSION)));
- meta.put(row, HConstants.CATALOG_FAMILY, edits);
+ meta.put(row, HConstants.CATALOG_FAMILY, cells);
}
/**
@@ -4819,15 +4824,15 @@ public class HRegion implements HeapSize
try {
long now = EnvironmentEdgeManager.currentTimeMillis();
// Process each family
- for (Map.Entry<byte[], List<KeyValue>> family : append.getFamilyMap()
- .entrySet()) {
+ for (Map.Entry<byte[], List<? extends Cell>> family : append.getFamilyMap().entrySet()) {
Store store = stores.get(family.getKey());
List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());
// Get previous values for all columns in this family
Get get = new Get(row);
- for (KeyValue kv : family.getValue()) {
+ for (Cell cell : family.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
get.addColumn(family.getKey(), kv.getQualifier());
}
List<KeyValue> results = get(get, false);
@@ -4839,7 +4844,8 @@ public class HRegion implements HeapSize
// once.
// Would be nice if KeyValue had scatter/gather logic
int idx = 0;
- for (KeyValue kv : family.getValue()) {
+ for (Cell cell : family.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
KeyValue newKV;
if (idx < results.size()
&& results.get(idx).matchingQualifier(kv.getBuffer(),
@@ -4913,7 +4919,8 @@ public class HRegion implements HeapSize
size += store.upsert(entry.getValue(), getSmallestReadPoint());
} else {
// otherwise keep older versions around
- for (KeyValue kv : entry.getValue()) {
+ for (Cell cell: entry.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
size += store.add(kv);
}
}
@@ -4962,7 +4969,7 @@ public class HRegion implements HeapSize
TimeRange tr = increment.getTimeRange();
boolean flush = false;
WALEdit walEdits = null;
- List<KeyValue> allKVs = new ArrayList<KeyValue>(increment.numColumns());
+ List<KeyValue> allKVs = new ArrayList<KeyValue>(increment.size());
Map<Store, List<KeyValue>> tempMemstore = new HashMap<Store, List<KeyValue>>();
long size = 0;
@@ -4984,16 +4991,17 @@ public class HRegion implements HeapSize
try {
long now = EnvironmentEdgeManager.currentTimeMillis();
// Process each family
- for (Map.Entry<byte [], NavigableMap<byte [], Long>> family :
- increment.getFamilyMap().entrySet()) {
+ for (Map.Entry<byte [], List<? extends Cell>> family:
+ increment.getFamilyMap().entrySet()) {
Store store = stores.get(family.getKey());
List<KeyValue> kvs = new ArrayList<KeyValue>(family.getValue().size());
// Get previous values for all columns in this family
Get get = new Get(row);
- for (Map.Entry<byte [], Long> column : family.getValue().entrySet()) {
- get.addColumn(family.getKey(), column.getKey());
+ for (Cell cell: family.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ get.addColumn(family.getKey(), kv.getQualifier());
}
get.setTimeRange(tr.getMin(), tr.getMax());
List<KeyValue> results = get(get, false);
@@ -5001,11 +5009,12 @@ public class HRegion implements HeapSize
// Iterate the input columns and update existing values if they were
// found, otherwise add new column initialized to the increment amount
int idx = 0;
- for (Map.Entry<byte [], Long> column : family.getValue().entrySet()) {
- long amount = column.getValue();
- if (idx < results.size() &&
- results.get(idx).matchingQualifier(column.getKey())) {
- KeyValue kv = results.get(idx);
+ for (Cell cell: family.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
+ long amount = Bytes.toLong(kv.getValue());
+ byte [] qualifier = kv.getQualifier();
+ if (idx < results.size() && results.get(idx).matchingQualifier(qualifier)) {
+ kv = results.get(idx);
if(kv.getValueLength() == Bytes.SIZEOF_LONG) {
amount += Bytes.toLong(kv.getBuffer(), kv.getValueOffset(), Bytes.SIZEOF_LONG);
} else {
@@ -5017,8 +5026,8 @@ public class HRegion implements HeapSize
}
// Append new incremented KeyValue to list
- KeyValue newKV = new KeyValue(row, family.getKey(), column.getKey(),
- now, Bytes.toBytes(amount));
+ KeyValue newKV =
+ new KeyValue(row, family.getKey(), qualifier, now, Bytes.toBytes(amount));
newKV.setMemstoreTS(w.getWriteNumber());
kvs.add(newKV);
@@ -5053,7 +5062,8 @@ public class HRegion implements HeapSize
size += store.upsert(entry.getValue(), getSmallestReadPoint());
} else {
// otherwise keep older versions around
- for (KeyValue kv : entry.getValue()) {
+ for (Cell cell : entry.getValue()) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
size += store.add(kv);
}
}
@@ -5444,7 +5454,7 @@ public class HRegion implements HeapSize
* Update counters for numer of puts without wal and the size of possible data loss.
* These information are exposed by the region server metrics.
*/
- private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
+ private void recordPutWithoutWal(final Map<byte [], List<? extends Cell>> familyMap) {
numPutsWithoutWAL.increment();
if (numPutsWithoutWAL.get() <= 1) {
LOG.info("writing data to region " + this +
@@ -5452,8 +5462,9 @@ public class HRegion implements HeapSize
}
long putSize = 0;
- for (List<KeyValue> edits : familyMap.values()) {
- for (KeyValue kv : edits) {
+ for (List<? extends Cell> cells: familyMap.values()) {
+ for (Cell cell : cells) {
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
putSize += kv.getKeyLength() + kv.getValueLength();
}
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java Tue Feb 26 04:33:36 2013
@@ -78,6 +78,7 @@ import org.apache.hadoop.hbase.util.Coll
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hbase.Cell;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableCollection;
@@ -1820,10 +1821,10 @@ public class HStore implements Store {
}
@Override
- public long upsert(Iterable<KeyValue> kvs, long readpoint) throws IOException {
+ public long upsert(Iterable<? extends Cell> cells, long readpoint) throws IOException {
this.lock.readLock().lock();
try {
- return this.memstore.upsert(kvs, readpoint);
+ return this.memstore.upsert(cells, readpoint);
} finally {
this.lock.readLock().unlock();
}
Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java?rev=1450017&r1=1450016&r2=1450017&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java Tue Feb 26 04:33:36 2013
@@ -22,7 +22,7 @@ package org.apache.hadoop.hbase.regionse
import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.rmi.UnexpectedException;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -38,11 +38,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.HeapSize;
import org.apache.hadoop.hbase.regionserver.MemStoreLAB.Allocation;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.ClassSize;
+import org.apache.hbase.Cell;
/**
* The MemStore holds in-memory modifications to the Store. Modifications
@@ -498,9 +500,9 @@ public class MemStore implements HeapSiz
// create or update (upsert) a new KeyValue with
// 'now' and a 0 memstoreTS == immediately visible
- return upsert(Arrays.asList(
- new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue))), 1L
- );
+ List<Cell> cells = new ArrayList<Cell>(1);
+ cells.add(new KeyValue(row, family, qualifier, now, Bytes.toBytes(newValue)));
+ return upsert(cells, 1L);
} finally {
this.lock.readLock().unlock();
}
@@ -520,16 +522,16 @@ public class MemStore implements HeapSiz
* This is called under row lock, so Get operations will still see updates
* atomically. Scans will only see each KeyValue update as atomic.
*
- * @param kvs
+ * @param cells
* @param readpoint readpoint below which we can safely remove duplicate KVs
* @return change in memstore size
*/
- public long upsert(Iterable<KeyValue> kvs, long readpoint) {
+ public long upsert(Iterable<? extends Cell> cells, long readpoint) {
this.lock.readLock().lock();
try {
long size = 0;
- for (KeyValue kv : kvs) {
- size += upsert(kv, readpoint);
+ for (Cell cell : cells) {
+ size += upsert(cell, readpoint);
}
return size;
} finally {
@@ -548,16 +550,17 @@ public class MemStore implements HeapSiz
* <p>
* Callers must hold the read lock.
*
- * @param kv
+ * @param cell
* @return change in size of MemStore
*/
- private long upsert(KeyValue kv, long readpoint) {
+ private long upsert(Cell cell, long readpoint) {
// Add the KeyValue to the MemStore
// Use the internalAdd method here since we (a) already have a lock
// and (b) cannot safely use the MSLAB here without potentially
// hitting OOME - see TestMemStore.testUpsertMSLAB for a
// test that triggers the pathological case if we don't avoid MSLAB
// here.
+ KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
long addedSize = internalAdd(kv);
// Get the KeyValues for the row/family/qualifier regardless of timestamp.