You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2009/06/26 00:12:25 UTC
svn commit: r788529 - in /hadoop/hbase/trunk: ./
src/java/org/apache/hadoop/hbase/ src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/ipc/
src/java/org/apache/hadoop/hbase/regionserver/
src/test/org/apache/hadoop/hbase/regionserver/
Author: stack
Date: Thu Jun 25 22:12:24 2009
New Revision: 788529
URL: http://svn.apache.org/viewvc?rev=788529&view=rev
Log:
HBASE-1563 incrementColumnValue does not write to WAL
Modified:
hadoop/hbase/trunk/CHANGES.txt
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/KeyValue.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Store.java
hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java
Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Thu Jun 25 22:12:24 2009
@@ -223,6 +223,7 @@
HBASE-1567 cant serialize new filters
HBASE-1585 More binary key/value log output cleanup
(Lars George via Stack)
+ HBASE-1563 incrementColumnValue does not write to WAL (Jon Gray via Stack)
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/KeyValue.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/KeyValue.java Thu Jun 25 22:12:24 2009
@@ -234,7 +234,7 @@
public KeyValue(final byte [] bytes) {
this(bytes, 0);
}
-
+
/**
* Creates a KeyValue from the specified byte array and offset.
* Presumes <code>bytes</code> content starting at <code>offset</code> is
@@ -566,6 +566,16 @@
//---------------------------------------------------------------------------
/**
+ * Clones a KeyValue. This creates a copy, re-allocating the buffer.
+ * @return Fully copied clone of this KeyValue
+ */
+ public KeyValue clone() {
+ byte [] bytes = new byte[this.length];
+ System.arraycopy(this.bytes, this.offset, bytes, 0, this.length);
+ return new KeyValue(bytes, 0, bytes.length);
+ }
+
+ /**
* Clones a row.
*
* @param timestamp The new time stamp for the row.
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/client/HTable.java Thu Jun 25 22:12:24 2009
@@ -472,8 +472,8 @@
}
/**
- * Atomically increments a column value. If the column value isn't long-like,
- * this could throw an exception.
+ * Atomically increments a column value. If the column value already exists
+ * and is not a big-endian long, this could throw an exception.<p>
*
* @param row
* @param family
@@ -485,6 +485,26 @@
public long incrementColumnValue(final byte [] row, final byte [] family,
final byte [] qualifier, final long amount)
throws IOException {
+ return incrementColumnValue(row, family, qualifier, amount, true);
+ }
+
+ /**
+ * Atomically increments a column value. If the column value already exists
+ * and is not a big-endian long, this could throw an exception.<p>
+ *
+ * Setting writeToWAL to false means that in a fail scenario, you will lose
+ * any increments that have not been flushed.
+ * @param row
+ * @param family
+ * @param qualifier
+ * @param amount
+ * @param writeToWAL true if increment should be applied to WAL, false if not
+ * @return The new value.
+ * @throws IOException
+ */
+ public long incrementColumnValue(final byte [] row, final byte [] family,
+ final byte [] qualifier, final long amount, final boolean writeToWAL)
+ throws IOException {
NullPointerException npe = null;
if (row == null) {
npe = new NullPointerException("row is null");
@@ -499,11 +519,9 @@
return connection.getRegionServerWithRetries(
new ServerCallable<Long>(connection, tableName, row) {
public Long call() throws IOException {
- Get get = new Get(row);
- get.addColumn(family, qualifier);
return server.incrementColumnValue(
location.getRegionInfo().getRegionName(), row, family,
- qualifier, amount);
+ qualifier, amount, writeToWAL);
}
}
);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/ipc/HRegionInterface.java Thu Jun 25 22:12:24 2009
@@ -147,11 +147,12 @@
* @param family
* @param qualifier
* @param amount
+ * @param writeToWAL whether to write the increment to the WAL
* @return new incremented column value
* @throws IOException
*/
public long incrementColumnValue(byte [] regionName, byte [] row,
- byte [] family, byte [] qualifier, long amount)
+ byte [] family, byte [] qualifier, long amount, boolean writeToWAL)
throws IOException;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegion.java Thu Jun 25 22:12:24 2009
@@ -2256,18 +2256,29 @@
* @throws IOException
*/
public long incrementColumnValue(byte [] row, byte [] family,
- byte [] qualifier, long amount)
+ byte [] qualifier, long amount, boolean writeToWAL)
throws IOException {
checkRow(row);
-
boolean flush = false;
// Lock row
Integer lid = obtainRowLock(row);
long result = 0L;
try {
Store store = stores.get(family);
- Store.ValueAndSize vas =
+ // Determine what to do and perform increment on returned KV, no insertion
+ Store.ICVResult vas =
store.incrementColumnValue(row, family, qualifier, amount);
+ // Write incremented value to WAL before inserting
+ if (writeToWAL) {
+ long now = System.currentTimeMillis();
+ List<KeyValue> edits = new ArrayList<KeyValue>(1);
+ edits.add(vas.kv);
+ this.log.append(regionInfo.getRegionName(),
+ regionInfo.getTableDesc().getName(), edits,
+ (regionInfo.isMetaRegion() || regionInfo.isRootRegion()), now);
+ }
+ // Insert to the Store
+ store.add(vas.kv);
result = vas.value;
long size = this.memstoreSize.addAndGet(vas.sizeAdded);
flush = isFlushSize(size);
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Thu Jun 25 22:12:24 2009
@@ -2392,7 +2392,7 @@
/** {@inheritDoc} */
public long incrementColumnValue(byte [] regionName, byte [] row,
- byte [] family, byte [] qualifier, long amount)
+ byte [] family, byte [] qualifier, long amount, boolean writeToWAL)
throws IOException {
checkOpen();
@@ -2403,7 +2403,8 @@
requestCount.incrementAndGet();
try {
HRegion region = getRegion(regionName);
- return region.incrementColumnValue(row, family, qualifier, amount);
+ return region.incrementColumnValue(row, family, qualifier, amount,
+ writeToWAL);
} catch (IOException e) {
checkFileSystem();
throw e;
Modified: hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Store.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Store.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Store.java (original)
+++ hadoop/hbase/trunk/src/java/org/apache/hadoop/hbase/regionserver/Store.java Thu Jun 25 22:12:24 2009
@@ -1493,15 +1493,21 @@
scanner.get(result);
}
- public static class ValueAndSize {
- public long value;
- public long sizeAdded;
- public ValueAndSize(long value, long sizeAdded) {
+ /*
+ * Data structure to hold incrementColumnValue result.
+ */
+ static class ICVResult {
+ final long value;
+ final long sizeAdded;
+ final KeyValue kv;
+
+ ICVResult(long value, long sizeAdded, KeyValue kv) {
this.value = value;
this.sizeAdded = sizeAdded;
+ this.kv = kv;
}
}
-
+
/**
* Increments the value for the given row/family/qualifier
* @param row
@@ -1511,8 +1517,9 @@
* @return The new value.
* @throws IOException
*/
- public ValueAndSize incrementColumnValue(byte [] row, byte [] f,
- byte [] qualifier, long amount) throws IOException {
+ public ICVResult incrementColumnValue(byte [] row, byte [] f,
+ byte [] qualifier, long amount)
+ throws IOException {
long value = 0;
List<KeyValue> result = new ArrayList<KeyValue>();
KeyComparator keyComparator = this.comparator.getRawComparator();
@@ -1526,19 +1533,20 @@
keyComparator, 1);
// Read from memstore
- if(this.memstore.get(matcher, result)) {
+ if (this.memstore.get(matcher, result)) {
// Received early-out from memstore
- KeyValue kv = result.get(0);
+ // Make a copy of the KV and increment it
+ KeyValue kv = result.get(0).clone();
byte [] buffer = kv.getBuffer();
int valueOffset = kv.getValueOffset();
value = Bytes.toLong(buffer, valueOffset, Bytes.SIZEOF_LONG) + amount;
- Bytes.putBytes(buffer, valueOffset, Bytes.toBytes(value), 0,
- Bytes.SIZEOF_LONG);
- return new ValueAndSize(value, 0);
+ Bytes.putBytes(buffer, valueOffset, Bytes.toBytes(value), 0,
+ Bytes.SIZEOF_LONG);
+ return new ICVResult(value, 0, kv);
}
// Check if we even have storefiles
if(this.storefiles.isEmpty()) {
- return addNewKeyValue(row, f, qualifier, value, amount);
+ return createNewKeyValue(row, f, qualifier, value, amount);
}
// Get storefiles for this store
@@ -1555,16 +1563,15 @@
if(result.size() > 0) {
value = Bytes.toLong(result.get(0).getValue());
}
- return addNewKeyValue(row, f, qualifier, value, amount);
+ return createNewKeyValue(row, f, qualifier, value, amount);
}
- private ValueAndSize addNewKeyValue(byte [] row, byte [] f, byte [] qualifier,
- long value, long amount) {
+ private ICVResult createNewKeyValue(byte [] row, byte [] f,
+ byte [] qualifier, long value, long amount) {
long newValue = value + amount;
KeyValue newKv = new KeyValue(row, f, qualifier,
System.currentTimeMillis(),
Bytes.toBytes(newValue));
- add(newKv);
- return new ValueAndSize(newValue, newKv.heapSize());
+ return new ICVResult(newValue, newKv.heapSize(), newKv);
}
}
\ No newline at end of file
Modified: hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java?rev=788529&r1=788528&r2=788529&view=diff
==============================================================================
--- hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java (original)
+++ hadoop/hbase/trunk/src/test/org/apache/hadoop/hbase/regionserver/TestStore.java Thu Jun 25 22:12:24 2009
@@ -208,7 +208,9 @@
long amount = 3L;
this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
- this.store.incrementColumnValue(row, family, qf1, amount);
+ Store.ICVResult vas = this.store.incrementColumnValue(row, family, qf1, amount);
+ assertEquals(vas.value, value+amount);
+ store.add(vas.kv);
Get get = new Get(row);
get.addColumn(family, qf1);
NavigableSet<byte[]> qualifiers =
@@ -232,7 +234,9 @@
long amount = -1L;
this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
- this.store.incrementColumnValue(row, family, qf1, amount);
+ Store.ICVResult vas = this.store.incrementColumnValue(row, family, qf1, amount);
+ assertEquals(vas.value, value+amount);
+ store.add(vas.kv);
Get get = new Get(row);
get.addColumn(family, qf1);
NavigableSet<byte[]> qualifiers =
@@ -256,7 +260,8 @@
this.store.add(new KeyValue(row, family, qf1, Bytes.toBytes(value)));
this.store.add(new KeyValue(row, family, qf2, Bytes.toBytes(value)));
- this.store.incrementColumnValue(row, family, qf3, amount);
+ Store.ICVResult vas = this.store.incrementColumnValue(row, family, qf3, amount);
+ store.add(vas.kv);
Get get = new Get(row);
get.addColumn(family, qf3);
NavigableSet<byte[]> qualifiers =
@@ -283,7 +288,8 @@
flush(1);
- this.store.incrementColumnValue(row, family, qf1, amount);
+ Store.ICVResult vas = this.store.incrementColumnValue(row, family, qf1, amount);
+ store.add(vas.kv);
Get get = new Get(row);
get.addColumn(family, qf1);
NavigableSet<byte[]> qualifiers =
@@ -311,7 +317,8 @@
flush(1);
- this.store.incrementColumnValue(row, family, qf3, amount);
+ Store.ICVResult vas = this.store.incrementColumnValue(row, family, qf3, amount);
+ store.add(vas.kv);
Get get = new Get(row);
get.addColumn(family, qf3);
NavigableSet<byte[]> qualifiers =