You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2009/07/09 08:47:10 UTC
svn commit: r792419 - in /hadoop/hbase/trunk_on_hadoop-0.18.3: ./ conf/
src/java/org/apache/hadoop/hbase/client/
src/java/org/apache/hadoop/hbase/regionserver/
src/test/org/apache/hadoop/hbase/client/
Author: apurtell
Date: Thu Jul 9 06:47:10 2009
New Revision: 792419
URL: http://svn.apache.org/viewvc?rev=792419&view=rev
Log:
HBASE-1618,HBASE-1625,HBASE-1624,HBASE-1626
Modified:
hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-env.sh
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/CHANGES.txt Thu Jul 9 06:47:10 2009
@@ -245,6 +245,10 @@
HBASE-1615 HBASE-1597 introduced a bug when compacting after a split
(Jon Gray via Stack)
HBASE-1616 Unit test of compacting referenced StoreFiles (Jon Gray via Stack)
+ HBASE-1618 Investigate further into the MemStoreFlusher StoreFile limit
+ (Jon Gray via Stack)
+ HBASE-1625 Adding check to Put.add(KeyValue kv), to see that it has the same
+ row as when instantiated (Erik Holstad via Stack)
IMPROVEMENTS
HBASE-1089 Add count of regions on filesystem to master UI; add percentage
@@ -445,6 +449,9 @@
HBASE-1575 HMaster does not handle ZK session expiration
HBASE-1620 Need to use special StoreScanner constructor for major compactions
(passed sf, no caching, etc) (Jon Gray via Stack)
+ HBASE-1624 Don't sort Puts if only one in list in HCM#processBatchOfRows
+ HBASE-1626 Allow emitting Deletes out of new TableReducer
+ (Lars George via Stack)
OPTIMIZATIONS
HBASE-1412 Change values for delete column and column family in KeyValue
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-env.sh?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-env.sh (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/conf/hbase-env.sh Thu Jul 9 06:47:10 2009
@@ -36,6 +36,9 @@
# see http://wiki.apache.org/hadoop/PerformanceTuning
export HBASE_OPTS="-XX:+HeapDumpOnOutOfMemoryError -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode"
+# Uncomment below to enable java garbage collection logging.
+#export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Delete.java Thu Jul 9 06:47:10 2009
@@ -70,7 +70,7 @@
/** Constructor for Writable. DO NOT USE */
public Delete() {
- this(null);
+ this((byte [])null);
}
/**
@@ -108,6 +108,16 @@
}
/**
+ * @param d Delete to clone.
+ */
+ public Delete(final Delete d) {
+ this.row = d.getRow();
+ this.ts = d.getTimeStamp();
+ this.lockId = d.getLockId();
+ this.familyMap.putAll(d.getFamilyMap());
+ }
+
+ /**
* Method to check if the familyMap is empty
* @return true if empty, false otherwise
*/
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/HConnectionManager.java Thu Jul 9 06:47:10 2009
@@ -797,7 +797,7 @@
server = (HRegionInterface)HBaseRPC.waitForProxy(
serverInterfaceClass, HBaseRPCProtocolVersion.versionID,
regionServer.getInetSocketAddress(), this.conf,
- this.maxRPCAttempts);
+ this.maxRPCAttempts, this.rpcTimeout);
} catch (RemoteException e) {
throw RemoteExceptionHandler.decodeRemoteException(e);
}
@@ -1003,7 +1003,9 @@
}
boolean retryOnlyOne = false;
int tries = 0;
- Collections.sort(list);
+ if (list.size() > 1) {
+ Collections.sort(list);
+ }
List<Put> currentPuts = new ArrayList<Put>();
HRegionLocation location =
getRegionLocationForRowWithRetries(tableName, list.get(0).getRow(),
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/client/Put.java Thu Jul 9 06:47:10 2009
@@ -144,12 +144,21 @@
* Add the specified KeyValue to this Put operation.
* @param kv
*/
- public Put add(KeyValue kv) {
+ public Put add(KeyValue kv) throws IOException{
byte [] family = kv.getFamily();
List<KeyValue> list = familyMap.get(family);
if(list == null) {
list = new ArrayList<KeyValue>();
}
+ //Checking that the row of the kv is the same as the put
+ int res = Bytes.compareTo(this.row, 0, row.length,
+ kv.getBuffer(), kv.getRowOffset(), kv.getRowLength());
+ if(res != 0) {
+ throw new IOException("The row in the recently added KeyValue " +
+ Bytes.toStringBinary(kv.getBuffer(), kv.getRowOffset(),
+ kv.getRowLength()) + " doesn't match the original one " +
+ Bytes.toStringBinary(this.row));
+ }
list.add(kv);
familyMap.put(family, list);
return this;
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Thu Jul 9 06:47:10 2009
@@ -222,10 +222,11 @@
* not flushed.
*/
private boolean flushRegion(HRegion region, boolean removeFromQueue) {
- // Wait until it is safe to flush
int count = 0;
boolean triggered = false;
+ boolean finished = false;
while (count++ < (blockingWaitTime / 500)) {
+ finished = true;
for (Store hstore: region.stores.values()) {
if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
// always request a compaction
@@ -233,22 +234,28 @@
// only log once
if (!triggered) {
LOG.info("Too many store files for region " + region + ": " +
- hstore.getStorefilesCount() + ", waiting");
+ hstore.getStorefilesCount() + ", requesting compaction and " +
+ "waiting");
triggered = true;
}
+ // pending compaction, not finished
+ finished = false;
try {
Thread.sleep(500);
} catch (InterruptedException e) {
// ignore
}
- continue;
}
}
- if (triggered) {
- LOG.info("Compaction completed on region " + region +
- ", proceeding");
+ if(triggered && finished) {
+ LOG.info("Compaction has completed, we waited " + (count * 500) + "ms, "
+ + "finishing flush of region " + region);
+ break;
}
- break;
+ }
+ if(triggered && !finished) {
+ LOG.warn("Tried to hold up flushing for compactions of region " + region +
+ " but have waited longer than " + blockingWaitTime + "ms, continuing");
}
synchronized (regionsInQueue) {
// See comment above for removeFromQueue on why we do not
Modified: hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java?rev=792419&r1=792418&r2=792419&view=diff
==============================================================================
--- hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java (original)
+++ hadoop/hbase/trunk_on_hadoop-0.18.3/src/test/org/apache/hadoop/hbase/client/TestPut.java Thu Jul 9 06:47:10 2009
@@ -22,17 +22,11 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Map;
import org.apache.hadoop.hbase.HBaseClusterTestCase;
import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.io.BatchUpdate;
-import org.apache.hadoop.hbase.io.Cell;
-import org.apache.hadoop.hbase.io.RowResult;
import org.apache.hadoop.hbase.util.Bytes;
/**
@@ -82,7 +76,6 @@
* @throws IOException
*/
public void testPut() throws IOException {
-
Put put = new Put(row1);
put.add(CONTENTS_FAMILY, null, value);
table.put(put);
@@ -198,5 +191,29 @@
}
}
+ public void testAddKeyValue() throws IOException {
+ byte [] qualifier = Bytes.toBytes("qf1");
+ Put put = new Put(row1);
+
+ //Adding KeyValue with the same row
+ KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
+ boolean ok = true;
+ try {
+ put.add(kv);
+ } catch (IOException e) {
+ ok = false;
+ }
+ assertEquals(true, ok);
+
+ //Adding KeyValue with the different row
+ kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
+ ok = false;
+ try {
+ put.add(kv);
+ } catch (IOException e) {
+ ok = true;
+ }
+ assertEquals(true, ok);
+ }
}