svn commit: r930143 - in /hadoop/hbase/trunk: CHANGES.txt core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java

[jd...@apache.org]

Author: jdcryans
Date: Fri Apr  2 00:55:03 2010
New Revision: 930143

URL: http://svn.apache.org/viewvc?rev=930143&view=rev
Log:
HBASE-2087  The wait on compaction because "Too many store files"
            holds up all flushing


Modified:
    hadoop/hbase/trunk/CHANGES.txt
    hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java

Modified: hadoop/hbase/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/CHANGES.txt?rev=930143&r1=930142&r2=930143&view=diff
==============================================================================
--- hadoop/hbase/trunk/CHANGES.txt (original)
+++ hadoop/hbase/trunk/CHANGES.txt Fri Apr  2 00:55:03 2010
@@ -479,6 +479,8 @@ Release 0.21.0 - Unreleased
    HBASE-2388  Give a very explicit message when we figure a big GC pause
    HBASE-2270  Improve how we handle recursive calls in ExplicitColumnTracker 
                and WildcardColumnTracker
+   HBASE-2087  The wait on compaction because "Too many store files" 
+               holds up all flushing
 
   NEW FEATURES
    HBASE-1961  HBase EC2 scripts

Modified: hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=930143&r1=930142&r2=930143&view=diff
==============================================================================
--- hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hadoop/hbase/trunk/core/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Fri Apr  2 00:55:03 2010
@@ -215,7 +215,25 @@ class MemStoreFlusher extends Thread imp
    * not flushed.
    */
   private boolean flushRegion(HRegion region, boolean removeFromQueue) {
-    checkStoreFileCount(region);
+    // if removeFromQueue, then we come from flushSomeRegions and we need
+    // to block if there's too many store files. Else, we don't want to hang
+    // the main flushing thread so we'll just the region at the end of the
+    // queue if there's too many files.
+    if (removeFromQueue) {
+      checkStoreFileCount(region);
+    } else if (isTooManyStoreFiles(region)) {
+      LOG.warn("Region " + region.getRegionNameAsString() + " has too many " +
+          "store files, putting it back at the end of the flush queue.");
+      server.compactSplitThread.compactionRequested(region, getName());
+      // If there's only this item in the queue or they are all in this
+      // situation, we will loop at lot. Sleep a bit.
+      try {
+        Thread.sleep(1000);
+      } catch (InterruptedException e) { } // just continue
+      flushQueue.add(region);
+      // Tell a lie, it's not flushed but it's ok
+      return true;
+    }
     synchronized (regionsInQueue) {
       // See comment above for removeFromQueue on why we do not
       // take the region out of the set. If removeFromQueue is true, remove it
@@ -303,6 +321,15 @@ class MemStoreFlusher extends Thread imp
     }
   }
 
+  private boolean isTooManyStoreFiles(HRegion region) {
+    for (Store hstore: region.stores.values()) {
+      if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+        return true;
+      }
+    }
+    return false;
+  }
+
   /**
    * Check if the regionserver's memstore memory usage is greater than the 
    * limit. If so, flush regions with the biggest memstores until we're down