svn commit: r769793 - in /hadoop/hbase/branches/0.19: CHANGES.txt src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java

[ap...@apache.org]

Author: apurtell
Date: Wed Apr 29 14:12:45 2009
New Revision: 769793

URL: http://svn.apache.org/viewvc?rev=769793&view=rev
Log:
HBASE-1058 Prevent runaway compactions

Modified:
    hadoop/hbase/branches/0.19/CHANGES.txt
    hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java

Modified: hadoop/hbase/branches/0.19/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/CHANGES.txt?rev=769793&r1=769792&r2=769793&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.19/CHANGES.txt Wed Apr 29 14:12:45 2009
@@ -14,6 +14,7 @@
                (Lars George and Billy Pearson via Stack)
    HBASE-1350  New method in HTable.java to return start and end keys for
                regions in a table (Vimal Mathew via Stack)
+   HBASE-1058  Prevent runaway compactions
 
 Release 0.19.1 - 03/19/2009
   BUG FIXES

Modified: hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java?rev=769793&r1=769792&r2=769793&view=diff
==============================================================================
--- hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java (original)
+++ hadoop/hbase/branches/0.19/src/java/org/apache/hadoop/hbase/regionserver/MemcacheFlusher.java Wed Apr 29 14:12:45 2009
@@ -67,7 +67,9 @@
     "hbase.regionserver.globalMemcache.upperLimit";
   public static final String LOWER_KEY =
     "hbase.regionserver.globalMemcache.lowerLimit";
-  
+  private long blockingStoreFilesNumber;
+  private long blockingWaitTime;
+
   /**
    * @param conf
    * @param server
@@ -88,6 +90,14 @@
         "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
     }
     this.globalMemcacheLimitLowMark = lower;
+    this.blockingStoreFilesNumber = 
+      conf.getInt("hbase.hstore.blockingStoreFiles", -1);
+    if (this.blockingStoreFilesNumber == -1) {
+      this.blockingStoreFilesNumber = 1 +
+        conf.getInt("hbase.hstore.compactionThreshold", 3);
+    }
+    this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime",
+      90000); // default of 180 seconds
     LOG.info("globalMemcacheLimit=" +
       StringUtils.humanReadableInt(this.globalMemcacheLimit) +
       ", globalMemcacheLimitLowMark=" +
@@ -203,19 +213,46 @@
    * it may have been determined that the region had a significant amount of 
    * memory in use and needed to be flushed to relieve memory pressure. In this
    * case, its flush may preempt the pending request in the queue, and if so,
-   * it needs to be removed from the queue to avoid flushing the region multiple
-   * times.
+   * it needs to be removed from the queue to avoid flushing the region
+   * multiple times.
    * 
    * @return true if the region was successfully flushed, false otherwise. If 
    * false, there will be accompanying log messages explaining why the log was
    * not flushed.
    */
   private boolean flushRegion(HRegion region, boolean removeFromQueue) {
+    // Wait until it is safe to flush
+    int count = 0;
+    boolean triggered = false;
+    while (count++ < (blockingWaitTime / 500)) {
+      for (HStore hstore: region.stores.values()) {
+        if (hstore.getStorefilesCount() > this.blockingStoreFilesNumber) {
+          if (!triggered) {
+            server.compactSplitThread.compactionRequested(region, getName());
+            LOG.info("Too many store files for region " + region + ": " +
+              hstore.getStorefilesCount() + ", waiting");
+            triggered = true;
+          }
+          try {
+            Thread.sleep(500);
+          } catch (InterruptedException e) {
+            // ignore
+          }
+          continue;
+        }
+      }
+      if (triggered) {
+        LOG.info("Compaction completed on region " + region +
+          ", proceeding");
+      }
+      break;
+    }
     synchronized (regionsInQueue) {
+      // See comment above for removeFromQueue on why we do not
       // take the region out of the set. If removeFromQueue is true, remove it
-      // from the queue too if it is there. This didn't used to be a constraint,
-      // but now that HBASE-512 is in play, we need to try and limit
-      // double-flushing of regions.
+      // from the queue too if it is there. This didn't used to be a
+      // constraint, but now that HBASE-512 is in play, we need to try and
+      // limit double-flushing of regions.
       if (regionsInQueue.remove(region) && removeFromQueue) {
         flushQueue.remove(region);
       }