You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2013/12/11 08:03:41 UTC

svn commit: r1550059 - in /hbase/trunk: hbase-common/src/main/java/org/apache/hadoop/hbase/ hbase-common/src/main/resources/ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ h...

Author: anoopsamjohn
Date: Wed Dec 11 07:03:40 2013
New Revision: 1550059

URL: http://svn.apache.org/r1550059
Log:
HBASE-5349 Automagically tweak global memstore and block cache sizes based on workload

Added:
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ResizableBlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequestListener.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryTuner.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
Modified:
    hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
    hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
    hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
    hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java

Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Wed Dec 11 07:03:40 2013
@@ -752,7 +752,7 @@ public final class HConstants {
   public static final String HFILE_BLOCK_CACHE_SIZE_KEY =
     "hfile.block.cache.size";
 
-  public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.25f;
+  public static final float HFILE_BLOCK_CACHE_SIZE_DEFAULT = 0.4f;
 
   /*
     * Minimum percentage of free heap necessary for a successful cluster startup.

Modified: hbase/trunk/hbase-common/src/main/resources/hbase-default.xml
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/resources/hbase-default.xml?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/resources/hbase-default.xml (original)
+++ hbase/trunk/hbase-common/src/main/resources/hbase-default.xml Wed Dec 11 07:03:40 2013
@@ -243,21 +243,20 @@ possible configurations would overwhelm 
     <description>The HLog file writer implementation.</description>
   </property>
   <property>
-    <name>hbase.regionserver.global.memstore.upperLimit</name>
+    <name>hbase.regionserver.global.memstore.size</name>
     <value>0.4</value>
     <description>Maximum size of all memstores in a region server before new
       updates are blocked and flushes are forced. Defaults to 40% of heap.
       Updates are blocked and flushes are forced until size of all memstores
-      in a region server hits hbase.regionserver.global.memstore.lowerLimit.</description>
+      in a region server hits hbase.regionserver.global.memstore.size.lower.limit.</description>
   </property>
   <property>
-    <name>hbase.regionserver.global.memstore.lowerLimit</name>
-    <value>0.38</value>
-    <description>Maximum size of all memstores in a region server before
-      flushes are forced. Defaults to 38% of heap.
-      This value equal to hbase.regionserver.global.memstore.upperLimit causes
-      the minimum possible flushing to occur when updates are blocked due to
-      memstore limiting.</description>
+    <name>hbase.regionserver.global.memstore.size.lower.limit</name>
+    <value>0.95</value>
+    <description>Maximum size of all memstores in a region server before flushes are forced.
+      Defaults to 95% of hbase.regionserver.global.memstore.size.
+      A 100% value for this value causes the minimum possible flushing to occur when updates are 
+      blocked due to memstore limiting.</description>
   </property>
   <property>
     <name>hbase.regionserver.optionalcacheflushinterval</name>

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java Wed Dec 11 07:03:40 2013
@@ -347,7 +347,7 @@ public class CacheConfig {
    * @param conf  The current configuration.
    * @return The block cache or <code>null</code>.
    */
-  private static synchronized BlockCache instantiateBlockCache(Configuration conf) {
+  public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
     if (globalBlockCache != null) return globalBlockCache;
     if (blockCacheDisabled) return null;
 

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java Wed Dec 11 07:03:40 2013
@@ -38,7 +38,7 @@ import org.apache.hadoop.util.StringUtil
  *
  **/
 @InterfaceAudience.Private
-public class DoubleBlockCache implements BlockCache, HeapSize {
+public class DoubleBlockCache implements ResizableBlockCache, HeapSize {
 
   static final Log LOG = LogFactory.getLog(DoubleBlockCache.class.getName());
 
@@ -172,4 +172,8 @@ public class DoubleBlockCache implements
     return onHeapCache.getBlockCount() + offHeapCache.getBlockCount();
   }
 
+  @Override
+  public void setMaxSize(long size) {
+    this.onHeapCache.setMaxSize(size);
+  }
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java Wed Dec 11 07:03:40 2013
@@ -95,7 +95,7 @@ import com.google.common.util.concurrent
  * to the relative sizes and usage.
  */
 @InterfaceAudience.Private
-public class LruBlockCache implements BlockCache, HeapSize {
+public class LruBlockCache implements ResizableBlockCache, HeapSize {
 
   static final Log LOG = LogFactory.getLog(LruBlockCache.class);
 
@@ -272,6 +272,7 @@ public class LruBlockCache implements Bl
         statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS);
   }
 
+  @Override
   public void setMaxSize(long maxSize) {
     this.maxSize = maxSize;
     if(this.size.get() > acceptableSize() && !evictionInProgress) {

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ResizableBlockCache.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ResizableBlockCache.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ResizableBlockCache.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/ResizableBlockCache.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,34 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * BlockCache which is resizable.
+ */
+@InterfaceAudience.Private
+public interface ResizableBlockCache extends BlockCache {
+
+  /**
+   * Sets the max heap size that can be used by the BlockCache.
+   * @param size The max heap size.
+   */
+  void setMaxSize(long size);
+}

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,114 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY;
+import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY;
+import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
+import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY;
+import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
+
+/**
+ * The default implementation for the HeapMemoryTuner. This will do simple checks to decide
+ * whether there should be changes in the heap size of memstore/block cache. When there is no block
+ * cache eviction at all but there are flushes because of global heap pressure, it will increase the
+ * memstore heap size and decrease block cache size. The step value for this heap size change can be
+ * specified using the config <i>hbase.regionserver.heapmemory.autotuner.step</i>. When there is no
+ * memstore flushes because of heap pressure but there is block cache evictions it will increase the
+ * block cache heap.
+ */
+@InterfaceAudience.Private
+class DefaultHeapMemoryTuner implements HeapMemoryTuner {
+
+  public static final String STEP_KEY = "hbase.regionserver.heapmemory.autotuner.step";
+  public static final float DEFAULT_STEP_VALUE = 0.02f; // 2%
+
+  private Configuration conf;
+  private float step = DEFAULT_STEP_VALUE;
+
+  private float globalMemStorePercentMinRange;
+  private float globalMemStorePercentMaxRange;
+  private float blockCachePercentMinRange;
+  private float blockCachePercentMaxRange;
+
+  @Override
+  public TunerResult tune(TunerContext context) {
+    long blockedFlushCount = context.getBlockedFlushCount();
+    long unblockedFlushCount = context.getUnblockedFlushCount();
+    long evictCount = context.getEvictCount();
+    boolean memstoreSufficient = blockedFlushCount == 0 && unblockedFlushCount == 0;
+    boolean blockCacheSufficient = evictCount == 0;
+    if (memstoreSufficient && blockCacheSufficient) {
+      return new TunerResult(false);
+    }
+    TunerResult result = new TunerResult(true);
+    float newMemstoreSize;
+    float newBlockCacheSize;
+    if (memstoreSufficient) {
+      // Increase the block cache size and corresponding decrease in memstore size
+      newBlockCacheSize = context.getCurBlockCacheSize() + step;
+      newMemstoreSize = context.getCurMemStoreSize() - step;
+    } else if (blockCacheSufficient) {
+      // Increase the memstore size and corresponding decrease in block cache size
+      newBlockCacheSize = context.getCurBlockCacheSize() - step;
+      newMemstoreSize = context.getCurMemStoreSize() + step;
+    } else {
+      return new TunerResult(false);
+      // As of now not making any tuning in write/read heavy scenario.
+    }
+    if (newMemstoreSize > globalMemStorePercentMaxRange) {
+      newMemstoreSize = globalMemStorePercentMaxRange;
+    } else if (newMemstoreSize < globalMemStorePercentMinRange) {
+      newMemstoreSize = globalMemStorePercentMinRange;
+    }
+    if (newBlockCacheSize > blockCachePercentMaxRange) {
+      newBlockCacheSize = blockCachePercentMaxRange;
+    } else if (newBlockCacheSize < blockCachePercentMinRange) {
+      newBlockCacheSize = blockCachePercentMinRange;
+    }
+    result.setBlockCacheSize(newBlockCacheSize);
+    result.setMemstoreSize(newMemstoreSize);
+    return result;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return this.conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+    this.conf = conf;
+    this.step = conf.getFloat(STEP_KEY, DEFAULT_STEP_VALUE);
+    this.blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY,
+        conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
+    this.blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY,
+        conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
+    this.globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
+        MemStoreFlusher.getGlobalMemStorePercent(conf));
+    this.globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
+        MemStoreFlusher.getGlobalMemStorePercent(conf));
+  }
+}

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequestListener.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequestListener.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequestListener.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequestListener.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,36 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Listener which will get notified regarding flush requests of regions.
+ */
+@InterfaceAudience.Private
+public interface FlushRequestListener {
+
+  /**
+   * Callback which will get called when a flush request is made for a region.
+   * 
+   * @param type The type of flush. (ie. Whether a normal flush or flush because of global heap preassure)
+   * @param region The region for which flush is requested
+   */
+  void flushRequested(FlushType type, HRegion region);
+}

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/FlushRequester.java Wed Dec 11 07:03:40 2013
@@ -39,4 +39,26 @@ public interface FlushRequester {
    * @param delay after how much time should the flush happen
    */
   void requestDelayedFlush(HRegion region, long delay);
+
+  /**
+   * Register a FlushRequestListener
+   * 
+   * @param listener
+   */
+  void registerFlushRequestListener(final FlushRequestListener listener);
+
+  /**
+   * Unregister the given FlushRequestListener
+   * 
+   * @param listener
+   * @return true when passed listener is unregistered successfully.
+   */
+  public boolean unregisterFlushRequestListener(final FlushRequestListener listener);
+
+  /**
+   * Sets the global memstore limit to a new size.
+   * 
+   * @param globalMemStoreSize
+   */
+  public void setGlobalMemstoreLimit(long globalMemStoreSize);
 }

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java Wed Dec 11 07:03:40 2013
@@ -274,6 +274,8 @@ public class HRegionServer implements Cl
   // Cache flushing
   protected MemStoreFlusher cacheFlusher;
 
+  protected HeapMemoryManager hMemManager;
+
   // catalog tracker
   protected CatalogTracker catalogTracker;
 
@@ -921,6 +923,7 @@ public class HRegionServer implements Cl
 
     // Send interrupts to wake up threads if sleeping so they notice shutdown.
     // TODO: Should we check they are alive? If OOME could have exited already
+    if(this.hMemManager != null) this.hMemManager.stop();
     if (this.cacheFlusher != null) this.cacheFlusher.interruptIfNecessary();
     if (this.compactSplitThread != null) this.compactSplitThread.interruptIfNecessary();
     if (this.hlogRoller != null) this.hlogRoller.interruptIfNecessary();
@@ -1240,6 +1243,7 @@ public class HRegionServer implements Cl
       spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration());
 
       startServiceThreads();
+      startHeapMemoryManager();
       LOG.info("Serving as " + this.serverNameFromMasterPOV +
         ", RpcServer on " + this.isa +
         ", sessionid=0x" +
@@ -1255,6 +1259,13 @@ public class HRegionServer implements Cl
     }
   }
 
+  private void startHeapMemoryManager() {
+    this.hMemManager = HeapMemoryManager.create(this.conf, this.cacheFlusher, this);
+    if (this.hMemManager != null) {
+      this.hMemManager.start();
+    }
+  }
+
   private void createMyEphemeralNode() throws KeeperException {
     ZKUtil.createEphemeralNodeAndWatch(this.zooKeeper, getMyEphemeralNodePath(),
       HConstants.EMPTY_BYTE_ARRAY);

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,384 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY;
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Chore;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.io.hfile.BlockCache;
+import org.apache.hadoop.hbase.io.hfile.CacheConfig;
+import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
+import org.apache.hadoop.hbase.util.Threads;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Manages tuning of Heap memory using <code>HeapMemoryTuner</code>.
+ */
+@InterfaceAudience.Private
+public class HeapMemoryManager {
+  private static final Log LOG = LogFactory.getLog(HeapMemoryManager.class);
+  private static final int CONVERT_TO_PERCENTAGE = 100;
+  private static final int CLUSTER_MINIMUM_MEMORY_THRESHOLD = 
+    (int) (CONVERT_TO_PERCENTAGE * HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD);
+
+  public static final String BLOCK_CACHE_SIZE_MAX_RANGE_KEY = "hfile.block.cache.size.max.range";
+  public static final String BLOCK_CACHE_SIZE_MIN_RANGE_KEY = "hfile.block.cache.size.min.range";
+  public static final String MEMSTORE_SIZE_MAX_RANGE_KEY = 
+      "hbase.regionserver.global.memstore.size.max.range";
+  public static final String MEMSTORE_SIZE_MIN_RANGE_KEY = 
+      "hbase.regionserver.global.memstore.size.min.range";
+  public static final String HBASE_RS_HEAP_MEMORY_TUNER_PERIOD = 
+      "hbase.regionserver.heapmemory.tuner.period";
+  public static final int HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD = 5 * 60 * 1000;
+  public static final String HBASE_RS_HEAP_MEMORY_TUNER_CLASS = 
+      "hbase.regionserver.heapmemory.tuner.class";
+
+  private float globalMemStorePercent;
+  private float globalMemStorePercentMinRange;
+  private float globalMemStorePercentMaxRange;
+
+  private float blockCachePercent;
+  private float blockCachePercentMinRange;
+  private float blockCachePercentMaxRange;
+
+  private final ResizableBlockCache blockCache;
+  private final FlushRequester memStoreFlusher;
+  private final Server server;
+
+  private HeapMemoryTunerChore heapMemTunerChore = null;
+  private final boolean tunerOn;
+
+  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+
+  public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher,
+      Server server) {
+    BlockCache blockCache = CacheConfig.instantiateBlockCache(conf);
+    if (blockCache instanceof ResizableBlockCache) {
+      return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server);
+    }
+    return null;
+  }
+
+  @VisibleForTesting
+  HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher,
+      Server server) {
+    this.blockCache = blockCache;
+    this.memStoreFlusher = memStoreFlusher;
+    this.server = server;
+    this.tunerOn = doInit(server.getConfiguration());
+  }
+
+  private boolean doInit(Configuration conf) {
+    globalMemStorePercent = MemStoreFlusher.getGlobalMemStorePercent(conf);
+    blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
+        HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+    int gml = (int) (globalMemStorePercent * CONVERT_TO_PERCENTAGE);
+    int bcul = (int) (blockCachePercent * CONVERT_TO_PERCENTAGE);
+    if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
+      throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+          + "the threshold required for successful cluster operation. "
+          + "The combined value cannot exceed 0.8. Please check " + "the settings for "
+          + MemStoreFlusher.MEMSTORE_SIZE_KEY + " and " + HFILE_BLOCK_CACHE_SIZE_KEY
+          + " in your configuration. " + MemStoreFlusher.MEMSTORE_SIZE_KEY + " is "
+          + globalMemStorePercent + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
+          + blockCachePercent);
+    }
+    // Initialize max and min range for memstore heap space
+    globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
+        globalMemStorePercent);
+    globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
+        globalMemStorePercent);
+    if (globalMemStorePercent < globalMemStorePercentMinRange) {
+      LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
+          + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+          + " because supplied value greater than initial memstore size value.");
+      globalMemStorePercentMinRange = globalMemStorePercent;
+      conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
+    }
+    if (globalMemStorePercent > globalMemStorePercentMaxRange) {
+      LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
+          + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+          + " because supplied value less than initial memstore size value.");
+      globalMemStorePercentMaxRange = globalMemStorePercent;
+      conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
+    }
+    if (globalMemStorePercent == globalMemStorePercentMinRange
+        && globalMemStorePercent == globalMemStorePercentMaxRange) {
+      return false;
+    }
+    // Initialize max and min range for block cache
+    blockCachePercentMinRange = conf.getFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercent);
+    blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercent);
+    if (blockCachePercent < blockCachePercentMinRange) {
+      LOG.warn("Setting " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " to " + blockCachePercent
+          + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
+          + " because supplied value greater than initial block cache size.");
+      blockCachePercentMinRange = blockCachePercent;
+      conf.setFloat(BLOCK_CACHE_SIZE_MIN_RANGE_KEY, blockCachePercentMinRange);
+    }
+    if (blockCachePercent > blockCachePercentMaxRange) {
+      LOG.warn("Setting " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " to " + blockCachePercent
+          + ", same value as " + HFILE_BLOCK_CACHE_SIZE_KEY
+          + " because supplied value less than initial block cache size.");
+      blockCachePercentMaxRange = blockCachePercent;
+      conf.setFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY, blockCachePercentMaxRange);
+    }
+    if (blockCachePercent == blockCachePercentMinRange
+        && blockCachePercent == blockCachePercentMaxRange) {
+      return false;
+    }
+
+    gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
+    bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE);
+    if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
+      throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+          + "the threshold required for successful cluster operation. "
+          + "The combined value cannot exceed 0.8. Please check the settings for "
+          + MEMSTORE_SIZE_MAX_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY
+          + " in your configuration. " + MEMSTORE_SIZE_MAX_RANGE_KEY + " is "
+          + globalMemStorePercentMaxRange + " and " + BLOCK_CACHE_SIZE_MIN_RANGE_KEY + " is "
+          + blockCachePercentMinRange);
+    }
+    gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
+    bcul = (int) (blockCachePercentMaxRange * CONVERT_TO_PERCENTAGE);
+    if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
+      throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+          + "the threshold required for successful cluster operation. "
+          + "The combined value cannot exceed 0.8. Please check the settings for "
+          + MEMSTORE_SIZE_MIN_RANGE_KEY + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY
+          + " in your configuration. " + MEMSTORE_SIZE_MIN_RANGE_KEY + " is "
+          + globalMemStorePercentMinRange + " and " + BLOCK_CACHE_SIZE_MAX_RANGE_KEY + " is "
+          + blockCachePercentMaxRange);
+    }
+    return true;
+  }
+
+  public void start() {
+    if (tunerOn) {
+      LOG.info("Starting HeapMemoryTuner chore.");
+      this.heapMemTunerChore = new HeapMemoryTunerChore();
+      Threads.setDaemonThreadRunning(heapMemTunerChore.getThread());
+      // Register HeapMemoryTuner as a memstore flush listener
+      memStoreFlusher.registerFlushRequestListener(heapMemTunerChore);
+    }
+  }
+
+  public void stop() {
+    // The thread is Daemon. Just interrupting the ongoing process.
+    if (tunerOn) {
+      LOG.info("Stoping HeapMemoryTuner chore.");
+      this.heapMemTunerChore.interrupt();
+    }
+  }
+
+  // Used by the test cases.
+  boolean isTunerOn() {
+    return this.tunerOn;
+  }
+
+  private class HeapMemoryTunerChore extends Chore implements FlushRequestListener {
+    private HeapMemoryTuner heapMemTuner;
+    private AtomicLong blockedFlushCount = new AtomicLong();
+    private AtomicLong unblockedFlushCount = new AtomicLong();
+    private long evictCount = 0L;
+    private TunerContext tunerContext = new TunerContext();
+
+    public HeapMemoryTunerChore() {
+      super(server.getServerName() + "-HeapMemoryTunerChore", server.getConfiguration().getInt(
+          HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, HBASE_RS_HEAP_MEMORY_TUNER_DEFAULT_PERIOD), server);
+      Class<? extends HeapMemoryTuner> tunerKlass = server.getConfiguration().getClass(
+          HBASE_RS_HEAP_MEMORY_TUNER_CLASS, DefaultHeapMemoryTuner.class, HeapMemoryTuner.class);
+      heapMemTuner = ReflectionUtils.newInstance(tunerKlass, server.getConfiguration());
+    }
+
+    @Override
+    protected void chore() {
+      evictCount = blockCache.getStats().getEvictedCount() - evictCount;
+      tunerContext.setBlockedFlushCount(blockedFlushCount.getAndSet(0));
+      tunerContext.setUnblockedFlushCount(unblockedFlushCount.getAndSet(0));
+      tunerContext.setEvictCount(evictCount);
+      tunerContext.setCurBlockCacheSize(blockCachePercent);
+      tunerContext.setCurMemStoreSize(globalMemStorePercent);
+      TunerResult result = null;
+      try {
+        result = this.heapMemTuner.tune(tunerContext);
+      } catch (Throwable t) {
+        LOG.error("Exception thrown from the HeapMemoryTuner implementation", t);
+      }
+      if (result != null && result.needsTuning()) {
+        float memstoreSize = result.getMemstoreSize();
+        float blockCacheSize = result.getBlockCacheSize();
+        LOG.debug("From HeapMemoryTuner new memstoreSize: " + memstoreSize
+            + ". new blockCacheSize: " + blockCacheSize);
+        if (memstoreSize < globalMemStorePercentMinRange) {
+          LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is below min level "
+              + globalMemStorePercentMinRange + ". Resetting memstoreSize to min size");
+          memstoreSize = globalMemStorePercentMinRange;
+        } else if (memstoreSize > globalMemStorePercentMaxRange) {
+          LOG.info("New memstoreSize from HeapMemoryTuner " + memstoreSize + " is above max level "
+              + globalMemStorePercentMaxRange + ". Resetting memstoreSize to max size");
+          memstoreSize = globalMemStorePercentMaxRange;
+        }
+        if (blockCacheSize < blockCachePercentMinRange) {
+          LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
+              + " is below min level " + blockCachePercentMinRange
+              + ". Resetting blockCacheSize to min size");
+          blockCacheSize = blockCachePercentMinRange;
+        } else if (blockCacheSize > blockCachePercentMaxRange) {
+          LOG.info("New blockCacheSize from HeapMemoryTuner " + blockCacheSize
+              + " is above max level " + blockCachePercentMaxRange
+              + ". Resetting blockCacheSize to min size");
+          blockCacheSize = blockCachePercentMaxRange;
+        }
+        int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
+        int bcul = (int) (blockCacheSize * CONVERT_TO_PERCENTAGE);
+        if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
+          LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
+              + "the threshold required for successful cluster operation. "
+              + "The combined value cannot exceed 0.8. " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+              + " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
+              + blockCacheSize);
+          // TODO can adjust the value so as not exceed 80%. Is that correct? may be.
+        } else {
+          long newBlockCacheSize = (long) (maxHeapSize * blockCacheSize);
+          long newMemstoreSize = (long) (maxHeapSize * memstoreSize);
+          LOG.info("Setting block cache heap size to " + newBlockCacheSize
+              + " and memstore heap size to " + newMemstoreSize);
+          blockCachePercent = blockCacheSize;
+          blockCache.setMaxSize(newBlockCacheSize);
+          globalMemStorePercent = memstoreSize;
+          memStoreFlusher.setGlobalMemstoreLimit(newMemstoreSize);
+        }
+      }
+    }
+
+    @Override
+    public void flushRequested(FlushType type, HRegion region) {
+      switch (type) {
+      case ABOVE_HIGHER_MARK:
+        blockedFlushCount.incrementAndGet();
+        break;
+      case ABOVE_LOWER_MARK:
+        unblockedFlushCount.incrementAndGet();
+        break;
+      default:
+        // In case of normal flush don't do any action.
+        break;
+      }
+    }
+  }
+
+  /**
+   * POJO to pass all the relevant information required to do the heap memory tuning. It holds the
+   * flush counts and block cache evictions happened within the interval. Also holds the current
+   * heap percentage allocated for memstore and block cache.
+   */
+  public static final class TunerContext {
+    private long blockedFlushCount;
+    private long unblockedFlushCount;
+    private long evictCount;
+    private float curMemStoreSize;
+    private float curBlockCacheSize;
+
+    public long getBlockedFlushCount() {
+      return blockedFlushCount;
+    }
+
+    public void setBlockedFlushCount(long blockedFlushCount) {
+      this.blockedFlushCount = blockedFlushCount;
+    }
+
+    public long getUnblockedFlushCount() {
+      return unblockedFlushCount;
+    }
+
+    public void setUnblockedFlushCount(long unblockedFlushCount) {
+      this.unblockedFlushCount = unblockedFlushCount;
+    }
+
+    public long getEvictCount() {
+      return evictCount;
+    }
+
+    public void setEvictCount(long evictCount) {
+      this.evictCount = evictCount;
+    }
+
+    public float getCurMemStoreSize() {
+      return curMemStoreSize;
+    }
+
+    public void setCurMemStoreSize(float curMemStoreSize) {
+      this.curMemStoreSize = curMemStoreSize;
+    }
+
+    public float getCurBlockCacheSize() {
+      return curBlockCacheSize;
+    }
+
+    public void setCurBlockCacheSize(float curBlockCacheSize) {
+      this.curBlockCacheSize = curBlockCacheSize;
+    }
+  }
+
+  /**
+   * POJO which holds the result of memory tuning done by HeapMemoryTuner implementation.
+   * It includes the new heap percentage for memstore and block cache.
+   */
+  public static final class TunerResult {
+    private float memstoreSize;
+    private float blockCacheSize;
+    private final boolean needsTuning;
+
+    public TunerResult(boolean needsTuning) {
+      this.needsTuning = needsTuning;
+    }
+
+    public float getMemstoreSize() {
+      return memstoreSize;
+    }
+
+    public void setMemstoreSize(float memstoreSize) {
+      this.memstoreSize = memstoreSize;
+    }
+
+    public float getBlockCacheSize() {
+      return blockCacheSize;
+    }
+
+    public void setBlockCacheSize(float blockCacheSize) {
+      this.blockCacheSize = blockCacheSize;
+    }
+
+    public boolean needsTuning() {
+      return needsTuning;
+    }
+  }
+}

Added: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryTuner.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryTuner.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryTuner.java (added)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryTuner.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,42 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
+
+/**
+ * Makes the decision regarding proper sizing of the heap memory. Decides what percentage of heap
+ * memory should be allocated for global memstore and BlockCache.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public interface HeapMemoryTuner extends Configurable {
+
+  /**
+   * Perform the heap memory tuning operation.
+   * 
+   * @param context
+   * @return <code>TunerResult</code> including the heap percentage for memstore and block cache
+   */
+  TunerResult tune(TunerContext context);
+}

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java Wed Dec 11 07:03:40 2013
@@ -194,8 +194,7 @@ public class MemStoreChunkPool {
     }
     long heapMax = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()
         .getMax();
-    long globalMemStoreLimit = MemStoreFlusher.globalMemStoreLimit(heapMax,
-        MemStoreFlusher.DEFAULT_UPPER, MemStoreFlusher.UPPER_KEY, conf);
+    long globalMemStoreLimit = (long) (heapMax * MemStoreFlusher.getGlobalMemStorePercent(conf));
     int chunkSize = conf.getInt(MemStoreLAB.CHUNK_SIZE_KEY,
         MemStoreLAB.CHUNK_SIZE_DEFAULT);
     int maxCount = (int) (globalMemStoreLimit * poolSizePercentage / chunkSize);

Modified: hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java (original)
+++ hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java Wed Dec 11 07:03:40 2013
@@ -21,9 +21,11 @@ package org.apache.hadoop.hbase.regionse
 import java.io.IOException;
 import java.lang.Thread.UncaughtExceptionHandler;
 import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
 import java.util.ConcurrentModificationException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.SortedMap;
@@ -65,6 +67,18 @@ import org.cloudera.htrace.TraceScope;
 @InterfaceAudience.Private
 class MemStoreFlusher implements FlushRequester {
   static final Log LOG = LogFactory.getLog(MemStoreFlusher.class);
+  static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
+  private static final String MEMSTORE_SIZE_OLD_KEY = 
+      "hbase.regionserver.global.memstore.upperLimit";
+  private static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY = 
+      "hbase.regionserver.global.memstore.size.lower.limit";
+  private static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY = 
+      "hbase.regionserver.global.memstore.lowerLimit";
+
+  private static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
+  // Default lower water mark limit is 95% size of memstore size.
+  private static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
+
   // These two data members go together.  Any entry in the one must have
   // a corresponding entry in the other.
   private final BlockingQueue<FlushQueueEntry> flushQueue =
@@ -78,19 +92,15 @@ class MemStoreFlusher implements FlushRe
   private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
   private final Object blockSignal = new Object();
 
-  protected final long globalMemStoreLimit;
-  protected final long globalMemStoreLimitLowMark;
+  protected long globalMemStoreLimit;
+  protected float globalMemStoreLimitLowMarkPercent;
+  protected long globalMemStoreLimitLowMark;
 
-  static final float DEFAULT_UPPER = 0.4f;
-  private static final float DEFAULT_LOWER = 0.35f;
-  static final String UPPER_KEY =
-    "hbase.regionserver.global.memstore.upperLimit";
-  private static final String LOWER_KEY =
-    "hbase.regionserver.global.memstore.lowerLimit";
   private long blockingWaitTime;
   private final Counter updatesBlockedMsHighWater = new Counter();
 
   private final FlushHandler[] flushHandlers;
+  private List<FlushRequestListener> flushRequestListeners = new ArrayList<FlushRequestListener>(1);
 
   /**
    * @param conf
@@ -103,15 +113,13 @@ class MemStoreFlusher implements FlushRe
     this.threadWakeFrequency =
       conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
     long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
-    this.globalMemStoreLimit = globalMemStoreLimit(max, DEFAULT_UPPER,
-      UPPER_KEY, conf);
-    long lower = globalMemStoreLimit(max, DEFAULT_LOWER, LOWER_KEY, conf);
-    if (lower > this.globalMemStoreLimit) {
-      lower = this.globalMemStoreLimit;
-      LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " +
-        "because supplied " + LOWER_KEY + " was > " + UPPER_KEY);
-    }
-    this.globalMemStoreLimitLowMark = lower;
+    float globalMemStorePercent = getGlobalMemStorePercent(conf);
+    this.globalMemStoreLimit = (long) (max * globalMemStorePercent);
+    this.globalMemStoreLimitLowMarkPercent = 
+        getGlobalMemStoreLowerMark(conf, globalMemStorePercent);
+    this.globalMemStoreLimitLowMark = 
+        (long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent);
+
     this.blockingWaitTime = conf.getInt("hbase.hstore.blockingWaitTime",
       90000);
     int handlerCount = conf.getInt("hbase.hstore.flusher.count", 1);
@@ -124,29 +132,40 @@ class MemStoreFlusher implements FlushRe
   }
 
   /**
-   * Calculate size using passed <code>key</code> for configured
-   * percentage of <code>max</code>.
+   * Calculate global memstore size for configured percentage of <code>max</code>.
    * @param max
-   * @param defaultLimit
-   * @param key
    * @param c
    * @return Limit.
    */
-  static long globalMemStoreLimit(final long max,
-     final float defaultLimit, final String key, final Configuration c) {
-    float limit = c.getFloat(key, defaultLimit);
-    return getMemStoreLimit(max, limit, defaultLimit);
-  }
-
-  static long getMemStoreLimit(final long max, final float limit,
-      final float defaultLimit) {
-    float effectiveLimit = limit;
-    if (limit >= 0.9f || limit < 0.1f) {
-      LOG.warn("Setting global memstore limit to default of " + defaultLimit +
-        " because supplied value outside allowed range of 0.1 -> 0.9");
-      effectiveLimit = defaultLimit;
+  static float getGlobalMemStorePercent(final Configuration c) {
+    float limit = c.getFloat(MEMSTORE_SIZE_KEY,
+        c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
+    if (limit > 0.8f || limit < 0.05f) {
+      LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
+          + " because supplied value outside allowed range of 0.05 -> 0.8");
+      limit = DEFAULT_MEMSTORE_SIZE;
+    }
+    return limit;
+  }
+
+  private static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
+    String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
+    if (lowMarkPercentStr != null) {
+      return Float.parseFloat(lowMarkPercentStr);
+    }
+    String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
+    if (lowerWaterMarkOldValStr != null) {
+      LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
+          + MEMSTORE_SIZE_LOWER_LIMIT_KEY);
+      float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
+      if (lowerWaterMarkOldVal > globalMemStorePercent) {
+        lowerWaterMarkOldVal = globalMemStorePercent;
+        LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
+            + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
+      }
+      return lowerWaterMarkOldVal / globalMemStorePercent;
     }
-    return (long)(max * effectiveLimit);
+    return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
   }
 
   public Counter getUpdatesBlockedMsHighWater() {
@@ -453,6 +472,7 @@ class MemStoreFlusher implements FlushRe
     }
     lock.readLock().lock();
     try {
+      notifyFlushRequest(region, emergencyFlush);
       boolean shouldCompact = region.flushcache();
       // We just want to check the size
       boolean shouldSplit = region.checkSplit() != null;
@@ -485,6 +505,16 @@ class MemStoreFlusher implements FlushRe
     return true;
   }
 
+  private void notifyFlushRequest(HRegion region, boolean emergencyFlush) {
+    FlushType type = FlushType.NORMAL;
+    if (emergencyFlush) {
+      type = isAboveHighWaterMark() ? FlushType.ABOVE_HIGHER_MARK : FlushType.ABOVE_LOWER_MARK;
+    }
+    for (FlushRequestListener listener : flushRequestListeners) {
+      listener.flushRequested(type, region);
+    }
+  }
+
   private void wakeUpIfBlocking() {
     synchronized (blockSignal) {
       blockSignal.notifyAll();
@@ -570,6 +600,38 @@ class MemStoreFlusher implements FlushRe
     return queueList.toString();
   }
 
+  /**
+   * Register a MemstoreFlushListener
+   * @param listener
+   */
+  public void registerFlushRequestListener(final FlushRequestListener listener) {
+    this.flushRequestListeners.add(listener);
+  }
+
+  /**
+   * Unregister the listener from MemstoreFlushListeners
+   * @param listener
+   * @return true when passed listener is unregistered successfully.
+   */
+  public boolean unregisterFlushRequestListener(final FlushRequestListener listener) {
+    return this.flushRequestListeners.remove(listener);
+  }
+
+  /**
+   * Sets the global memstore limit to a new size.
+   * @param globalMemStoreSize
+   */
+  public void setGlobalMemstoreLimit(long globalMemStoreSize) {
+    this.globalMemStoreLimit = globalMemStoreSize;
+    this.globalMemStoreLimitLowMark = 
+        (long) (this.globalMemStoreLimitLowMarkPercent * globalMemStoreSize);
+    reclaimMemStoreMemory();
+  }
+
+  public long getMemoryLimit() {
+    return this.globalMemStoreLimit;
+  }
+
   interface FlushQueueEntry extends Delayed {}
 
   /**
@@ -672,3 +734,7 @@ class MemStoreFlusher implements FlushRe
     }
   }
 }
+
+enum FlushType {
+  NORMAL, ABOVE_LOWER_MARK, ABOVE_HIGHER_MARK;
+}
\ No newline at end of file

Added: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java?rev=1550059&view=auto
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java (added)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java Wed Dec 11 07:03:40 2013
@@ -0,0 +1,453 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.regionserver;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.SmallTests;
+import org.apache.hadoop.hbase.catalog.CatalogTracker;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary;
+import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
+import org.apache.hadoop.hbase.io.hfile.CacheStats;
+import org.apache.hadoop.hbase.io.hfile.Cacheable;
+import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
+import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
+import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(SmallTests.class)
+public class TestHeapMemoryManager {
+
+  private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+
+  @Test
+  public void testAutoTunerShouldBeOffWhenMaxMinRangesForMemstoreIsNotGiven() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
+    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
+        new MemstoreFlusherStub(0), new RegionServerStub(conf));
+    assertFalse(manager.isTunerOn());
+  }
+
+  @Test
+  public void testAutoTunerShouldBeOffWhenMaxMinRangesForBlockCacheIsNotGiven() throws Exception {
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
+    HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0),
+        new MemstoreFlusherStub(0), new RegionServerStub(conf));
+    assertFalse(manager.isTunerOn());
+  }
+
+  @Test
+  public void testWhenMemstoreAndBlockCacheMaxMinChecksFails() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub(0);
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub(0);
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f);
+    try {
+      new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf));
+      fail();
+    } catch (RuntimeException e) {
+    }
+    conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    try {
+      new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf));
+      fail();
+    } catch (RuntimeException e) {
+    }
+  }
+
+  @Test
+  public void testWhenClusterIsWriteHeavy() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    // Let the system start with default values for memstore heap and block cache size.
+    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
+        new RegionServerStub(conf));
+    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
+    long oldBlockCacheSize = blockCache.maxSize;
+    heapMemoryManager.start();
+    memStoreFlusher.flushType = FlushType.ABOVE_HIGHER_MARK;
+    memStoreFlusher.requestFlush(null);
+    memStoreFlusher.requestFlush(null);
+    memStoreFlusher.requestFlush(null);
+    memStoreFlusher.flushType = FlushType.ABOVE_LOWER_MARK;
+    memStoreFlusher.requestFlush(null);
+    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
+    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE, oldMemstoreHeapSize,
+        memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE), oldBlockCacheSize,
+        blockCache.maxSize);
+    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
+    oldBlockCacheSize = blockCache.maxSize;
+    // Do some more flushes before the next run of HeapMemoryTuner
+    memStoreFlusher.flushType = FlushType.ABOVE_HIGHER_MARK;
+    memStoreFlusher.requestFlush(null);
+    memStoreFlusher.requestFlush(null);
+    Thread.sleep(1500);
+    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE, oldMemstoreHeapSize,
+        memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE), oldBlockCacheSize,
+        blockCache.maxSize);
+  }
+
+  @Test
+  public void testWhenClusterIsReadHeavy() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    // Let the system start with default values for memstore heap and block cache size.
+    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
+        new RegionServerStub(conf));
+    long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
+    long oldBlockCacheSize = blockCache.maxSize;
+    heapMemoryManager.start();
+    blockCache.evictBlock(null);
+    blockCache.evictBlock(null);
+    blockCache.evictBlock(null);
+    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
+    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE), oldMemstoreHeapSize,
+        memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE, oldBlockCacheSize,
+        blockCache.maxSize);
+    oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
+    oldBlockCacheSize = blockCache.maxSize;
+    // Do some more evictions before the next run of HeapMemoryTuner
+    blockCache.evictBlock(null);
+    Thread.sleep(1500);
+    assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE), oldMemstoreHeapSize,
+        memStoreFlusher.memstoreSize);
+    assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_STEP_VALUE, oldBlockCacheSize,
+        blockCache.maxSize);
+  }
+
+  @Test
+  public void testPluggingInHeapMemoryTuner() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
+        HeapMemoryTuner.class);
+    // Let the system start with default values for memstore heap and block cache size.
+    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
+        new RegionServerStub(conf));
+    heapMemoryManager.start();
+    // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
+    CustomHeapMemoryTuner.memstoreSize = 0.78f;
+    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
+    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
+    assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
+    assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
+    // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
+    CustomHeapMemoryTuner.blockCacheSize = 0.75f;
+    CustomHeapMemoryTuner.memstoreSize = 0.05f;
+    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
+    assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
+    assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
+  }
+
+  @Test
+  public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
+        HeapMemoryTuner.class);
+    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
+        new RegionServerStub(conf));
+    heapMemoryManager.start();
+    CustomHeapMemoryTuner.memstoreSize = 0.78f;
+    CustomHeapMemoryTuner.blockCacheSize = 0.02f;
+    Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
+    // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
+    // upper bound. Same with block cache as 10% is the lower bound.
+    assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
+    assertHeapSpace(0.1f, blockCache.maxSize);
+  }
+
+  @Test
+  public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
+        HeapMemoryTuner.class);
+    HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
+        new RegionServerStub(conf));
+    long oldMemstoreSize = memStoreFlusher.memstoreSize;
+    long oldBlockCacheSize = blockCache.maxSize;
+    heapMemoryManager.start();
+    CustomHeapMemoryTuner.memstoreSize = 0.7f;
+    CustomHeapMemoryTuner.blockCacheSize = 0.3f;
+    Thread.sleep(1500);
+    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
+    assertEquals(oldBlockCacheSize, blockCache.maxSize);
+  }
+
+  private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
+    long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
+    assertEquals(expected, currentHeapSpace);
+  }
+
+  private void assertHeapSpaceDelta(float expectedDeltaPercent, long oldHeapSpace, long newHeapSpace) {
+    long expctedMinDelta = (long) (this.maxHeapSize * expectedDeltaPercent);
+    if (expectedDeltaPercent > 0) {
+      assertTrue(expctedMinDelta <= (newHeapSpace - oldHeapSpace));
+    } else {
+      assertTrue(expctedMinDelta <= (oldHeapSpace - newHeapSpace));
+    }
+  }
+
+  private static class BlockCacheStub implements ResizableBlockCache {
+
+    CacheStats stats = new CacheStats();
+    long maxSize = 0;
+    
+    public BlockCacheStub(long size){
+      this.maxSize = size;
+    }
+
+    @Override
+    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) {
+
+    }
+
+    @Override
+    public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
+
+    }
+
+    @Override
+    public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) {
+      return null;
+    }
+
+    @Override
+    public boolean evictBlock(BlockCacheKey cacheKey) {
+      stats.evicted();
+      return false;
+    }
+
+    @Override
+    public int evictBlocksByHfileName(String hfileName) {
+      stats.evicted(); // Just assuming only one block for file here.
+      return 0;
+    }
+
+    @Override
+    public CacheStats getStats() {
+      return this.stats;
+    }
+
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public long size() {
+      return 0;
+    }
+
+    @Override
+    public long getFreeSize() {
+      return 0;
+    }
+
+    @Override
+    public long getCurrentSize() {
+      return 0;
+    }
+
+    @Override
+    public long getEvictedCount() {
+      return 0;
+    }
+
+    @Override
+    public long getBlockCount() {
+      return 0;
+    }
+
+    @Override
+    public List<BlockCacheColumnFamilySummary> getBlockCacheColumnFamilySummaries(Configuration conf)
+        throws IOException {
+      return null;
+    }
+
+    @Override
+    public void setMaxSize(long size) {
+      this.maxSize = size;
+    }
+  }
+
+  private static class MemstoreFlusherStub implements FlushRequester {
+
+    long memstoreSize;
+
+    FlushRequestListener listener;
+
+    FlushType flushType = FlushType.NORMAL;
+
+    public MemstoreFlusherStub(long memstoreSize) {
+      this.memstoreSize = memstoreSize;
+    }
+
+    @Override
+    public void requestFlush(HRegion region) {
+      this.listener.flushRequested(flushType, region);
+    }
+
+    @Override
+    public void requestDelayedFlush(HRegion region, long delay) {
+
+    }
+
+    @Override
+    public void registerFlushRequestListener(FlushRequestListener listener) {
+      this.listener = listener;
+    }
+
+    @Override
+    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
+      return false;
+    }
+
+    @Override
+    public void setGlobalMemstoreLimit(long globalMemStoreSize) {
+      this.memstoreSize = globalMemStoreSize;
+    }
+  }
+
+  private static class RegionServerStub implements Server {
+    private Configuration conf;
+    private boolean stopped = false;
+
+    public RegionServerStub(Configuration conf) {
+      this.conf = conf;
+    }
+
+    @Override
+    public void abort(String why, Throwable e) {
+
+    }
+
+    @Override
+    public boolean isAborted() {
+      return false;
+    }
+
+    @Override
+    public void stop(String why) {
+      this.stopped = true;
+    }
+
+    @Override
+    public boolean isStopped() {
+      return this.stopped;
+    }
+
+    @Override
+    public Configuration getConfiguration() {
+      return this.conf;
+    }
+
+    @Override
+    public ZooKeeperWatcher getZooKeeper() {
+      return null;
+    }
+
+    @Override
+    public CatalogTracker getCatalogTracker() {
+      return null;
+    }
+
+    @Override
+    public ServerName getServerName() {
+      return ServerName.valueOf("server1",4000,12345);
+    }
+  }
+
+  static class CustomHeapMemoryTuner implements HeapMemoryTuner {
+    static float blockCacheSize = 0.4f;
+    static float memstoreSize = 0.4f;
+
+    @Override
+    public Configuration getConf() {
+      return null;
+    }
+
+    @Override
+    public void setConf(Configuration arg0) {
+
+    }
+
+    @Override
+    public TunerResult tune(TunerContext context) {
+      TunerResult result = new TunerResult(true);
+      result.setBlockCacheSize(blockCacheSize);
+      result.setMemstoreSize(memstoreSize);
+      return result;
+    }
+  }
+}

Modified: hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java?rev=1550059&r1=1550058&r2=1550059&view=diff
==============================================================================
--- hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java (original)
+++ hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALReplay.java Wed Dec 11 07:03:40 2013
@@ -65,6 +65,7 @@ import org.apache.hadoop.hbase.master.HM
 import org.apache.hadoop.hbase.monitoring.MonitoredTask;
 import org.apache.hadoop.hbase.regionserver.DefaultStoreEngine;
 import org.apache.hadoop.hbase.regionserver.DefaultStoreFlusher;
+import org.apache.hadoop.hbase.regionserver.FlushRequestListener;
 import org.apache.hadoop.hbase.regionserver.FlushRequester;
 import org.apache.hadoop.hbase.regionserver.HRegion;
 import org.apache.hadoop.hbase.regionserver.HRegionServer;
@@ -867,6 +868,21 @@ public class TestWALReplay {
       // TODO Auto-generated method stub
       
     }
+
+    @Override
+    public void registerFlushRequestListener(FlushRequestListener listener) {
+
+    }
+
+    @Override
+    public boolean unregisterFlushRequestListener(FlushRequestListener listener) {
+      return false;
+    }
+
+    @Override
+    public void setGlobalMemstoreLimit(long globalMemStoreSize) {
+
+    }
   }
 
   private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,