You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by an...@apache.org on 2014/08/07 13:46:55 UTC
git commit: HBASE-11527 Cluster free memory limit check should
consider L2 block cache size also when L2 cache is onheap. (Anoop)
Repository: hbase
Updated Branches:
refs/heads/master 17dff6818 -> 12d9697d9
HBASE-11527 Cluster free memory limit check should consider L2 block cache size also when L2 cache is onheap. (Anoop)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/12d9697d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/12d9697d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/12d9697d
Branch: refs/heads/master
Commit: 12d9697d934df90e0ed0261aa20446120c1086a6
Parents: 17dff68
Author: anoopsjohn <an...@gmail.com>
Authored: Thu Aug 7 17:16:16 2014 +0530
Committer: anoopsjohn <an...@gmail.com>
Committed: Thu Aug 7 17:16:16 2014 +0530
----------------------------------------------------------------------
.../apache/hadoop/hbase/HBaseConfiguration.java | 33 +----
.../org/apache/hadoop/hbase/HConstants.java | 22 +++
.../hbase/io/util/HeapMemorySizeUtil.java | 136 +++++++++++++++++++
.../hadoop/hbase/io/hfile/CacheConfig.java | 18 +--
.../hbase/mapreduce/TableMapReduceUtil.java | 2 +-
.../regionserver/DefaultHeapMemoryTuner.java | 5 +-
.../hbase/regionserver/HeapMemoryManager.java | 25 ++--
.../hbase/regionserver/MemStoreChunkPool.java | 4 +-
.../hbase/regionserver/MemStoreFlusher.java | 50 +------
.../hbase/io/hfile/TestBlockCacheReporting.java | 5 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 16 +--
11 files changed, 192 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
index a8f9cf9..9a0b38b 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java
@@ -24,6 +24,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.VersionInfo;
/**
@@ -35,9 +36,6 @@ public class HBaseConfiguration extends Configuration {
private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class);
- // a constant to convert a fraction to a percentage
- private static final int CONVERT_TO_PERCENTAGE = 100;
-
/**
* Instantinating HBaseConfiguration() is deprecated. Please use
* HBaseConfiguration#create() to construct a plain Configuration
@@ -73,39 +71,12 @@ public class HBaseConfiguration extends Configuration {
}
}
- private static void checkForClusterFreeMemoryLimit(Configuration conf) {
- if (conf.get("hbase.regionserver.global.memstore.upperLimit") != null) {
- LOG.warn("hbase.regionserver.global.memstore.upperLimit is deprecated by "
- + "hbase.regionserver.global.memstore.size");
- }
- float globalMemstoreSize = conf.getFloat("hbase.regionserver.global.memstore.size",
- conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f));
- int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
- float blockCacheUpperLimit =
- conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
- HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
- int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
- if (CONVERT_TO_PERCENTAGE - (gml + bcul)
- < (int)(CONVERT_TO_PERCENTAGE *
- HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
- throw new RuntimeException(
- "Current heap configuration for MemStore and BlockCache exceeds " +
- "the threshold required for successful cluster operation. " +
- "The combined value cannot exceed 0.8. Please check " +
- "the settings for hbase.regionserver.global.memstore.size and " +
- "hfile.block.cache.size in your configuration. " +
- "hbase.regionserver.global.memstore.size is " +
- globalMemstoreSize +
- " hfile.block.cache.size is " + blockCacheUpperLimit);
- }
- }
-
public static Configuration addHbaseResources(Configuration conf) {
conf.addResource("hbase-default.xml");
conf.addResource("hbase-site.xml");
checkDefaultsVersion(conf);
- checkForClusterFreeMemoryLimit(conf);
+ HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
return conf;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 93209fd..24fa8e5 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1017,6 +1017,28 @@ public final class HConstants {
public static final String HBASE_COORDINATED_STATE_MANAGER_CLASS =
"hbase.coordinated.state.manager.class";
+ /**
+ * Configuration keys for Bucket cache
+ */
+ // TODO moving these bucket cache implementation specific configs to this level is violation of
+ // encapsulation. But as these has to be referred from hbase-common and bucket cache
+ // sits in hbase-server, there were no other go! Can we move the cache implementation to
+ // hbase-common?
+
+ /**
+ * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
+ * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
+ * list of supported ioengine options.
+ * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
+ */
+ public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
+
+ /**
+ * When using bucket cache, this is a float that EITHER represents a percentage of total heap
+ * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache.
+ */
+ public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
+
private HConstants() {
// Can't be instantiated with this ctor.
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
new file mode 100644
index 0000000..ab7b715
--- /dev/null
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.util;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryUsage;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HConstants;
+
+@InterfaceAudience.Private
+public class HeapMemorySizeUtil {
+
+ public static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
+ public static final String MEMSTORE_SIZE_OLD_KEY =
+ "hbase.regionserver.global.memstore.upperLimit";
+ public static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
+ "hbase.regionserver.global.memstore.size.lower.limit";
+ public static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
+ "hbase.regionserver.global.memstore.lowerLimit";
+
+ public static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
+ // Default lower water mark limit is 95% size of memstore size.
+ public static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
+
+ private static final Log LOG = LogFactory.getLog(HeapMemorySizeUtil.class);
+ // a constant to convert a fraction to a percentage
+ private static final int CONVERT_TO_PERCENTAGE = 100;
+
+ /**
+ * Checks whether we have enough heap memory left out after portion for Memstore and Block cache.
+ * We need atleast 20% of heap left out for other RS functions.
+ * @param conf
+ */
+ public static void checkForClusterFreeMemoryLimit(Configuration conf) {
+ if (conf.get(MEMSTORE_SIZE_OLD_KEY) != null) {
+ LOG.warn(MEMSTORE_SIZE_OLD_KEY + " is deprecated by " + MEMSTORE_SIZE_KEY);
+ }
+ float globalMemstoreSize = getGlobalMemStorePercent(conf, false);
+ int gml = (int)(globalMemstoreSize * CONVERT_TO_PERCENTAGE);
+ float blockCacheUpperLimit = getBlockCacheHeapPercent(conf);
+ int bcul = (int)(blockCacheUpperLimit * CONVERT_TO_PERCENTAGE);
+ if (CONVERT_TO_PERCENTAGE - (gml + bcul)
+ < (int)(CONVERT_TO_PERCENTAGE *
+ HConstants.HBASE_CLUSTER_MINIMUM_MEMORY_THRESHOLD)) {
+ throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ + "the threshold required for successful cluster operation. "
+ + "The combined value cannot exceed 0.8. Please check "
+ + "the settings for hbase.regionserver.global.memstore.size and "
+ + "hfile.block.cache.size in your configuration. "
+ + "hbase.regionserver.global.memstore.size is " + globalMemstoreSize
+ + " hfile.block.cache.size is " + blockCacheUpperLimit);
+ }
+ }
+
+ /**
+ * Retrieve global memstore configured size as percentage of total heap.
+ * @param conf
+ * @param logInvalid
+ */
+ public static float getGlobalMemStorePercent(final Configuration c, final boolean logInvalid) {
+ float limit = c.getFloat(MEMSTORE_SIZE_KEY,
+ c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
+ if (limit > 0.8f || limit < 0.05f) {
+ if (logInvalid) {
+ LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
+ + " because supplied value outside allowed range of 0.05 -> 0.8");
+ }
+ limit = DEFAULT_MEMSTORE_SIZE;
+ }
+ return limit;
+ }
+
+ /**
+ * Retrieve configured size for global memstore lower water mark as percentage of total heap.
+ * @param conf
+ * @param globalMemStorePercent
+ */
+ public static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
+ String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
+ if (lowMarkPercentStr != null) {
+ return Float.parseFloat(lowMarkPercentStr);
+ }
+ String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
+ if (lowerWaterMarkOldValStr != null) {
+ LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
+ + MEMSTORE_SIZE_LOWER_LIMIT_KEY);
+ float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
+ if (lowerWaterMarkOldVal > globalMemStorePercent) {
+ lowerWaterMarkOldVal = globalMemStorePercent;
+ LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
+ + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
+ }
+ return lowerWaterMarkOldVal / globalMemStorePercent;
+ }
+ return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
+ }
+
+ /**
+ * Retrieve configured size for on heap block cache as percentage of total heap.
+ * @param conf
+ */
+ public static float getBlockCacheHeapPercent(final Configuration conf) {
+ // L1 block cache is always on heap
+ float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
+ HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+ float l2CachePercent = 0.0F;
+ String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null);
+ // L2 block cache can be on heap when IOEngine is "heap"
+ if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) {
+ float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F);
+ MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+ l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage
+ : (bucketCachePercentage * 1024 * 1024) / mu.getMax();
+ }
+ return l1CachePercent + l2CachePercent;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 25f9727..51f3543 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY;
+import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
+
import java.io.IOException;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryUsage;
@@ -79,21 +82,6 @@ public class CacheConfig {
*/
/**
- * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path
- * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for
- * list of supported ioengine options.
- *
- * <p>Set this option and a non-zero {@link #BUCKET_CACHE_SIZE_KEY} to enable bucket cache.
- */
- public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine";
-
- /**
- * When using bucket cache, this is a float that EITHER represents a percentage of total heap
- * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache.
- */
- public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
-
- /**
* If the chosen ioengine can persist its state across restarts, the path to the file to
* persist to.
*/
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
index 71c60b2..2337a96 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
@@ -281,7 +281,7 @@ public class TableMapReduceUtil {
public static void resetCacheConfig(Configuration conf) {
conf.setFloat(
HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
- conf.setFloat(CacheConfig.BUCKET_CACHE_SIZE_KEY, 0f);
+ conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0f);
conf.setFloat("hbase.bucketcache.size", 0f);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
index 75caf1e..4d8ddb2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
@@ -27,6 +27,7 @@ import static org.apache.hadoop.hbase.regionserver.HeapMemoryManager.MEMSTORE_SI
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
@@ -109,8 +110,8 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
this.blockCachePercentMaxRange = conf.getFloat(BLOCK_CACHE_SIZE_MAX_RANGE_KEY,
conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT));
this.globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
- MemStoreFlusher.getGlobalMemStorePercent(conf));
+ HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
this.globalMemStorePercentMaxRange = conf.getFloat(MEMSTORE_SIZE_MAX_RANGE_KEY,
- MemStoreFlusher.getGlobalMemStorePercent(conf));
+ HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false));
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
index 3869fad..b90a0c5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hbase.Server;
import org.apache.hadoop.hbase.io.hfile.BlockCache;
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.util.ReflectionUtils;
@@ -96,20 +97,10 @@ public class HeapMemoryManager {
}
private boolean doInit(Configuration conf) {
- globalMemStorePercent = MemStoreFlusher.getGlobalMemStorePercent(conf);
+ globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, false);
blockCachePercent = conf.getFloat(HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
- int gml = (int) (globalMemStorePercent * CONVERT_TO_PERCENTAGE);
- int bcul = (int) (blockCachePercent * CONVERT_TO_PERCENTAGE);
- if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
- throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
- + "the threshold required for successful cluster operation. "
- + "The combined value cannot exceed 0.8. Please check " + "the settings for "
- + MemStoreFlusher.MEMSTORE_SIZE_KEY + " and " + HFILE_BLOCK_CACHE_SIZE_KEY
- + " in your configuration. " + MemStoreFlusher.MEMSTORE_SIZE_KEY + " is "
- + globalMemStorePercent + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
- + blockCachePercent);
- }
+ HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf);
// Initialize max and min range for memstore heap space
globalMemStorePercentMinRange = conf.getFloat(MEMSTORE_SIZE_MIN_RANGE_KEY,
globalMemStorePercent);
@@ -117,14 +108,14 @@ public class HeapMemoryManager {
globalMemStorePercent);
if (globalMemStorePercent < globalMemStorePercentMinRange) {
LOG.warn("Setting " + MEMSTORE_SIZE_MIN_RANGE_KEY + " to " + globalMemStorePercent
- + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " because supplied value greater than initial memstore size value.");
globalMemStorePercentMinRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MIN_RANGE_KEY, globalMemStorePercentMinRange);
}
if (globalMemStorePercent > globalMemStorePercentMaxRange) {
LOG.warn("Setting " + MEMSTORE_SIZE_MAX_RANGE_KEY + " to " + globalMemStorePercent
- + ", same value as " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ + ", same value as " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " because supplied value less than initial memstore size value.");
globalMemStorePercentMaxRange = globalMemStorePercent;
conf.setFloat(MEMSTORE_SIZE_MAX_RANGE_KEY, globalMemStorePercentMaxRange);
@@ -155,8 +146,8 @@ public class HeapMemoryManager {
return false;
}
- gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
- bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE);
+ int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
+ int bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE);
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
+ "the threshold required for successful cluster operation. "
@@ -262,7 +253,7 @@ public class HeapMemoryManager {
if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
+ "the threshold required for successful cluster operation. "
- + "The combined value cannot exceed 0.8. " + MemStoreFlusher.MEMSTORE_SIZE_KEY
+ + "The combined value cannot exceed 0.8. " + HeapMemorySizeUtil.MEMSTORE_SIZE_KEY
+ " is " + memstoreSize + " and " + HFILE_BLOCK_CACHE_SIZE_KEY + " is "
+ blockCacheSize);
// TODO can adjust the value so as not exceed 80%. Is that correct? may be.
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
index 615b860..48bf221 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.regionserver.HeapMemStoreLAB.Chunk;
import org.apache.hadoop.util.StringUtils;
@@ -192,7 +193,8 @@ public class MemStoreChunkPool {
throw new IllegalArgumentException(CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0");
}
long heapMax = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
- long globalMemStoreLimit = (long) (heapMax * MemStoreFlusher.getGlobalMemStorePercent(conf));
+ long globalMemStoreLimit = (long) (heapMax * HeapMemorySizeUtil.getGlobalMemStorePercent(conf,
+ false));
int chunkSize = conf.getInt(HeapMemStoreLAB.CHUNK_SIZE_KEY,
HeapMemStoreLAB.CHUNK_SIZE_DEFAULT);
int maxCount = (int) (globalMemStoreLimit * poolSizePercentage / chunkSize);
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
index c9d54cc..eed98e5 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.DroppedSnapshotException;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Counter;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -67,17 +68,6 @@ import com.google.common.base.Preconditions;
@InterfaceAudience.Private
class MemStoreFlusher implements FlushRequester {
static final Log LOG = LogFactory.getLog(MemStoreFlusher.class);
- static final String MEMSTORE_SIZE_KEY = "hbase.regionserver.global.memstore.size";
- private static final String MEMSTORE_SIZE_OLD_KEY =
- "hbase.regionserver.global.memstore.upperLimit";
- private static final String MEMSTORE_SIZE_LOWER_LIMIT_KEY =
- "hbase.regionserver.global.memstore.size.lower.limit";
- private static final String MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY =
- "hbase.regionserver.global.memstore.lowerLimit";
-
- private static final float DEFAULT_MEMSTORE_SIZE = 0.4f;
- // Default lower water mark limit is 95% size of memstore size.
- private static final float DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT = 0.95f;
// These two data members go together. Any entry in the one must have
// a corresponding entry in the other.
@@ -113,10 +103,10 @@ class MemStoreFlusher implements FlushRequester {
this.threadWakeFrequency =
conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000);
long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
- float globalMemStorePercent = getGlobalMemStorePercent(conf);
+ float globalMemStorePercent = HeapMemorySizeUtil.getGlobalMemStorePercent(conf, true);
this.globalMemStoreLimit = (long) (max * globalMemStorePercent);
this.globalMemStoreLimitLowMarkPercent =
- getGlobalMemStoreLowerMark(conf, globalMemStorePercent);
+ HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf, globalMemStorePercent);
this.globalMemStoreLimitLowMark =
(long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent);
@@ -131,40 +121,6 @@ class MemStoreFlusher implements FlushRequester {
", maxHeap=" + StringUtils.humanReadableInt(max));
}
- /**
- * Retrieve global memstore configured size as percentage of total heap.
- */
- static float getGlobalMemStorePercent(final Configuration c) {
- float limit = c.getFloat(MEMSTORE_SIZE_KEY,
- c.getFloat(MEMSTORE_SIZE_OLD_KEY, DEFAULT_MEMSTORE_SIZE));
- if (limit > 0.8f || limit < 0.05f) {
- LOG.warn("Setting global memstore limit to default of " + DEFAULT_MEMSTORE_SIZE
- + " because supplied value outside allowed range of 0.05 -> 0.8");
- limit = DEFAULT_MEMSTORE_SIZE;
- }
- return limit;
- }
-
- private static float getGlobalMemStoreLowerMark(final Configuration c, float globalMemStorePercent) {
- String lowMarkPercentStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_KEY);
- if (lowMarkPercentStr != null) {
- return Float.parseFloat(lowMarkPercentStr);
- }
- String lowerWaterMarkOldValStr = c.get(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY);
- if (lowerWaterMarkOldValStr != null) {
- LOG.warn(MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " is deprecated. Instead use "
- + MEMSTORE_SIZE_LOWER_LIMIT_KEY);
- float lowerWaterMarkOldVal = Float.parseFloat(lowerWaterMarkOldValStr);
- if (lowerWaterMarkOldVal > globalMemStorePercent) {
- lowerWaterMarkOldVal = globalMemStorePercent;
- LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + "because supplied "
- + MEMSTORE_SIZE_LOWER_LIMIT_OLD_KEY + " was > " + MEMSTORE_SIZE_OLD_KEY);
- }
- return lowerWaterMarkOldVal / globalMemStorePercent;
- }
- return DEFAULT_MEMSTORE_SIZE_LOWER_LIMIT;
- }
-
public Counter getUpdatesBlockedMsHighWater() {
return this.updatesBlockedMsHighWater;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
index bbf385b..aca0021 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.SmallTests;
import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.DataCacheEntry;
import org.apache.hadoop.hbase.io.hfile.TestCacheConfig.IndexCacheEntry;
@@ -76,8 +77,8 @@ public class TestBlockCacheReporting {
@Test
public void testBucketCache() throws JsonGenerationException, JsonMappingException, IOException {
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
- this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100);
CacheConfig cc = new CacheConfig(this.conf);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
logPerBlock(cc.getBlockCache());
http://git-wip-us.apache.org/repos/asf/hbase/blob/12d9697d/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index be86abb..e6a476c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -214,13 +214,13 @@ public class TestCacheConfig {
*/
@Test
public void testOffHeapBucketCacheConfig() {
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
doBucketCacheConfigTest();
}
@Test
public void testOnHeapBucketCacheConfig() {
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "heap");
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap");
doBucketCacheConfigTest();
}
@@ -231,7 +231,7 @@ public class TestCacheConfig {
Path p = new Path(htu.getDataTestDir(), "bc.txt");
FileSystem fs = FileSystem.get(this.conf);
fs.create(p).close();
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "file:" + p);
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "file:" + p);
doBucketCacheConfigTest();
} finally {
htu.cleanupTestDir();
@@ -240,7 +240,7 @@ public class TestCacheConfig {
private void doBucketCacheConfigTest() {
final int bcSize = 100;
- this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize);
+ this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
CacheConfig cc = new CacheConfig(this.conf);
basicBlockCacheOps(cc, false, false);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
@@ -263,7 +263,7 @@ public class TestCacheConfig {
*/
@Test (timeout=10000)
public void testBucketCacheConfigL1L2Setup() {
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
// Make lru size is smaller than bcSize for sure. Need this to be true so when eviction
// from L1 happens, it does not fail because L2 can't take the eviction because block too big.
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f);
@@ -272,7 +272,7 @@ public class TestCacheConfig {
final int bcSize = 100;
long bcExpectedSize = 100 * 1024 * 1024; // MB.
assertTrue(lruExpectedSize < bcExpectedSize);
- this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize);
+ this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, bcSize);
this.conf.setBoolean(CacheConfig.BUCKET_CACHE_COMBINED_KEY, false);
CacheConfig cc = new CacheConfig(this.conf);
basicBlockCacheOps(cc, false, false);
@@ -317,8 +317,8 @@ public class TestCacheConfig {
*/
@Test
public void testCacheDataInL1() {
- this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap");
- this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100);
+ this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
+ this.conf.setInt(HConstants.BUCKET_CACHE_SIZE_KEY, 100);
CacheConfig cc = new CacheConfig(this.conf);
assertTrue(cc.getBlockCache() instanceof CombinedBlockCache);
CombinedBlockCache cbc = (CombinedBlockCache)cc.getBlockCache();