You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2016/01/19 21:24:23 UTC
[1/2] hbase git commit: HBASE-15102 Fix HeapMemoryTuner overtuning
memstore
Repository: hbase
Updated Branches:
refs/heads/branch-1.2 4fb7babad -> e59536947
HBASE-15102 Fix HeapMemoryTuner overtuning memstore
HeapMemoryTuner often over tunes memstore without looking at
the lower limit of the previous memstore size and causing a
situation in which memstore used size suddenly exceeds the
total memstore size.
Signed-off-by: Elliott Clark <ec...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e5953694
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e5953694
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e5953694
Branch: refs/heads/branch-1.2
Commit: e5953694750bd64f26a4bed927d095755b7dd105
Parents: e738e69
Author: Ashu Pachauri <as...@gmail.com>
Authored: Wed Jan 13 13:49:43 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Tue Jan 19 12:23:52 2016 -0800
----------------------------------------------------------------------
.../regionserver/DefaultHeapMemoryTuner.java | 218 +++++++++++--------
.../regionserver/TestHeapMemoryManager.java | 23 +-
2 files changed, 143 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e5953694/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
index b6e81dd..1d237d0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
@@ -124,30 +124,113 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
@Override
public TunerResult tune(TunerContext context) {
- long blockedFlushCount = context.getBlockedFlushCount();
- long unblockedFlushCount = context.getUnblockedFlushCount();
- long evictCount = context.getEvictCount();
- long cacheMissCount = context.getCacheMissCount();
- long totalFlushCount = blockedFlushCount+unblockedFlushCount;
- rollingStatsForCacheMisses.insertDataValue(cacheMissCount);
- rollingStatsForFlushes.insertDataValue(totalFlushCount);
- rollingStatsForEvictions.insertDataValue(evictCount);
- StepDirection newTuneDirection = StepDirection.NEUTRAL;
+ float curMemstoreSize = context.getCurMemStoreSize();
+ float curBlockCacheSize = context.getCurBlockCacheSize();
+ addToRollingStats(context);
+
if (ignoreInitialPeriods < numPeriodsToIgnore) {
// Ignoring the first few tuner periods
ignoreInitialPeriods++;
rollingStatsForTunerSteps.insertDataValue(0);
return NO_OP_TUNER_RESULT;
}
- String tunerLog = "";
+ StepDirection newTuneDirection = getTuneDirection(context);
+
+ float newMemstoreSize;
+ float newBlockCacheSize;
+
+ // Adjusting step size for tuning to get to steady state or restart from steady state.
+ // Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back and forth
+ // per tuner operation and it can affect the performance of cluster so we keep on decreasing
+ // step size until everything settles.
+ if (prevTuneDirection == StepDirection.NEUTRAL
+ && newTuneDirection != StepDirection.NEUTRAL
+ && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
+ // Restarting the tuning from steady state and setting step size to maximum.
+ // The deviation cannot be that low if last period was neutral and some recent periods were
+ // not neutral.
+ step = maximumStepSize;
+ } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
+ && decayingTunerStepSizeSum < 0) ||
+ (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
+ && decayingTunerStepSizeSum > 0)) {
+ // Current step is opposite of past tuner actions so decrease the step size to reach steady
+ // state.
+ step = step/2.00f;
+ }
+ if (step < minimumStepSize) {
+ // If step size is too small then we do nothing.
+ LOG.debug("Tuner step size is too low; we will not perform any tuning this time.");
+ step = 0.0f;
+ newTuneDirection = StepDirection.NEUTRAL;
+ }
+ // Increase / decrease the memstore / block cahce sizes depending on new tuner step.
+ float globalMemstoreLowerMark = HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf,
+ curMemstoreSize);
+ // We don't want to exert immediate pressure on memstore. So, we decrease its size gracefully;
+ // we set a minimum bar in the middle of the total memstore size and the lower limit.
+ float minMemstoreSize = ((globalMemstoreLowerMark + 1) * curMemstoreSize) / 2.00f;
+
+ switch (newTuneDirection) {
+ case INCREASE_BLOCK_CACHE_SIZE:
+ if (curMemstoreSize - step < minMemstoreSize) {
+ step = curMemstoreSize - minMemstoreSize;
+ }
+ newMemstoreSize = curMemstoreSize - step;
+ newBlockCacheSize = curBlockCacheSize + step;
+ rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
+ break;
+ case INCREASE_MEMSTORE_SIZE:
+ newBlockCacheSize = curBlockCacheSize - step;
+ newMemstoreSize = curMemstoreSize + step;
+ rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
+ break;
+ default:
+ prevTuneDirection = StepDirection.NEUTRAL;
+ rollingStatsForTunerSteps.insertDataValue(0);
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
+ return NO_OP_TUNER_RESULT;
+ }
+ // Check we are within max/min bounds.
+ if (newMemstoreSize > globalMemStorePercentMaxRange) {
+ newMemstoreSize = globalMemStorePercentMaxRange;
+ } else if (newMemstoreSize < globalMemStorePercentMinRange) {
+ newMemstoreSize = globalMemStorePercentMinRange;
+ }
+ if (newBlockCacheSize > blockCachePercentMaxRange) {
+ newBlockCacheSize = blockCachePercentMaxRange;
+ } else if (newBlockCacheSize < blockCachePercentMinRange) {
+ newBlockCacheSize = blockCachePercentMinRange;
+ }
+ TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
+ TUNER_RESULT.setMemstoreSize(newMemstoreSize);
+ prevTuneDirection = newTuneDirection;
+ return TUNER_RESULT;
+ }
+
+ /**
+ * Determine best direction of tuning base on given context.
+ * @param context The tuner context.
+ * @return tuning direction.
+ */
+ private StepDirection getTuneDirection(TunerContext context) {
+ StepDirection newTuneDirection = StepDirection.NEUTRAL;
+ long blockedFlushCount = context.getBlockedFlushCount();
+ long unblockedFlushCount = context.getUnblockedFlushCount();
+ long evictCount = context.getEvictCount();
+ long cacheMissCount = context.getCacheMissCount();
+ long totalFlushCount = blockedFlushCount+unblockedFlushCount;
+ float curMemstoreSize = context.getCurMemStoreSize();
+ float curBlockCacheSize = context.getCurBlockCacheSize();
+ StringBuilder tunerLog = new StringBuilder();
// We can consider memstore or block cache to be sufficient if
// we are using only a minor fraction of what have been already provided to it.
boolean earlyMemstoreSufficientCheck = totalFlushCount == 0
- || context.getCurMemStoreUsed() < context.getCurMemStoreSize()*sufficientMemoryLevel;
+ || context.getCurMemStoreUsed() < curMemstoreSize * sufficientMemoryLevel;
boolean earlyBlockCacheSufficientCheck = evictCount == 0 ||
- context.getCurBlockCacheUsed() < context.getCurBlockCacheSize()*sufficientMemoryLevel;
- float newMemstoreSize;
- float newBlockCacheSize;
+ context.getCurBlockCacheUsed() < curBlockCacheSize * sufficientMemoryLevel;
if (earlyMemstoreSufficientCheck && earlyBlockCacheSufficientCheck) {
// Both memstore and block cache memory seems to be sufficient. No operation required.
newTuneDirection = StepDirection.NEUTRAL;
@@ -168,15 +251,15 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
case INCREASE_BLOCK_CACHE_SIZE:
if ((double)evictCount > rollingStatsForEvictions.getMean() ||
(double)totalFlushCount > rollingStatsForFlushes.getMean() +
- rollingStatsForFlushes.getDeviation()/2.00) {
+ rollingStatsForFlushes.getDeviation()/2.00) {
// Reverting previous step as it was not useful.
// Tuning failed to decrease evictions or tuning resulted in large number of flushes.
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
- tunerLog += "Reverting previous tuning.";
+ tunerLog.append("We will revert previous tuning");
if ((double)evictCount > rollingStatsForEvictions.getMean()) {
- tunerLog += " As could not decrease evctions sufficiently.";
+ tunerLog.append(" because we could not decrease evictions sufficiently.");
} else {
- tunerLog += " As number of flushes rose significantly.";
+ tunerLog.append(" because the number of flushes rose significantly.");
}
isReverting = true;
}
@@ -184,15 +267,15 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
case INCREASE_MEMSTORE_SIZE:
if ((double)totalFlushCount > rollingStatsForFlushes.getMean() ||
(double)evictCount > rollingStatsForEvictions.getMean() +
- rollingStatsForEvictions.getDeviation()/2.00) {
+ rollingStatsForEvictions.getDeviation()/2.00) {
// Reverting previous step as it was not useful.
// Tuning failed to decrease flushes or tuning resulted in large number of evictions.
newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
- tunerLog += "Reverting previous tuning.";
+ tunerLog.append("We will revert previous tuning");
if ((double)totalFlushCount > rollingStatsForFlushes.getMean()) {
- tunerLog += " As could not decrease flushes sufficiently.";
+ tunerLog.append(" because we could not decrease flushes sufficiently.");
} else {
- tunerLog += " As number of evictions rose significantly.";
+ tunerLog.append(" because number of evictions rose significantly.");
}
isReverting = true;
}
@@ -215,97 +298,52 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount < rollingStatsForFlushes.getMean() -
- rollingStatsForFlushes.getDeviation()*0.80) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// Everything is fine no tuning required
newTuneDirection = StepDirection.NEUTRAL;
} else if ((double)cacheMissCount > rollingStatsForCacheMisses.getMean() +
rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount < rollingStatsForFlushes.getMean() -
- rollingStatsForFlushes.getDeviation()*0.80) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// more misses , increasing cache size
newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
- tunerLog +=
- "Increasing block cache size as observed increase in number of cache misses.";
+ tunerLog.append(
+ "Going to increase block cache size due to increase in number of cache misses.");
} else if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount > rollingStatsForFlushes.getMean() +
- rollingStatsForFlushes.getDeviation()*0.80) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// more flushes , increasing memstore size
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
- tunerLog += "Increasing memstore size as observed increase in number of flushes.";
+ tunerLog.append("Going to increase memstore size due to increase in number of flushes.");
} else if (blockedFlushCount > 0 && prevTuneDirection == StepDirection.NEUTRAL) {
// we do not want blocked flushes
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
- tunerLog += "Increasing memstore size as observed "
- + blockedFlushCount + " blocked flushes.";
+ tunerLog.append("Going to increase memstore size due to"
+ + blockedFlushCount + " blocked flushes.");
} else {
// Default. Not enough facts to do tuning.
+ tunerLog.append("Going to do nothing because we "
+ + "could not determine best tuning direction");
newTuneDirection = StepDirection.NEUTRAL;
}
}
}
- // Adjusting step size for tuning to get to steady state or restart from steady state.
- // Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back and forth
- // per tuner operation and it can affect the performance of cluster so we keep on decreasing
- // step size until everything settles.
- if (prevTuneDirection == StepDirection.NEUTRAL
- && newTuneDirection != StepDirection.NEUTRAL
- && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
- // Restarting the tuning from steady state and setting step size to maximum.
- // The deviation cannot be that low if last period was neutral and some recent periods were
- // not neutral.
- step = maximumStepSize;
- } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
- && decayingTunerStepSizeSum < 0) ||
- (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
- && decayingTunerStepSizeSum > 0)) {
- // Current step is opposite of past tuner actions so decrease the step size to reach steady
- // state.
- step = step/2.00f;
- }
- if (step < minimumStepSize) {
- // If step size is too small then we do nothing.
- step = 0.0f;
- newTuneDirection = StepDirection.NEUTRAL;
- }
- // Increase / decrease the memstore / block cahce sizes depending on new tuner step.
- switch (newTuneDirection) {
- case INCREASE_BLOCK_CACHE_SIZE:
- newBlockCacheSize = context.getCurBlockCacheSize() + step;
- newMemstoreSize = context.getCurMemStoreSize() - step;
- rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
- decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
- break;
- case INCREASE_MEMSTORE_SIZE:
- newBlockCacheSize = context.getCurBlockCacheSize() - step;
- newMemstoreSize = context.getCurMemStoreSize() + step;
- rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
- decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
- break;
- default:
- prevTuneDirection = StepDirection.NEUTRAL;
- rollingStatsForTunerSteps.insertDataValue(0);
- decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
- return NO_OP_TUNER_RESULT;
- }
- // Check we are within max/min bounds.
- if (newMemstoreSize > globalMemStorePercentMaxRange) {
- newMemstoreSize = globalMemStorePercentMaxRange;
- } else if (newMemstoreSize < globalMemStorePercentMinRange) {
- newMemstoreSize = globalMemStorePercentMinRange;
- }
- if (newBlockCacheSize > blockCachePercentMaxRange) {
- newBlockCacheSize = blockCachePercentMaxRange;
- } else if (newBlockCacheSize < blockCachePercentMinRange) {
- newBlockCacheSize = blockCachePercentMinRange;
- }
- TUNER_RESULT.setBlockCacheSize(newBlockCacheSize);
- TUNER_RESULT.setMemstoreSize(newMemstoreSize);
if (LOG.isDebugEnabled()) {
- LOG.debug(tunerLog);
+ LOG.debug(tunerLog.toString());
}
- prevTuneDirection = newTuneDirection;
- return TUNER_RESULT;
+ return newTuneDirection;
+ }
+
+ /**
+ * Add the given context to the rolling tuner stats.
+ * @param context The tuner context.
+ */
+ private void addToRollingStats(TunerContext context) {
+ rollingStatsForCacheMisses.insertDataValue(context.getCacheMissCount());
+ rollingStatsForFlushes.insertDataValue(context.getBlockedFlushCount() +
+ context.getUnblockedFlushCount());
+ rollingStatsForEvictions.insertDataValue(context.getEvictCount());
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e5953694/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index 99acf0f..0e72d0d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -227,6 +227,7 @@ public class TestHeapMemoryManager {
blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8));
regionServerAccounting.setTestMemstoreSize(0);
Configuration conf = HBaseConfiguration.create();
+ conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f);
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
@@ -238,6 +239,11 @@ public class TestHeapMemoryManager {
new RegionServerStub(conf), new RegionServerAccountingStub());
long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
long oldBlockCacheSize = blockCache.maxSize;
+ long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
+ long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
+ float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
+ maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
+ DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
heapMemoryManager.start(choreService);
blockCache.evictBlock(null);
@@ -245,20 +251,21 @@ public class TestHeapMemoryManager {
blockCache.evictBlock(null);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
- assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
- memStoreFlusher.memstoreSize);
- assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
- blockCache.maxSize);
+ assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
+ assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
oldBlockCacheSize = blockCache.maxSize;
+ oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10;
+ maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2;
+ maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize;
+ maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ?
+ DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue;
// Do some more evictions before the next run of HeapMemoryTuner
blockCache.evictBlock(null);
// Allow the tuner to run once and do necessary memory up
waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
- assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize,
- memStoreFlusher.memstoreSize);
- assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize,
- blockCache.maxSize);
+ assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
+ assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize);
}
@Test
[2/2] hbase git commit: HBASE-14058 Stabilizing default heap memory
tuner
Posted by ec...@apache.org.
HBASE-14058 Stabilizing default heap memory tuner
Signed-off-by: Elliott Clark <ec...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e738e69f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e738e69f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e738e69f
Branch: refs/heads/branch-1.2
Commit: e738e69f8cc59581a454207483aca42e7f314396
Parents: 4fb7bab
Author: abhilak <ab...@gmail.com>
Authored: Wed Jul 22 11:30:18 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Tue Jan 19 12:23:52 2016 -0800
----------------------------------------------------------------------
.../regionserver/DefaultHeapMemoryTuner.java | 93 +++++++++++++-------
1 file changed, 63 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e738e69f/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
index 62db9e2..b6e81dd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java
@@ -48,20 +48,24 @@ import org.apache.hadoop.hbase.util.RollingStatCalculator;
* same amount. If none of them is sufficient we do statistical analysis on number of cache misses
* and flushes to determine tuner direction. Based on these statistics we decide the tuner
* direction. If we are not confident about which step direction to take we do nothing and wait for
- * next iteration. On expectation we will be tuning for at least 22% tuner calls. The number of
+ * next iteration. On expectation we will be tuning for at least 10% tuner calls. The number of
* past periods to consider for statistics calculation can be specified in config by
* <i>hbase.regionserver.heapmemory.autotuner.lookup.periods</i>. Also these many initial calls to
* tuner will be ignored (cache is warming up and we leave the system to reach steady state).
* After the tuner takes a step, in next call we insure that last call was indeed helpful and did
* not do us any harm. If not then we revert the previous step. The step size is dynamic and it
- * changes based on current and previous tuning direction. When last tuner step was NEUTRAL
- * and current tuning step is not NEUTRAL then we assume we are restarting the tuning process and
- * step size is changed to maximum allowed size which can be specified in config by
- * <i>hbase.regionserver.heapmemory.autotuner.step.max</i>. If we are reverting the previous step
- * then we decrease step size to half. This decrease is similar to binary search where we try to
- * reach the most desired value. The minimum step size can be specified in config by
- * <i>hbase.regionserver.heapmemory.autotuner.step.min</i>. In other cases we leave step size
- * unchanged.
+ * changes based on current and past few tuning directions and their step sizes. We maintain a
+ * parameter <i>decayingAvgTunerStepSize</i> which is sum of past tuner steps with
+ * sign(positive for increase in memstore and negative for increase in block cache). But rather
+ * than simple sum it is calculated by giving more priority to the recent tuning steps.
+ * When last few tuner steps were NETURAL then we assume we are restarting the tuning process and
+ * step size is updated to maximum allowed size which can be specified in config by
+ * <i>hbase.regionserver.heapmemory.autotuner.step.max</i>. If in a particular tuning operation
+ * the step direction is opposite to what indicated by <i>decayingTunerStepSizeSum</i>
+ * we decrease the step size by half. Step size does not change in other tuning operations.
+ * When step size gets below a certain threshold then the following tuner operations are
+ * considered to be neutral. The minimum step size can be specified in config by
+ * <i>hbase.regionserver.heapmemory.autotuner.step.min</i>.
*/
@InterfaceAudience.Private
class DefaultHeapMemoryTuner implements HeapMemoryTuner {
@@ -74,9 +78,9 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
public static final String NUM_PERIODS_TO_IGNORE =
"hbase.regionserver.heapmemory.autotuner.ignored.periods";
// Maximum step size that the tuner can take
- public static final float DEFAULT_MAX_STEP_VALUE = 0.08f; // 8%
+ public static final float DEFAULT_MAX_STEP_VALUE = 0.04f; // 4%
// Minimum step size that the tuner can take
- public static final float DEFAULT_MIN_STEP_VALUE = 0.005f; // 0.5%
+ public static final float DEFAULT_MIN_STEP_VALUE = 0.00125f; // 0.125%
// If current block cache size or memstore size in use is below this level relative to memory
// provided to it then corresponding component will be considered to have sufficient memory
public static final float DEFAULT_SUFFICIENT_MEMORY_LEVEL_VALUE = 0.5f; // 50%
@@ -85,6 +89,9 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
public static final int DEFAULT_LOOKUP_PERIODS = 60;
public static final int DEFAULT_NUM_PERIODS_IGNORED = 60;
private static final TunerResult NO_OP_TUNER_RESULT = new TunerResult(false);
+ // If deviation of tuner step size gets below this value then it means past few periods were
+ // NEUTRAL(given that last tuner period was also NEUTRAL).
+ private static final double TUNER_STEP_EPS = 1e-6;
private Log LOG = LogFactory.getLog(DefaultHeapMemoryTuner.class);
private TunerResult TUNER_RESULT = new TunerResult(true);
@@ -106,9 +113,14 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
private RollingStatCalculator rollingStatsForCacheMisses;
private RollingStatCalculator rollingStatsForFlushes;
private RollingStatCalculator rollingStatsForEvictions;
+ private RollingStatCalculator rollingStatsForTunerSteps;
// Set step size to max value for tuning, this step size will adjust dynamically while tuning
private float step = DEFAULT_MAX_STEP_VALUE;
private StepDirection prevTuneDirection = StepDirection.NEUTRAL;
+ //positive means memstore's size was increased
+ //It is not just arithmetic sum of past tuner periods. More priority is given to recent
+ //tuning steps.
+ private double decayingTunerStepSizeSum = 0;
@Override
public TunerResult tune(TunerContext context) {
@@ -124,6 +136,7 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
if (ignoreInitialPeriods < numPeriodsToIgnore) {
// Ignoring the first few tuner periods
ignoreInitialPeriods++;
+ rollingStatsForTunerSteps.insertDataValue(0);
return NO_OP_TUNER_RESULT;
}
String tunerLog = "";
@@ -190,30 +203,33 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
}
// If we are not reverting. We try to tune memory sizes by looking at cache misses / flushes.
if (!isReverting){
- // mean +- deviation/2 is considered to be normal
+ // mean +- deviation*0.8 is considered to be normal
// below it its consider low and above it is considered high.
// We can safely assume that the number cache misses, flushes are normally distributed over
// past periods and hence on all the above mentioned classes (normal, high and low)
- // are equally likely with 33% probability each. Hence there is very good probability that
- // we will not always fall in default step.
+ // are likely to occur with probability 56%, 22%, 22% respectively. Hence there is at
+ // least ~10% probability that we will not fall in NEUTRAL step.
+ // This optimization solution is feedback based and we revert when we
+ // dont find our steps helpful. Hence we want to do tuning only when we have clear
+ // indications because too many unnecessary tuning may affect the performance of cluster.
if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
- rollingStatsForCacheMisses.getDeviation()/2.00 &&
+ rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount < rollingStatsForFlushes.getMean() -
- rollingStatsForFlushes.getDeviation()/2.00) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// Everything is fine no tuning required
newTuneDirection = StepDirection.NEUTRAL;
} else if ((double)cacheMissCount > rollingStatsForCacheMisses.getMean() +
- rollingStatsForCacheMisses.getDeviation()/2.00 &&
+ rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount < rollingStatsForFlushes.getMean() -
- rollingStatsForFlushes.getDeviation()/2.00) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// more misses , increasing cache size
newTuneDirection = StepDirection.INCREASE_BLOCK_CACHE_SIZE;
tunerLog +=
"Increasing block cache size as observed increase in number of cache misses.";
} else if ((double)cacheMissCount < rollingStatsForCacheMisses.getMean() -
- rollingStatsForCacheMisses.getDeviation()/2.00 &&
+ rollingStatsForCacheMisses.getDeviation()*0.80 &&
(double)totalFlushCount > rollingStatsForFlushes.getMean() +
- rollingStatsForFlushes.getDeviation()/2.00) {
+ rollingStatsForFlushes.getDeviation()*0.80) {
// more flushes , increasing memstore size
newTuneDirection = StepDirection.INCREASE_MEMSTORE_SIZE;
tunerLog += "Increasing memstore size as observed increase in number of flushes.";
@@ -228,32 +244,48 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
}
}
}
- // Adjusting step size for tuning to get to steady state.
+ // Adjusting step size for tuning to get to steady state or restart from steady state.
// Even if the step size was 4% and 32 GB memory size, we will be shifting 1 GB back and forth
- // per tuner operation and it can affect the performance of cluster
- if (prevTuneDirection == StepDirection.NEUTRAL && newTuneDirection != StepDirection.NEUTRAL) {
- // Restarting the tuning from steady state.
+ // per tuner operation and it can affect the performance of cluster so we keep on decreasing
+ // step size until everything settles.
+ if (prevTuneDirection == StepDirection.NEUTRAL
+ && newTuneDirection != StepDirection.NEUTRAL
+ && rollingStatsForTunerSteps.getDeviation() < TUNER_STEP_EPS) {
+ // Restarting the tuning from steady state and setting step size to maximum.
+ // The deviation cannot be that low if last period was neutral and some recent periods were
+ // not neutral.
step = maximumStepSize;
- } else if (prevTuneDirection != newTuneDirection) {
- // Decrease the step size to reach the steady state. Similar procedure as binary search.
+ } else if ((newTuneDirection == StepDirection.INCREASE_MEMSTORE_SIZE
+ && decayingTunerStepSizeSum < 0) ||
+ (newTuneDirection == StepDirection.INCREASE_BLOCK_CACHE_SIZE
+ && decayingTunerStepSizeSum > 0)) {
+ // Current step is opposite of past tuner actions so decrease the step size to reach steady
+ // state.
step = step/2.00f;
- if (step < minimumStepSize) {
- // Ensure step size does not gets too small.
- step = minimumStepSize;
- }
+ }
+ if (step < minimumStepSize) {
+ // If step size is too small then we do nothing.
+ step = 0.0f;
+ newTuneDirection = StepDirection.NEUTRAL;
}
// Increase / decrease the memstore / block cahce sizes depending on new tuner step.
switch (newTuneDirection) {
case INCREASE_BLOCK_CACHE_SIZE:
newBlockCacheSize = context.getCurBlockCacheSize() + step;
newMemstoreSize = context.getCurMemStoreSize() - step;
+ rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000));
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f;
break;
case INCREASE_MEMSTORE_SIZE:
newBlockCacheSize = context.getCurBlockCacheSize() - step;
newMemstoreSize = context.getCurMemStoreSize() + step;
+ rollingStatsForTunerSteps.insertDataValue((int)(step*100000));
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum + step)/2.00f;
break;
default:
prevTuneDirection = StepDirection.NEUTRAL;
+ rollingStatsForTunerSteps.insertDataValue(0);
+ decayingTunerStepSizeSum = (decayingTunerStepSizeSum)/2.00f;
return NO_OP_TUNER_RESULT;
}
// Check we are within max/min bounds.
@@ -303,6 +335,7 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner {
this.rollingStatsForCacheMisses = new RollingStatCalculator(this.tunerLookupPeriods);
this.rollingStatsForFlushes = new RollingStatCalculator(this.tunerLookupPeriods);
this.rollingStatsForEvictions = new RollingStatCalculator(this.tunerLookupPeriods);
+ this.rollingStatsForTunerSteps = new RollingStatCalculator(this.tunerLookupPeriods);
}
private enum StepDirection{