You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ch...@apache.org on 2016/08/04 13:22:41 UTC
hbase git commit: HBASE-16287 LruBlockCache size should not exceed
acceptableSize too many(Yu Sun)
Repository: hbase
Updated Branches:
refs/heads/master 544dc1e55 -> 550b937bc
HBASE-16287 LruBlockCache size should not exceed acceptableSize too many(Yu Sun)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/550b937b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/550b937b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/550b937b
Branch: refs/heads/master
Commit: 550b937bcf0b5a7e030194cca7d90524a0fc0f3d
Parents: 544dc1e
Author: chenheng <ch...@apache.org>
Authored: Thu Aug 4 21:13:42 2016 +0800
Committer: chenheng <ch...@apache.org>
Committed: Thu Aug 4 21:13:42 2016 +0800
----------------------------------------------------------------------
.../hadoop/hbase/io/hfile/LruBlockCache.java | 36 ++++++++++++++++++--
.../hadoop/hbase/io/hfile/TestCacheOnWrite.java | 2 ++
.../hbase/io/hfile/TestLruBlockCache.java | 5 +++
3 files changed, 40 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/550b937b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 29c5922..2fd9fdf 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -110,6 +110,10 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
*/
static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor";
+ /**
+ * Hard capacity limit of cache, will reject any put if size > this * acceptable
+ */
+ static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.hard.capacity.limit.factor";
static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage";
static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage";
static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage";
@@ -136,6 +140,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
static final float DEFAULT_MULTI_FACTOR = 0.50f;
static final float DEFAULT_MEMORY_FACTOR = 0.25f;
+ /** default hard capacity limit */
+ static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f;
+
static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false;
/** Statistics thread */
@@ -169,6 +176,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
/** Cache access count (sequential ID) */
private final AtomicLong count;
+ /** hard capacity limit */
+ private float hardCapacityLimitFactor;
+
/** Cache statistics */
private final CacheStats stats;
@@ -226,6 +236,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
DEFAULT_SINGLE_FACTOR,
DEFAULT_MULTI_FACTOR,
DEFAULT_MEMORY_FACTOR,
+ DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
false,
DEFAULT_MAX_BLOCK_SIZE
);
@@ -241,6 +252,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR),
conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR),
conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
+ conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE),
conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
);
@@ -267,7 +279,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
public LruBlockCache(long maxSize, long blockSize, boolean evictionThread,
int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel,
float minFactor, float acceptableFactor, float singleFactor,
- float multiFactor, float memoryFactor, boolean forceInMemory, long maxBlockSize) {
+ float multiFactor, float memoryFactor, float hardLimitFactor,
+ boolean forceInMemory, long maxBlockSize) {
this.maxBlockSize = maxBlockSize;
if(singleFactor + multiFactor + memoryFactor != 1 ||
singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) {
@@ -295,6 +308,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
this.elements = new AtomicLong(0);
this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
this.size = new AtomicLong(this.overhead);
+ this.hardCapacityLimitFactor = hardLimitFactor;
if(evictionThread) {
this.evictionThread = new EvictionThread(this);
this.evictionThread.start(); // FindBugs SC_START_IN_CTOR
@@ -357,6 +371,22 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
LOG.warn(msg);
return;
}
+ long currentSize = size.get();
+ long currentAcceptableSize = acceptableSize();
+ long hardLimitSize = (long) (hardCapacityLimitFactor * currentAcceptableSize);
+ if (currentSize >= hardLimitSize) {
+ stats.failInsert();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("LruBlockCache current size " + StringUtils.byteDesc(currentSize)
+ + " has exceeded acceptable size " + StringUtils.byteDesc(currentAcceptableSize) + " too many."
+ + " the hard limit size is " + StringUtils.byteDesc(hardLimitSize) + ", failed to put cacheKey:"
+ + cacheKey + " into LruBlockCache.");
+ }
+ if (!evictionInProgress) {
+ runEviction();
+ }
+ return;
+ }
cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory);
long newSize = updateSizeMetrics(cb, false);
map.put(cacheKey, cb);
@@ -365,7 +395,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
long size = map.size();
assertCounterSanity(size, val);
}
- if (newSize > acceptableSize() && !evictionInProgress) {
+ if (newSize > currentAcceptableSize && !evictionInProgress) {
runEviction();
}
}
@@ -915,7 +945,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
public final static long CACHE_FIXED_OVERHEAD = ClassSize.align(
(4 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) +
- (5 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN)
+ (6 * Bytes.SIZEOF_FLOAT) + (2 * Bytes.SIZEOF_BOOLEAN)
+ ClassSize.OBJECT);
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/550b937b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
index 32db448..8f9c4f7 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java
@@ -158,6 +158,8 @@ public class TestCacheOnWrite {
// default
blockcaches.add(new CacheConfig(conf).getBlockCache());
+ //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287
+ TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f);
// memory
BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration());
blockcaches.add(lru);
http://git-wip-us.apache.org/repos/asf/hbase/blob/550b937b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
index 0d8a3bb..b4dfc0c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java
@@ -311,6 +311,7 @@ public class TestLruBlockCache {
0.33f, // single
0.33f, // multi
0.34f, // memory
+ 1.2f, // limit
false,
16 * 1024 * 1024);
@@ -432,6 +433,7 @@ public class TestLruBlockCache {
0.2f, // single
0.3f, // multi
0.5f, // memory
+ 1.2f, // limit
true,
16 * 1024 * 1024);
@@ -538,6 +540,7 @@ public class TestLruBlockCache {
0.33f, // single
0.33f, // multi
0.34f, // memory
+ 1.2f, // limit
false,
16 * 1024 * 1024);
@@ -601,6 +604,7 @@ public class TestLruBlockCache {
0.33f, // single
0.33f, // multi
0.34f, // memory
+ 1.2f, // limit
false,
1024);
CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long");
@@ -640,6 +644,7 @@ public class TestLruBlockCache {
0.33f, // single
0.33f, // multi
0.34f, // memory
+ 1.2f, // limit
false,
16 * 1024 * 1024);