You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by sy...@apache.org on 2016/10/07 19:15:23 UTC
[55/77] [abbrv] hbase git commit: HBASE-15560 TinyLFU-based
BlockCache - revert pending performance verification
HBASE-15560 TinyLFU-based BlockCache - revert pending performance verification
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b952e647
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b952e647
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b952e647
Branch: refs/heads/hbase-12439
Commit: b952e64751d309e920bf6e44caa2b3d5801e3be8
Parents: b5d34cf
Author: tedyu <yu...@gmail.com>
Authored: Tue Oct 4 08:37:29 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Oct 4 08:37:29 2016 -0700
----------------------------------------------------------------------
.../src/main/resources/hbase-default.xml | 5 -
.../src/main/resources/supplemental-models.xml | 16 +-
hbase-server/pom.xml | 4 -
.../hadoop/hbase/io/hfile/CacheConfig.java | 91 ++---
.../hbase/io/hfile/CombinedBlockCache.java | 48 +--
.../hbase/io/hfile/FirstLevelBlockCache.java | 45 ---
.../io/hfile/InclusiveCombinedBlockCache.java | 6 +-
.../hadoop/hbase/io/hfile/LruBlockCache.java | 36 +-
.../hbase/io/hfile/TinyLfuBlockCache.java | 402 -------------------
.../hbase/io/hfile/bucket/BucketCache.java | 5 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 8 +-
.../hbase/io/hfile/TestTinyLfuBlockCache.java | 304 --------------
pom.xml | 6 -
13 files changed, 79 insertions(+), 897 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 3b7b05e..4f769cb 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -807,11 +807,6 @@ possible configurations would overwhelm and obscure the important.
The default thread pool size if parallel-seeking feature enabled.</description>
</property>
<property>
- <name>hfile.block.cache.policy</name>
- <value>LRU</value>
- <description>The eviction policy for the L1 block cache (LRU or TinyLFU).</description>
- </property>
- <property>
<name>hfile.block.cache.size</name>
<value>0.4</value>
<description>Percentage of maximum heap (-Xmx setting) to allocate to block cache
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-resource-bundle/src/main/resources/supplemental-models.xml
----------------------------------------------------------------------
diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
index 11c405e..0979b5f 100644
--- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml
+++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
@@ -152,20 +152,6 @@ under the License.
</supplement>
<supplement>
<project>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
-
- <licenses>
- <license>
- <name>Apache License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
- </project>
- </supplement>
- <supplement>
- <project>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
@@ -1682,7 +1668,7 @@ Mozilla Public License Version 2.0
means any form of the work other than Source Code Form.
1.7. "Larger Work"
- means a work that combines Covered Software with other material, in
+ means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/pom.xml
----------------------------------------------------------------------
diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml
index d036357..a431006 100644
--- a/hbase-server/pom.xml
+++ b/hbase-server/pom.xml
@@ -435,10 +435,6 @@
<optional>true</optional>
</dependency>
<dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- </dependency>
- <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 0933e82..321f72c 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -22,14 +22,13 @@ import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.util.concurrent.ForkJoinPool;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.util.ReflectionUtils;
@@ -45,12 +44,6 @@ public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
- * Configuration key to cache block policy (Lru, TinyLfu).
- */
- public static final String HFILE_BLOCK_CACHE_POLICY_KEY = "hfile.block.cache.policy";
- public static final String HFILE_BLOCK_CACHE_POLICY_DEFAULT = "LRU";
-
- /**
* Configuration key to cache data blocks on read. Bloom blocks and index blocks are always be
* cached if the block cache is enabled.
*/
@@ -103,7 +96,7 @@ public class CacheConfig {
* is an in-memory map that needs to be persisted across restarts. Where to store this
* in-memory state is what you supply here: e.g. <code>/tmp/bucketcache.map</code>.
*/
- public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
+ public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
"hbase.bucketcache.persistent.path";
/**
@@ -111,11 +104,11 @@ public class CacheConfig {
* as indices and blooms are kept in the lru blockcache and the data blocks in the
* bucket cache).
*/
- public static final String BUCKET_CACHE_COMBINED_KEY =
+ public static final String BUCKET_CACHE_COMBINED_KEY =
"hbase.bucketcache.combinedcache.enabled";
public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
- public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
+ public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
"hbase.bucketcache.writer.queuelength";
/**
@@ -161,7 +154,6 @@ public class CacheConfig {
memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache");
// TODO(eclark): Consider more. Redis, etc.
Class<? extends BlockCache> clazz;
- @SuppressWarnings("unchecked")
ExternalBlockCaches(String clazzName) {
try {
clazz = (Class<? extends BlockCache>) Class.forName(clazzName);
@@ -457,9 +449,7 @@ public class CacheConfig {
* @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
*/
public boolean shouldCacheCompressed(BlockCategory category) {
- if (!isBlockCacheEnabled()) {
- return false;
- }
+ if (!isBlockCacheEnabled()) return false;
switch (category) {
case DATA:
return this.cacheDataOnRead && this.cacheDataCompressed;
@@ -541,13 +531,13 @@ public class CacheConfig {
// Clear this if in tests you'd make more than one block cache instance.
@VisibleForTesting
static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
- private static FirstLevelBlockCache GLOBAL_L1_CACHE_INSTANCE;
+ private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE;
/** Boolean whether we have disabled the block cache entirely. */
@VisibleForTesting
static boolean blockCacheDisabled = false;
- static long getFirstLevelCacheSize(final Configuration conf, final long xmx) {
+ static long getLruCacheSize(final Configuration conf, final long xmx) {
float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
if (cachePercentage <= 0.0001f) {
@@ -565,37 +555,26 @@ public class CacheConfig {
/**
* @param c Configuration to use.
- * @return An L1 instance
+ * @return An L1 instance. Currently an instance of LruBlockCache.
*/
- public static FirstLevelBlockCache getL1(final Configuration c) {
- long xmx = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
- long l1CacheSize = getFirstLevelCacheSize(c, xmx);
- return getL1(l1CacheSize, c);
+ public static LruBlockCache getL1(final Configuration c) {
+ return getL1(c, ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax());
}
/**
* @param c Configuration to use.
* @param xmx Max heap memory
- * @return An L1 instance.
+ * @return An L1 instance. Currently an instance of LruBlockCache.
*/
- private synchronized static FirstLevelBlockCache getL1(long cacheSize, Configuration c) {
+ private synchronized static LruBlockCache getL1(final Configuration c, final long xmx) {
if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE;
if (blockCacheDisabled) return null;
- if (cacheSize < 0) return null;
-
- String policy = c.get(HFILE_BLOCK_CACHE_POLICY_KEY, HFILE_BLOCK_CACHE_POLICY_DEFAULT);
+ long lruCacheSize = getLruCacheSize(c, xmx);
+ if (lruCacheSize < 0) return null;
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
- LOG.info("Allocating BlockCache size=" +
- StringUtils.byteDesc(cacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
-
- if (policy.equalsIgnoreCase("LRU")) {
- GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(cacheSize, blockSize, true, c);
- } else if (policy.equalsIgnoreCase("TinyLFU")) {
- GLOBAL_L1_CACHE_INSTANCE = new TinyLfuBlockCache(
- cacheSize, blockSize, ForkJoinPool.commonPool(), c);
- } else {
- throw new IllegalArgumentException("Unknown policy: " + policy);
- }
+ LOG.info("Allocating LruBlockCache size=" +
+ StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
+ GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c);
return GLOBAL_L1_CACHE_INSTANCE;
}
@@ -622,7 +601,7 @@ public class CacheConfig {
}
private static BlockCache getExternalBlockcache(Configuration c) {
- Class<?> klass = null;
+ Class klass = null;
// Get the class, from the config. s
try {
@@ -650,9 +629,7 @@ public class CacheConfig {
private static BlockCache getBucketCache(Configuration c, long xmx) {
// Check for L2. ioengine name must be non-null.
String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
- if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) {
- return null;
- }
+ if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
float bucketCachePercentage = c.getFloat(BUCKET_CACHE_SIZE_KEY, 0F);
@@ -702,37 +679,29 @@ public class CacheConfig {
* @return The block cache or <code>null</code>.
*/
public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
- if (GLOBAL_BLOCK_CACHE_INSTANCE != null) {
- return GLOBAL_BLOCK_CACHE_INSTANCE;
- }
- if (blockCacheDisabled) {
- return null;
- }
- // blockCacheDisabled is set as a side-effect of getFirstLevelCacheSize()
- // so check it again after the call
+ if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
+ if (blockCacheDisabled) return null;
long xmx = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
- long l1CacheSize = getFirstLevelCacheSize(conf, xmx);
- if (blockCacheDisabled) {
- return null;
- }
+ LruBlockCache l1 = getL1(conf, xmx);
+ // blockCacheDisabled is set as a side-effect of getL1(), so check it again after the call.
+ if (blockCacheDisabled) return null;
BlockCache l2 = getL2(conf, xmx);
- FirstLevelBlockCache l1 = getL1(l1CacheSize, conf);
if (l2 == null) {
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
} else {
boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
- boolean combinedWithL1 = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
+ boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
DEFAULT_BUCKET_CACHE_COMBINED);
if (useExternal) {
GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
} else {
- if (combinedWithL1) {
+ if (combinedWithLru) {
GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
} else {
- // L1 and L2 are not 'combined'. They are connected via the FirstLevelBlockCache
- // victimhandler mechanism. It is a little ugly but works according to the following:
- // when the background eviction thread runs, blocks evicted from L1 will go to L2 AND when
- // we get a block from the L1 cache, if not in L1, we will search L2.
+ // L1 and L2 are not 'combined'. They are connected via the LruBlockCache victimhandler
+ // mechanism. It is a little ugly but works according to the following: when the
+ // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
+ // a block from the L1 cache, if not in L1, we will search L2.
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index 188900c..4ceda39 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -30,24 +30,24 @@ import com.google.common.annotations.VisibleForTesting;
/**
* CombinedBlockCache is an abstraction layer that combines
- * {@link FirstLevelBlockCache} and {@link BucketCache}. The smaller lruCache is used
+ * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used
* to cache bloom blocks and index blocks. The larger l2Cache is used to
* cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean, boolean)} reads
- * first from the smaller l1Cache before looking for the block in the l2Cache. Blocks evicted
- * from l1Cache are put into the bucket cache.
+ * first from the smaller lruCache before looking for the block in the l2Cache. Blocks evicted
+ * from lruCache are put into the bucket cache.
* Metrics are the combined size and hits and misses of both caches.
- *
+ *
*/
@InterfaceAudience.Private
public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
- protected final FirstLevelBlockCache l1Cache;
+ protected final LruBlockCache lruCache;
protected final BlockCache l2Cache;
protected final CombinedCacheStats combinedCacheStats;
- public CombinedBlockCache(FirstLevelBlockCache l1Cache, BlockCache l2Cache) {
- this.l1Cache = l1Cache;
+ public CombinedBlockCache(LruBlockCache lruCache, BlockCache l2Cache) {
+ this.lruCache = lruCache;
this.l2Cache = l2Cache;
- this.combinedCacheStats = new CombinedCacheStats(l1Cache.getStats(),
+ this.combinedCacheStats = new CombinedCacheStats(lruCache.getStats(),
l2Cache.getStats());
}
@@ -57,7 +57,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
if (l2Cache instanceof HeapSize) {
l2size = ((HeapSize) l2Cache).heapSize();
}
- return l1Cache.heapSize() + l2size;
+ return lruCache.heapSize() + l2size;
}
@Override
@@ -65,7 +65,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
final boolean cacheDataInL1) {
boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
if (metaBlock || cacheDataInL1) {
- l1Cache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
+ lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
} else {
l2Cache.cacheBlock(cacheKey, buf, inMemory, false);
}
@@ -81,19 +81,19 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
boolean repeat, boolean updateCacheMetrics) {
// TODO: is there a hole here, or just awkwardness since in the lruCache getBlock
// we end up calling l2Cache.getBlock.
- return l1Cache.containsBlock(cacheKey)?
- l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
+ return lruCache.containsBlock(cacheKey)?
+ lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
@Override
public boolean evictBlock(BlockCacheKey cacheKey) {
- return l1Cache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
+ return lruCache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
}
@Override
public int evictBlocksByHfileName(String hfileName) {
- return l1Cache.evictBlocksByHfileName(hfileName)
+ return lruCache.evictBlocksByHfileName(hfileName)
+ l2Cache.evictBlocksByHfileName(hfileName);
}
@@ -104,28 +104,28 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public void shutdown() {
- l1Cache.shutdown();
+ lruCache.shutdown();
l2Cache.shutdown();
}
@Override
public long size() {
- return l1Cache.size() + l2Cache.size();
+ return lruCache.size() + l2Cache.size();
}
@Override
public long getFreeSize() {
- return l1Cache.getFreeSize() + l2Cache.getFreeSize();
+ return lruCache.getFreeSize() + l2Cache.getFreeSize();
}
@Override
public long getCurrentSize() {
- return l1Cache.getCurrentSize() + l2Cache.getCurrentSize();
+ return lruCache.getCurrentSize() + l2Cache.getCurrentSize();
}
@Override
public long getBlockCount() {
- return l1Cache.getBlockCount() + l2Cache.getBlockCount();
+ return lruCache.getBlockCount() + l2Cache.getBlockCount();
}
public static class CombinedCacheStats extends CacheStats {
@@ -310,7 +310,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
lruCacheStats.rollMetricsPeriod();
bucketCacheStats.rollMetricsPeriod();
}
-
+
@Override
public long getFailedInserts() {
return lruCacheStats.getFailedInserts() + bucketCacheStats.getFailedInserts();
@@ -321,13 +321,13 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
return lruCacheStats.getSumHitCountsPastNPeriods()
+ bucketCacheStats.getSumHitCountsPastNPeriods();
}
-
+
@Override
public long getSumRequestCountsPastNPeriods() {
return lruCacheStats.getSumRequestCountsPastNPeriods()
+ bucketCacheStats.getSumRequestCountsPastNPeriods();
}
-
+
@Override
public long getSumHitCachingCountsPastNPeriods() {
return lruCacheStats.getSumHitCachingCountsPastNPeriods()
@@ -348,12 +348,12 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public BlockCache[] getBlockCaches() {
- return new BlockCache [] {this.l1Cache, this.l2Cache};
+ return new BlockCache [] {this.lruCache, this.l2Cache};
}
@Override
public void setMaxSize(long size) {
- this.l1Cache.setMaxSize(size);
+ this.lruCache.setMaxSize(size);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
deleted file mode 100644
index 6becd0e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.io.hfile;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.io.HeapSize;
-
-/**
- * In-memory BlockCache that may be backed by secondary layer(s).
- */
-@InterfaceAudience.Private
-public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize {
-
- /**
- * Whether the cache contains the block with specified cacheKey
- *
- * @param cacheKey
- * @return true if it contains the block
- */
- boolean containsBlock(BlockCacheKey cacheKey);
-
- /**
- * Specifies the secondary cache. An entry that is evicted from this cache due to a size
- * constraint will be inserted into the victim cache.
- *
- * @param victimCache the second level cache
- * @throws IllegalArgumentException if the victim cache had already been set
- */
- void setVictimCache(BlockCache victimCache);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
index 160714b..667e7b4 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class InclusiveCombinedBlockCache extends CombinedBlockCache implements BlockCache {
- public InclusiveCombinedBlockCache(FirstLevelBlockCache l1, BlockCache l2) {
+ public InclusiveCombinedBlockCache(LruBlockCache l1, BlockCache l2) {
super(l1,l2);
}
@@ -34,7 +34,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
// On all external cache set ups the lru should have the l2 cache set as the victimHandler
// Because of that all requests that miss inside of the lru block cache will be
// tried in the l2 block cache.
- return l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+ return lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
/**
@@ -50,7 +50,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
final boolean cacheDataInL1) {
// This is the inclusive part of the combined block cache.
// Every block is placed into both block caches.
- l1Cache.cacheBlock(cacheKey, buf, inMemory, true);
+ lruCache.cacheBlock(cacheKey, buf, inMemory, true);
// This assumes that insertion into the L2 block cache is either async or very fast.
l2Cache.cacheBlock(cacheKey, buf, inMemory, true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 04700b9..f454549 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -18,8 +18,6 @@
*/
package org.apache.hadoop.hbase.io.hfile;
-import static java.util.Objects.requireNonNull;
-
import java.lang.ref.WeakReference;
import java.nio.ByteBuffer;
import java.util.EnumMap;
@@ -59,15 +57,15 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
* {@link ConcurrentHashMap} and with a non-blocking eviction thread giving
* constant-time {@link #cacheBlock} and {@link #getBlock} operations.<p>
*
- * Contains three levels of block priority to allow for scan-resistance and in-memory families
- * {@link org.apache.hadoop.hbase.HColumnDescriptor#setInMemory(boolean)} (An in-memory column
+ * Contains three levels of block priority to allow for scan-resistance and in-memory families
+ * {@link org.apache.hadoop.hbase.HColumnDescriptor#setInMemory(boolean)} (An in-memory column
* family is a column family that should be served from memory if possible):
* single-access, multiple-accesses, and in-memory priority.
* A block is added with an in-memory priority flag if
* {@link org.apache.hadoop.hbase.HColumnDescriptor#isInMemory()}, otherwise a block becomes a
* single access priority the first time it is read into this block cache. If a block is
* accessed again while in cache, it is marked as a multiple access priority block. This
- * delineation of blocks is used to prevent scans from thrashing the cache adding a
+ * delineation of blocks is used to prevent scans from thrashing the cache adding a
* least-frequently-used element to the eviction algorithm.<p>
*
* Each priority is given its own chunk of the total cache to ensure
@@ -97,7 +95,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
@InterfaceAudience.Private
@JsonIgnoreProperties({"encodingCountsForTest"})
-public class LruBlockCache implements FirstLevelBlockCache {
+public class LruBlockCache implements ResizableBlockCache, HeapSize {
private static final Log LOG = LogFactory.getLog(LruBlockCache.class);
@@ -240,7 +238,8 @@ public class LruBlockCache implements FirstLevelBlockCache {
DEFAULT_MEMORY_FACTOR,
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
false,
- DEFAULT_MAX_BLOCK_SIZE);
+ DEFAULT_MAX_BLOCK_SIZE
+ );
}
public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
@@ -255,7 +254,8 @@ public class LruBlockCache implements FirstLevelBlockCache {
conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE),
- conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE));
+ conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
+ );
}
public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
@@ -322,14 +322,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
}
@Override
- public void setVictimCache(BlockCache victimCache) {
- if (victimHandler != null) {
- throw new IllegalArgumentException("The victim cache has already been set");
- }
- victimHandler = requireNonNull(victimCache);
- }
-
- @Override
public void setMaxSize(long maxSize) {
this.maxSize = maxSize;
if(this.size.get() > acceptableSize() && !evictionInProgress) {
@@ -442,7 +434,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
* @param cacheKey block's cache key
* @param buf block buffer
*/
- @Override
public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) {
cacheBlock(cacheKey, buf, false, false);
}
@@ -507,7 +498,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
* @param cacheKey
* @return true if contains the block
*/
- @Override
public boolean containsBlock(BlockCacheKey cacheKey) {
return map.containsKey(cacheKey);
}
@@ -795,7 +785,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
return totalSize;
}
- @Override
public int compareTo(BlockBucket that) {
return Long.compare(this.overflow(), that.overflow());
}
@@ -957,7 +946,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
* <p>Includes: total accesses, hits, misses, evicted blocks, and runs
* of the eviction processes.
*/
- @Override
public CacheStats getStats() {
return this.stats;
}
@@ -1086,7 +1074,6 @@ public class LruBlockCache implements FirstLevelBlockCache {
return (long)Math.floor(this.maxSize * this.memoryFactor * this.minFactor);
}
- @Override
public void shutdown() {
if (victimHandler != null)
victimHandler.shutdown();
@@ -1135,7 +1122,7 @@ public class LruBlockCache implements FirstLevelBlockCache {
Map<BlockType, Integer> counts =
new EnumMap<BlockType, Integer>(BlockType.class);
for (LruCachedBlock cb : map.values()) {
- BlockType blockType = cb.getBuffer().getBlockType();
+ BlockType blockType = ((Cacheable)cb.getBuffer()).getBlockType();
Integer count = counts.get(blockType);
counts.put(blockType, (count == null ? 0 : count) + 1);
}
@@ -1155,6 +1142,11 @@ public class LruBlockCache implements FirstLevelBlockCache {
return counts;
}
+ public void setVictimCache(BlockCache handler) {
+ assert victimHandler == null;
+ victimHandler = handler;
+ }
+
@VisibleForTesting
Map<BlockCacheKey, LruCachedBlock> getMapForTests() {
return map;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
deleted file mode 100644
index 5ef8887..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.io.hfile;
-
-import static java.util.Objects.requireNonNull;
-
-import java.util.Comparator;
-import java.util.Iterator;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
-import org.apache.hadoop.util.StringUtils;
-
-import com.github.benmanes.caffeine.cache.Cache;
-import com.github.benmanes.caffeine.cache.Caffeine;
-import com.github.benmanes.caffeine.cache.Policy.Eviction;
-import com.github.benmanes.caffeine.cache.RemovalCause;
-import com.github.benmanes.caffeine.cache.RemovalListener;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Objects;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
-/**
- * A block cache that is memory-aware using {@link HeapSize}, memory bounded using the W-TinyLFU
- * eviction algorithm, and concurrent. This implementation delegates to a Caffeine cache to provide
- * O(1) read and write operations.
- * <ul>
- * <li>W-TinyLFU: http://arxiv.org/pdf/1512.00727.pdf</li>
- * <li>Caffeine: https://github.com/ben-manes/caffeine</li>
- * <li>Cache design: http://highscalability.com/blog/2016/1/25/design-of-a-modern-cache.html</li>
- * </ul>
- */
-@InterfaceAudience.Private
-public final class TinyLfuBlockCache implements FirstLevelBlockCache {
- private static final Log LOG = LogFactory.getLog(TinyLfuBlockCache.class);
-
- private static final String MAX_BLOCK_SIZE = "hbase.tinylfu.max.block.size";
- private static final long DEFAULT_MAX_BLOCK_SIZE = 16L * 1024L * 1024L;
- private static final int STAT_THREAD_PERIOD_SECONDS = 5 * 60;
-
- private final Eviction<BlockCacheKey, Cacheable> policy;
- private final ScheduledExecutorService statsThreadPool;
- private final long maxBlockSize;
- private final CacheStats stats;
-
- private BlockCache victimCache;
-
- @VisibleForTesting
- final Cache<BlockCacheKey, Cacheable> cache;
-
- /**
- * Creates a block cache.
- *
- * @param maximumSizeInBytes maximum size of this cache, in bytes
- * @param avgBlockSize expected average size of blocks, in bytes
- * @param executor the cache's executor
- * @param conf additional configuration
- */
- public TinyLfuBlockCache(long maximumSizeInBytes, long avgBlockSize,
- Executor executor, Configuration conf) {
- this(maximumSizeInBytes, avgBlockSize,
- conf.getLong(MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE), executor);
- }
-
- /**
- * Creates a block cache.
- *
- * @param maximumSizeInBytes maximum size of this cache, in bytes
- * @param avgBlockSize expected average size of blocks, in bytes
- * @param maxBlockSize maximum size of a block, in bytes
- * @param executor the cache's executor
- */
- public TinyLfuBlockCache(long maximumSizeInBytes,
- long avgBlockSize, long maxBlockSize, Executor executor) {
- this.cache = Caffeine.newBuilder()
- .executor(executor)
- .maximumWeight(maximumSizeInBytes)
- .removalListener(new EvictionListener())
- .weigher((BlockCacheKey key, Cacheable value) ->
- (int) Math.min(value.heapSize(), Integer.MAX_VALUE))
- .initialCapacity((int) Math.ceil((1.2 * maximumSizeInBytes) / avgBlockSize))
- .build();
- this.maxBlockSize = maxBlockSize;
- this.policy = cache.policy().eviction().get();
- this.stats = new CacheStats(getClass().getSimpleName());
-
- statsThreadPool = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
- .setNameFormat("TinyLfuBlockCacheStatsExecutor").setDaemon(true).build());
- statsThreadPool.scheduleAtFixedRate(this::logStats,
- STAT_THREAD_PERIOD_SECONDS, STAT_THREAD_PERIOD_SECONDS, TimeUnit.SECONDS);
- }
-
- @Override
- public void setVictimCache(BlockCache victimCache) {
- if (this.victimCache != null) {
- throw new IllegalArgumentException("The victim cache has already been set");
- }
- this.victimCache = requireNonNull(victimCache);
- }
-
- @Override
- public long size() {
- return policy.getMaximum();
- }
-
- @Override
- public long getFreeSize() {
- return size() - getCurrentSize();
- }
-
- @Override
- public long getCurrentSize() {
- return policy.weightedSize().getAsLong();
- }
-
- @Override
- public long getBlockCount() {
- return cache.estimatedSize();
- }
-
- @Override
- public long heapSize() {
- return getCurrentSize();
- }
-
- @Override
- public void setMaxSize(long size) {
- policy.setMaximum(size);
- }
-
- @Override
- public boolean containsBlock(BlockCacheKey cacheKey) {
- return cache.asMap().containsKey(cacheKey);
- }
-
- @Override
- public Cacheable getBlock(BlockCacheKey cacheKey,
- boolean caching, boolean repeat, boolean updateCacheMetrics) {
- Cacheable value = cache.getIfPresent(cacheKey);
- if (value == null) {
- if (repeat) {
- return null;
- }
- if (updateCacheMetrics) {
- stats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
- }
- if (victimCache != null) {
- value = victimCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
- if ((value != null) && caching) {
- if ((value instanceof HFileBlock) && ((HFileBlock) value).usesSharedMemory()) {
- value = ((HFileBlock) value).deepClone();
- }
- cacheBlock(cacheKey, value);
- }
- }
- } else if (updateCacheMetrics) {
- stats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
- }
- return value;
- }
-
- @Override
- public void cacheBlock(BlockCacheKey cacheKey, Cacheable value,
- boolean inMemory, boolean cacheDataInL1) {
- cacheBlock(cacheKey, value);
- }
-
- @Override
- public void cacheBlock(BlockCacheKey key, Cacheable value) {
- if (value.heapSize() > maxBlockSize) {
- // If there are a lot of blocks that are too big this can make the logs too noisy (2% logged)
- if (stats.failInsert() % 50 == 0) {
- LOG.warn(String.format(
- "Trying to cache too large a block %s @ %,d is %,d which is larger than %,d",
- key.getHfileName(), key.getOffset(), value.heapSize(), DEFAULT_MAX_BLOCK_SIZE));
- }
- } else {
- cache.put(key, value);
- }
- }
-
- @Override
- public boolean evictBlock(BlockCacheKey cacheKey) {
- Cacheable value = cache.asMap().remove(cacheKey);
- return (value != null);
- }
-
- @Override
- public int evictBlocksByHfileName(String hfileName) {
- int evicted = 0;
- for (BlockCacheKey key : cache.asMap().keySet()) {
- if (key.getHfileName().equals(hfileName) && evictBlock(key)) {
- evicted++;
- }
- }
- if (victimCache != null) {
- evicted += victimCache.evictBlocksByHfileName(hfileName);
- }
- return evicted;
- }
-
- @Override
- public CacheStats getStats() {
- return stats;
- }
-
- @Override
- public void shutdown() {
- if (victimCache != null) {
- victimCache.shutdown();
- }
- statsThreadPool.shutdown();
- }
-
- @Override
- public BlockCache[] getBlockCaches() {
- return null;
- }
-
- @Override
- public Iterator<CachedBlock> iterator() {
- long now = System.nanoTime();
- return cache.asMap().entrySet().stream()
- .map(entry -> (CachedBlock) new CachedBlockView(entry.getKey(), entry.getValue(), now))
- .iterator();
- }
-
- @Override
- public void returnBlock(BlockCacheKey cacheKey, Cacheable block) {
- // There is no SHARED type here in L1. But the block might have been served from the L2 victim
- // cache (when the Combined mode = false). So just try return this block to the victim cache.
- // Note : In case of CombinedBlockCache we will have this victim cache configured for L1
- // cache. But CombinedBlockCache will only call returnBlock on L2 cache.
- if (victimCache != null) {
- victimCache.returnBlock(cacheKey, block);
- }
- }
-
- private void logStats() {
- LOG.info(
- "totalSize=" + StringUtils.byteDesc(heapSize()) + ", " +
- "freeSize=" + StringUtils.byteDesc(getFreeSize()) + ", " +
- "max=" + StringUtils.byteDesc(size()) + ", " +
- "blockCount=" + getBlockCount() + ", " +
- "accesses=" + stats.getRequestCount() + ", " +
- "hits=" + stats.getHitCount() + ", " +
- "hitRatio=" + (stats.getHitCount() == 0 ?
- "0," : StringUtils.formatPercent(stats.getHitRatio(), 2) + ", ") +
- "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
- "cachingHits=" + stats.getHitCachingCount() + ", " +
- "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
- "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
- "evictions=" + stats.getEvictionCount() + ", " +
- "evicted=" + stats.getEvictedCount());
- }
-
- @Override
- public String toString() {
- return Objects.toStringHelper(this)
- .add("blockCount", getBlockCount())
- .add("currentSize", getCurrentSize())
- .add("freeSize", getFreeSize())
- .add("maxSize", size())
- .add("heapSize", heapSize())
- .add("victimCache", (victimCache != null))
- .toString();
- }
-
- /** A removal listener to asynchronously record evictions and populate the victim cache. */
- private final class EvictionListener implements RemovalListener<BlockCacheKey, Cacheable> {
-
- @Override
- public void onRemoval(BlockCacheKey key, Cacheable value, RemovalCause cause) {
- if (!cause.wasEvicted()) {
- // An explicit eviction (invalidation) is not added to the victim cache as the data may
- // no longer be valid for subsequent queries.
- return;
- }
-
- recordEviction();
-
- if (victimCache == null) {
- return;
- } else if (victimCache instanceof BucketCache) {
- BucketCache victimBucketCache = (BucketCache) victimCache;
- victimBucketCache.cacheBlockWithWait(key, value, /* inMemory */ true, /* wait */ true);
- } else {
- victimCache.cacheBlock(key, value);
- }
- }
- }
-
- /**
- * Records an eviction. The number of eviction operations and evicted blocks are identical, as
- * an eviction is triggered immediately when the capacity has been exceeded. An eviction is
- * performed asynchronously. See the library's documentation for details on write buffers,
- * batching, and maintenance behavior.
- */
- private void recordEviction() {
- // FIXME: Currently does not capture the insertion time
- stats.evicted(Long.MAX_VALUE, true);
- stats.evict();
- }
-
- private static final class CachedBlockView implements CachedBlock {
- private static final Comparator<CachedBlock> COMPARATOR = Comparator
- .comparing(CachedBlock::getFilename)
- .thenComparing(CachedBlock::getOffset)
- .thenComparing(CachedBlock::getCachedTime);
-
- private final BlockCacheKey key;
- private final Cacheable value;
- private final long now;
-
- public CachedBlockView(BlockCacheKey key, Cacheable value, long now) {
- this.now = now;
- this.key = key;
- this.value = value;
- }
-
- @Override
- public BlockPriority getBlockPriority() {
- // This does not appear to be used in any meaningful way and is irrelevant to this cache
- return BlockPriority.MEMORY;
- }
-
- @Override
- public BlockType getBlockType() {
- return value.getBlockType();
- }
-
- @Override
- public long getOffset() {
- return key.getOffset();
- }
-
- @Override
- public long getSize() {
- return value.heapSize();
- }
-
- @Override
- public long getCachedTime() {
- // This does not appear to be used in any meaningful way, so not captured
- return 0L;
- }
-
- @Override
- public String getFilename() {
- return key.getHfileName();
- }
-
- @Override
- public int compareTo(CachedBlock other) {
- return COMPARATOR.compare(this, other);
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- } else if (!(obj instanceof CachedBlock)) {
- return false;
- }
- CachedBlock other = (CachedBlock) obj;
- return compareTo(other) == 0;
- }
-
- @Override
- public int hashCode() {
- return key.hashCode();
- }
-
- @Override
- public String toString() {
- return BlockCacheUtil.toString(this, now);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 39a5e78..ec7a71f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -88,10 +88,11 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*
* <p>BucketCache can be used as mainly a block cache (see
* {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}), combined with
- * a BlockCache to decrease CMS GC and heap fragmentation.
+ * LruBlockCache to decrease CMS GC and heap fragmentation.
*
* <p>It also can be used as a secondary cache (e.g. using a file on ssd/fusionio to store
- * blocks) to enlarge cache space via a victim cache.
+ * blocks) to enlarge cache space via
+ * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache}
*/
@InterfaceAudience.Private
public class BucketCache implements BlockCache, HeapSize {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index bf44d33..d9d5261 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -37,12 +37,12 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.testclassification.IOTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory;
import org.apache.hadoop.hbase.io.hfile.Cacheable.MemoryType;
import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
import org.apache.hadoop.hbase.nio.ByteBuff;
-import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.util.Threads;
import org.junit.After;
import org.junit.Before;
@@ -328,7 +328,7 @@ public class TestCacheConfig {
BlockCache [] bcs = cbc.getBlockCaches();
assertTrue(bcs[0] instanceof LruBlockCache);
LruBlockCache lbc = (LruBlockCache)bcs[0];
- assertEquals(CacheConfig.getFirstLevelCacheSize(this.conf,
+ assertEquals(CacheConfig.getLruCacheSize(this.conf,
ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()), lbc.getMaxSize());
assertTrue(bcs[1] instanceof BucketCache);
BucketCache bc = (BucketCache)bcs[1];
@@ -347,7 +347,7 @@ public class TestCacheConfig {
// from L1 happens, it does not fail because L2 can't take the eviction because block too big.
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f);
MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
- long lruExpectedSize = CacheConfig.getFirstLevelCacheSize(this.conf, mu.getMax());
+ long lruExpectedSize = CacheConfig.getLruCacheSize(this.conf, mu.getMax());
final int bcSize = 100;
long bcExpectedSize = 100 * 1024 * 1024; // MB.
assertTrue(lruExpectedSize < bcExpectedSize);
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
deleted file mode 100644
index f06da8c..0000000
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.io.hfile;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.apache.hadoop.hbase.io.HeapSize;
-import org.apache.hadoop.hbase.testclassification.IOTests;
-import org.apache.hadoop.hbase.testclassification.SmallTests;
-import org.apache.hadoop.hbase.util.ClassSize;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
-
-/**
- * Tests the concurrent TinyLfuBlockCache.
- */
-@Category({IOTests.class, SmallTests.class})
-public class TestTinyLfuBlockCache {
-
- @Test
- public void testCacheSimple() throws Exception {
-
- long maxSize = 1000000;
- long blockSize = calculateBlockSizeDefault(maxSize, 101);
-
- TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
-
- CachedItem [] blocks = generateRandomBlocks(100, blockSize);
-
- long expectedCacheSize = cache.heapSize();
-
- // Confirm empty
- for (CachedItem block : blocks) {
- assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
- }
-
- // Add blocks
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- expectedCacheSize += block.heapSize();
- }
-
- // Verify correctly calculated cache heap size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // Check if all blocks are properly cached and retrieved
- for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
- assertTrue(buf != null);
- assertEquals(buf.heapSize(), block.heapSize());
- }
-
- // Re-add same blocks and ensure nothing has changed
- long expectedBlockCount = cache.getBlockCount();
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- }
- assertEquals(
- "Cache should ignore cache requests for blocks already in cache",
- expectedBlockCount, cache.getBlockCount());
-
- // Verify correctly calculated cache heap size
- assertEquals(expectedCacheSize, cache.heapSize());
-
- // Check if all blocks are properly cached and retrieved
- for (CachedItem block : blocks) {
- HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
- assertTrue(buf != null);
- assertEquals(buf.heapSize(), block.heapSize());
- }
-
- // Expect no evictions
- assertEquals(0, cache.getStats().getEvictionCount());
- }
-
- @Test
- public void testCacheEvictionSimple() throws Exception {
-
- long maxSize = 100000;
- long blockSize = calculateBlockSizeDefault(maxSize, 10);
-
- TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
-
- CachedItem [] blocks = generateFixedBlocks(11, blockSize, "block");
-
- // Add all the blocks
- for (CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- }
-
- // A single eviction run should have occurred
- assertEquals(1, cache.getStats().getEvictionCount());
-
- // The cache did not grow beyond max
- assertTrue(cache.heapSize() < maxSize);
-
- // All blocks except one should be in the cache
- assertEquals(10, cache.getBlockCount());
- }
-
- @Test
- public void testScanResistance() throws Exception {
-
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
-
- CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
- CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
-
- // Add 5 blocks from each
- for(int i=0; i<5; i++) {
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
- }
-
- // Add frequency
- for (int i = 0; i < 5; i++) {
- for (int j = 0; j < 10; j++) {
- CachedItem block = multiBlocks[i];
- cache.getBlock(block.cacheKey, true, false, true);
- }
- }
-
- // Let's keep "scanning" by adding single blocks. From here on we only
- // expect evictions from the single bucket.
-
- for(int i=5;i<18;i++) {
- cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
- }
-
- for (CachedItem block : multiBlocks) {
- assertTrue(cache.cache.asMap().containsKey(block.cacheKey));
- }
-
- assertEquals(10, cache.getBlockCount());
- assertEquals(13, cache.getStats().getEvictionCount());
-
- }
-
- @Test
- public void testMaxBlockSize() throws Exception {
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
- CachedItem [] tooLong = generateFixedBlocks(10, 2 * blockSize, "long");
- CachedItem [] small = generateFixedBlocks(15, blockSize / 2, "small");
-
- for (CachedItem i:tooLong) {
- cache.cacheBlock(i.cacheKey, i);
- }
- for (CachedItem i:small) {
- cache.cacheBlock(i.cacheKey, i);
- }
- assertEquals(15,cache.getBlockCount());
- for (CachedItem i:small) {
- assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
- }
- for (CachedItem i:tooLong) {
- assertNull(cache.getBlock(i.cacheKey, true, false, false));
- }
-
- assertEquals(10, cache.getStats().getFailedInserts());
- }
-
- @Test
- public void testResizeBlockCache() throws Exception {
-
- long maxSize = 100000;
- long blockSize = calculateBlockSize(maxSize, 10);
-
- TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
-
- CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
-
- for(CachedItem block : blocks) {
- cache.cacheBlock(block.cacheKey, block);
- }
-
- // Do not expect any evictions yet
- assertEquals(10, cache.getBlockCount());
- assertEquals(0, cache.getStats().getEvictionCount());
-
- // Resize to half capacity plus an extra block (otherwise we evict an extra)
- cache.setMaxSize(maxSize / 2);
-
- // And we expect 1/2 of the blocks to be evicted
- assertEquals(5, cache.getBlockCount());
- assertEquals(5, cache.getStats().getEvictedCount());
- }
-
- private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
- CachedItem [] blocks = new CachedItem[numBlocks];
- for(int i=0;i<numBlocks;i++) {
- blocks[i] = new CachedItem(pfx + i, size);
- }
- return blocks;
- }
-
- private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
- return generateFixedBlocks(numBlocks, (int)size, pfx);
- }
-
- private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
- CachedItem [] blocks = new CachedItem[numBlocks];
- Random r = new Random();
- for(int i=0;i<numBlocks;i++) {
- blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
- }
- return blocks;
- }
-
- private long calculateBlockSize(long maxSize, int numBlocks) {
- long roughBlockSize = maxSize / numBlocks;
- int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
- long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
- ClassSize.CONCURRENT_HASHMAP +
- (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
- (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
- long negateBlockSize = totalOverhead/numEntries;
- negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
- return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
- }
-
- private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
- long roughBlockSize = maxSize / numBlocks;
- int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
- long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
- ClassSize.CONCURRENT_HASHMAP +
- (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
- (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
- long negateBlockSize = totalOverhead / numEntries;
- negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
- return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
- LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
- }
-
- private static class CachedItem implements Cacheable {
- BlockCacheKey cacheKey;
- int size;
-
- CachedItem(String blockName, int size) {
- this.cacheKey = new BlockCacheKey(blockName, 0);
- this.size = size;
- }
-
- /** The size of this item reported to the block cache layer */
- @Override
- public long heapSize() {
- return ClassSize.align(size);
- }
-
- @Override
- public int getSerializedLength() {
- return 0;
- }
-
- @Override
- public CacheableDeserializer<Cacheable> getDeserializer() {
- return null;
- }
-
- @Override
- public void serialize(ByteBuffer destination) {
- }
-
- @Override
- public BlockType getBlockType() {
- return BlockType.DATA;
- }
-
- @Override
- public MemoryType getMemoryType() {
- return MemoryType.EXCLUSIVE;
- }
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/b952e647/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 028cbf7..a796abb 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1198,7 +1198,6 @@
<httpclient.version>4.5.2</httpclient.version>
<httpcore.version>4.4.4</httpcore.version>
<metrics-core.version>3.1.2</metrics-core.version>
- <caffeine.version>2.3.3</caffeine.version>
<guava.version>12.0.1</guava.version>
<jackson.version>1.9.13</jackson.version>
<jasper.version>5.5.23</jasper.version>
@@ -1483,11 +1482,6 @@
<version>${slf4j.version}</version>
</dependency>
<dependency>
- <groupId>com.github.ben-manes.caffeine</groupId>
- <artifactId>caffeine</artifactId>
- <version>${caffeine.version}</version>
- </dependency>
- <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>${metrics-core.version}</version>