You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2019/08/20 17:01:08 UTC
[hbase] 01/01: HBASE-22114 Port HBASE-15560 (TinyLFU-based
BlockCache) to branch-1
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch HBASE-22114-branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 942f4f84ce4739729d2f31cbe4ad4d2cae7417b0
Author: Andrew Purtell <ap...@apache.org>
AuthorDate: Tue Aug 20 09:48:51 2019 -0700
HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1
HBASE-15560 W-TinyLFU based BlockCache (Ben Manes)
---
.../hadoop/hbase/io/util/HeapMemorySizeUtil.java | 2 +-
hbase-common/src/main/resources/hbase-default.xml | 9 +
hbase-it/pom.xml | 15 +
.../src/main/resources/supplemental-models.xml | 29 +-
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 108 ++++--
.../hadoop/hbase/io/hfile/CombinedBlockCache.java | 54 +--
.../hbase/io/hfile/FirstLevelBlockCache.java | 45 +++
.../io/hfile/InclusiveCombinedBlockCache.java | 6 +-
.../hadoop/hbase/io/hfile/LruBlockCache.java | 27 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 5 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 7 +-
hbase-tinylfu-blockcache/pom.xml | 413 +++++++++++++++++++++
.../hadoop/hbase/io/hfile/TinyLfuBlockCache.java | 404 ++++++++++++++++++++
.../hbase/io/hfile/TestTinyLfuBlockCache.java | 298 +++++++++++++++
pom.xml | 9 +
15 files changed, 1352 insertions(+), 79 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
index 4369305..864a54c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
@@ -180,7 +180,7 @@ public class HeapMemorySizeUtil {
* @return the number of bytes to use for LRU, negative if disabled.
* @throws IllegalArgumentException if HFILE_BLOCK_CACHE_SIZE_KEY is > 1.0
*/
- public static long getLruCacheSize(final Configuration conf) {
+ public static long getFirstLevelCacheSize(final Configuration conf, final long xmx) {
float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
if (cachePercentage <= 0.0001f) {
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 3f409d7..82c43d9 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -807,6 +807,15 @@ possible configurations would overwhelm and obscure the important.
The default thread pool size if parallel-seeking feature enabled.</description>
</property>
<property>
+ <name>hfile.block.cache.policy</name>
+ <value>LRU</value>
+ <description>The eviction policy for the L1 block cache (LRU or TinyLFU). If you want to
+ use TinyLFU you must build with JDK8+, run with JRE8+, and have both the
+ hbase-tinylfu-blockcache module and its dependency the caffiene library installed into
+ the classpath.
+ </description>
+ </property>
+ <property>
<name>hfile.block.cache.size</name>
<value>0.4</value>
<description>Percentage of maximum heap (-Xmx setting) to allocate to block cache
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index e8a15e7..aec3ce5 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -363,6 +363,21 @@
</dependencies>
<profiles>
+
+ <profile>
+ <id>build-with-jdk8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-tinylfu-blockcache</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
<profile>
<id>rsgroup</id>
<activation>
diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
index 165e758..8911eeb 100644
--- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml
+++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
@@ -344,6 +344,20 @@ under the License.
</supplement>
<supplement>
<project>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
@@ -2162,7 +2176,7 @@ Mozilla Public License Version 2.0
means any form of the work other than Source Code Form.
1.7. "Larger Work"
- means a work that combines Covered Software with other material, in
+ means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
@@ -2593,4 +2607,17 @@ Copyright (c) 2007-2011 The JRuby project
</licenses>
</project>
</supplement>
+ <supplement>
+ <project>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_annotations</artifactId>
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ </project>
+ </supplement>
</supplementalDataModels>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 87eae04..cdd304d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -22,7 +22,8 @@ import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ForkJoinPool;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,6 +47,12 @@ public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
+ * Configuration key to cache block policy (Lru, TinyLfu).
+ */
+ public static final String HFILE_BLOCK_CACHE_POLICY_KEY = "hfile.block.cache.policy";
+ public static final String HFILE_BLOCK_CACHE_POLICY_DEFAULT = "LRU";
+
+ /**
* Configuration key to cache data blocks on write. There are separate
* switches for bloom blocks and non-root index blocks.
*/
@@ -92,7 +99,7 @@ public class CacheConfig {
* is an in-memory map that needs to be persisted across restarts. Where to store this
* in-memory state is what you supply here: e.g. <code>/tmp/bucketcache.map</code>.
*/
- public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
+ public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
"hbase.bucketcache.persistent.path";
/**
@@ -100,11 +107,11 @@ public class CacheConfig {
* as indices and blooms are kept in the lru blockcache and the data blocks in the
* bucket cache).
*/
- public static final String BUCKET_CACHE_COMBINED_KEY =
+ public static final String BUCKET_CACHE_COMBINED_KEY =
"hbase.bucketcache.combinedcache.enabled";
public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
- public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
+ public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
"hbase.bucketcache.writer.queuelength";
/**
@@ -162,6 +169,7 @@ public class CacheConfig {
memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache");
// TODO(eclark): Consider more. Redis, etc.
Class<? extends BlockCache> clazz;
+ @SuppressWarnings("unchecked")
ExternalBlockCaches(String clazzName) {
try {
clazz = (Class<? extends BlockCache>) Class.forName(clazzName);
@@ -456,7 +464,9 @@ public class CacheConfig {
* @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
*/
public boolean shouldCacheCompressed(BlockCategory category) {
- if (!isBlockCacheEnabled()) return false;
+ if (!isBlockCacheEnabled()) {
+ return false;
+ }
switch (category) {
case DATA:
return this.cacheDataCompressed;
@@ -538,28 +548,62 @@ public class CacheConfig {
// Clear this if in tests you'd make more than one block cache instance.
@VisibleForTesting
static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
- private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE = null;
- private static BlockCache GLOBAL_L2_CACHE_INSTANCE = null;
+ private static FirstLevelBlockCache GLOBAL_L1_CACHE_INSTANCE;
+ private static BlockCache GLOBAL_L2_CACHE_INSTANCE;
+ private static ForkJoinPool GLOBAL_FORKJOIN_POOL;
/** Boolean whether we have disabled the block cache entirely. */
@VisibleForTesting
static boolean blockCacheDisabled = false;
/**
- * @param c Configuration to use.
- * @return An L1 instance. Currently an instance of LruBlockCache.
+ * @param c Configuration to use
+ * @return An L1 instance
+ */
+ public static FirstLevelBlockCache getL1(final Configuration c) {
+ long xmx = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+ long l1CacheSize = HeapMemorySizeUtil.getFirstLevelCacheSize(c, xmx);
+ return getL1(l1CacheSize, c);
+ }
+
+ /**
+ * @param c Configuration to use
+ * @param xmx Max heap memory
+ * @return An L1 instance
*/
- private static synchronized LruBlockCache getL1(final Configuration c) {
+
+ private synchronized static FirstLevelBlockCache getL1(long cacheSize, Configuration c) {
if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE;
- final long lruCacheSize = HeapMemorySizeUtil.getLruCacheSize(c);
- if (lruCacheSize < 0) {
- blockCacheDisabled = true;
+ if (cacheSize < 0) {
+ return null;
}
- if (blockCacheDisabled) return null;
+ String policy = c.get(HFILE_BLOCK_CACHE_POLICY_KEY, HFILE_BLOCK_CACHE_POLICY_DEFAULT);
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
- LOG.info("Allocating LruBlockCache size=" +
- StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
- GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c);
+ LOG.info("Allocating BlockCache size=" +
+ StringUtils.byteDesc(cacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
+ if (policy.equalsIgnoreCase("LRU")) {
+ GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(cacheSize, blockSize, true, c);
+ } else if (policy.equalsIgnoreCase("TinyLFU")) {
+ if (GLOBAL_FORKJOIN_POOL == null) {
+ GLOBAL_FORKJOIN_POOL = new ForkJoinPool();
+ }
+ Class<?> tinyLFUClass;
+ try {
+ tinyLFUClass = Class.forName("org.apache.hadoop.hbase.io.hfile.TinyLfuBlockCache");
+ GLOBAL_L1_CACHE_INSTANCE = (FirstLevelBlockCache)
+ tinyLFUClass.getDeclaredConstructor(long.class, long.class, Executor.class,
+ Configuration.class)
+ .newInstance(cacheSize, blockSize, GLOBAL_FORKJOIN_POOL, c);
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Unable to instantiate the TinyLfuBlockCache block cache policy." +
+ "If you want to use TinyLFU you must build with JDK8+, run with JRE8+, and have both " +
+ "the hbase-tinylfu-blockcache module and its dependency the caffiene library " +
+ "installed into the classpath.", e);
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown block cache policy " + policy);
+ }
return GLOBAL_L1_CACHE_INSTANCE;
}
@@ -600,7 +644,7 @@ public class CacheConfig {
}
private static BlockCache getExternalBlockcache(Configuration c) {
- Class klass = null;
+ Class<?> klass = null;
// Get the class, from the config. s
try {
@@ -628,7 +672,9 @@ public class CacheConfig {
private static BlockCache getBucketCache(Configuration c) {
// Check for L2. ioengine name must be non-null.
String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
- if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
+ if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) {
+ return null;
+ }
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
final long bucketCacheSize = HeapMemorySizeUtil.getBucketCacheSize(c);
@@ -686,33 +732,35 @@ public class CacheConfig {
* @return The block cache or <code>null</code>.
*/
public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
- if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
- if (blockCacheDisabled) return null;
+ if (GLOBAL_BLOCK_CACHE_INSTANCE != null) {
+ return GLOBAL_BLOCK_CACHE_INSTANCE;
+ }
+ if (blockCacheDisabled) {
+ return null;
+ }
if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
LOG.warn("The config key " + DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY +
" is deprecated now, instead please use " + BLOCKCACHE_BLOCKSIZE_KEY +". "
+ "In future release we will remove the deprecated config.");
}
- LruBlockCache l1 = getL1(conf);
- // blockCacheDisabled is set as a side-effect of getL1Internal(), so check it again after the call.
- if (blockCacheDisabled) return null;
+ FirstLevelBlockCache l1 = getL1(conf);
BlockCache l2 = getL2(conf);
if (l2 == null) {
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
} else {
boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
- boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
+ boolean combinedWithL1 = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
DEFAULT_BUCKET_CACHE_COMBINED);
if (useExternal) {
GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
} else {
- if (combinedWithLru) {
+ if (combinedWithL1) {
GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
} else {
- // L1 and L2 are not 'combined'. They are connected via the LruBlockCache victimhandler
- // mechanism. It is a little ugly but works according to the following: when the
- // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
- // a block from the L1 cache, if not in L1, we will search L2.
+ // L1 and L2 are not 'combined'. They are connected via the FirstLevelBlockCache
+ // victimhandler mechanism. It is a little ugly but works according to the following:
+ // when the background eviction thread runs, blocks evicted from L1 will go to L2 AND when
+ // we get a block from the L1 cache, if not in L1, we will search L2.
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index ec8e5a0..3cd786f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -28,24 +28,24 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
/**
* CombinedBlockCache is an abstraction layer that combines
- * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used
+ * {@link FirstLevelBlockCache} and {@link BucketCache}. The smaller lruCache is used
* to cache bloom blocks and index blocks. The larger l2Cache is used to
* cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean, boolean)} reads
- * first from the smaller lruCache before looking for the block in the l2Cache. Blocks evicted
- * from lruCache are put into the bucket cache.
+ * first from the smaller l1Cache before looking for the block in the l2Cache. Blocks evicted
+ * from l1Cache are put into the bucket cache.
* Metrics are the combined size and hits and misses of both caches.
- *
+ *
*/
@InterfaceAudience.Private
public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
- protected final LruBlockCache lruCache;
+ protected final FirstLevelBlockCache l1Cache;
protected final BlockCache l2Cache;
protected final CombinedCacheStats combinedCacheStats;
- public CombinedBlockCache(LruBlockCache lruCache, BlockCache l2Cache) {
- this.lruCache = lruCache;
+ public CombinedBlockCache(FirstLevelBlockCache l1Cache, BlockCache l2Cache) {
+ this.l1Cache = l1Cache;
this.l2Cache = l2Cache;
- this.combinedCacheStats = new CombinedCacheStats(lruCache.getStats(),
+ this.combinedCacheStats = new CombinedCacheStats(l1Cache.getStats(),
l2Cache.getStats());
}
@@ -55,7 +55,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
if (l2Cache instanceof HeapSize) {
l2size = ((HeapSize) l2Cache).heapSize();
}
- return lruCache.heapSize() + l2size;
+ return l1Cache.heapSize() + l2size;
}
@Override
@@ -63,7 +63,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
final boolean cacheDataInL1) {
boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
if (metaBlock || cacheDataInL1) {
- lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
+ l1Cache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
} else {
l2Cache.cacheBlock(cacheKey, buf, inMemory, false);
}
@@ -79,19 +79,19 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
boolean repeat, boolean updateCacheMetrics) {
// TODO: is there a hole here, or just awkwardness since in the lruCache getBlock
// we end up calling l2Cache.getBlock.
- return lruCache.containsBlock(cacheKey)?
- lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
+ return l1Cache.containsBlock(cacheKey)?
+ l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
@Override
public boolean evictBlock(BlockCacheKey cacheKey) {
- return lruCache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
+ return l1Cache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
}
@Override
public int evictBlocksByHfileName(String hfileName) {
- return lruCache.evictBlocksByHfileName(hfileName)
+ return l1Cache.evictBlocksByHfileName(hfileName)
+ l2Cache.evictBlocksByHfileName(hfileName);
}
@@ -102,43 +102,43 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public void shutdown() {
- lruCache.shutdown();
+ l1Cache.shutdown();
l2Cache.shutdown();
}
@Override
public long size() {
- return lruCache.size() + l2Cache.size();
+ return l1Cache.size() + l2Cache.size();
}
@Override
public long getMaxSize() {
- return lruCache.getMaxSize() + l2Cache.getMaxSize();
+ return l1Cache.getMaxSize() + l2Cache.getMaxSize();
}
@Override
public long getFreeSize() {
- return lruCache.getFreeSize() + l2Cache.getFreeSize();
+ return l1Cache.getFreeSize() + l2Cache.getFreeSize();
}
@Override
public long getCurrentSize() {
- return lruCache.getCurrentSize() + l2Cache.getCurrentSize();
+ return l1Cache.getCurrentSize() + l2Cache.getCurrentSize();
}
@Override
public long getCurrentDataSize() {
- return lruCache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
+ return l1Cache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
}
@Override
public long getBlockCount() {
- return lruCache.getBlockCount() + l2Cache.getBlockCount();
+ return l1Cache.getBlockCount() + l2Cache.getBlockCount();
}
@Override
public long getDataBlockCount() {
- return lruCache.getDataBlockCount() + l2Cache.getDataBlockCount();
+ return l1Cache.getDataBlockCount() + l2Cache.getDataBlockCount();
}
public static class CombinedCacheStats extends CacheStats {
@@ -323,7 +323,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
lruCacheStats.rollMetricsPeriod();
bucketCacheStats.rollMetricsPeriod();
}
-
+
@Override
public long getFailedInserts() {
return lruCacheStats.getFailedInserts() + bucketCacheStats.getFailedInserts();
@@ -334,13 +334,13 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
return lruCacheStats.getSumHitCountsPastNPeriods()
+ bucketCacheStats.getSumHitCountsPastNPeriods();
}
-
+
@Override
public long getSumRequestCountsPastNPeriods() {
return lruCacheStats.getSumRequestCountsPastNPeriods()
+ bucketCacheStats.getSumRequestCountsPastNPeriods();
}
-
+
@Override
public long getSumHitCachingCountsPastNPeriods() {
return lruCacheStats.getSumHitCachingCountsPastNPeriods()
@@ -361,11 +361,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public BlockCache[] getBlockCaches() {
- return new BlockCache [] {this.lruCache, this.l2Cache};
+ return new BlockCache [] {this.l1Cache, this.l2Cache};
}
@Override
public void setMaxSize(long size) {
- this.lruCache.setMaxSize(size);
+ this.l1Cache.setMaxSize(size);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
new file mode 100644
index 0000000..65c71e2
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.HeapSize;
+
+/**
+ * In-memory BlockCache that may be backed by secondary layer(s).
+ */
+@InterfaceAudience.Private
+public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize {
+
+ /**
+ * Whether the cache contains the block with specified cacheKey
+ *
+ * @param cacheKey block key
+ * @return true if it contains the block
+ */
+ boolean containsBlock(BlockCacheKey cacheKey);
+
+ /**
+ * Specifies the secondary cache. An entry that is evicted from this cache due to a size
+ * constraint will be inserted into the victim cache.
+ *
+ * @param victimCache the second level cache
+ * @throws IllegalArgumentException if the victim cache had already been set
+ */
+ void setVictimCache(BlockCache victimCache);
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
index 667e7b4..160714b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class InclusiveCombinedBlockCache extends CombinedBlockCache implements BlockCache {
- public InclusiveCombinedBlockCache(LruBlockCache l1, BlockCache l2) {
+ public InclusiveCombinedBlockCache(FirstLevelBlockCache l1, BlockCache l2) {
super(l1,l2);
}
@@ -34,7 +34,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
// On all external cache set ups the lru should have the l2 cache set as the victimHandler
// Because of that all requests that miss inside of the lru block cache will be
// tried in the l2 block cache.
- return lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+ return l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
/**
@@ -50,7 +50,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
final boolean cacheDataInL1) {
// This is the inclusive part of the combined block cache.
// Every block is placed into both block caches.
- lruCache.cacheBlock(cacheKey, buf, inMemory, true);
+ l1Cache.cacheBlock(cacheKey, buf, inMemory, true);
// This assumes that insertion into the L2 block cache is either async or very fast.
l2Cache.cacheBlock(cacheKey, buf, inMemory, true);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index b854079..78967d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -18,8 +18,11 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import static java.util.Objects.requireNonNull;
+
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
import java.lang.ref.WeakReference;
import java.util.EnumMap;
import java.util.Iterator;
@@ -97,7 +100,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
@InterfaceAudience.Private
@JsonIgnoreProperties({"encodingCountsForTest"})
-public class LruBlockCache implements ResizableBlockCache, HeapSize {
+public class LruBlockCache implements FirstLevelBlockCache {
private static final Log LOG = LogFactory.getLog(LruBlockCache.class);
@@ -246,8 +249,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
DEFAULT_MEMORY_FACTOR,
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
false,
- DEFAULT_MAX_BLOCK_SIZE
- );
+ DEFAULT_MAX_BLOCK_SIZE);
}
public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
@@ -262,8 +264,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE),
- conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
- );
+ conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE));
}
public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
@@ -332,6 +333,14 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
}
@Override
+ public void setVictimCache(BlockCache victimCache) {
+ if (victimHandler != null) {
+ throw new IllegalArgumentException("The victim cache has already been set");
+ }
+ victimHandler = requireNonNull(victimCache);
+ }
+
+ @Override
public void setMaxSize(long maxSize) {
this.maxSize = maxSize;
if(this.size.get() > acceptableSize() && !evictionInProgress) {
@@ -497,6 +506,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
* @param cacheKey
* @return true if contains the block
*/
+ @Override
public boolean containsBlock(BlockCacheKey cacheKey) {
return map.containsKey(cacheKey);
}
@@ -1139,7 +1149,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
Map<BlockType, Integer> counts =
new EnumMap<BlockType, Integer>(BlockType.class);
for (LruCachedBlock cb : map.values()) {
- BlockType blockType = ((Cacheable)cb.getBuffer()).getBlockType();
+ BlockType blockType = cb.getBuffer().getBlockType();
Integer count = counts.get(blockType);
counts.put(blockType, (count == null ? 0 : count) + 1);
}
@@ -1159,11 +1169,6 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
return counts;
}
- public void setVictimCache(BlockCache handler) {
- assert victimHandler == null;
- victimHandler = handler;
- }
-
@VisibleForTesting
Map<BlockCacheKey, LruCachedBlock> getMapForTests() {
return map;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index c5a1b21..889c976 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -89,11 +89,10 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*
* <p>BucketCache can be used as mainly a block cache (see
* {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}), combined with
- * combined with LruBlockCache to decrease CMS GC and heap fragmentation.
+ * a BlockCache to decrease CMS GC and heap fragmentation.
*
* <p>It also can be used as a secondary cache (e.g. using a file on ssd/fusionio to store
- * blocks) to enlarge cache space via
- * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache}
+ * blocks) to enlarge cache space via a victim cache.
*/
@InterfaceAudience.Private
public class BucketCache implements BlockCache, HeapSize {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index b1cc58c..061ed4f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -253,7 +252,8 @@ public class TestCacheConfig {
BlockCache [] bcs = cbc.getBlockCaches();
assertTrue(bcs[0] instanceof LruBlockCache);
LruBlockCache lbc = (LruBlockCache)bcs[0];
- assertEquals(HeapMemorySizeUtil.getLruCacheSize(this.conf), lbc.getMaxSize());
+ assertEquals(HeapMemorySizeUtil.getFirstLevelCacheSize(this.conf,
+ ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()), lbc.getMaxSize());
assertTrue(bcs[1] instanceof BucketCache);
BucketCache bc = (BucketCache)bcs[1];
// getMaxSize comes back in bytes but we specified size in MB
@@ -270,7 +270,8 @@ public class TestCacheConfig {
// Make lru size is smaller than bcSize for sure. Need this to be true so when eviction
// from L1 happens, it does not fail because L2 can't take the eviction because block too big.
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f);
- long lruExpectedSize = HeapMemorySizeUtil.getLruCacheSize(this.conf);
+ long lruExpectedSize = HeapMemorySizeUtil.getFirstLevelCacheSize(this.conf,
+ ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax());
final int bcSize = 100;
long bcExpectedSize = 100 * 1024 * 1024; // MB.
assertTrue(lruExpectedSize < bcExpectedSize);
diff --git a/hbase-tinylfu-blockcache/pom.xml b/hbase-tinylfu-blockcache/pom.xml
new file mode 100644
index 0000000..48e4a29
--- /dev/null
+++ b/hbase-tinylfu-blockcache/pom.xml
@@ -0,0 +1,413 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>1.5.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <artifactId>hbase-tinylfu-blockcache</artifactId>
+ <name>Apache HBase - TinyLFU Block Cache</name>
+ <description>
+ HBase module that provides the alternative L1 block cache policy TinyLFU.
+ </description>
+
+ <properties>
+ <compileSource>1.8</compileSource>
+ <!-- Enforcer plugin rules activated by root POM will check this -->
+ <java.min.version>1.8</java.min.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <!-- Add the generated sources -->
+ <execution>
+ <id>versionInfo-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.hbase.ResourceCheckerJUnitListener</value>
+ </property>
+ </properties>
+ </configuration>
+ <!-- Always skip the second part executions, since we only run
+ simple unit tests in this module -->
+ <executions>
+ <execution>
+ <id>secondPartTestsExecution</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Make a jar and put the sources in the jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings
+ only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <versionRange>[${maven.antrun.version}]</versionRange>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute/>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[2.8,)</versionRange>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <versionRange>[3.2,)</versionRange>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${caffeine.version}</version>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+
+ <profile>
+ <id>build-with-jdk8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <properties>
+ <compileSource>1.8</compileSource>
+ </properties>
+ </profile>
+
+ <!-- Needs to make the profile in apache parent pom -->
+ <profile>
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>license-javadocs</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/apidocs</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/javadoc/META-INF/</directory>
+ <targetPath>META-INF/</targetPath>
+ <includes>
+ <include>NOTICE</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <!-- Skip the tests in this module -->
+ <profile>
+ <id>skipCommonTests</id>
+ <activation>
+ <property>
+ <name>skipCommonTests</name>
+ </property>
+ </activation>
+ <properties>
+ <surefire.skipFirstPart>true</surefire.skipFirstPart>
+ </properties>
+ </profile>
+
+ <!-- profile for building against Hadoop 2.x: This is the default -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate
+ the required classpath that is required in the env
+ of the launch container in the mini mr/yarn cluster
+ -->
+ <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop-three.version}</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate
+ the required classpath that is required in the env
+ of the launch container in the mini mr/yarn cluster
+ -->
+ <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>errorProne</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <!-- Turn on error-prone -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven.compiler.version}</version>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ <showWarnings>true</showWarnings>
+ <compilerArgs>
+ <arg>-XepDisableWarningsInGeneratedCode</arg>
+ <arg>-Xep:FallThrough:OFF</arg> <!-- already in findbugs -->
+ </compilerArgs>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-compiler-javac-errorprone</artifactId>
+ <version>${plexus.errorprone.javac.version}</version>
+ </dependency>
+ <!-- override plexus-compiler-javac-errorprone's dependency on
+ Error Prone with the latest version -->
+ <dependency>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_core</artifactId>
+ <version>${error-prone.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+</project>
diff --git a/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java b/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
new file mode 100644
index 0000000..38a6aa9
--- /dev/null
+++ b/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Policy.Eviction;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.RemovalListener;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * A block cache that is memory-aware using {@link HeapSize}, memory bounded using the W-TinyLFU
+ * eviction algorithm, and concurrent. This implementation delegates to a Caffeine cache to provide
+ * O(1) read and write operations.
+ * <ul>
+ * <li>W-TinyLFU: http://arxiv.org/pdf/1512.00727.pdf</li>
+ * <li>Caffeine: https://github.com/ben-manes/caffeine</li>
+ * <li>Cache design: http://highscalability.com/blog/2016/1/25/design-of-a-modern-cache.html</li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+public final class TinyLfuBlockCache implements FirstLevelBlockCache {
+ private static final Log LOG = LogFactory.getLog(TinyLfuBlockCache.class);
+
+ private static final String MAX_BLOCK_SIZE = "hbase.tinylfu.max.block.size";
+ private static final long DEFAULT_MAX_BLOCK_SIZE = 16L * 1024L * 1024L;
+ private static final int STAT_THREAD_PERIOD_SECONDS = 5 * 60;
+
+ private final Eviction<BlockCacheKey, Cacheable> policy;
+ private final ScheduledExecutorService statsThreadPool;
+ private final long maxBlockSize;
+ private final CacheStats stats;
+
+ private BlockCache victimCache;
+
+ @VisibleForTesting
+ final Cache<BlockCacheKey, Cacheable> cache;
+
+ /**
+ * Creates a block cache.
+ *
+ * @param maximumSizeInBytes maximum size of this cache, in bytes
+ * @param avgBlockSize expected average size of blocks, in bytes
+ * @param executor the cache's executor
+ * @param conf additional configuration
+ */
+ public TinyLfuBlockCache(long maximumSizeInBytes, long avgBlockSize,
+ Executor executor, Configuration conf) {
+ this(maximumSizeInBytes, avgBlockSize,
+ conf.getLong(MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE), executor);
+ }
+
+ /**
+ * Creates a block cache.
+ *
+ * @param maximumSizeInBytes maximum size of this cache, in bytes
+ * @param avgBlockSize expected average size of blocks, in bytes
+ * @param maxBlockSize maximum size of a block, in bytes
+ * @param executor the cache's executor
+ */
+ public TinyLfuBlockCache(long maximumSizeInBytes,
+ long avgBlockSize, long maxBlockSize, Executor executor) {
+ this.cache = Caffeine.newBuilder()
+ .executor(executor)
+ .maximumWeight(maximumSizeInBytes)
+ .removalListener(new EvictionListener())
+ .weigher((BlockCacheKey key, Cacheable value) ->
+ (int) Math.min(value.heapSize(), Integer.MAX_VALUE))
+ .initialCapacity((int) Math.ceil((1.2 * maximumSizeInBytes) / avgBlockSize))
+ .build();
+ this.maxBlockSize = maxBlockSize;
+ this.policy = cache.policy().eviction().get();
+ this.stats = new CacheStats(getClass().getSimpleName());
+
+ statsThreadPool = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
+ .setNameFormat("TinyLfuBlockCacheStatsExecutor").setDaemon(true).build());
+ statsThreadPool.scheduleAtFixedRate(this::logStats,
+ STAT_THREAD_PERIOD_SECONDS, STAT_THREAD_PERIOD_SECONDS, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void setVictimCache(BlockCache victimCache) {
+ if (this.victimCache != null) {
+ throw new IllegalArgumentException("The victim cache has already been set");
+ }
+ this.victimCache = requireNonNull(victimCache);
+ }
+
+ @Override
+ public long size() {
+ return policy.getMaximum();
+ }
+
+ @Override
+ public long getFreeSize() {
+ return size() - getCurrentSize();
+ }
+
+ @Override
+ public long getCurrentSize() {
+ return policy.weightedSize().getAsLong();
+ }
+
+ @Override
+ public long getBlockCount() {
+ return cache.estimatedSize();
+ }
+
+ @Override
+ public long heapSize() {
+ return getCurrentSize();
+ }
+
+ @Override
+ public void setMaxSize(long size) {
+ policy.setMaximum(size);
+ }
+
+ @Override
+ public boolean containsBlock(BlockCacheKey cacheKey) {
+ return cache.asMap().containsKey(cacheKey);
+ }
+
+ @Override
+ public Cacheable getBlock(BlockCacheKey cacheKey,
+ boolean caching, boolean repeat, boolean updateCacheMetrics) {
+ Cacheable value = cache.getIfPresent(cacheKey);
+ if (value == null) {
+ if (repeat) {
+ return null;
+ }
+ if (updateCacheMetrics) {
+ stats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
+ }
+ if (victimCache != null) {
+ value = victimCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+ if ((value != null) && caching) {
+ cacheBlock(cacheKey, value);
+ }
+ }
+ } else if (updateCacheMetrics) {
+ stats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
+ }
+ return value;
+ }
+
+ @Override
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable value,
+ boolean inMemory, boolean cacheDataInL1) {
+ cacheBlock(cacheKey, value);
+ }
+
+ @Override
+ public void cacheBlock(BlockCacheKey key, Cacheable value) {
+ if (value.heapSize() > maxBlockSize) {
+ // If there are a lot of blocks that are too big this can make the logs too noisy (2% logged)
+ if (stats.failInsert() % 50 == 0) {
+ LOG.warn(String.format(
+ "Trying to cache too large a block %s @ %,d is %,d which is larger than %,d",
+ key.getHfileName(), key.getOffset(), value.heapSize(), DEFAULT_MAX_BLOCK_SIZE));
+ }
+ } else {
+ cache.put(key, value);
+ }
+ }
+
+ @Override
+ public boolean evictBlock(BlockCacheKey cacheKey) {
+ Cacheable value = cache.asMap().remove(cacheKey);
+ return (value != null);
+ }
+
+ @Override
+ public int evictBlocksByHfileName(String hfileName) {
+ int evicted = 0;
+ for (BlockCacheKey key : cache.asMap().keySet()) {
+ if (key.getHfileName().equals(hfileName) && evictBlock(key)) {
+ evicted++;
+ }
+ }
+ if (victimCache != null) {
+ evicted += victimCache.evictBlocksByHfileName(hfileName);
+ }
+ return evicted;
+ }
+
+ @Override
+ public CacheStats getStats() {
+ return stats;
+ }
+
+ @Override
+ public void shutdown() {
+ if (victimCache != null) {
+ victimCache.shutdown();
+ }
+ statsThreadPool.shutdown();
+ }
+
+ @Override
+ public BlockCache[] getBlockCaches() {
+ return null;
+ }
+
+ @Override
+ public Iterator<CachedBlock> iterator() {
+ long now = System.nanoTime();
+ return cache.asMap().entrySet().stream()
+ .map(entry -> (CachedBlock) new CachedBlockView(entry.getKey(), entry.getValue(), now))
+ .iterator();
+ }
+
+ private void logStats() {
+ LOG.info(
+ "totalSize=" + StringUtils.byteDesc(heapSize()) + ", " +
+ "freeSize=" + StringUtils.byteDesc(getFreeSize()) + ", " +
+ "max=" + StringUtils.byteDesc(size()) + ", " +
+ "blockCount=" + getBlockCount() + ", " +
+ "accesses=" + stats.getRequestCount() + ", " +
+ "hits=" + stats.getHitCount() + ", " +
+ "hitRatio=" + (stats.getHitCount() == 0 ?
+ "0," : StringUtils.formatPercent(stats.getHitRatio(), 2) + ", ") +
+ "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
+ "cachingHits=" + stats.getHitCachingCount() + ", " +
+ "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
+ "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
+ "evictions=" + stats.getEvictionCount() + ", " +
+ "evicted=" + stats.getEvictedCount());
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("blockCount", getBlockCount())
+ .add("currentSize", getCurrentSize())
+ .add("freeSize", getFreeSize())
+ .add("maxSize", size())
+ .add("heapSize", heapSize())
+ .add("victimCache", (victimCache != null))
+ .toString();
+ }
+
+ /** A removal listener to asynchronously record evictions and populate the victim cache. */
+ private final class EvictionListener implements RemovalListener<BlockCacheKey, Cacheable> {
+
+ @Override
+ public void onRemoval(BlockCacheKey key, Cacheable value, RemovalCause cause) {
+ if (!cause.wasEvicted()) {
+ // An explicit eviction (invalidation) is not added to the victim cache as the data may
+ // no longer be valid for subsequent queries.
+ return;
+ }
+
+ recordEviction();
+
+ if (victimCache == null) {
+ return;
+ } else if (victimCache instanceof BucketCache) {
+ BucketCache victimBucketCache = (BucketCache) victimCache;
+ victimBucketCache.cacheBlockWithWait(key, value, /* inMemory */ true,
+ /* cacheDataInL1 */ true, /* wait */ true);
+ } else {
+ victimCache.cacheBlock(key, value);
+ }
+ }
+ }
+
+ /**
+ * Records an eviction. The number of eviction operations and evicted blocks are identical, as
+ * an eviction is triggered immediately when the capacity has been exceeded. An eviction is
+ * performed asynchronously. See the library's documentation for details on write buffers,
+ * batching, and maintenance behavior.
+ */
+ private void recordEviction() {
+ // FIXME: Currently does not capture the insertion time
+ stats.evicted(Long.MAX_VALUE, true);
+ stats.evict();
+ }
+
+ private static final class CachedBlockView implements CachedBlock {
+ private static final Comparator<CachedBlock> COMPARATOR = Comparator
+ .comparing(CachedBlock::getFilename)
+ .thenComparing(CachedBlock::getOffset)
+ .thenComparing(CachedBlock::getCachedTime);
+
+ private final BlockCacheKey key;
+ private final Cacheable value;
+ private final long now;
+
+ public CachedBlockView(BlockCacheKey key, Cacheable value, long now) {
+ this.now = now;
+ this.key = key;
+ this.value = value;
+ }
+
+ @Override
+ public BlockPriority getBlockPriority() {
+ // This does not appear to be used in any meaningful way and is irrelevant to this cache
+ return BlockPriority.MEMORY;
+ }
+
+ @Override
+ public BlockType getBlockType() {
+ return value.getBlockType();
+ }
+
+ @Override
+ public long getOffset() {
+ return key.getOffset();
+ }
+
+ @Override
+ public long getSize() {
+ return value.heapSize();
+ }
+
+ @Override
+ public long getCachedTime() {
+ // This does not appear to be used in any meaningful way, so not captured
+ return 0L;
+ }
+
+ @Override
+ public String getFilename() {
+ return key.getHfileName();
+ }
+
+ @Override
+ public int compareTo(CachedBlock other) {
+ return COMPARATOR.compare(this, other);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ } else if (!(obj instanceof CachedBlock)) {
+ return false;
+ }
+ CachedBlock other = (CachedBlock) obj;
+ return compareTo(other) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return key.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return BlockCacheUtil.toString(this, now);
+ }
+ }
+
+ @Override
+ public long getMaxSize() {
+ return size();
+ }
+
+ @Override
+ public long getCurrentDataSize() {
+ return getCurrentSize();
+ }
+
+ @Override
+ public long getDataBlockCount() {
+ return getBlockCount();
+ }
+}
diff --git a/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java b/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
new file mode 100644
index 0000000..3d5047e
--- /dev/null
+++ b/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
@@ -0,0 +1,298 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.testclassification.IOTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests the concurrent TinyLfuBlockCache.
+ */
+@Category({IOTests.class, SmallTests.class})
+public class TestTinyLfuBlockCache {
+
+ @Test
+ public void testCacheSimple() throws Exception {
+
+ long maxSize = 1000000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 101);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateRandomBlocks(100, blockSize);
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Confirm empty
+ for (CachedItem block : blocks) {
+ assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
+ }
+
+ // Add blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.heapSize();
+ }
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ // Re-add same blocks and ensure nothing has changed
+ long expectedBlockCount = cache.getBlockCount();
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+ assertEquals(
+ "Cache should ignore cache requests for blocks already in cache",
+ expectedBlockCount, cache.getBlockCount());
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ // Expect no evictions
+ assertEquals(0, cache.getStats().getEvictionCount());
+ }
+
+ @Test
+ public void testCacheEvictionSimple() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateFixedBlocks(11, blockSize, "block");
+
+ // Add all the blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+
+ // A single eviction run should have occurred
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // The cache did not grow beyond max
+ assertTrue(cache.heapSize() < maxSize);
+
+ // All blocks except one should be in the cache
+ assertEquals(10, cache.getBlockCount());
+ }
+
+ @Test
+ public void testScanResistance() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
+ CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+
+ // Add 5 blocks from each
+ for(int i=0; i<5; i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ }
+
+ // Add frequency
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 10; j++) {
+ CachedItem block = multiBlocks[i];
+ cache.getBlock(block.cacheKey, true, false, true);
+ }
+ }
+
+ // Let's keep "scanning" by adding single blocks. From here on we only
+ // expect evictions from the single bucket.
+
+ for(int i=5;i<18;i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ }
+
+ for (CachedItem block : multiBlocks) {
+ assertTrue(cache.cache.asMap().containsKey(block.cacheKey));
+ }
+
+ assertEquals(10, cache.getBlockCount());
+ assertEquals(13, cache.getStats().getEvictionCount());
+
+ }
+
+ @Test
+ public void testMaxBlockSize() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+ CachedItem [] tooLong = generateFixedBlocks(10, 2 * blockSize, "long");
+ CachedItem [] small = generateFixedBlocks(15, blockSize / 2, "small");
+
+ for (CachedItem i:tooLong) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ for (CachedItem i:small) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ assertEquals(15,cache.getBlockCount());
+ for (CachedItem i:small) {
+ assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+ for (CachedItem i:tooLong) {
+ assertNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+
+ assertEquals(10, cache.getStats().getFailedInserts());
+ }
+
+ @Test
+ public void testResizeBlockCache() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
+
+ for(CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+
+ // Do not expect any evictions yet
+ assertEquals(10, cache.getBlockCount());
+ assertEquals(0, cache.getStats().getEvictionCount());
+
+ // Resize to half capacity plus an extra block (otherwise we evict an extra)
+ cache.setMaxSize(maxSize / 2);
+
+ // And we expect 1/2 of the blocks to be evicted
+ assertEquals(5, cache.getBlockCount());
+ assertEquals(5, cache.getStats().getEvictedCount());
+ }
+
+ private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
+ CachedItem [] blocks = new CachedItem[numBlocks];
+ for(int i=0;i<numBlocks;i++) {
+ blocks[i] = new CachedItem(pfx + i, size);
+ }
+ return blocks;
+ }
+
+ private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
+ return generateFixedBlocks(numBlocks, (int)size, pfx);
+ }
+
+ private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
+ CachedItem [] blocks = new CachedItem[numBlocks];
+ Random r = new Random();
+ for(int i=0;i<numBlocks;i++) {
+ blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
+ }
+ return blocks;
+ }
+
+ private long calculateBlockSize(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
+ long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
+ ClassSize.CONCURRENT_HASHMAP +
+ (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
+ (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = totalOverhead/numEntries;
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
+ }
+
+ private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
+ long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
+ ClassSize.CONCURRENT_HASHMAP +
+ (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
+ (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = totalOverhead / numEntries;
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
+ LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
+ }
+
+ private static class CachedItem implements Cacheable {
+ BlockCacheKey cacheKey;
+ int size;
+
+ CachedItem(String blockName, int size) {
+ this.cacheKey = new BlockCacheKey(blockName, 0);
+ this.size = size;
+ }
+
+ /** The size of this item reported to the block cache layer */
+ @Override
+ public long heapSize() {
+ return ClassSize.align(size);
+ }
+
+ @Override
+ public int getSerializedLength() {
+ return 0;
+ }
+
+ @Override
+ public CacheableDeserializer<Cacheable> getDeserializer() {
+ return null;
+ }
+
+ @Override
+ public BlockType getBlockType() {
+ return BlockType.DATA;
+ }
+
+ @Override
+ public void serialize(ByteBuffer destination, boolean includeNextBlockOnDiskSize) {
+ }
+ }
+
+}
+
diff --git a/pom.xml b/pom.xml
index 874188c..24f8921 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1263,6 +1263,7 @@
<assembly.file>src/main/assembly/hadoop-two-compat.xml</assembly.file>
<!-- end HBASE-15925 default hadoop compatibility values -->
<avro.version>1.7.7</avro.version>
+ <caffeine.version>2.7.0</caffeine.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-codec.version>1.9</commons-codec.version>
<!-- pretty outdated -->
@@ -1603,6 +1604,11 @@
<version>${metrics-core.version}</version>
</dependency>
<dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${caffeine.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>3.1.2</version>
@@ -2010,6 +2016,9 @@
<activation>
<jdk>1.8</jdk>
</activation>
+ <modules>
+ <module>hbase-tinylfu-blockcache</module>
+ </modules>
<build>
<pluginManagement>
<plugins>