You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2019/08/20 17:01:07 UTC
[hbase] branch HBASE-22114-branch-1 created (now 942f4f8)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a change to branch HBASE-22114-branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git.
at 942f4f8 HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1
This branch includes the following new commits:
new bb31ffa Revert "HBASE-22823 Mark Canary as Public/Evolving"
new 942f4f8 HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hbase] 01/01: HBASE-22114 Port HBASE-15560 (TinyLFU-based
BlockCache) to branch-1
Posted by ap...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch HBASE-22114-branch-1
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 942f4f84ce4739729d2f31cbe4ad4d2cae7417b0
Author: Andrew Purtell <ap...@apache.org>
AuthorDate: Tue Aug 20 09:48:51 2019 -0700
HBASE-22114 Port HBASE-15560 (TinyLFU-based BlockCache) to branch-1
HBASE-15560 W-TinyLFU based BlockCache (Ben Manes)
---
.../hadoop/hbase/io/util/HeapMemorySizeUtil.java | 2 +-
hbase-common/src/main/resources/hbase-default.xml | 9 +
hbase-it/pom.xml | 15 +
.../src/main/resources/supplemental-models.xml | 29 +-
.../apache/hadoop/hbase/io/hfile/CacheConfig.java | 108 ++++--
.../hadoop/hbase/io/hfile/CombinedBlockCache.java | 54 +--
.../hbase/io/hfile/FirstLevelBlockCache.java | 45 +++
.../io/hfile/InclusiveCombinedBlockCache.java | 6 +-
.../hadoop/hbase/io/hfile/LruBlockCache.java | 27 +-
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 5 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 7 +-
hbase-tinylfu-blockcache/pom.xml | 413 +++++++++++++++++++++
.../hadoop/hbase/io/hfile/TinyLfuBlockCache.java | 404 ++++++++++++++++++++
.../hbase/io/hfile/TestTinyLfuBlockCache.java | 298 +++++++++++++++
pom.xml | 9 +
15 files changed, 1352 insertions(+), 79 deletions(-)
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
index 4369305..864a54c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java
@@ -180,7 +180,7 @@ public class HeapMemorySizeUtil {
* @return the number of bytes to use for LRU, negative if disabled.
* @throws IllegalArgumentException if HFILE_BLOCK_CACHE_SIZE_KEY is > 1.0
*/
- public static long getLruCacheSize(final Configuration conf) {
+ public static long getFirstLevelCacheSize(final Configuration conf, final long xmx) {
float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
if (cachePercentage <= 0.0001f) {
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 3f409d7..82c43d9 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -807,6 +807,15 @@ possible configurations would overwhelm and obscure the important.
The default thread pool size if parallel-seeking feature enabled.</description>
</property>
<property>
+ <name>hfile.block.cache.policy</name>
+ <value>LRU</value>
+ <description>The eviction policy for the L1 block cache (LRU or TinyLFU). If you want to
+ use TinyLFU you must build with JDK8+, run with JRE8+, and have both the
+ hbase-tinylfu-blockcache module and its dependency the caffiene library installed into
+ the classpath.
+ </description>
+ </property>
+ <property>
<name>hfile.block.cache.size</name>
<value>0.4</value>
<description>Percentage of maximum heap (-Xmx setting) to allocate to block cache
diff --git a/hbase-it/pom.xml b/hbase-it/pom.xml
index e8a15e7..aec3ce5 100644
--- a/hbase-it/pom.xml
+++ b/hbase-it/pom.xml
@@ -363,6 +363,21 @@
</dependencies>
<profiles>
+
+ <profile>
+ <id>build-with-jdk8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-tinylfu-blockcache</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+
<profile>
<id>rsgroup</id>
<activation>
diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
index 165e758..8911eeb 100644
--- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml
+++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml
@@ -344,6 +344,20 @@ under the License.
</supplement>
<supplement>
<project>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ </project>
+ </supplement>
+ <supplement>
+ <project>
<groupId>com.lmax</groupId>
<artifactId>disruptor</artifactId>
@@ -2162,7 +2176,7 @@ Mozilla Public License Version 2.0
means any form of the work other than Source Code Form.
1.7. "Larger Work"
- means a work that combines Covered Software with other material, in
+ means a work that combines Covered Software with other material, in
a separate file or files, that is not Covered Software.
1.8. "License"
@@ -2593,4 +2607,17 @@ Copyright (c) 2007-2011 The JRuby project
</licenses>
</project>
</supplement>
+ <supplement>
+ <project>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_annotations</artifactId>
+ <licenses>
+ <license>
+ <name>Apache License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ <distribution>repo</distribution>
+ </license>
+ </licenses>
+ </project>
+ </supplement>
</supplementalDataModels>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 87eae04..cdd304d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -22,7 +22,8 @@ import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_SIZE_KEY;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ForkJoinPool;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -46,6 +47,12 @@ public class CacheConfig {
private static final Log LOG = LogFactory.getLog(CacheConfig.class.getName());
/**
+ * Configuration key to cache block policy (Lru, TinyLfu).
+ */
+ public static final String HFILE_BLOCK_CACHE_POLICY_KEY = "hfile.block.cache.policy";
+ public static final String HFILE_BLOCK_CACHE_POLICY_DEFAULT = "LRU";
+
+ /**
* Configuration key to cache data blocks on write. There are separate
* switches for bloom blocks and non-root index blocks.
*/
@@ -92,7 +99,7 @@ public class CacheConfig {
* is an in-memory map that needs to be persisted across restarts. Where to store this
* in-memory state is what you supply here: e.g. <code>/tmp/bucketcache.map</code>.
*/
- public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
+ public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
"hbase.bucketcache.persistent.path";
/**
@@ -100,11 +107,11 @@ public class CacheConfig {
* as indices and blooms are kept in the lru blockcache and the data blocks in the
* bucket cache).
*/
- public static final String BUCKET_CACHE_COMBINED_KEY =
+ public static final String BUCKET_CACHE_COMBINED_KEY =
"hbase.bucketcache.combinedcache.enabled";
public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
- public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
+ public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
"hbase.bucketcache.writer.queuelength";
/**
@@ -162,6 +169,7 @@ public class CacheConfig {
memcached("org.apache.hadoop.hbase.io.hfile.MemcachedBlockCache");
// TODO(eclark): Consider more. Redis, etc.
Class<? extends BlockCache> clazz;
+ @SuppressWarnings("unchecked")
ExternalBlockCaches(String clazzName) {
try {
clazz = (Class<? extends BlockCache>) Class.forName(clazzName);
@@ -456,7 +464,9 @@ public class CacheConfig {
* @return true if this {@link BlockCategory} should be compressed in blockcache, false otherwise
*/
public boolean shouldCacheCompressed(BlockCategory category) {
- if (!isBlockCacheEnabled()) return false;
+ if (!isBlockCacheEnabled()) {
+ return false;
+ }
switch (category) {
case DATA:
return this.cacheDataCompressed;
@@ -538,28 +548,62 @@ public class CacheConfig {
// Clear this if in tests you'd make more than one block cache instance.
@VisibleForTesting
static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
- private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE = null;
- private static BlockCache GLOBAL_L2_CACHE_INSTANCE = null;
+ private static FirstLevelBlockCache GLOBAL_L1_CACHE_INSTANCE;
+ private static BlockCache GLOBAL_L2_CACHE_INSTANCE;
+ private static ForkJoinPool GLOBAL_FORKJOIN_POOL;
/** Boolean whether we have disabled the block cache entirely. */
@VisibleForTesting
static boolean blockCacheDisabled = false;
/**
- * @param c Configuration to use.
- * @return An L1 instance. Currently an instance of LruBlockCache.
+ * @param c Configuration to use
+ * @return An L1 instance
+ */
+ public static FirstLevelBlockCache getL1(final Configuration c) {
+ long xmx = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
+ long l1CacheSize = HeapMemorySizeUtil.getFirstLevelCacheSize(c, xmx);
+ return getL1(l1CacheSize, c);
+ }
+
+ /**
+ * @param c Configuration to use
+ * @param xmx Max heap memory
+ * @return An L1 instance
*/
- private static synchronized LruBlockCache getL1(final Configuration c) {
+
+ private synchronized static FirstLevelBlockCache getL1(long cacheSize, Configuration c) {
if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE;
- final long lruCacheSize = HeapMemorySizeUtil.getLruCacheSize(c);
- if (lruCacheSize < 0) {
- blockCacheDisabled = true;
+ if (cacheSize < 0) {
+ return null;
}
- if (blockCacheDisabled) return null;
+ String policy = c.get(HFILE_BLOCK_CACHE_POLICY_KEY, HFILE_BLOCK_CACHE_POLICY_DEFAULT);
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
- LOG.info("Allocating LruBlockCache size=" +
- StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
- GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c);
+ LOG.info("Allocating BlockCache size=" +
+ StringUtils.byteDesc(cacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
+ if (policy.equalsIgnoreCase("LRU")) {
+ GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(cacheSize, blockSize, true, c);
+ } else if (policy.equalsIgnoreCase("TinyLFU")) {
+ if (GLOBAL_FORKJOIN_POOL == null) {
+ GLOBAL_FORKJOIN_POOL = new ForkJoinPool();
+ }
+ Class<?> tinyLFUClass;
+ try {
+ tinyLFUClass = Class.forName("org.apache.hadoop.hbase.io.hfile.TinyLfuBlockCache");
+ GLOBAL_L1_CACHE_INSTANCE = (FirstLevelBlockCache)
+ tinyLFUClass.getDeclaredConstructor(long.class, long.class, Executor.class,
+ Configuration.class)
+ .newInstance(cacheSize, blockSize, GLOBAL_FORKJOIN_POOL, c);
+ } catch (Exception e) {
+ throw new RuntimeException(
+ "Unable to instantiate the TinyLfuBlockCache block cache policy." +
+ "If you want to use TinyLFU you must build with JDK8+, run with JRE8+, and have both " +
+ "the hbase-tinylfu-blockcache module and its dependency the caffiene library " +
+ "installed into the classpath.", e);
+ }
+ } else {
+ throw new IllegalArgumentException("Unknown block cache policy " + policy);
+ }
return GLOBAL_L1_CACHE_INSTANCE;
}
@@ -600,7 +644,7 @@ public class CacheConfig {
}
private static BlockCache getExternalBlockcache(Configuration c) {
- Class klass = null;
+ Class<?> klass = null;
// Get the class, from the config. s
try {
@@ -628,7 +672,9 @@ public class CacheConfig {
private static BlockCache getBucketCache(Configuration c) {
// Check for L2. ioengine name must be non-null.
String bucketCacheIOEngineName = c.get(BUCKET_CACHE_IOENGINE_KEY, null);
- if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) return null;
+ if (bucketCacheIOEngineName == null || bucketCacheIOEngineName.length() <= 0) {
+ return null;
+ }
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
final long bucketCacheSize = HeapMemorySizeUtil.getBucketCacheSize(c);
@@ -686,33 +732,35 @@ public class CacheConfig {
* @return The block cache or <code>null</code>.
*/
public static synchronized BlockCache instantiateBlockCache(Configuration conf) {
- if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE;
- if (blockCacheDisabled) return null;
+ if (GLOBAL_BLOCK_CACHE_INSTANCE != null) {
+ return GLOBAL_BLOCK_CACHE_INSTANCE;
+ }
+ if (blockCacheDisabled) {
+ return null;
+ }
if (conf.get(DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY) != null) {
LOG.warn("The config key " + DEPRECATED_BLOCKCACHE_BLOCKSIZE_KEY +
" is deprecated now, instead please use " + BLOCKCACHE_BLOCKSIZE_KEY +". "
+ "In future release we will remove the deprecated config.");
}
- LruBlockCache l1 = getL1(conf);
- // blockCacheDisabled is set as a side-effect of getL1Internal(), so check it again after the call.
- if (blockCacheDisabled) return null;
+ FirstLevelBlockCache l1 = getL1(conf);
BlockCache l2 = getL2(conf);
if (l2 == null) {
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
} else {
boolean useExternal = conf.getBoolean(EXTERNAL_BLOCKCACHE_KEY, EXTERNAL_BLOCKCACHE_DEFAULT);
- boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
+ boolean combinedWithL1 = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY,
DEFAULT_BUCKET_CACHE_COMBINED);
if (useExternal) {
GLOBAL_BLOCK_CACHE_INSTANCE = new InclusiveCombinedBlockCache(l1, l2);
} else {
- if (combinedWithLru) {
+ if (combinedWithL1) {
GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(l1, l2);
} else {
- // L1 and L2 are not 'combined'. They are connected via the LruBlockCache victimhandler
- // mechanism. It is a little ugly but works according to the following: when the
- // background eviction thread runs, blocks evicted from L1 will go to L2 AND when we get
- // a block from the L1 cache, if not in L1, we will search L2.
+ // L1 and L2 are not 'combined'. They are connected via the FirstLevelBlockCache
+ // victimhandler mechanism. It is a little ugly but works according to the following:
+ // when the background eviction thread runs, blocks evicted from L1 will go to L2 AND when
+ // we get a block from the L1 cache, if not in L1, we will search L2.
GLOBAL_BLOCK_CACHE_INSTANCE = l1;
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index ec8e5a0..3cd786f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -28,24 +28,24 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
/**
* CombinedBlockCache is an abstraction layer that combines
- * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used
+ * {@link FirstLevelBlockCache} and {@link BucketCache}. The smaller lruCache is used
* to cache bloom blocks and index blocks. The larger l2Cache is used to
* cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean, boolean)} reads
- * first from the smaller lruCache before looking for the block in the l2Cache. Blocks evicted
- * from lruCache are put into the bucket cache.
+ * first from the smaller l1Cache before looking for the block in the l2Cache. Blocks evicted
+ * from l1Cache are put into the bucket cache.
* Metrics are the combined size and hits and misses of both caches.
- *
+ *
*/
@InterfaceAudience.Private
public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
- protected final LruBlockCache lruCache;
+ protected final FirstLevelBlockCache l1Cache;
protected final BlockCache l2Cache;
protected final CombinedCacheStats combinedCacheStats;
- public CombinedBlockCache(LruBlockCache lruCache, BlockCache l2Cache) {
- this.lruCache = lruCache;
+ public CombinedBlockCache(FirstLevelBlockCache l1Cache, BlockCache l2Cache) {
+ this.l1Cache = l1Cache;
this.l2Cache = l2Cache;
- this.combinedCacheStats = new CombinedCacheStats(lruCache.getStats(),
+ this.combinedCacheStats = new CombinedCacheStats(l1Cache.getStats(),
l2Cache.getStats());
}
@@ -55,7 +55,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
if (l2Cache instanceof HeapSize) {
l2size = ((HeapSize) l2Cache).heapSize();
}
- return lruCache.heapSize() + l2size;
+ return l1Cache.heapSize() + l2size;
}
@Override
@@ -63,7 +63,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
final boolean cacheDataInL1) {
boolean metaBlock = buf.getBlockType().getCategory() != BlockCategory.DATA;
if (metaBlock || cacheDataInL1) {
- lruCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
+ l1Cache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1);
} else {
l2Cache.cacheBlock(cacheKey, buf, inMemory, false);
}
@@ -79,19 +79,19 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
boolean repeat, boolean updateCacheMetrics) {
// TODO: is there a hole here, or just awkwardness since in the lruCache getBlock
// we end up calling l2Cache.getBlock.
- return lruCache.containsBlock(cacheKey)?
- lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
+ return l1Cache.containsBlock(cacheKey)?
+ l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics):
l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
@Override
public boolean evictBlock(BlockCacheKey cacheKey) {
- return lruCache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
+ return l1Cache.evictBlock(cacheKey) || l2Cache.evictBlock(cacheKey);
}
@Override
public int evictBlocksByHfileName(String hfileName) {
- return lruCache.evictBlocksByHfileName(hfileName)
+ return l1Cache.evictBlocksByHfileName(hfileName)
+ l2Cache.evictBlocksByHfileName(hfileName);
}
@@ -102,43 +102,43 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public void shutdown() {
- lruCache.shutdown();
+ l1Cache.shutdown();
l2Cache.shutdown();
}
@Override
public long size() {
- return lruCache.size() + l2Cache.size();
+ return l1Cache.size() + l2Cache.size();
}
@Override
public long getMaxSize() {
- return lruCache.getMaxSize() + l2Cache.getMaxSize();
+ return l1Cache.getMaxSize() + l2Cache.getMaxSize();
}
@Override
public long getFreeSize() {
- return lruCache.getFreeSize() + l2Cache.getFreeSize();
+ return l1Cache.getFreeSize() + l2Cache.getFreeSize();
}
@Override
public long getCurrentSize() {
- return lruCache.getCurrentSize() + l2Cache.getCurrentSize();
+ return l1Cache.getCurrentSize() + l2Cache.getCurrentSize();
}
@Override
public long getCurrentDataSize() {
- return lruCache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
+ return l1Cache.getCurrentDataSize() + l2Cache.getCurrentDataSize();
}
@Override
public long getBlockCount() {
- return lruCache.getBlockCount() + l2Cache.getBlockCount();
+ return l1Cache.getBlockCount() + l2Cache.getBlockCount();
}
@Override
public long getDataBlockCount() {
- return lruCache.getDataBlockCount() + l2Cache.getDataBlockCount();
+ return l1Cache.getDataBlockCount() + l2Cache.getDataBlockCount();
}
public static class CombinedCacheStats extends CacheStats {
@@ -323,7 +323,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
lruCacheStats.rollMetricsPeriod();
bucketCacheStats.rollMetricsPeriod();
}
-
+
@Override
public long getFailedInserts() {
return lruCacheStats.getFailedInserts() + bucketCacheStats.getFailedInserts();
@@ -334,13 +334,13 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
return lruCacheStats.getSumHitCountsPastNPeriods()
+ bucketCacheStats.getSumHitCountsPastNPeriods();
}
-
+
@Override
public long getSumRequestCountsPastNPeriods() {
return lruCacheStats.getSumRequestCountsPastNPeriods()
+ bucketCacheStats.getSumRequestCountsPastNPeriods();
}
-
+
@Override
public long getSumHitCachingCountsPastNPeriods() {
return lruCacheStats.getSumHitCachingCountsPastNPeriods()
@@ -361,11 +361,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
@Override
public BlockCache[] getBlockCaches() {
- return new BlockCache [] {this.lruCache, this.l2Cache};
+ return new BlockCache [] {this.l1Cache, this.l2Cache};
}
@Override
public void setMaxSize(long size) {
- this.lruCache.setMaxSize(size);
+ this.l1Cache.setMaxSize(size);
}
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
new file mode 100644
index 0000000..65c71e2
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/FirstLevelBlockCache.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.HeapSize;
+
+/**
+ * In-memory BlockCache that may be backed by secondary layer(s).
+ */
+@InterfaceAudience.Private
+public interface FirstLevelBlockCache extends ResizableBlockCache, HeapSize {
+
+ /**
+ * Whether the cache contains the block with specified cacheKey
+ *
+ * @param cacheKey block key
+ * @return true if it contains the block
+ */
+ boolean containsBlock(BlockCacheKey cacheKey);
+
+ /**
+ * Specifies the secondary cache. An entry that is evicted from this cache due to a size
+ * constraint will be inserted into the victim cache.
+ *
+ * @param victimCache the second level cache
+ * @throws IllegalArgumentException if the victim cache had already been set
+ */
+ void setVictimCache(BlockCache victimCache);
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
index 667e7b4..160714b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java
@@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
@InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG)
public class InclusiveCombinedBlockCache extends CombinedBlockCache implements BlockCache {
- public InclusiveCombinedBlockCache(LruBlockCache l1, BlockCache l2) {
+ public InclusiveCombinedBlockCache(FirstLevelBlockCache l1, BlockCache l2) {
super(l1,l2);
}
@@ -34,7 +34,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
// On all external cache set ups the lru should have the l2 cache set as the victimHandler
// Because of that all requests that miss inside of the lru block cache will be
// tried in the l2 block cache.
- return lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+ return l1Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
}
/**
@@ -50,7 +50,7 @@ public class InclusiveCombinedBlockCache extends CombinedBlockCache implements B
final boolean cacheDataInL1) {
// This is the inclusive part of the combined block cache.
// Every block is placed into both block caches.
- lruCache.cacheBlock(cacheKey, buf, inMemory, true);
+ l1Cache.cacheBlock(cacheKey, buf, inMemory, true);
// This assumes that insertion into the L2 block cache is either async or very fast.
l2Cache.cacheBlock(cacheKey, buf, inMemory, true);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index b854079..78967d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -18,8 +18,11 @@
*/
package org.apache.hadoop.hbase.io.hfile;
+import static java.util.Objects.requireNonNull;
+
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
import java.lang.ref.WeakReference;
import java.util.EnumMap;
import java.util.Iterator;
@@ -97,7 +100,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
@InterfaceAudience.Private
@JsonIgnoreProperties({"encodingCountsForTest"})
-public class LruBlockCache implements ResizableBlockCache, HeapSize {
+public class LruBlockCache implements FirstLevelBlockCache {
private static final Log LOG = LogFactory.getLog(LruBlockCache.class);
@@ -246,8 +249,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
DEFAULT_MEMORY_FACTOR,
DEFAULT_HARD_CAPACITY_LIMIT_FACTOR,
false,
- DEFAULT_MAX_BLOCK_SIZE
- );
+ DEFAULT_MAX_BLOCK_SIZE);
}
public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) {
@@ -262,8 +264,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR),
conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR),
conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE),
- conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE)
- );
+ conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE));
}
public LruBlockCache(long maxSize, long blockSize, Configuration conf) {
@@ -332,6 +333,14 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
}
@Override
+ public void setVictimCache(BlockCache victimCache) {
+ if (victimHandler != null) {
+ throw new IllegalArgumentException("The victim cache has already been set");
+ }
+ victimHandler = requireNonNull(victimCache);
+ }
+
+ @Override
public void setMaxSize(long maxSize) {
this.maxSize = maxSize;
if(this.size.get() > acceptableSize() && !evictionInProgress) {
@@ -497,6 +506,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
* @param cacheKey
* @return true if contains the block
*/
+ @Override
public boolean containsBlock(BlockCacheKey cacheKey) {
return map.containsKey(cacheKey);
}
@@ -1139,7 +1149,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
Map<BlockType, Integer> counts =
new EnumMap<BlockType, Integer>(BlockType.class);
for (LruCachedBlock cb : map.values()) {
- BlockType blockType = ((Cacheable)cb.getBuffer()).getBlockType();
+ BlockType blockType = cb.getBuffer().getBlockType();
Integer count = counts.get(blockType);
counts.put(blockType, (count == null ? 0 : count) + 1);
}
@@ -1159,11 +1169,6 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
return counts;
}
- public void setVictimCache(BlockCache handler) {
- assert victimHandler == null;
- victimHandler = handler;
- }
-
@VisibleForTesting
Map<BlockCacheKey, LruCachedBlock> getMapForTests() {
return map;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index c5a1b21..889c976 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -89,11 +89,10 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*
* <p>BucketCache can be used as mainly a block cache (see
* {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}), combined with
- * combined with LruBlockCache to decrease CMS GC and heap fragmentation.
+ * a BlockCache to decrease CMS GC and heap fragmentation.
*
* <p>It also can be used as a secondary cache (e.g. using a file on ssd/fusionio to store
- * blocks) to enlarge cache space via
- * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache}
+ * blocks) to enlarge cache space via a victim cache.
*/
@InterfaceAudience.Private
public class BucketCache implements BlockCache, HeapSize {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index b1cc58c..061ed4f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -24,7 +24,6 @@ import static org.junit.Assert.fail;
import java.io.IOException;
import java.lang.management.ManagementFactory;
-import java.lang.management.MemoryUsage;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -253,7 +252,8 @@ public class TestCacheConfig {
BlockCache [] bcs = cbc.getBlockCaches();
assertTrue(bcs[0] instanceof LruBlockCache);
LruBlockCache lbc = (LruBlockCache)bcs[0];
- assertEquals(HeapMemorySizeUtil.getLruCacheSize(this.conf), lbc.getMaxSize());
+ assertEquals(HeapMemorySizeUtil.getFirstLevelCacheSize(this.conf,
+ ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax()), lbc.getMaxSize());
assertTrue(bcs[1] instanceof BucketCache);
BucketCache bc = (BucketCache)bcs[1];
// getMaxSize comes back in bytes but we specified size in MB
@@ -270,7 +270,8 @@ public class TestCacheConfig {
// Make lru size is smaller than bcSize for sure. Need this to be true so when eviction
// from L1 happens, it does not fail because L2 can't take the eviction because block too big.
this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f);
- long lruExpectedSize = HeapMemorySizeUtil.getLruCacheSize(this.conf);
+ long lruExpectedSize = HeapMemorySizeUtil.getFirstLevelCacheSize(this.conf,
+ ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax());
final int bcSize = 100;
long bcExpectedSize = 100 * 1024 * 1024; // MB.
assertTrue(lruExpectedSize < bcExpectedSize);
diff --git a/hbase-tinylfu-blockcache/pom.xml b/hbase-tinylfu-blockcache/pom.xml
new file mode 100644
index 0000000..48e4a29
--- /dev/null
+++ b/hbase-tinylfu-blockcache/pom.xml
@@ -0,0 +1,413 @@
+<?xml version="1.0"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>hbase</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <version>1.5.0-SNAPSHOT</version>
+ <relativePath>..</relativePath>
+ </parent>
+ <artifactId>hbase-tinylfu-blockcache</artifactId>
+ <name>Apache HBase - TinyLFU Block Cache</name>
+ <description>
+ HBase module that provides the alternative L1 block cache policy TinyLFU.
+ </description>
+
+ <properties>
+ <compileSource>1.8</compileSource>
+ <!-- Enforcer plugin rules activated by root POM will check this -->
+ <java.min.version>1.8</java.min.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-remote-resources-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <!--Make it so assembly:single does nothing in here-->
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <skipAssembly>true</skipAssembly>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <!-- Add the generated sources -->
+ <execution>
+ <id>versionInfo-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>${project.build.directory}/generated-sources/java</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <properties>
+ <property>
+ <name>listener</name>
+ <value>org.apache.hadoop.hbase.ResourceCheckerJUnitListener</value>
+ </property>
+ </properties>
+ </configuration>
+ <!-- Always skip the second part executions, since we only run
+ simple unit tests in this module -->
+ <executions>
+ <execution>
+ <id>secondPartTestsExecution</id>
+ <phase>test</phase>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <!-- Make a jar and put the sources in the jar -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings
+ only. It has no influence on the Maven build itself. -->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <versionRange>[${maven.antrun.version}]</versionRange>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <execute/>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[2.8,)</versionRange>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <versionRange>[3.2,)</versionRange>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-annotations</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${caffeine.version}</version>
+ </dependency>
+ </dependencies>
+
+ <profiles>
+
+ <profile>
+ <id>build-with-jdk8</id>
+ <activation>
+ <jdk>1.8</jdk>
+ </activation>
+ <properties>
+ <compileSource>1.8</compileSource>
+ </properties>
+ </profile>
+
+ <!-- Needs to make the profile in apache parent pom -->
+ <profile>
+ <id>apache-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-resources-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>license-javadocs</id>
+ <phase>prepare-package</phase>
+ <goals>
+ <goal>copy-resources</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/apidocs</outputDirectory>
+ <resources>
+ <resource>
+ <directory>src/main/javadoc/META-INF/</directory>
+ <targetPath>META-INF/</targetPath>
+ <includes>
+ <include>NOTICE</include>
+ </includes>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <!-- Skip the tests in this module -->
+ <profile>
+ <id>skipCommonTests</id>
+ <activation>
+ <property>
+ <name>skipCommonTests</name>
+ </property>
+ </activation>
+ <properties>
+ <surefire.skipFirstPart>true</surefire.skipFirstPart>
+ </properties>
+ </profile>
+
+ <!-- profile for building against Hadoop 2.x: This is the default -->
+ <profile>
+ <id>hadoop-2.0</id>
+ <activation>
+ <property>
+ <!--Below formatting for dev-support/generate-hadoopX-poms.sh-->
+ <!--h2--><name>!hadoop.profile</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate
+ the required classpath that is required in the env
+ of the launch container in the mini mr/yarn cluster
+ -->
+ <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <!--
+ profile for building against Hadoop 3.0.x. Activate using:
+ mvn -Dhadoop.profile=3.0
+ -->
+ <profile>
+ <id>hadoop-3.0</id>
+ <activation>
+ <property>
+ <name>hadoop.profile</name>
+ <value>3.0</value>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>${hadoop-three.version}</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate
+ the required classpath that is required in the env
+ of the launch container in the mini mr/yarn cluster
+ -->
+ <outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ <profile>
+ <id>errorProne</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <!-- Turn on error-prone -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>${maven.compiler.version}</version>
+ <configuration>
+ <compilerId>javac-with-errorprone</compilerId>
+ <forceJavacCompilerUse>true</forceJavacCompilerUse>
+ <showWarnings>true</showWarnings>
+ <compilerArgs>
+ <arg>-XepDisableWarningsInGeneratedCode</arg>
+ <arg>-Xep:FallThrough:OFF</arg> <!-- already in findbugs -->
+ </compilerArgs>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </path>
+ </annotationProcessorPaths>
+ </configuration>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-compiler-javac-errorprone</artifactId>
+ <version>${plexus.errorprone.javac.version}</version>
+ </dependency>
+ <!-- override plexus-compiler-javac-errorprone's dependency on
+ Error Prone with the latest version -->
+ <dependency>
+ <groupId>com.google.errorprone</groupId>
+ <artifactId>error_prone_core</artifactId>
+ <version>${error-prone.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-error-prone</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+</project>
diff --git a/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java b/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
new file mode 100644
index 0000000..38a6aa9
--- /dev/null
+++ b/hbase-tinylfu-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/TinyLfuBlockCache.java
@@ -0,0 +1,404 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static java.util.Objects.requireNonNull;
+
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
+import com.github.benmanes.caffeine.cache.Policy.Eviction;
+import com.github.benmanes.caffeine.cache.RemovalCause;
+import com.github.benmanes.caffeine.cache.RemovalListener;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Objects;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * A block cache that is memory-aware using {@link HeapSize}, memory bounded using the W-TinyLFU
+ * eviction algorithm, and concurrent. This implementation delegates to a Caffeine cache to provide
+ * O(1) read and write operations.
+ * <ul>
+ * <li>W-TinyLFU: http://arxiv.org/pdf/1512.00727.pdf</li>
+ * <li>Caffeine: https://github.com/ben-manes/caffeine</li>
+ * <li>Cache design: http://highscalability.com/blog/2016/1/25/design-of-a-modern-cache.html</li>
+ * </ul>
+ */
+@InterfaceAudience.Private
+public final class TinyLfuBlockCache implements FirstLevelBlockCache {
+ private static final Log LOG = LogFactory.getLog(TinyLfuBlockCache.class);
+
+ private static final String MAX_BLOCK_SIZE = "hbase.tinylfu.max.block.size";
+ private static final long DEFAULT_MAX_BLOCK_SIZE = 16L * 1024L * 1024L;
+ private static final int STAT_THREAD_PERIOD_SECONDS = 5 * 60;
+
+ private final Eviction<BlockCacheKey, Cacheable> policy;
+ private final ScheduledExecutorService statsThreadPool;
+ private final long maxBlockSize;
+ private final CacheStats stats;
+
+ private BlockCache victimCache;
+
+ @VisibleForTesting
+ final Cache<BlockCacheKey, Cacheable> cache;
+
+ /**
+ * Creates a block cache.
+ *
+ * @param maximumSizeInBytes maximum size of this cache, in bytes
+ * @param avgBlockSize expected average size of blocks, in bytes
+ * @param executor the cache's executor
+ * @param conf additional configuration
+ */
+ public TinyLfuBlockCache(long maximumSizeInBytes, long avgBlockSize,
+ Executor executor, Configuration conf) {
+ this(maximumSizeInBytes, avgBlockSize,
+ conf.getLong(MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE), executor);
+ }
+
+ /**
+ * Creates a block cache.
+ *
+ * @param maximumSizeInBytes maximum size of this cache, in bytes
+ * @param avgBlockSize expected average size of blocks, in bytes
+ * @param maxBlockSize maximum size of a block, in bytes
+ * @param executor the cache's executor
+ */
+ public TinyLfuBlockCache(long maximumSizeInBytes,
+ long avgBlockSize, long maxBlockSize, Executor executor) {
+ this.cache = Caffeine.newBuilder()
+ .executor(executor)
+ .maximumWeight(maximumSizeInBytes)
+ .removalListener(new EvictionListener())
+ .weigher((BlockCacheKey key, Cacheable value) ->
+ (int) Math.min(value.heapSize(), Integer.MAX_VALUE))
+ .initialCapacity((int) Math.ceil((1.2 * maximumSizeInBytes) / avgBlockSize))
+ .build();
+ this.maxBlockSize = maxBlockSize;
+ this.policy = cache.policy().eviction().get();
+ this.stats = new CacheStats(getClass().getSimpleName());
+
+ statsThreadPool = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryBuilder()
+ .setNameFormat("TinyLfuBlockCacheStatsExecutor").setDaemon(true).build());
+ statsThreadPool.scheduleAtFixedRate(this::logStats,
+ STAT_THREAD_PERIOD_SECONDS, STAT_THREAD_PERIOD_SECONDS, TimeUnit.SECONDS);
+ }
+
+ @Override
+ public void setVictimCache(BlockCache victimCache) {
+ if (this.victimCache != null) {
+ throw new IllegalArgumentException("The victim cache has already been set");
+ }
+ this.victimCache = requireNonNull(victimCache);
+ }
+
+ @Override
+ public long size() {
+ return policy.getMaximum();
+ }
+
+ @Override
+ public long getFreeSize() {
+ return size() - getCurrentSize();
+ }
+
+ @Override
+ public long getCurrentSize() {
+ return policy.weightedSize().getAsLong();
+ }
+
+ @Override
+ public long getBlockCount() {
+ return cache.estimatedSize();
+ }
+
+ @Override
+ public long heapSize() {
+ return getCurrentSize();
+ }
+
+ @Override
+ public void setMaxSize(long size) {
+ policy.setMaximum(size);
+ }
+
+ @Override
+ public boolean containsBlock(BlockCacheKey cacheKey) {
+ return cache.asMap().containsKey(cacheKey);
+ }
+
+ @Override
+ public Cacheable getBlock(BlockCacheKey cacheKey,
+ boolean caching, boolean repeat, boolean updateCacheMetrics) {
+ Cacheable value = cache.getIfPresent(cacheKey);
+ if (value == null) {
+ if (repeat) {
+ return null;
+ }
+ if (updateCacheMetrics) {
+ stats.miss(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
+ }
+ if (victimCache != null) {
+ value = victimCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics);
+ if ((value != null) && caching) {
+ cacheBlock(cacheKey, value);
+ }
+ }
+ } else if (updateCacheMetrics) {
+ stats.hit(caching, cacheKey.isPrimary(), cacheKey.getBlockType());
+ }
+ return value;
+ }
+
+ @Override
+ public void cacheBlock(BlockCacheKey cacheKey, Cacheable value,
+ boolean inMemory, boolean cacheDataInL1) {
+ cacheBlock(cacheKey, value);
+ }
+
+ @Override
+ public void cacheBlock(BlockCacheKey key, Cacheable value) {
+ if (value.heapSize() > maxBlockSize) {
+ // If there are a lot of blocks that are too big this can make the logs too noisy (2% logged)
+ if (stats.failInsert() % 50 == 0) {
+ LOG.warn(String.format(
+ "Trying to cache too large a block %s @ %,d is %,d which is larger than %,d",
+ key.getHfileName(), key.getOffset(), value.heapSize(), DEFAULT_MAX_BLOCK_SIZE));
+ }
+ } else {
+ cache.put(key, value);
+ }
+ }
+
+ @Override
+ public boolean evictBlock(BlockCacheKey cacheKey) {
+ Cacheable value = cache.asMap().remove(cacheKey);
+ return (value != null);
+ }
+
+ @Override
+ public int evictBlocksByHfileName(String hfileName) {
+ int evicted = 0;
+ for (BlockCacheKey key : cache.asMap().keySet()) {
+ if (key.getHfileName().equals(hfileName) && evictBlock(key)) {
+ evicted++;
+ }
+ }
+ if (victimCache != null) {
+ evicted += victimCache.evictBlocksByHfileName(hfileName);
+ }
+ return evicted;
+ }
+
+ @Override
+ public CacheStats getStats() {
+ return stats;
+ }
+
+ @Override
+ public void shutdown() {
+ if (victimCache != null) {
+ victimCache.shutdown();
+ }
+ statsThreadPool.shutdown();
+ }
+
+ @Override
+ public BlockCache[] getBlockCaches() {
+ return null;
+ }
+
+ @Override
+ public Iterator<CachedBlock> iterator() {
+ long now = System.nanoTime();
+ return cache.asMap().entrySet().stream()
+ .map(entry -> (CachedBlock) new CachedBlockView(entry.getKey(), entry.getValue(), now))
+ .iterator();
+ }
+
+ private void logStats() {
+ LOG.info(
+ "totalSize=" + StringUtils.byteDesc(heapSize()) + ", " +
+ "freeSize=" + StringUtils.byteDesc(getFreeSize()) + ", " +
+ "max=" + StringUtils.byteDesc(size()) + ", " +
+ "blockCount=" + getBlockCount() + ", " +
+ "accesses=" + stats.getRequestCount() + ", " +
+ "hits=" + stats.getHitCount() + ", " +
+ "hitRatio=" + (stats.getHitCount() == 0 ?
+ "0," : StringUtils.formatPercent(stats.getHitRatio(), 2) + ", ") +
+ "cachingAccesses=" + stats.getRequestCachingCount() + ", " +
+ "cachingHits=" + stats.getHitCachingCount() + ", " +
+ "cachingHitsRatio=" + (stats.getHitCachingCount() == 0 ?
+ "0,": (StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + ", ")) +
+ "evictions=" + stats.getEvictionCount() + ", " +
+ "evicted=" + stats.getEvictedCount());
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("blockCount", getBlockCount())
+ .add("currentSize", getCurrentSize())
+ .add("freeSize", getFreeSize())
+ .add("maxSize", size())
+ .add("heapSize", heapSize())
+ .add("victimCache", (victimCache != null))
+ .toString();
+ }
+
+ /** A removal listener to asynchronously record evictions and populate the victim cache. */
+ private final class EvictionListener implements RemovalListener<BlockCacheKey, Cacheable> {
+
+ @Override
+ public void onRemoval(BlockCacheKey key, Cacheable value, RemovalCause cause) {
+ if (!cause.wasEvicted()) {
+ // An explicit eviction (invalidation) is not added to the victim cache as the data may
+ // no longer be valid for subsequent queries.
+ return;
+ }
+
+ recordEviction();
+
+ if (victimCache == null) {
+ return;
+ } else if (victimCache instanceof BucketCache) {
+ BucketCache victimBucketCache = (BucketCache) victimCache;
+ victimBucketCache.cacheBlockWithWait(key, value, /* inMemory */ true,
+ /* cacheDataInL1 */ true, /* wait */ true);
+ } else {
+ victimCache.cacheBlock(key, value);
+ }
+ }
+ }
+
+ /**
+ * Records an eviction. The number of eviction operations and evicted blocks are identical, as
+ * an eviction is triggered immediately when the capacity has been exceeded. An eviction is
+ * performed asynchronously. See the library's documentation for details on write buffers,
+ * batching, and maintenance behavior.
+ */
+ private void recordEviction() {
+ // FIXME: Currently does not capture the insertion time
+ stats.evicted(Long.MAX_VALUE, true);
+ stats.evict();
+ }
+
+ private static final class CachedBlockView implements CachedBlock {
+ private static final Comparator<CachedBlock> COMPARATOR = Comparator
+ .comparing(CachedBlock::getFilename)
+ .thenComparing(CachedBlock::getOffset)
+ .thenComparing(CachedBlock::getCachedTime);
+
+ private final BlockCacheKey key;
+ private final Cacheable value;
+ private final long now;
+
+ public CachedBlockView(BlockCacheKey key, Cacheable value, long now) {
+ this.now = now;
+ this.key = key;
+ this.value = value;
+ }
+
+ @Override
+ public BlockPriority getBlockPriority() {
+ // This does not appear to be used in any meaningful way and is irrelevant to this cache
+ return BlockPriority.MEMORY;
+ }
+
+ @Override
+ public BlockType getBlockType() {
+ return value.getBlockType();
+ }
+
+ @Override
+ public long getOffset() {
+ return key.getOffset();
+ }
+
+ @Override
+ public long getSize() {
+ return value.heapSize();
+ }
+
+ @Override
+ public long getCachedTime() {
+ // This does not appear to be used in any meaningful way, so not captured
+ return 0L;
+ }
+
+ @Override
+ public String getFilename() {
+ return key.getHfileName();
+ }
+
+ @Override
+ public int compareTo(CachedBlock other) {
+ return COMPARATOR.compare(this, other);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ } else if (!(obj instanceof CachedBlock)) {
+ return false;
+ }
+ CachedBlock other = (CachedBlock) obj;
+ return compareTo(other) == 0;
+ }
+
+ @Override
+ public int hashCode() {
+ return key.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return BlockCacheUtil.toString(this, now);
+ }
+ }
+
+ @Override
+ public long getMaxSize() {
+ return size();
+ }
+
+ @Override
+ public long getCurrentDataSize() {
+ return getCurrentSize();
+ }
+
+ @Override
+ public long getDataBlockCount() {
+ return getBlockCount();
+ }
+}
diff --git a/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java b/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
new file mode 100644
index 0000000..3d5047e
--- /dev/null
+++ b/hbase-tinylfu-blockcache/src/test/java/org/apache/hadoop/hbase/io/hfile/TestTinyLfuBlockCache.java
@@ -0,0 +1,298 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.ByteBuffer;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.io.HeapSize;
+import org.apache.hadoop.hbase.testclassification.IOTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.ClassSize;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+/**
+ * Tests the concurrent TinyLfuBlockCache.
+ */
+@Category({IOTests.class, SmallTests.class})
+public class TestTinyLfuBlockCache {
+
+ @Test
+ public void testCacheSimple() throws Exception {
+
+ long maxSize = 1000000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 101);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateRandomBlocks(100, blockSize);
+
+ long expectedCacheSize = cache.heapSize();
+
+ // Confirm empty
+ for (CachedItem block : blocks) {
+ assertTrue(cache.getBlock(block.cacheKey, true, false, true) == null);
+ }
+
+ // Add blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ expectedCacheSize += block.heapSize();
+ }
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ // Re-add same blocks and ensure nothing has changed
+ long expectedBlockCount = cache.getBlockCount();
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+ assertEquals(
+ "Cache should ignore cache requests for blocks already in cache",
+ expectedBlockCount, cache.getBlockCount());
+
+ // Verify correctly calculated cache heap size
+ assertEquals(expectedCacheSize, cache.heapSize());
+
+ // Check if all blocks are properly cached and retrieved
+ for (CachedItem block : blocks) {
+ HeapSize buf = cache.getBlock(block.cacheKey, true, false, true);
+ assertTrue(buf != null);
+ assertEquals(buf.heapSize(), block.heapSize());
+ }
+
+ // Expect no evictions
+ assertEquals(0, cache.getStats().getEvictionCount());
+ }
+
+ @Test
+ public void testCacheEvictionSimple() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSizeDefault(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateFixedBlocks(11, blockSize, "block");
+
+ // Add all the blocks
+ for (CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+
+ // A single eviction run should have occurred
+ assertEquals(1, cache.getStats().getEvictionCount());
+
+ // The cache did not grow beyond max
+ assertTrue(cache.heapSize() < maxSize);
+
+ // All blocks except one should be in the cache
+ assertEquals(10, cache.getBlockCount());
+ }
+
+ @Test
+ public void testScanResistance() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single");
+ CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi");
+
+ // Add 5 blocks from each
+ for(int i=0; i<5; i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ cache.cacheBlock(multiBlocks[i].cacheKey, multiBlocks[i]);
+ }
+
+ // Add frequency
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 10; j++) {
+ CachedItem block = multiBlocks[i];
+ cache.getBlock(block.cacheKey, true, false, true);
+ }
+ }
+
+ // Let's keep "scanning" by adding single blocks. From here on we only
+ // expect evictions from the single bucket.
+
+ for(int i=5;i<18;i++) {
+ cache.cacheBlock(singleBlocks[i].cacheKey, singleBlocks[i]);
+ }
+
+ for (CachedItem block : multiBlocks) {
+ assertTrue(cache.cache.asMap().containsKey(block.cacheKey));
+ }
+
+ assertEquals(10, cache.getBlockCount());
+ assertEquals(13, cache.getStats().getEvictionCount());
+
+ }
+
+ @Test
+ public void testMaxBlockSize() throws Exception {
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+ CachedItem [] tooLong = generateFixedBlocks(10, 2 * blockSize, "long");
+ CachedItem [] small = generateFixedBlocks(15, blockSize / 2, "small");
+
+ for (CachedItem i:tooLong) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ for (CachedItem i:small) {
+ cache.cacheBlock(i.cacheKey, i);
+ }
+ assertEquals(15,cache.getBlockCount());
+ for (CachedItem i:small) {
+ assertNotNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+ for (CachedItem i:tooLong) {
+ assertNull(cache.getBlock(i.cacheKey, true, false, false));
+ }
+
+ assertEquals(10, cache.getStats().getFailedInserts());
+ }
+
+ @Test
+ public void testResizeBlockCache() throws Exception {
+
+ long maxSize = 100000;
+ long blockSize = calculateBlockSize(maxSize, 10);
+
+ TinyLfuBlockCache cache = new TinyLfuBlockCache(maxSize, blockSize, blockSize, Runnable::run);
+
+ CachedItem [] blocks = generateFixedBlocks(10, blockSize, "block");
+
+ for(CachedItem block : blocks) {
+ cache.cacheBlock(block.cacheKey, block);
+ }
+
+ // Do not expect any evictions yet
+ assertEquals(10, cache.getBlockCount());
+ assertEquals(0, cache.getStats().getEvictionCount());
+
+ // Resize to half capacity plus an extra block (otherwise we evict an extra)
+ cache.setMaxSize(maxSize / 2);
+
+ // And we expect 1/2 of the blocks to be evicted
+ assertEquals(5, cache.getBlockCount());
+ assertEquals(5, cache.getStats().getEvictedCount());
+ }
+
+ private CachedItem [] generateFixedBlocks(int numBlocks, int size, String pfx) {
+ CachedItem [] blocks = new CachedItem[numBlocks];
+ for(int i=0;i<numBlocks;i++) {
+ blocks[i] = new CachedItem(pfx + i, size);
+ }
+ return blocks;
+ }
+
+ private CachedItem [] generateFixedBlocks(int numBlocks, long size, String pfx) {
+ return generateFixedBlocks(numBlocks, (int)size, pfx);
+ }
+
+ private CachedItem [] generateRandomBlocks(int numBlocks, long maxSize) {
+ CachedItem [] blocks = new CachedItem[numBlocks];
+ Random r = new Random();
+ for(int i=0;i<numBlocks;i++) {
+ blocks[i] = new CachedItem("block" + i, r.nextInt((int)maxSize)+1);
+ }
+ return blocks;
+ }
+
+ private long calculateBlockSize(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
+ long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
+ ClassSize.CONCURRENT_HASHMAP +
+ (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
+ (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = totalOverhead/numEntries;
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*0.99f));
+ }
+
+ private long calculateBlockSizeDefault(long maxSize, int numBlocks) {
+ long roughBlockSize = maxSize / numBlocks;
+ int numEntries = (int)Math.ceil((1.2)*maxSize/roughBlockSize);
+ long totalOverhead = LruBlockCache.CACHE_FIXED_OVERHEAD +
+ ClassSize.CONCURRENT_HASHMAP +
+ (numEntries * ClassSize.CONCURRENT_HASHMAP_ENTRY) +
+ (LruBlockCache.DEFAULT_CONCURRENCY_LEVEL * ClassSize.CONCURRENT_HASHMAP_SEGMENT);
+ long negateBlockSize = totalOverhead / numEntries;
+ negateBlockSize += LruCachedBlock.PER_BLOCK_OVERHEAD;
+ return ClassSize.align((long)Math.floor((roughBlockSize - negateBlockSize)*
+ LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR));
+ }
+
+ private static class CachedItem implements Cacheable {
+ BlockCacheKey cacheKey;
+ int size;
+
+ CachedItem(String blockName, int size) {
+ this.cacheKey = new BlockCacheKey(blockName, 0);
+ this.size = size;
+ }
+
+ /** The size of this item reported to the block cache layer */
+ @Override
+ public long heapSize() {
+ return ClassSize.align(size);
+ }
+
+ @Override
+ public int getSerializedLength() {
+ return 0;
+ }
+
+ @Override
+ public CacheableDeserializer<Cacheable> getDeserializer() {
+ return null;
+ }
+
+ @Override
+ public BlockType getBlockType() {
+ return BlockType.DATA;
+ }
+
+ @Override
+ public void serialize(ByteBuffer destination, boolean includeNextBlockOnDiskSize) {
+ }
+ }
+
+}
+
diff --git a/pom.xml b/pom.xml
index 874188c..24f8921 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1263,6 +1263,7 @@
<assembly.file>src/main/assembly/hadoop-two-compat.xml</assembly.file>
<!-- end HBASE-15925 default hadoop compatibility values -->
<avro.version>1.7.7</avro.version>
+ <caffeine.version>2.7.0</caffeine.version>
<commons-cli.version>1.2</commons-cli.version>
<commons-codec.version>1.9</commons-codec.version>
<!-- pretty outdated -->
@@ -1603,6 +1604,11 @@
<version>${metrics-core.version}</version>
</dependency>
<dependency>
+ <groupId>com.github.ben-manes.caffeine</groupId>
+ <artifactId>caffeine</artifactId>
+ <version>${caffeine.version}</version>
+ </dependency>
+ <dependency>
<groupId>io.dropwizard.metrics</groupId>
<artifactId>metrics-core</artifactId>
<version>3.1.2</version>
@@ -2010,6 +2016,9 @@
<activation>
<jdk>1.8</jdk>
</activation>
+ <modules>
+ <module>hbase-tinylfu-blockcache</module>
+ </modules>
<build>
<pluginManagement>
<plugins>