You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/09/26 03:56:13 UTC
hbase git commit: HBASE-18652 Expose individual cache stats in a
CombinedCache through JMX (Biju Nair)
Repository: hbase
Updated Branches:
refs/heads/branch-1 9d68535e8 -> eeaa9aee6
HBASE-18652 Expose individual cache stats in a CombinedCache through JMX (Biju Nair)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eeaa9aee
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eeaa9aee
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eeaa9aee
Branch: refs/heads/branch-1
Commit: eeaa9aee66edf6b4e65d517756f61bfc19fc4fe3
Parents: 9d68535
Author: Michael Stack <st...@apache.org>
Authored: Mon Sep 25 20:51:11 2017 -0700
Committer: Michael Stack <st...@apache.org>
Committed: Mon Sep 25 20:51:11 2017 -0700
----------------------------------------------------------------------
.../regionserver/MetricsRegionServerSource.java | 17 +++++
.../MetricsRegionServerWrapper.java | 40 +++++++++++
.../MetricsRegionServerSourceImpl.java | 16 +++++
.../hadoop/hbase/io/hfile/CacheConfig.java | 37 ++++++++--
.../MetricsRegionServerWrapperImpl.java | 71 +++++++++++++++++++-
.../hbase/io/hfile/TestBlockCacheReporting.java | 2 +-
.../hadoop/hbase/io/hfile/TestCacheConfig.java | 4 +-
.../io/hfile/TestForceCacheImportantBlocks.java | 2 +-
.../MetricsRegionServerWrapperStub.java | 40 +++++++++++
.../regionserver/TestMetricsRegionServer.java | 8 +++
10 files changed, 226 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 3ac678e..b72deb8 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -345,6 +345,23 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT = "blockCacheDeleteFamilyBloomHitCount";
String BLOCK_CACHE_TRAILER_HIT_COUNT = "blockCacheTrailerHitCount";
+ String L1_CACHE_HIT_COUNT = "l1CacheHitCount";
+ String L1_CACHE_HIT_COUNT_DESC = "L1 cache hit count.";
+ String L1_CACHE_MISS_COUNT = "l1CacheMissCount";
+ String L1_CACHE_MISS_COUNT_DESC = "L1 cache miss count.";
+ String L1_CACHE_HIT_RATIO = "l1CacheHitRatio";
+ String L1_CACHE_HIT_RATIO_DESC = "L1 cache hit ratio.";
+ String L1_CACHE_MISS_RATIO = "l1CacheMissRatio";
+ String L1_CACHE_MISS_RATIO_DESC = "L1 cache miss ratio.";
+ String L2_CACHE_HIT_COUNT = "l2CacheHitCount";
+ String L2_CACHE_HIT_COUNT_DESC = "L2 cache hit count.";
+ String L2_CACHE_MISS_COUNT = "l2CacheMissCount";
+ String L2_CACHE_MISS_COUNT_DESC = "L2 cache miss count.";
+ String L2_CACHE_HIT_RATIO = "l2CacheHitRatio";
+ String L2_CACHE_HIT_RATIO_DESC = "L2 cache hit ratio.";
+ String L2_CACHE_MISS_RATIO = "l2CacheMissRatio";
+ String L2_CACHE_MISS_RATIO_DESC = "L2 cache miss ratio.";
+
String RS_START_TIME_NAME = "regionServerStartTime";
String ZOOKEEPER_QUORUM_NAME = "zookeeperQuorum";
String SERVER_NAME_NAME = "serverName";
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 7d7f66d..3344dce 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -259,6 +259,46 @@ public interface MetricsRegionServerWrapper {
long getBlockCacheFailedInsertions();
/**
+ * Hit count of L1 cache.
+ */
+ public long getL1CacheHitCount();
+
+ /**
+ * Miss count of L1 cache.
+ */
+ public long getL1CacheMissCount();
+
+ /**
+ * Hit ratio of L1 cache.
+ */
+ public double getL1CacheHitRatio();
+
+ /**
+ * Miss ratio of L1 cache.
+ */
+ public double getL1CacheMissRatio();
+
+ /**
+ * Hit count of L2 cache.
+ */
+ public long getL2CacheHitCount();
+
+ /**
+ * Miss count of L2 cache.
+ */
+ public long getL2CacheMissCount();
+
+ /**
+ * Hit ratio of L2 cache.
+ */
+ public double getL2CacheHitRatio();
+
+ /**
+ * Miss ratio of L2 cache.
+ */
+ public double getL2CacheMissRatio();
+
+ /**
* Force a re-computation of the metrics.
*/
void forceRecompute();
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
----------------------------------------------------------------------
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 628a564..208188e 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -458,6 +458,22 @@ public class MetricsRegionServerSourceImpl
.addCounter(Interns.info(BLOCK_CACHE_DELETE_FAMILY_BLOOM_HIT_COUNT, ""),
rsWrap.getDeleteFamilyBloomHitCount())
.addCounter(Interns.info(BLOCK_CACHE_TRAILER_HIT_COUNT, ""), rsWrap.getTrailerHitCount())
+ .addGauge(Interns.info(L1_CACHE_HIT_COUNT, L1_CACHE_HIT_COUNT_DESC),
+ rsWrap.getL1CacheHitCount())
+ .addGauge(Interns.info(L1_CACHE_MISS_COUNT, L1_CACHE_MISS_COUNT_DESC),
+ rsWrap.getL1CacheMissCount())
+ .addGauge(Interns.info(L1_CACHE_HIT_RATIO, L1_CACHE_HIT_RATIO_DESC),
+ rsWrap.getL1CacheHitRatio())
+ .addGauge(Interns.info(L1_CACHE_MISS_RATIO, L1_CACHE_MISS_RATIO_DESC),
+ rsWrap.getL1CacheMissRatio())
+ .addGauge(Interns.info(L2_CACHE_HIT_COUNT, L2_CACHE_HIT_COUNT_DESC),
+ rsWrap.getL2CacheHitCount())
+ .addGauge(Interns.info(L2_CACHE_MISS_COUNT, L2_CACHE_MISS_COUNT_DESC),
+ rsWrap.getL2CacheMissCount())
+ .addGauge(Interns.info(L2_CACHE_HIT_RATIO, L2_CACHE_HIT_RATIO_DESC),
+ rsWrap.getL2CacheHitRatio())
+ .addGauge(Interns.info(L2_CACHE_MISS_RATIO, L2_CACHE_MISS_RATIO_DESC),
+ rsWrap.getL2CacheMissRatio())
.addCounter(Interns.info(UPDATES_BLOCKED_TIME, UPDATES_BLOCKED_DESC),
rsWrap.getUpdatesBlockedTime())
.addCounter(Interns.info(FLUSHED_CELLS, FLUSHED_CELLS_DESC),
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
index 1d68b99..d020eb0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java
@@ -525,6 +525,8 @@ public class CacheConfig {
// Clear this if in tests you'd make more than one block cache instance.
@VisibleForTesting
static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE;
+ private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE = null;
+ private static BlockCache GLOBAL_L2_CACHE_INSTANCE = null;
/** Boolean whether we have disabled the block cache entirely. */
@VisibleForTesting
@@ -534,7 +536,8 @@ public class CacheConfig {
* @param c Configuration to use.
* @return An L1 instance. Currently an instance of LruBlockCache.
*/
- private static LruBlockCache getL1(final Configuration c) {
+ private static synchronized LruBlockCache getL1(final Configuration c) {
+ if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE;
final long lruCacheSize = HeapMemorySizeUtil.getLruCacheSize(c);
if (lruCacheSize < 0) {
blockCacheDisabled = true;
@@ -543,7 +546,8 @@ public class CacheConfig {
int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE);
LOG.info("Allocating LruBlockCache size=" +
StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize));
- return new LruBlockCache(lruCacheSize, blockSize, true, c);
+ GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c);
+ return GLOBAL_L1_CACHE_INSTANCE;
}
/**
@@ -560,10 +564,26 @@ public class CacheConfig {
// If we want to use an external block cache then create that.
if (useExternal) {
- return getExternalBlockcache(c);
+ GLOBAL_L2_CACHE_INSTANCE = getExternalBlockcache(c);
+ } else {
+ // otherwise use the bucket cache.
+ GLOBAL_L2_CACHE_INSTANCE = getBucketCache(c);
+ }
+ return GLOBAL_L2_CACHE_INSTANCE;
+ }
+
+ public CacheStats getL1Stats() {
+ if (GLOBAL_L1_CACHE_INSTANCE != null) {
+ return GLOBAL_L1_CACHE_INSTANCE.getStats();
}
- // otherwise use the bucket cache.
- return getBucketCache(c);
+ return null;
+ }
+
+ public CacheStats getL2Stats() {
+ if (GLOBAL_L2_CACHE_INSTANCE != null) {
+ return GLOBAL_L2_CACHE_INSTANCE.getStats();
+ }
+ return null;
}
private static BlockCache getExternalBlockcache(Configuration c) {
@@ -682,4 +702,11 @@ public class CacheConfig {
}
return GLOBAL_BLOCK_CACHE_INSTANCE;
}
+
+ @VisibleForTesting
+ static synchronized void clearGlobalInstances() {
+ GLOBAL_L1_CACHE_INSTANCE = null;
+ GLOBAL_L2_CACHE_INSTANCE = null;
+ GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index e805e75..31cfbd1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -84,6 +84,8 @@ class MetricsRegionServerWrapperImpl
private volatile long averageRegionSize = 0L;
private CacheStats cacheStats;
+ private CacheStats l1Stats = null;
+ private CacheStats l2Stats = null;
private ScheduledExecutorService executor;
private Runnable runnable;
private long period;
@@ -113,8 +115,12 @@ class MetricsRegionServerWrapperImpl
*/
private synchronized void initBlockCache() {
CacheConfig cacheConfig = this.regionServer.cacheConfig;
- if (cacheConfig != null && this.blockCache == null) {
- this.blockCache = cacheConfig.getBlockCache();
+ if (cacheConfig != null) {
+ l1Stats = cacheConfig.getL1Stats();
+ l2Stats = cacheConfig.getL2Stats();
+ if (this.blockCache == null) {
+ this.blockCache = cacheConfig.getBlockCache();
+ }
}
if (this.blockCache != null && this.cacheStats == null) {
@@ -328,6 +334,67 @@ class MetricsRegionServerWrapperImpl
return this.cacheStats.getFailedInserts();
}
+ @Override
+ public long getL1CacheHitCount() {
+ return 200;
+ }
+
+ @Override
+ public long getL1CacheMissCount() {
+ if (this.l1Stats == null) {
+ return 0;
+ }
+ return this.l1Stats.getMissCount();
+ }
+
+ @Override
+ public double getL1CacheHitRatio() {
+ if (this.l1Stats == null) {
+ return 0;
+ }
+ return this.l1Stats.getHitRatio();
+ }
+
+ @Override
+ public double getL1CacheMissRatio() {
+ if (this.l1Stats == null) {
+ return 0;
+ }
+ return this.l1Stats.getMissRatio();
+ }
+
+ @Override
+ public long getL2CacheHitCount() {
+ if (this.l2Stats == null) {
+ return 0;
+ }
+ return this.l2Stats.getHitCount();
+ }
+
+ @Override
+ public long getL2CacheMissCount() {
+ if (this.l2Stats == null) {
+ return 0;
+ }
+ return this.l2Stats.getMissCount();
+ }
+
+ @Override
+ public double getL2CacheHitRatio() {
+ if (this.l2Stats == null) {
+ return 0;
+ }
+ return this.l2Stats.getHitRatio();
+ }
+
+ @Override
+ public double getL2CacheMissRatio() {
+ if (this.l2Stats == null) {
+ return 0;
+ }
+ return this.l2Stats.getMissRatio();
+ }
+
@Override public void forceRecompute() {
this.runnable.run();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
index 7ce9679..b8fe745 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java
@@ -45,7 +45,7 @@ public class TestBlockCacheReporting {
@Before
public void setUp() throws Exception {
- CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ CacheConfig.clearGlobalInstances();
this.conf = HBaseConfiguration.create();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
index 3f60cd7..eea6558 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java
@@ -150,14 +150,14 @@ public class TestCacheConfig {
@Before
public void setUp() throws Exception {
- CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ CacheConfig.clearGlobalInstances();
this.conf = HBaseConfiguration.create();
}
@After
public void tearDown() throws Exception {
// Let go of current block cache.
- CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ CacheConfig.clearGlobalInstances();
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
index 16583d9..6c4585a 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestForceCacheImportantBlocks.java
@@ -97,7 +97,7 @@ public class TestForceCacheImportantBlocks {
@Before
public void setup() {
// Make sure we make a new one each time.
- CacheConfig.GLOBAL_BLOCK_CACHE_INSTANCE = null;
+ CacheConfig.clearGlobalInstances();
HFile.DATABLOCK_READ_COUNT.set(0);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index b573323..21d06cf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -261,6 +261,46 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
}
@Override
+ public long getL1CacheHitCount() {
+ return 200;
+ }
+
+ @Override
+ public long getL1CacheMissCount() {
+ return 100;
+ }
+
+ @Override
+ public double getL1CacheHitRatio() {
+ return 80;
+ }
+
+ @Override
+ public double getL1CacheMissRatio() {
+ return 20;
+ }
+
+ @Override
+ public long getL2CacheHitCount() {
+ return 800;
+ }
+
+ @Override
+ public long getL2CacheMissCount() {
+ return 200;
+ }
+
+ @Override
+ public double getL2CacheHitRatio() {
+ return 90;
+ }
+
+ @Override
+ public double getL2CacheMissRatio() {
+ return 10;
+ }
+
+ @Override
public long getUpdatesBlockedTime() {
return 419;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eeaa9aee/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
index e1ef971..92acad4 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
@@ -97,6 +97,14 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("blockCacheCountHitPercent", 98, serverSource);
HELPER.assertGauge("blockCacheExpressHitPercent", 97, serverSource);
HELPER.assertCounter("blockCacheFailedInsertionCount", 36, serverSource);
+ HELPER.assertGauge("l1CacheHitCount", 200, serverSource);
+ HELPER.assertGauge("l1CacheMissCount", 100, serverSource);
+ HELPER.assertGauge("l1CacheHitRatio", 80, serverSource);
+ HELPER.assertGauge("l1CacheMissRatio", 20, serverSource);
+ HELPER.assertGauge("l2CacheHitCount", 800, serverSource);
+ HELPER.assertGauge("l2CacheMissCount", 200, serverSource);
+ HELPER.assertGauge("l2CacheHitRatio", 90, serverSource);
+ HELPER.assertGauge("l2CacheMissRatio", 10, serverSource);
HELPER.assertCounter("updatesBlockedTime", 419, serverSource);
}