You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2023/06/29 07:14:30 UTC
[hbase] branch master updated: HBASE-27948 Report memstore on-heap and off-heap size as jmx metrics in sub=Memory bean (#5293)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 25455b6fe3c HBASE-27948 Report memstore on-heap and off-heap size as jmx metrics in sub=Memory bean (#5293)
25455b6fe3c is described below
commit 25455b6fe3cbd8f093fd9bc8c51a1bab95353a62
Author: Jing Yu <yu...@salesforce.com>
AuthorDate: Thu Jun 29 03:14:23 2023 -0400
HBASE-27948 Report memstore on-heap and off-heap size as jmx metrics in sub=Memory bean (#5293)
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../regionserver/MetricsHeapMemoryManagerSource.java | 19 +++++++++++++++++++
.../MetricsHeapMemoryManagerSourceImpl.java | 16 ++++++++++++++++
.../hadoop/hbase/regionserver/HeapMemoryManager.java | 7 +++++--
.../hbase/regionserver/MetricsHeapMemoryManager.java | 16 ++++++++++++++++
.../hbase/regionserver/RegionServerAccounting.java | 4 ++--
.../regionserver/TestMetricsHeapMemoryManager.java | 4 ++++
6 files changed, 62 insertions(+), 4 deletions(-)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java
index d9e972ba1fb..ef130fbece4 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSource.java
@@ -70,6 +70,18 @@ public interface MetricsHeapMemoryManagerSource extends BaseSource {
*/
void setCurMemStoreSizeGauge(long memStoreSize);
+ /**
+ * Set the current global memstore on-heap size used gauge
+ * @param memStoreOnHeapSize the current memory usage in memstore on-heap, in bytes.
+ */
+ void setCurMemStoreOnHeapSizeGauge(long memStoreOnHeapSize);
+
+ /**
+ * Set the current global memstore off-heap size used gauge
+ * @param memStoreOffHeapSize the current memory usage in memstore off-heap, in bytes.
+ */
+ void setCurMemStoreOffHeapSizeGauge(long memStoreOffHeapSize);
+
/**
* Update the increase/decrease memstore size histogram
* @param memStoreDeltaSize the tuning result of memstore.
@@ -118,6 +130,13 @@ public interface MetricsHeapMemoryManagerSource extends BaseSource {
String UNBLOCKED_FLUSH_GAUGE_DESC = "Gauge for the unblocked flush count before tuning";
String MEMSTORE_SIZE_GAUGE_NAME = "memStoreSize";
String MEMSTORE_SIZE_GAUGE_DESC = "Global MemStore used in bytes by the RegionServer";
+ String MEMSTORE_ONHEAP_SIZE_GAUGE_NAME = "memStoreOnHeapSize";
+ String MEMSTORE_ONHEAP_SIZE_GAUGE_DESC =
+ "Global MemStore On-heap size in bytes by the RegionServer";
+ String MEMSTORE_OFFHEAP_SIZE_GAUGE_NAME = "memStoreOffHeapSize";
+ String MEMSTORE_OFFHEAP_SIZE_GAUGE_DESC =
+ "Global MemStore Off-heap size in bytes by the RegionServer";
+
String BLOCKCACHE_SIZE_GAUGE_NAME = "blockCacheSize";
String BLOCKCACHE_SIZE_GAUGE_DESC = "BlockCache used in bytes by the RegionServer";
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java
index c2e8d329143..e8967246dd7 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManagerSourceImpl.java
@@ -41,6 +41,8 @@ public class MetricsHeapMemoryManagerSourceImpl extends BaseSourceImpl
private final MutableGaugeLong blockedFlushGauge;
private final MutableGaugeLong unblockedFlushGauge;
private final MutableGaugeLong memStoreSizeGauge;
+ private final MutableGaugeLong memStoreOnHeapSizeGauge;
+ private final MutableGaugeLong memStoreOffHeapSizeGauge;
private final MutableGaugeLong blockCacheSizeGauge;
private final MutableFastCounter doNothingCounter;
@@ -75,6 +77,10 @@ public class MetricsHeapMemoryManagerSourceImpl extends BaseSourceImpl
getMetricsRegistry().newGauge(UNBLOCKED_FLUSH_GAUGE_NAME, UNBLOCKED_FLUSH_GAUGE_DESC, 0L);
memStoreSizeGauge =
getMetricsRegistry().newGauge(MEMSTORE_SIZE_GAUGE_NAME, MEMSTORE_SIZE_GAUGE_DESC, 0L);
+ memStoreOnHeapSizeGauge = getMetricsRegistry().newGauge(MEMSTORE_ONHEAP_SIZE_GAUGE_NAME,
+ MEMSTORE_ONHEAP_SIZE_GAUGE_DESC, 0L);
+ memStoreOffHeapSizeGauge = getMetricsRegistry().newGauge(MEMSTORE_OFFHEAP_SIZE_GAUGE_NAME,
+ MEMSTORE_OFFHEAP_SIZE_GAUGE_DESC, 0L);
blockCacheSizeGauge =
getMetricsRegistry().newGauge(BLOCKCACHE_SIZE_GAUGE_NAME, BLOCKCACHE_SIZE_GAUGE_DESC, 0L);
@@ -111,6 +117,16 @@ public class MetricsHeapMemoryManagerSourceImpl extends BaseSourceImpl
memStoreSizeGauge.set(memstoreSize);
}
+ @Override
+ public void setCurMemStoreOnHeapSizeGauge(long memstoreOnHeapSize) {
+ memStoreOnHeapSizeGauge.set(memstoreOnHeapSize);
+ }
+
+ @Override
+ public void setCurMemStoreOffHeapSizeGauge(long memstoreOffHeapSize) {
+ memStoreOffHeapSizeGauge.set(memstoreOffHeapSize);
+ }
+
@Override
public void updateMemStoreDeltaSizeHistogram(int memStoreDeltaSize) {
if (memStoreDeltaSize >= 0) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
index 7a72b9af41c..fe2737b0a7d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java
@@ -317,12 +317,15 @@ public class HeapMemoryManager {
unblockedFlushCnt = unblockedFlushCount.getAndSet(0);
tunerContext.setUnblockedFlushCount(unblockedFlushCnt);
metricsHeapMemoryManager.updateUnblockedFlushCount(unblockedFlushCnt);
- // TODO : add support for offheap metrics
tunerContext.setCurBlockCacheUsed((float) blockCache.getCurrentSize() / maxHeapSize);
metricsHeapMemoryManager.setCurBlockCacheSizeGauge(blockCache.getCurrentSize());
+ long globalMemstoreDataSize = regionServerAccounting.getGlobalMemStoreDataSize();
long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemStoreHeapSize();
+ long globalMemStoreOffHeapSize = regionServerAccounting.getGlobalMemStoreOffHeapSize();
tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize);
- metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize);
+ metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreDataSize);
+ metricsHeapMemoryManager.setCurMemStoreOnHeapSizeGauge(globalMemstoreHeapSize);
+ metricsHeapMemoryManager.setCurMemStoreOffHeapSizeGauge(globalMemStoreOffHeapSize);
tunerContext.setCurBlockCacheSize(blockCachePercent);
tunerContext.setCurMemStoreSize(globalMemStorePercent);
TunerResult result = null;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java
index e781bddb227..2f3e43fccc9 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsHeapMemoryManager.java
@@ -73,6 +73,22 @@ public class MetricsHeapMemoryManager {
source.setCurMemStoreSizeGauge(memStoreSize);
}
+ /**
+ * Set the current global memstore on-heap size gauge
+ * @param memStoreOnHeapSize the current memory on-heap size in memstore, in bytes.
+ */
+ public void setCurMemStoreOnHeapSizeGauge(final long memStoreOnHeapSize) {
+ source.setCurMemStoreOnHeapSizeGauge(memStoreOnHeapSize);
+ }
+
+ /**
+ * Set the current global memstore off-heap size gauge
+ * @param memStoreOffHeapSize the current memory off-heap size in memstore, in bytes.
+ */
+ public void setCurMemStoreOffHeapSizeGauge(final long memStoreOffHeapSize) {
+ source.setCurMemStoreOffHeapSizeGauge(memStoreOffHeapSize);
+ }
+
/**
* Update the increase/decrease memstore size histogram
* @param memStoreDeltaSize the tuning result of memstore.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index c1706c995e7..b26611cb426 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -121,12 +121,12 @@ public class RegionServerAccounting {
return globalMemStoreDataSize.sum();
}
- /** Returns the global memstore heap size in the RegionServer */
+ /** Returns the global memstore on-heap size in the RegionServer */
public long getGlobalMemStoreHeapSize() {
return this.globalMemStoreHeapSize.sum();
}
- /** Returns the global memstore heap size in the RegionServer */
+ /** Returns the global memstore off-heap size in the RegionServer */
public long getGlobalMemStoreOffHeapSize() {
return this.globalMemStoreOffHeapSize.sum();
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java
index 2d6a47fc212..974d6485377 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsHeapMemoryManager.java
@@ -74,11 +74,15 @@ public class TestMetricsHeapMemoryManager {
hmm.updateBlockedFlushCount(200);
hmm.updateUnblockedFlushCount(50);
hmm.setCurMemStoreSizeGauge(256 * 1024 * 1024);
+ hmm.setCurMemStoreOnHeapSizeGauge(512 * 1024 * 1024);
+ hmm.setCurMemStoreOffHeapSizeGauge(128 * 1024 * 1024);
hmm.setCurBlockCacheSizeGauge(100 * 1024 * 1024);
HELPER.assertGauge("blockedFlushGauge", 200, source);
HELPER.assertGauge("unblockedFlushGauge", 50, source);
HELPER.assertGauge("memStoreSize", 256 * 1024 * 1024, source);
+ HELPER.assertGauge("memStoreOnHeapSize", 512 * 1024 * 1024, source);
+ HELPER.assertGauge("memStoreOffHeapSize", 128 * 1024 * 1024, source);
HELPER.assertGauge("blockCacheSize", 100 * 1024 * 1024, source);
}
}