You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zh...@apache.org on 2022/04/28 15:16:24 UTC
[hbase] branch branch-2 updated: HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)
This is an automated email from the ASF dual-hosted git repository.
zhangduo pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 895e0f474a4 HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)
895e0f474a4 is described below
commit 895e0f474a4454f14b74e85b680a7ddf74429d41
Author: liangxs <li...@gmail.com>
AuthorDate: Thu Apr 28 23:08:53 2022 +0800
HBASE-26975 Add on heap and off heap memstore info in rs web UI (#4368)
Signed-off-by: Duo Zhang <zh...@apache.org>
(cherry picked from commit cdf81ea5cca16cec3b854ac67bc08c43c489918b)
---
.../regionserver/MetricsRegionServerWrapper.java | 25 +++++++++++++++++++
.../tmpl/regionserver/ServerMetricsTmpl.jamon | 14 ++++++++---
.../hadoop/hbase/io/util/MemorySizeUtil.java | 2 +-
.../MetricsRegionServerWrapperImpl.java | 29 +++++++++++++++++++++-
.../hbase/regionserver/RegionServerAccounting.java | 8 ++++++
.../MetricsRegionServerWrapperStub.java | 20 +++++++++++++++
6 files changed, 92 insertions(+), 6 deletions(-)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index ab2697c34f8..ca0c6d9a31d 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -101,6 +101,16 @@ public interface MetricsRegionServerWrapper {
*/
long getMemStoreSize();
+ /**
+ * Get the size of the on heap memstore on this region server.
+ */
+ long getOnHeapMemStoreSize();
+
+ /**
+ * Get the size of the off heap memstore on this region server.
+ */
+ long getOffHeapMemStoreSize();
+
/**
* Get the total size of the store files this region server is serving from.
*/
@@ -226,7 +236,22 @@ public interface MetricsRegionServerWrapper {
*/
int getFlushQueueSize();
+ /**
+ * Get the limit size of the off heap memstore (if enabled), otherwise
+ * get the limit size of the on heap memstore.
+ */
long getMemStoreLimit();
+
+ /**
+ * Get the limit size of the on heap memstore.
+ */
+ long getOnHeapMemStoreLimit();
+
+ /**
+ * Get the limit size of the off heap memstore.
+ */
+ long getOffHeapMemStoreLimit();
+
/**
* Get the size (in bytes) of the block cache that is free.
*/
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
index 23752b29774..0e01f8f4e97 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon
@@ -117,8 +117,9 @@ MetricsRegionServerWrapper mWrap;
<th>Max Heap</th>
<th>Direct Memory Used</th>
<th>Direct Memory Configured</th>
- <th>Memstore Size</th>
- <th>Memstore Limit</th>
+ <th>Memstore On-Heap Size / Limit</th>
+ <th>Memstore Off-Heap Size / Limit</th>
+ <th>Memstore Data Size (On&&Off Heap)</th>
</tr>
</tr>
<tr>
@@ -135,10 +136,15 @@ MetricsRegionServerWrapper mWrap;
<% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(), "B", 1) %>
</td>
<td>
- <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreSize(), "B", 1) + " / "
+ + TraditionalBinaryPrefix.long2String(mWrap.getOnHeapMemStoreLimit(), "B", 1) %>
+ </td>
+ <td>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreSize(), "B", 1) + " / "
+ + TraditionalBinaryPrefix.long2String(mWrap.getOffHeapMemStoreLimit(), "B", 1) %>
</td>
<td>
- <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreLimit(), "B", 1) %>
+ <% TraditionalBinaryPrefix.long2String(mWrap.getMemStoreSize(), "B", 1) %>
</td>
</tr>
</table>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
index b1f298e3772..4a0191fc0be 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/util/MemorySizeUtil.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.regionserver.MemStoreLAB;
import org.apache.hadoop.hbase.util.Pair;
/**
- * Util class to calculate memory size for memstore, block cache(L1, L2) of RS.
+ * Util class to calculate memory size for memstore(on heap, off heap), block cache(L1, L2) of RS.
*/
@InterfaceAudience.Private
public class MemorySizeUtil {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index b8f00f9778f..bafeed0e07d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -78,6 +78,8 @@ class MetricsRegionServerWrapperImpl
private volatile long walFileSize = 0;
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
+ private volatile long onHeapMemstoreSize = 0;
+ private volatile long offHeapMemstoreSize = 0;
private volatile long storeFileSize = 0;
private volatile long maxStoreFileAge = 0;
private volatile long minStoreFileAge = 0;
@@ -281,6 +283,16 @@ class MetricsRegionServerWrapperImpl
return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
}
+ @Override
+ public long getOnHeapMemStoreLimit() {
+ return this.regionServer.getRegionServerAccounting().getGlobalOnHeapMemStoreLimit();
+ }
+
+ @Override
+ public long getOffHeapMemStoreLimit() {
+ return this.regionServer.getRegionServerAccounting().getGlobalOffHeapMemStoreLimit();
+ }
+
@Override
public long getBlockCacheSize() {
return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
@@ -449,6 +461,16 @@ class MetricsRegionServerWrapperImpl
return memstoreSize;
}
+ @Override
+ public long getOnHeapMemStoreSize() {
+ return onHeapMemstoreSize;
+ }
+
+ @Override
+ public long getOffHeapMemStoreSize() {
+ return offHeapMemstoreSize;
+ }
+
@Override
public long getStoreFileSize() {
return storeFileSize;
@@ -683,7 +705,8 @@ class MetricsRegionServerWrapperImpl
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
new HDFSBlocksDistribution();
- long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
+ long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0;
+ long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0;
long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
long avgAgeNumerator = 0, numHFiles = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
@@ -762,6 +785,8 @@ class MetricsRegionServerWrapperImpl
for (Store store : storeList) {
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize().getDataSize();
+ tempOnHeapMemstoreSize += store.getMemStoreSize().getHeapSize();
+ tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -856,6 +881,8 @@ class MetricsRegionServerWrapperImpl
numStores = tempNumStores;
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
+ onHeapMemstoreSize = tempOnHeapMemstoreSize;
+ offHeapMemstoreSize = tempOffHeapMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (regionCount > 0) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
index c406221f4f9..50ba94c3b9f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java
@@ -81,6 +81,14 @@ public class RegionServerAccounting {
return this.globalMemStoreLimit;
}
+ long getGlobalOffHeapMemStoreLimit() {
+ if (isOffheap()) {
+ return this.globalMemStoreLimit;
+ } else {
+ return 0;
+ }
+ }
+
long getGlobalOnHeapMemStoreLimit() {
return this.globalOnHeapMemstoreLimit;
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 1c9ddf6b2c8..a57afa1c3db 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -68,6 +68,16 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 1025;
}
+ @Override
+ public long getOnHeapMemStoreSize() {
+ return 500;
+ }
+
+ @Override
+ public long getOffHeapMemStoreSize() {
+ return 600;
+ }
+
@Override
public long getStoreFileSize() {
return 1900;
@@ -253,6 +263,16 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 419;
}
+ @Override
+ public long getOnHeapMemStoreLimit() {
+ return 311;
+ }
+
+ @Override
+ public long getOffHeapMemStoreLimit() {
+ return 419;
+ }
+
@Override
public long getBlockCacheFreeSize() {
return 413;