You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by rm...@apache.org on 2025/01/17 15:29:47 UTC
(hbase) 01/01: HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596) (#6606)
This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a commit to branch HBASE-29068-branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f4f57870da9e7bf41e082758b024c044a3c88eaf
Author: Ray Mattingly <rm...@apache.org>
AuthorDate: Fri Jan 17 10:26:45 2025 -0500
HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596) (#6606)
Signed-off-by: Ray Mattingly <rm...@apache.org>
Co-authored-by: Charles Connell <cc...@hubspot.com>
---
.../hbase/regionserver/MetricsRegionServerSource.java | 10 ++++++++--
.../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++
.../hbase/regionserver/MetricsRegionServerSourceImpl.java | 3 +++
.../regionserver/MetricsRegionServerWrapperImpl.java | 15 +++++++++++++++
.../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++
5 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 75269e57181..c68809a1fdd 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -307,10 +307,16 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
"Size of data that has been sent by clients with the write ahead logging turned off.";
String PERCENT_FILES_LOCAL = "percentFilesLocal";
String PERCENT_FILES_LOCAL_DESC =
- "The percent of HFiles that are stored on the local hdfs data node.";
+ "The percent of HFiles that are stored on the local hdfs data node. If not using "
+ + "region replicas, this should equal percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS = "percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC =
+ "The percent of HFiles used by primary regions that are stored on the local hdfs data node. "
+ + "This is the category of locality that you want to reach 100% when using region replicas";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS = "percentFilesLocalSecondaryRegions";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC =
- "The percent of HFiles used by secondary regions that are stored on the local hdfs data node.";
+ "The percent of HFiles used by secondary regions that are stored on the local hdfs data node. "
+ + "This is not likely to reach 100%";
String SPLIT_QUEUE_LENGTH = "splitQueueLength";
String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits.";
String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 1e5c6c409ed..10e71d091f5 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -219,6 +219,11 @@ public interface MetricsRegionServerWrapper {
*/
double getPercentFileLocal();
+ /**
+ * Get the percent of HFiles' that are local for primary region replicas.
+ */
+ double getPercentFileLocalPrimaryRegions();
+
/**
* Get the percent of HFiles' that are local for secondary region replicas.
*/
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index 9e857a49997..e0429cfb55d 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -553,6 +553,9 @@ public class MetricsRegionServerSourceImpl extends BaseSourceImpl
rsWrap.getDataInMemoryWithoutWAL())
.addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
rsWrap.getPercentFileLocal())
+ .addGauge(
+ Interns.info(PERCENT_FILES_LOCAL_PRIMARY_REGIONS, PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC),
+ rsWrap.getPercentFileLocalPrimaryRegions())
.addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS,
PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions())
.addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead())
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 6efc4df2361..2bd396242a1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -602,6 +602,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
return aggregate.percentFileLocal;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return aggregate.percentFileLocalPrimaryRegions;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return aggregate.percentFileLocalSecondaryRegions;
@@ -750,6 +755,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
private long numMutationsWithoutWAL = 0;
private long dataInMemoryWithoutWAL = 0;
private double percentFileLocal = 0;
+ private double percentFileLocalPrimaryRegions = 0;
private double percentFileLocalSecondaryRegions = 0;
private long flushedCellsCount = 0;
private long compactedCellsCount = 0;
@@ -782,6 +788,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
private void aggregate(HRegionServer regionServer,
Map<String, ArrayList<Long>> requestsCountCache) {
HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
+ HDFSBlocksDistribution hdfsBlocksDistributionPrimaryRegions = new HDFSBlocksDistribution();
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new HDFSBlocksDistribution();
long avgAgeNumerator = 0;
@@ -808,6 +815,9 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
hdfsBlocksDistribution.add(distro);
+ if (r.getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID) {
+ hdfsBlocksDistributionPrimaryRegions.add(distro);
+ }
if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
hdfsBlocksDistributionSecondaryRegions.add(distro);
}
@@ -819,6 +829,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex * 100);
+ float localityIndexPrimaryRegions = hdfsBlocksDistributionPrimaryRegions
+ .getBlockLocalityIndex(regionServer.getServerName().getHostname());
+ percentFileLocalPrimaryRegions =
+ Double.isNaN(localityIndexPrimaryRegions) ? 0 : (localityIndexPrimaryRegions * 100);
+
float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocalSecondaryRegions =
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 3ed0dab52e7..0e77ae89fef 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -252,6 +252,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 99;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return 99;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return 99;