You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by rm...@apache.org on 2025/01/16 13:46:44 UTC
(hbase) branch HBASE-29068-branch-3 created (now caf3d655417)
This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a change to branch HBASE-29068-branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
at caf3d655417 HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596)
This branch includes the following new commits:
new caf3d655417 HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(hbase) 01/01: HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596)
Posted by rm...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
rmattingly pushed a commit to branch HBASE-29068-branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit caf3d655417ab9e13736b7c6b345d0bed19da785
Author: Charles Connell <cc...@hubspot.com>
AuthorDate: Thu Jan 16 08:45:11 2025 -0500
HBASE-29068 Report percentFilesLocalPrimaryRegions metric (#6596)
Signed-off-by: Ray Mattingly <rm...@apache.org>
---
.../hbase/regionserver/MetricsRegionServerSource.java | 10 ++++++++--
.../hbase/regionserver/MetricsRegionServerSourceImpl.java | 3 +++
.../hbase/regionserver/MetricsRegionServerWrapper.java | 5 +++++
.../regionserver/MetricsRegionServerWrapperImpl.java | 15 +++++++++++++++
.../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++
5 files changed, 36 insertions(+), 2 deletions(-)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
index 8b64e793bed..c88a77b5140 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
@@ -314,10 +314,16 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
"Size of data that has been sent by clients with the write ahead logging turned off.";
String PERCENT_FILES_LOCAL = "percentFilesLocal";
String PERCENT_FILES_LOCAL_DESC =
- "The percent of HFiles that are stored on the local hdfs data node.";
+ "The percent of HFiles that are stored on the local hdfs data node. If not using "
+ + "region replicas, this should equal percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS = "percentFilesLocalPrimaryRegions";
+ String PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC =
+ "The percent of HFiles used by primary regions that are stored on the local hdfs data node. "
+ + "This is the category of locality that you want to reach 100% when using region replicas";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS = "percentFilesLocalSecondaryRegions";
String PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC =
- "The percent of HFiles used by secondary regions that are stored on the local hdfs data node.";
+ "The percent of HFiles used by secondary regions that are stored on the local hdfs data node. "
+ + "This is not likely to reach 100%";
String SPLIT_QUEUE_LENGTH = "splitQueueLength";
String SPLIT_QUEUE_LENGTH_DESC = "Length of the queue for splits.";
String COMPACTION_QUEUE_LENGTH = "compactionQueueLength";
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
index bfac0843e33..b214c8f8f4e 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
@@ -557,6 +557,9 @@ public class MetricsRegionServerSourceImpl extends BaseSourceImpl
rsWrap.getDataInMemoryWithoutWAL())
.addGauge(Interns.info(PERCENT_FILES_LOCAL, PERCENT_FILES_LOCAL_DESC),
rsWrap.getPercentFileLocal())
+ .addGauge(
+ Interns.info(PERCENT_FILES_LOCAL_PRIMARY_REGIONS, PERCENT_FILES_LOCAL_PRIMARY_REGIONS_DESC),
+ rsWrap.getPercentFileLocalPrimaryRegions())
.addGauge(Interns.info(PERCENT_FILES_LOCAL_SECONDARY_REGIONS,
PERCENT_FILES_LOCAL_SECONDARY_REGIONS_DESC), rsWrap.getPercentFileLocalSecondaryRegions())
.addGauge(Interns.info(TOTAL_BYTES_READ, TOTAL_BYTES_READ_DESC), rsWrap.getTotalBytesRead())
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
index 26612e14825..5b957d9bf08 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
@@ -229,6 +229,11 @@ public interface MetricsRegionServerWrapper {
*/
double getPercentFileLocal();
+ /**
+ * Get the percent of HFiles' that are local for primary region replicas.
+ */
+ double getPercentFileLocalPrimaryRegions();
+
/**
* Get the percent of HFiles' that are local for secondary region replicas.
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
index 5d1e51c11ec..ef6bbd1f4b6 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
@@ -612,6 +612,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
return aggregate.percentFileLocal;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return aggregate.percentFileLocalPrimaryRegions;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return aggregate.percentFileLocalSecondaryRegions;
@@ -762,6 +767,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
private long numMutationsWithoutWAL = 0;
private long dataInMemoryWithoutWAL = 0;
private double percentFileLocal = 0;
+ private double percentFileLocalPrimaryRegions = 0;
private double percentFileLocalSecondaryRegions = 0;
private long flushedCellsCount = 0;
private long compactedCellsCount = 0;
@@ -794,6 +800,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
private void aggregate(HRegionServer regionServer,
Map<String, ArrayList<Long>> requestsCountCache) {
HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution();
+ HDFSBlocksDistribution hdfsBlocksDistributionPrimaryRegions = new HDFSBlocksDistribution();
HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions = new HDFSBlocksDistribution();
long avgAgeNumerator = 0;
@@ -821,6 +828,9 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
hdfsBlocksDistribution.add(distro);
+ if (r.getRegionInfo().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
+ hdfsBlocksDistributionPrimaryRegions.add(distro);
+ }
if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
hdfsBlocksDistributionSecondaryRegions.add(distro);
}
@@ -832,6 +842,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper {
hdfsBlocksDistribution.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocal = Double.isNaN(localityIndex) ? 0 : (localityIndex * 100);
+ float localityIndexPrimaryRegions = hdfsBlocksDistributionPrimaryRegions
+ .getBlockLocalityIndex(regionServer.getServerName().getHostname());
+ percentFileLocalPrimaryRegions =
+ Double.isNaN(localityIndexPrimaryRegions) ? 0 : (localityIndexPrimaryRegions * 100);
+
float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
.getBlockLocalityIndex(regionServer.getServerName().getHostname());
percentFileLocalSecondaryRegions =
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
index 32c80e18004..f1b6efe50a9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
@@ -262,6 +262,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 99;
}
+ @Override
+ public double getPercentFileLocalPrimaryRegions() {
+ return 99;
+ }
+
@Override
public double getPercentFileLocalSecondaryRegions() {
return 99;