You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by me...@apache.org on 2021/03/11 07:46:50 UTC
[hbase] branch branch-2.4 updated: HBASE-25636 Expose HBCK report
as metrics (#3031)
This is an automated email from the ASF dual-hosted git repository.
meiyi pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 35d44c6 HBASE-25636 Expose HBCK report as metrics (#3031)
35d44c6 is described below
commit 35d44c6ce6b8973a2e5f75bd78f92b60a03572e1
Author: meiyi <my...@gmail.com>
AuthorDate: Thu Mar 11 15:12:30 2021 +0800
HBASE-25636 Expose HBCK report as metrics (#3031)
Signed-off-by: zhangduo <zh...@apache.org>
---
.../master/MetricsAssignmentManagerSource.java | 70 ++++++++++++++++++++++
.../master/MetricsAssignmentManagerSourceImpl.java | 58 ++++++++++++++++++
.../org/apache/hadoop/hbase/master/HbckChore.java | 10 ++++
.../hbase/master/MetricsAssignmentManager.java | 28 +++++++++
.../hbase/master/janitor/CatalogJanitor.java | 12 ++++
5 files changed, 178 insertions(+)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java
index 822e407..2794542 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSource.java
@@ -60,6 +60,27 @@ public interface MetricsAssignmentManagerSource extends BaseSource {
String RIT_DURATION_DESC =
"Total durations in milliseconds for all Regions in Transition (Histogram).";
+ // HBCK report metrics
+ String ORPHAN_REGIONS_ON_RS = "orphanRegionsOnRS";
+ String ORPHAN_REGIONS_ON_FS = "orphanRegionsOnFS";
+ String INCONSISTENT_REGIONS = "inconsistentRegions";
+
+ String ORPHAN_REGIONS_ON_RS_DESC = "Current number of Orphan Regions on RS (Gauge).";
+ String ORPHAN_REGIONS_ON_FS_DESC = "Current number of Orphan Regions on FS (Gauge).";
+ String INCONSISTENT_REGIONS_DESC = "Current number of Inconsistent Regions (Gauge).";
+
+ // CatalogJanitor Consistency report metrics
+ String HOLES = "holes";
+ String OVERLAPS = "overlaps";
+ String UNKNOWN_SERVER_REGIONS = "unknownServerRegions";
+ String EMPTY_REGION_INFO_REGIONS = "emptyRegionInfoRegions";
+
+ String HOLES_DESC = "Current number of Holes (Gauge).";
+ String OVERLAPS_DESC = "Current number of Overlaps (Gauge).";
+ String UNKNOWN_SERVER_REGIONS_DESC = "Current number of Unknown Server Regions (Gauge).";
+ String EMPTY_REGION_INFO_REGIONS_DESC =
+ "Current number of Regions with Empty Region Info (Gauge).";
+
String ASSIGN_METRIC_PREFIX = "assign";
String UNASSIGN_METRIC_PREFIX = "unassign";
String MOVE_METRIC_PREFIX = "move";
@@ -99,6 +120,55 @@ public interface MetricsAssignmentManagerSource extends BaseSource {
void updateUnknownServerOpenRegions(int unknownRegions);
/**
+ * Set the number of orphan regions on RS.
+ *
+ * @param orphanRegionsOnRs count of the orphan regions on RS in HBCK chore report.
+ */
+ void setOrphanRegionsOnRs(int orphanRegionsOnRs);
+
+ /**
+ * Set the number of orphan regions on FS.
+ *
+ * @param orphanRegionsOnFs count of the orphan regions on FS in HBCK chore report.
+ */
+ void setOrphanRegionsOnFs(int orphanRegionsOnFs);
+
+ /**
+ * Set the number of inconsistent regions.
+ *
+ * @param inconsistentRegions count of the inconsistent regions in HBCK chore report.
+ */
+ void setInconsistentRegions(int inconsistentRegions);
+
+ /**
+ * Set the number of holes.
+ *
+ * @param holes count of the holes in CatalogJanitor Consistency report.
+ */
+ void setHoles(int holes);
+
+ /**
+ * Set the number of overlaps.
+ *
+ * @param overlaps count of the overlaps in CatalogJanitor Consistency report.
+ */
+ void setOverlaps(int overlaps);
+
+ /**
+ * Set the number of unknown server regions.
+ * @param unknownServerRegions count of the unknown server regions in CatalogJanitor Consistency
+ * report.
+ */
+ void setUnknownServerRegions(int unknownServerRegions);
+
+ /**
+ * Set the number of regions with empty region info.
+ * @param emptyRegionInfoRegions count of the regions with empty region info in CatalogJanitor
+ * Consistency report.
+ */
+ void setEmptyRegionInfoRegions(int emptyRegionInfoRegions);
+
+ /**
* TODO: Remove. This may not be needed now as assign and unassign counts are tracked separately
* Increment the count of operations (assign/unassign).
*/
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
index 59b3647..c78464d 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManagerSourceImpl.java
@@ -39,6 +39,15 @@ public class MetricsAssignmentManagerSourceImpl
private MutableGaugeLong deadServerOpenRegions;
private MutableGaugeLong unknownServerOpenRegions;
+ private MutableGaugeLong orphanRegionsOnRsGauge;
+ private MutableGaugeLong orphanRegionsOnFsGauge;
+ private MutableGaugeLong inconsistentRegionsGauge;
+
+ private MutableGaugeLong holesGauge;
+ private MutableGaugeLong overlapsGauge;
+ private MutableGaugeLong unknownServerRegionsGauge;
+ private MutableGaugeLong emptyRegionInfoRegionsGauge;
+
private MutableFastCounter operationCounter;
private OperationMetrics assignMetrics;
@@ -70,6 +79,20 @@ public class MetricsAssignmentManagerSourceImpl
deadServerOpenRegions = metricsRegistry.newGauge(DEAD_SERVER_OPEN_REGIONS, "", 0);
unknownServerOpenRegions = metricsRegistry.newGauge(UNKNOWN_SERVER_OPEN_REGIONS, "", 0);
+ orphanRegionsOnRsGauge =
+ metricsRegistry.newGauge(ORPHAN_REGIONS_ON_RS, ORPHAN_REGIONS_ON_RS_DESC, 0L);
+ orphanRegionsOnFsGauge =
+ metricsRegistry.newGauge(ORPHAN_REGIONS_ON_FS, ORPHAN_REGIONS_ON_FS_DESC, 0L);
+ inconsistentRegionsGauge =
+ metricsRegistry.newGauge(INCONSISTENT_REGIONS, INCONSISTENT_REGIONS_DESC, 0L);
+
+ holesGauge = metricsRegistry.newGauge(HOLES, HOLES_DESC, 0L);
+ overlapsGauge = metricsRegistry.newGauge(OVERLAPS, OVERLAPS_DESC, 0L);
+ unknownServerRegionsGauge =
+ metricsRegistry.newGauge(UNKNOWN_SERVER_REGIONS, UNKNOWN_SERVER_REGIONS_DESC, 0L);
+ emptyRegionInfoRegionsGauge =
+ metricsRegistry.newGauge(EMPTY_REGION_INFO_REGIONS, EMPTY_REGION_INFO_REGIONS_DESC, 0L);
+
/**
* NOTE: Please refer to HBASE-9774 and HBASE-14282. Based on these two issues, HBase is
* moving away from using Hadoop's metric2 to having independent HBase specific Metrics. Use
@@ -121,6 +144,41 @@ public class MetricsAssignmentManagerSourceImpl
}
@Override
+ public void setOrphanRegionsOnRs(int orphanRegionsOnRs) {
+ orphanRegionsOnRsGauge.set(orphanRegionsOnRs);
+ }
+
+ @Override
+ public void setOrphanRegionsOnFs(int orphanRegionsOnFs) {
+ orphanRegionsOnFsGauge.set(orphanRegionsOnFs);
+ }
+
+ @Override
+ public void setInconsistentRegions(int inconsistentRegions) {
+ inconsistentRegionsGauge.set(inconsistentRegions);
+ }
+
+ @Override
+ public void setHoles(int holes) {
+ holesGauge.set(holes);
+ }
+
+ @Override
+ public void setOverlaps(int overlaps) {
+ overlapsGauge.set(overlaps);
+ }
+
+ @Override
+ public void setUnknownServerRegions(int unknownServerRegions) {
+ unknownServerRegionsGauge.set(unknownServerRegions);
+ }
+
+ @Override
+ public void setEmptyRegionInfoRegions(int emptyRegionInfoRegions) {
+ emptyRegionInfoRegionsGauge.set(emptyRegionInfoRegions);
+ }
+
+ @Override
public OperationMetrics getAssignMetrics() {
return assignMetrics;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java
index 63fb5dd..050c03a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HbckChore.java
@@ -144,6 +144,7 @@ public class HbckChore extends ScheduledChore {
LOG.warn("Unexpected", t);
}
running = false;
+ updateAssignmentManagerMetrics();
}
// This function does the sanity checks of making sure the chore is not run when it is
@@ -310,6 +311,15 @@ public class HbckChore extends ScheduledChore {
tableDirs.size(), numRegions, orphanRegionsOnFS.size());
}
+ private void updateAssignmentManagerMetrics() {
+ master.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateOrphanRegionsOnRs(getOrphanRegionsOnRS().size());
+ master.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateOrphanRegionsOnFs(getOrphanRegionsOnFS().size());
+ master.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateInconsistentRegions(getInconsistentRegions().size());
+ }
+
/**
* When running, the HBCK report may be changed later.
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java
index 38aeef2..4c89263 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsAssignmentManager.java
@@ -104,6 +104,34 @@ public class MetricsAssignmentManager {
assignmentManagerSource.updateUnknownServerOpenRegions(unknownRegions);
}
+ public void updateOrphanRegionsOnRs(int orphanRegionsOnRs) {
+ assignmentManagerSource.setOrphanRegionsOnRs(orphanRegionsOnRs);
+ }
+
+ public void updateOrphanRegionsOnFs(int orphanRegionsOnFs) {
+ assignmentManagerSource.setOrphanRegionsOnFs(orphanRegionsOnFs);
+ }
+
+ public void updateInconsistentRegions(int inconsistentRegions) {
+ assignmentManagerSource.setInconsistentRegions(inconsistentRegions);
+ }
+
+ public void updateHoles(int holes) {
+ assignmentManagerSource.setHoles(holes);
+ }
+
+ public void updateOverlaps(int overlaps) {
+ assignmentManagerSource.setOverlaps(overlaps);
+ }
+
+ public void updateUnknownServerRegions(int unknownServerRegions) {
+ assignmentManagerSource.setUnknownServerRegions(unknownServerRegions);
+ }
+
+ public void updateEmptyRegionInfoRegions(int emptyRegionInfoRegions) {
+ assignmentManagerSource.setEmptyRegionInfoRegions(emptyRegionInfoRegions);
+ }
+
/**
* @return Set of common metrics for assign procedure
*/
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
index f0189ad..c23ccf0 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/janitor/CatalogJanitor.java
@@ -165,6 +165,7 @@ public class CatalogJanitor extends ScheduledChore {
if (!this.lastReport.isEmpty()) {
LOG.warn(this.lastReport.toString());
}
+ updateAssignmentManagerMetrics();
if (isRIT(this.services.getAssignmentManager())) {
LOG.warn("Playing-it-safe skipping merge/split gc'ing of regions from hbase:meta while " +
@@ -404,6 +405,17 @@ public class CatalogJanitor extends ScheduledChore {
return this.services.getTableDescriptors().get(tableName);
}
+ private void updateAssignmentManagerMetrics() {
+ services.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateHoles(lastReport.getHoles().size());
+ services.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateOverlaps(lastReport.getOverlaps().size());
+ services.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateUnknownServerRegions(lastReport.getUnknownServers().size());
+ services.getAssignmentManager().getAssignmentManagerMetrics()
+ .updateEmptyRegionInfoRegions(lastReport.getEmptyRegionInfo().size());
+ }
+
private static void checkLog4jProperties() {
String filename = "log4j.properties";
try (final InputStream inStream =