You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2026/09/24 05:37:25 UTC
(hbase) branch branch-2.6 updated: HBASE-29640 Add metric for oldest procedure age (#8689) (#8563)
This is an automated email from the ASF dual-hosted git repository.
virajjasani pushed a commit to branch branch-2.6
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.6 by this push:
new cb8515e7b12 HBASE-29640 Add metric for oldest procedure age (#8689) (#8563)
cb8515e7b12 is described below
commit cb8515e7b12ab480ddce7d20dec14e45cecb94cf
Author: Ma Zhengxuan <49...@users.noreply.github.com>
AuthorDate: Thu Sep 24 13:36:38 2026 +0800
HBASE-29640 Add metric for oldest procedure age (#8689) (#8563)
Signed-off-by: Viraj Jasani <vj...@apache.org>
Signed-off-by: Umesh <94...@gmail.com>
Signed-off-by: Aman Poonia <ap...@salesforce.com>
---
.../hbase/master/MetricsMasterProcSource.java | 5 +
.../hadoop/hbase/master/MetricsMasterWrapper.java | 6 ++
.../hbase/master/MetricsMasterProcSourceImpl.java | 8 ++
.../hbase/master/MetricsMasterWrapperImpl.java | 21 +++++
.../hbase/master/TestOldestProcedureMetrics.java | 101 +++++++++++++++++++++
5 files changed, 141 insertions(+)
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java
index 07ceaaf2e24..3e6968a29e7 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSource.java
@@ -51,4 +51,9 @@ public interface MetricsMasterProcSource extends BaseSource {
String NUM_MASTER_WALS_DESC = "Number of master WAL files";
+ String OLDEST_PROCEDURE_AGE_NAME = "oldestProcedureAge";
+
+ String OLDEST_PROCEDURE_AGE_DESC =
+ "Age in milliseconds of the oldest active master procedure (Gauge).";
+
}
diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
index ab8b4f5d949..51311124eed 100644
--- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
+++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapper.java
@@ -122,6 +122,12 @@ public interface MetricsMasterWrapper {
*/
long getNumWALFiles();
+ /**
+ * Get the age in milliseconds of the oldest active master procedure for each procedure type,
+ * keyed by the procedure's simple class name.
+ */
+ Map<String, Long> getOldestProcedureAgeByType();
+
/**
* Get the number of region split plans executed.
*/
diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
index 001ab016814..5e1bcc0df3d 100644
--- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
+++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterProcSourceImpl.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.master;
+import java.util.Map;
import org.apache.hadoop.hbase.metrics.BaseSourceImpl;
import org.apache.hadoop.hbase.metrics.Interns;
import org.apache.hadoop.metrics2.MetricsCollector;
@@ -56,6 +57,13 @@ public class MetricsMasterProcSourceImpl extends BaseSourceImpl implements Metri
if (masterWrapper != null) {
metricsRecordBuilder.addGauge(Interns.info(NUM_MASTER_WALS_NAME, NUM_MASTER_WALS_DESC),
masterWrapper.getNumWALFiles());
+ Map<String, Long> oldestProcedureAgeByType = masterWrapper.getOldestProcedureAgeByType();
+ metricsRecordBuilder.addGauge(
+ Interns.info(OLDEST_PROCEDURE_AGE_NAME, OLDEST_PROCEDURE_AGE_DESC),
+ oldestProcedureAgeByType.values().stream().mapToLong(Long::longValue).max().orElse(0L));
+ oldestProcedureAgeByType.forEach((procedureType, age) -> metricsRecordBuilder.addGauge(
+ Interns.info(OLDEST_PROCEDURE_AGE_NAME + "_" + procedureType, OLDEST_PROCEDURE_AGE_DESC),
+ age));
}
metricsRegistry.snapshot(metricsRecordBuilder, all);
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
index e60f3111a93..0057f2d9d4f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MetricsMasterWrapperImpl.java
@@ -25,8 +25,12 @@ import java.util.Map.Entry;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.quotas.QuotaObserverChore;
import org.apache.hadoop.hbase.quotas.SpaceQuotaSnapshot;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
import org.apache.yetus.audience.InterfaceAudience;
@@ -175,6 +179,23 @@ public class MetricsMasterWrapperImpl implements MetricsMasterWrapper {
return master.getNumWALFiles();
}
+ @Override
+ public Map<String, Long> getOldestProcedureAgeByType() {
+ ProcedureExecutor<MasterProcedureEnv> procedureExecutor = master.getMasterProcedureExecutor();
+ if (procedureExecutor == null) {
+ return Collections.emptyMap();
+ }
+ long now = EnvironmentEdgeManager.currentTime();
+ Map<String, Long> oldestProcedureAgeByType = new HashMap<>();
+ for (Procedure<MasterProcedureEnv> procedure : procedureExecutor.getProcedures()) {
+ if (!procedure.isFinished()) {
+ long age = Math.max(0L, now - procedure.getSubmittedTime());
+ oldestProcedureAgeByType.merge(procedure.getClass().getSimpleName(), age, Math::max);
+ }
+ }
+ return oldestProcedureAgeByType;
+ }
+
@Override
public Map<String, Entry<Long, Long>> getTableSpaceUtilization() {
if (master == null) {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java
new file mode 100644
index 00000000000..70fcfd0185b
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestOldestProcedureMetrics.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.hadoop.hbase.CompatibilityFactory;
+import org.apache.hadoop.hbase.master.locking.LockProcedure;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.test.MetricsAssertHelper;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+@Tag(MasterTests.TAG)
+@Tag(SmallTests.TAG)
+public class TestOldestProcedureMetrics {
+
+ private static final MetricsAssertHelper METRICS_HELPER =
+ CompatibilityFactory.getInstance(MetricsAssertHelper.class);
+
+ @AfterEach
+ public void resetEnvironmentEdge() {
+ EnvironmentEdgeManager.reset();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testOldestProcedureAge() {
+ ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
+ edge.setValue(10_000L);
+ EnvironmentEdgeManager.injectEdge(edge);
+
+ HMaster master = mock(HMaster.class);
+ ProcedureExecutor<MasterProcedureEnv> procedureExecutor = mock(ProcedureExecutor.class);
+ LockProcedure oldestLock = mock(LockProcedure.class);
+ LockProcedure newerLock = mock(LockProcedure.class);
+ ServerCrashProcedure serverCrash = mock(ServerCrashProcedure.class);
+ ServerCrashProcedure finished = mock(ServerCrashProcedure.class);
+
+ when(master.getMasterProcedureExecutor()).thenReturn(procedureExecutor);
+ when(procedureExecutor.getProcedures())
+ .thenReturn(Arrays.asList(oldestLock, newerLock, serverCrash, finished));
+ when(oldestLock.getSubmittedTime()).thenReturn(1_000L);
+ when(newerLock.getSubmittedTime()).thenReturn(4_000L);
+ when(serverCrash.getSubmittedTime()).thenReturn(2_000L);
+ when(finished.isFinished()).thenReturn(true);
+ when(finished.getSubmittedTime()).thenReturn(100L);
+
+ MetricsMasterWrapperImpl wrapper = new MetricsMasterWrapperImpl(master);
+ Map<String, Long> expected = new HashMap<>();
+ expected.put("LockProcedure", 9_000L);
+ expected.put("ServerCrashProcedure", 8_000L);
+ assertEquals(expected, wrapper.getOldestProcedureAgeByType());
+
+ MetricsMasterProcSource source = new MetricsMasterProcSourceImpl(wrapper);
+ METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME, 9_000L, source);
+ METRICS_HELPER.assertGauge(MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_LockProcedure",
+ 9_000L, source);
+ METRICS_HELPER.assertGauge(
+ MetricsMasterProcSource.OLDEST_PROCEDURE_AGE_NAME + "_ServerCrashProcedure", 8_000L, source);
+
+ when(procedureExecutor.getProcedures()).thenReturn(Collections.emptyList());
+ assertEquals(Collections.emptyMap(), wrapper.getOldestProcedureAgeByType());
+
+ when(procedureExecutor.getProcedures()).thenReturn(Collections.singletonList(newerLock));
+ when(newerLock.getSubmittedTime()).thenReturn(11_000L);
+ assertEquals(Collections.singletonMap("LockProcedure", 0L),
+ wrapper.getOldestProcedureAgeByType());
+
+ when(master.getMasterProcedureExecutor()).thenReturn(null);
+ assertEquals(Collections.emptyMap(), wrapper.getOldestProcedureAgeByType());
+ }
+}