You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ps...@apache.org on 2026/10/14 09:38:12 UTC
(hbase) branch master updated: HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)
This is an automated email from the ASF dual-hosted git repository.
petersomogyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/master by this push:
new 27396d67553 HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)
27396d67553 is described below
commit 27396d6755341e3cbe3e5519ba74306506b6631f
Author: Bram Schuur <br...@suse.com>
AuthorDate: Wed Oct 14 11:38:05 2026 +0200
HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)
Merge duplicate SCPs per server with Math::min instead of throwing on the duplicate key.
Co-authored-by: Claude Opus 5.5 (1M context) <no...@anthropic.com>
Signed-off-by: Peter Somogyi <ps...@apache.org>
---
.../org/apache/hadoop/hbase/master/HMaster.java | 7 +-
.../master/TestMasterStartupWithDuplicateSCPs.java | 94 ++++++++++++++++++++++
2 files changed, 99 insertions(+), 2 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9ba1ee5aa16..8d9acf005e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1070,10 +1070,13 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
// it out).
// We also pass dirs that are already 'splitting'... so we can do some checks down in tracker.
// TODO: Generate the splitting and live Set in one pass instead of two as we currently do.
+ // The procedure store may hold more than one SCP for the same server; keep the earliest
+ // submit time as the crash time, matching DeadServer#putIfAbsent.
this.regionServerTracker.upgrade(
procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
- .map(p -> (ServerCrashProcedure) p).collect(
- Collectors.toMap(ServerCrashProcedure::getServerName, Procedure::getSubmittedTime)),
+ .map(p -> (ServerCrashProcedure) p)
+ .collect(Collectors.toMap(ServerCrashProcedure::getServerName, Procedure::getSubmittedTime,
+ Math::min)),
Sets.union(rsListStorage.getAll(), walManager.getLiveServersFromWALDir()),
walManager.getSplittingServersFromWALDir());
// This manager must be accessed AFTER hbase:meta is confirmed on line..
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java
new file mode 100644
index 00000000000..a83ec2d509f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The master must initialize when its procedure store holds more than one unfinished
+ * ServerCrashProcedure for the same server.
+ */
+@Tag(MasterTests.TAG)
+@Tag(MediumTests.TAG)
+public class TestMasterStartupWithDuplicateSCPs {
+
+ private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+ @BeforeAll
+ public static void setUp() throws Exception {
+ // the restarted master gets a new port, so the default rpc connection registry can not work
+ UTIL.getConfiguration().set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
+ HConstants.ZK_CONNECTION_REGISTRY_CLASS);
+ UTIL.startMiniCluster(1);
+ }
+
+ @AfterAll
+ public static void tearDown() throws IOException {
+ UTIL.shutdownMiniCluster();
+ }
+
+ @Test
+ public void testRestartWithDuplicateSCPs() throws Exception {
+ HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+ ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
+ MasterProcedureEnv env = procExec.getEnvironment();
+ ServerName deadServer =
+ ServerName.valueOf("dead.example.org", 16020, EnvironmentEdgeManager.currentTime());
+
+ // Hold the server lock so both SCPs stay queued, and therefore unfinished, in the store.
+ NoopProcedure<MasterProcedureEnv> lockHolder = new NoopProcedure<>();
+ assertFalse(env.getProcedureScheduler().waitServerExclusiveLock(lockHolder, deadServer));
+ long pid1 = procExec.submitProcedure(new ServerCrashProcedure(env, deadServer, false, false));
+ long pid2 = procExec.submitProcedure(new ServerCrashProcedure(env, deadServer, false, false));
+ assertEquals(2,
+ procExec
+ .getActiveProceduresNoCopy().stream().filter(p -> p instanceof ServerCrashProcedure
+ && !p.isFinished() && ((ServerCrashProcedure) p).getServerName().equals(deadServer))
+ .count());
+
+ UTIL.getMiniHBaseCluster().stopMaster(0).join();
+ UTIL.getMiniHBaseCluster().startMaster();
+ UTIL.waitFor(60000, () -> {
+ HMaster m = UTIL.getMiniHBaseCluster().getMaster();
+ return m != null && m.isInitialized();
+ });
+
+ HMaster newMaster = UTIL.getMiniHBaseCluster().getMaster();
+ assertTrue(newMaster.getServerManager().getDeadServers().isDeadServer(deadServer));
+ ProcedureExecutor<MasterProcedureEnv> newProcExec = newMaster.getMasterProcedureExecutor();
+ UTIL.waitFor(60000, () -> newProcExec.isFinished(pid1) && newProcExec.isFinished(pid2));
+ }
+}