You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ps...@apache.org on 2026/10/14 09:38:12 UTC

(hbase) branch master updated: HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)

This is an automated email from the ASF dual-hosted git repository.

petersomogyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/master by this push:
     new 27396d67553 HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)
27396d67553 is described below

commit 27396d6755341e3cbe3e5519ba74306506b6631f
Author: Bram Schuur <br...@suse.com>
AuthorDate: Wed Oct 14 11:38:05 2026 +0200

    HBASE-30463 HMaster fails to initialize when the procedure store contains more than one ServerCrashProcedure for the same server (#8765)
    
    Merge duplicate SCPs per server with Math::min instead of throwing on the duplicate key.
    
    Co-authored-by: Claude Opus 5.5 (1M context) <no...@anthropic.com>
    Signed-off-by: Peter Somogyi <ps...@apache.org>
---
 .../org/apache/hadoop/hbase/master/HMaster.java    |  7 +-
 .../master/TestMasterStartupWithDuplicateSCPs.java | 94 ++++++++++++++++++++++
 2 files changed, 99 insertions(+), 2 deletions(-)

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 9ba1ee5aa16..8d9acf005e3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -1070,10 +1070,13 @@ public class HMaster extends HBaseServerBase<MasterRpcServices> implements Maste
     // it out).
     // We also pass dirs that are already 'splitting'... so we can do some checks down in tracker.
     // TODO: Generate the splitting and live Set in one pass instead of two as we currently do.
+    // The procedure store may hold more than one SCP for the same server; keep the earliest
+    // submit time as the crash time, matching DeadServer#putIfAbsent.
     this.regionServerTracker.upgrade(
       procsByType.getOrDefault(ServerCrashProcedure.class, Collections.emptyList()).stream()
-        .map(p -> (ServerCrashProcedure) p).collect(
-          Collectors.toMap(ServerCrashProcedure::getServerName, Procedure::getSubmittedTime)),
+        .map(p -> (ServerCrashProcedure) p)
+        .collect(Collectors.toMap(ServerCrashProcedure::getServerName, Procedure::getSubmittedTime,
+          Math::min)),
       Sets.union(rsListStorage.getAll(), walManager.getLiveServersFromWALDir()),
       walManager.getSplittingServersFromWALDir());
     // This manager must be accessed AFTER hbase:meta is confirmed on line..
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java
new file mode 100644
index 00000000000..a83ec2d509f
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterStartupWithDuplicateSCPs.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.io.IOException;
+import org.apache.hadoop.hbase.HBaseTestingUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
+import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.NoopProcedure;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+
+/**
+ * The master must initialize when its procedure store holds more than one unfinished
+ * ServerCrashProcedure for the same server.
+ */
+@Tag(MasterTests.TAG)
+@Tag(MediumTests.TAG)
+public class TestMasterStartupWithDuplicateSCPs {
+
+  private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
+
+  @BeforeAll
+  public static void setUp() throws Exception {
+    // the restarted master gets a new port, so the default rpc connection registry can not work
+    UTIL.getConfiguration().set(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY,
+      HConstants.ZK_CONNECTION_REGISTRY_CLASS);
+    UTIL.startMiniCluster(1);
+  }
+
+  @AfterAll
+  public static void tearDown() throws IOException {
+    UTIL.shutdownMiniCluster();
+  }
+
+  @Test
+  public void testRestartWithDuplicateSCPs() throws Exception {
+    HMaster master = UTIL.getMiniHBaseCluster().getMaster();
+    ProcedureExecutor<MasterProcedureEnv> procExec = master.getMasterProcedureExecutor();
+    MasterProcedureEnv env = procExec.getEnvironment();
+    ServerName deadServer =
+      ServerName.valueOf("dead.example.org", 16020, EnvironmentEdgeManager.currentTime());
+
+    // Hold the server lock so both SCPs stay queued, and therefore unfinished, in the store.
+    NoopProcedure<MasterProcedureEnv> lockHolder = new NoopProcedure<>();
+    assertFalse(env.getProcedureScheduler().waitServerExclusiveLock(lockHolder, deadServer));
+    long pid1 = procExec.submitProcedure(new ServerCrashProcedure(env, deadServer, false, false));
+    long pid2 = procExec.submitProcedure(new ServerCrashProcedure(env, deadServer, false, false));
+    assertEquals(2,
+      procExec
+        .getActiveProceduresNoCopy().stream().filter(p -> p instanceof ServerCrashProcedure
+          && !p.isFinished() && ((ServerCrashProcedure) p).getServerName().equals(deadServer))
+        .count());
+
+    UTIL.getMiniHBaseCluster().stopMaster(0).join();
+    UTIL.getMiniHBaseCluster().startMaster();
+    UTIL.waitFor(60000, () -> {
+      HMaster m = UTIL.getMiniHBaseCluster().getMaster();
+      return m != null && m.isInitialized();
+    });
+
+    HMaster newMaster = UTIL.getMiniHBaseCluster().getMaster();
+    assertTrue(newMaster.getServerManager().getDeadServers().isDeadServer(deadServer));
+    ProcedureExecutor<MasterProcedureEnv> newProcExec = newMaster.getMasterProcedureExecutor();
+    UTIL.waitFor(60000, () -> newProcExec.isFinished(pid1) && newProcExec.isFinished(pid2));
+  }
+}