You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by ch...@apache.org on 2026/10/19 11:41:59 UTC
(ozone) branch master updated: HDDS-16922. Exit SCM with failure status on fatal shutdown (#11528)
This is an automated email from the ASF dual-hosted git repository.
chihsuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 52c26aad185 HDDS-16922. Exit SCM with failure status on fatal shutdown (#11528)
52c26aad185 is described below
commit 52c26aad1855045cbbdd3c17feebc40878d162f2
Author: Andy Huang <ky...@gmail.com>
AuthorDate: Mon Oct 19 19:41:53 2026 +0800
HDDS-16922. Exit SCM with failure status on fatal shutdown (#11528)
---
.../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 8 +++++
.../hdds/scm/server/StorageContainerManager.java | 2 +-
.../hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java | 15 ++++++++--
.../hdds/scm/TestStorageContainerManager.java | 35 ++++++++++++++++++++++
.../hdds/scm/TestStorageContainerManagerHA.java | 21 +++++++++++++
5 files changed, 77 insertions(+), 4 deletions(-)
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
index 546f5b63dfe..5ec902dbebc 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMStateMachine.java
@@ -696,6 +696,14 @@ public void close() throws IOException {
shutdown(installSnapshotExecutor, LOG, 5, TimeUnit.SECONDS);
stopDNServerStartRetry();
} else if (!scm.isStopped()) {
+ // A removed SCM may only have received the joint configuration, which still lists it among the old peers.
+ boolean decommissioned = scm.getScmHAManager().getRatisServer().getDivision().getRaftConf()
+ .getCurrentPeers().stream().noneMatch(peer -> peer.getId().equals(RaftPeerId.valueOf(scm.getScmId())));
+ if (decommissioned) {
+ scm.stop();
+ ExitUtils.terminate(0, "SCM is decommissioned, terminate SCM", LOG);
+ return;
+ }
scm.shutDown("scm statemachine is closed by ratis, terminate SCM");
}
}
diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
index cef2adca52e..a3a61dfb374 100644
--- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
+++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java
@@ -1799,7 +1799,7 @@ private void stopContainerExportManager() {
@Override
public void shutDown(String message) {
stop();
- ExitUtils.terminate(0, message, LOG);
+ ExitUtils.terminate(1, message, LOG);
}
public boolean isStopped() {
diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
index 0ed2e957041..60725942fcc 100644
--- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
+++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestSCMHAManagerImpl.java
@@ -61,6 +61,7 @@
import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.ozone.test.GenericTestUtils;
import org.apache.ratis.server.DivisionInfo;
+import org.apache.ratis.util.ExitUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.MethodOrderer;
@@ -89,6 +90,8 @@ class TestSCMHAManagerImpl {
@BeforeAll
void setup(@TempDir Path tempDir) throws IOException, InterruptedException,
TimeoutException {
+ ExitUtils.disableSystemExit();
+ ExitUtils.clear();
storageBaseDir = tempDir;
clusterID = UUID.randomUUID().toString();
final StorageContainerManager scm = getMockStorageContainerManager(LEADER_SCM_ID, LEADER_PORT);
@@ -123,8 +126,12 @@ private void waitForSCMToBeReady(DivisionInfo ratisDivision)
@AfterAll
void cleanup() throws IOException {
- follower.stop();
- primarySCMHAManager.stop();
+ try {
+ follower.stop();
+ primarySCMHAManager.stop();
+ } finally {
+ ExitUtils.clear();
+ }
}
@Test
@@ -141,13 +148,15 @@ void testAddSCM() throws IOException {
@Test
@Order(2) // requires testAddSCM
- void testRemoveSCM() throws IOException {
+ void testRemoveSCM() throws IOException, InterruptedException, TimeoutException {
assumeThat(getPeerCount()).isEqualTo(2);
final RemoveSCMRequest removeSCMRequest = new RemoveSCMRequest(
clusterID, FOLLOWER_SCM_ID, getFollowerAddress());
primarySCMHAManager.removeSCM(removeSCMRequest);
assertEquals(1, getPeerCount());
+ GenericTestUtils.waitFor(ExitUtils::isTerminated, 100, 10_000);
+ assertEquals(0, ExitUtils.getFirstExitException().getStatus());
}
private int getPeerCount() {
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
index 4eae67ae3c7..c5b14b4405f 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
@@ -95,6 +95,7 @@
import org.apache.hadoop.hdds.scm.ha.RatisUtil;
import org.apache.hadoop.hdds.scm.ha.SCMHAUtils;
import org.apache.hadoop.hdds.scm.ha.SCMRatisServerImpl;
+import org.apache.hadoop.hdds.scm.ha.SCMStateMachine;
import org.apache.hadoop.hdds.scm.node.DatanodeInfo;
import org.apache.hadoop.hdds.scm.node.NodeManager;
import org.apache.hadoop.hdds.scm.node.NodeStatus;
@@ -146,6 +147,7 @@
import org.apache.ratis.conf.RaftProperties;
import org.apache.ratis.protocol.RaftGroupId;
import org.apache.ratis.server.RaftServerConfigKeys;
+import org.apache.ratis.util.ExitUtils;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
@@ -625,6 +627,39 @@ public void testScmInfo(@TempDir Path tempDir) throws Exception {
}
}
+ @Test
+ void testStateMachineCloseTerminatesWithFailureStatus(@TempDir Path tempDir) throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempDir.resolve("scm-meta").toString());
+ StorageContainerManager.scmInit(conf, UUID.randomUUID().toString());
+ ExitUtils.disableSystemExit();
+ StorageContainerManager scm = HddsTestUtils.getScmSimple(conf);
+ try {
+ scm.start();
+ waitForStateMachineReady(scm);
+ scm.stop();
+ scm.join();
+ assertThat(ExitUtils.isTerminated()).isFalse();
+
+ scm = HddsTestUtils.getScmSimple(conf);
+ scm.start();
+ waitForStateMachineReady(scm);
+ SCMStateMachine stateMachine = scm.getScmHAManager().getRatisServer().getSCMStateMachine();
+ ExitUtils.ExitException exit = assertThrows(ExitUtils.ExitException.class, stateMachine::close);
+ assertThat(exit.getStatus()).isEqualTo(1);
+ assertThat(scm.isStopped()).isTrue();
+ } finally {
+ scm.stop();
+ ExitUtils.clear();
+ }
+ }
+
+ /** Waits until the state machine is ready. */
+ private static void waitForStateMachineReady(StorageContainerManager scm) throws Exception {
+ SCMStateMachine stateMachine = scm.getScmHAManager().getRatisServer().getSCMStateMachine();
+ GenericTestUtils.waitFor(stateMachine::getIsStateMachineReady, 100, 30_000);
+ }
+
/**
* Test datanode heartbeat well processed with a 4-layer network topology.
*/
diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java
index ae10947d52c..9ded335c96c 100644
--- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java
+++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManagerHA.java
@@ -32,6 +32,8 @@
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.ozone.test.GenericTestUtils;
+import org.apache.ratis.util.ExitUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.slf4j.Logger;
@@ -69,6 +71,25 @@ public void shutdown() {
}
}
+ @Test
+ public void testDecommissionSCMTerminatesWithSuccessStatus() throws Exception {
+ ExitUtils.disableSystemExit();
+ ExitUtils.clear();
+ try {
+ init();
+ StorageContainerManager leader = cluster.getScmLeader(true);
+ StorageContainerManager removed = cluster.getStorageContainerManagers().stream()
+ .filter(scm -> scm != leader).findFirst().orElseThrow(IllegalStateException::new);
+ assertTrue(cluster.getStorageContainerLocationClient().decommissionScm(removed.getScmId()).getSuccess());
+ GenericTestUtils.waitFor(ExitUtils::isTerminated, 100, 120_000);
+ assertEquals(0, ExitUtils.getFirstExitException().getStatus());
+ assertTrue(removed.isStopped());
+ } finally {
+ shutdown();
+ ExitUtils.clear();
+ }
+ }
+
@Test
public void testPrimordialSCM() throws Exception {
init();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org