You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2026/10/09 14:03:23 UTC
(iotdb) branch master updated: Extend the node state machine and persist node statuses (#18801)
This is an automated email from the ASF dual-hosted git repository.
CRZbulabula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 94f99d6516a Extend the node state machine and persist node statuses (#18801)
94f99d6516a is described below
commit 94f99d6516aa337fa43676a3e04063869ace707e
Author: d-wang-commit <d....@gmail.com>
AuthorDate: Fri Oct 9 22:03:16 2026 +0800
Extend the node state machine and persist node statuses (#18801)
* feat: add and persist Stopped node status
Report graceful shutdown as Stopped, apply shared node status transition rules, and persist Stopped/Removing statuses through ConfigNode consensus for recovery after leadership changes.
* test: cover Stopped node status and persistence
Cover shutdown reporting, status transitions, scheduling and removal behavior, and node status persistence and recovery with unit and integration tests.
* feature: add ReadOnly status reasons
Classify local ReadOnly reasons and propagate them through heartbeats and node display APIs.
Keep reasons separate from status and append StatusReason to SHOW results when present.
* test: cover ReadOnly status reasons
Cover reason precedence, explicit updates, removal rollback, SHOW layouts and table node metadata.
Verify reason recovery after leader changes and cleanup after node failure.
* feat: persist ReadOnly node status and reasons
---
.../org/apache/iotdb/it/env/cluster/EnvUtils.java | 12 +
.../it/cluster/IoTDBClusterNodeErrorStartUpIT.java | 31 +-
.../it/cluster/IoTDBClusterNodeShutdownHookIT.java | 60 +-
.../it/cluster/IoTDBNodeStatusFailoverIT.java | 743 +++++++++++++++++
.../it/cluster/IoTDBNodeStatusPersistenceIT.java | 379 +++++++++
.../confignode/it/cluster/NodeStatusTestAgent.java | 203 +++++
.../load/IoTDBRegionGroupLeaderDistributionIT.java | 3 +-
.../it/partition/IoTDBPartitionCreationIT.java | 58 +-
.../it/partition/IoTDBPartitionDurableIT.java | 49 +-
.../iotdb/pipe/it/single/IoTDBShowReceiversIT.java | 6 +-
.../it/db/it/IoTDBSetSystemStatusTableIT.java | 70 +-
...ableAggregationQueryWithNetworkPartitionIT.java | 5 +-
.../IoTDBTableViewQueryWithCachedDeviceIT.java | 3 +-
.../relational/it/schema/IoTDBDatabaseIT.java | 1 +
.../iotdb/session/it/IoTDBConnectionsIT.java | 53 +-
.../iotdb/session/it/pool/SessionPoolIT.java | 21 +-
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 3 +
.../iotdb/confignode/i18n/ManagerMessages.java | 8 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 7 +
.../iotdb/confignode/i18n/ConfigNodeMessages.java | 3 +
.../iotdb/confignode/i18n/ManagerMessages.java | 8 +-
.../iotdb/confignode/i18n/ProcedureMessages.java | 7 +
.../handlers/heartbeat/AINodeHeartbeatHandler.java | 4 +-
.../heartbeat/ConfigNodeHeartbeatHandler.java | 4 +-
.../heartbeat/DataNodeHeartbeatHandler.java | 7 +-
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 2 +
.../write/confignode/UpdateNodeStatusPlan.java | 104 +++
.../iotdb/confignode/manager/ConfigManager.java | 54 +-
.../apache/iotdb/confignode/manager/IManager.java | 18 +-
.../iotdb/confignode/manager/ProcedureManager.java | 13 +-
.../confignode/manager/RetryFailedTasksThread.java | 3 +-
.../iotdb/confignode/manager/load/LoadManager.java | 76 +-
.../manager/load/balancer/RegionBalancer.java | 12 +-
.../manager/load/cache/AbstractLoadCache.java | 9 +-
.../confignode/manager/load/cache/LoadCache.java | 140 +++-
.../load/cache/consensus/ConsensusGroupCache.java | 3 +-
.../load/cache/node/AINodeHeartbeatCache.java | 48 --
.../manager/load/cache/node/BaseNodeCache.java | 84 +-
.../load/cache/node/ConfigNodeHeartbeatCache.java | 39 +-
.../load/cache/node/DataNodeHeartbeatCache.java | 57 +-
.../load/cache/node/NodeHeartbeatSample.java | 7 +-
.../manager/load/cache/node/NodeStatistics.java | 17 +
.../manager/load/cache/region/RegionCache.java | 3 +-
.../load/cache/region/RegionGroupCache.java | 2 +-
.../manager/load/service/StatisticsService.java | 2 +-
.../iotdb/confignode/manager/node/NodeManager.java | 25 +-
.../runtime/SubscriptionRuntimeCoordinator.java | 7 +-
.../persistence/executor/ConfigPlanExecutor.java | 3 +
.../confignode/persistence/node/NodeInfo.java | 115 ++-
.../procedure/env/ConfigNodeProcedureEnv.java | 7 +-
.../procedure/env/RegionMaintainHandler.java | 2 +-
.../procedure/env/RemoveDataNodeHandler.java | 71 +-
.../procedure/impl/node/RemoveAINodeProcedure.java | 1 +
.../impl/node/RemoveDataNodesProcedure.java | 6 +-
.../procedure/impl/schema/SchemaUtils.java | 9 +-
.../confignode/service/ConfigNodeShutdownHook.java | 162 +++-
.../heartbeat/DataNodeHeartbeatHandlerTest.java | 100 +++
.../request/ConfigPhysicalPlanSerDeTest.java | 67 ++
.../manager/ConfigManagerShutdownReportTest.java | 161 ++++
.../confignode/manager/ProcedureManagerTest.java | 25 +
.../manager/RetryFailedTasksThreadTest.java | 107 +++
.../confignode/manager/load/LoadManagerTest.java | 115 ++-
.../manager/load/balancer/RegionBalancerTest.java | 144 ++++
.../router/priority/GreedyPriorityTest.java | 2 +-
.../priority/LeaderPriorityBalancerTest.java | 2 +-
.../load/cache/ConsensusGroupCacheTest.java | 2 +-
.../load/cache/LoadCacheNodeStatusTest.java | 336 ++++++++
.../cache/LoadCachePersistedNodeStatusTest.java | 898 +++++++++++++++++++++
.../manager/load/cache/NodeCacheTest.java | 228 +++++-
.../manager/node/NodeManagerStatusReasonTest.java | 133 +++
.../SubscriptionRuntimeCoordinatorTest.java | 119 +++
.../iotdb/confignode/persistence/NodeInfoTest.java | 269 ++++++
.../procedure/env/ConfigNodeProcedureEnvTest.java | 93 +++
.../procedure/env/RegionMaintainHandlerTest.java | 68 ++
.../impl/node/RemoveDataNodePersistenceTest.java | 581 +++++++++++++
.../procedure/impl/schema/SchemaUtilsTest.java | 91 +++
.../service/ConfigNodeShutdownHookTest.java | 346 ++++++++
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 3 +-
.../apache/iotdb/db/i18n/DataNodeMiscMessages.java | 2 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 35 +-
.../InformationSchemaContentSupplierFactory.java | 31 +-
.../config/executor/ClusterConfigTaskExecutor.java | 8 +-
.../config/metadata/ShowClusterDetailsTask.java | 63 +-
.../execution/config/metadata/ShowClusterTask.java | 40 +-
.../config/metadata/ShowConfigNodesTask.java | 28 +-
.../config/metadata/ShowDataNodesTask.java | 30 +-
.../iotdb/db/service/DataNodeShutdownHook.java | 4 +-
.../db/storageengine/dataregion/DataRegion.java | 6 +-
.../dataregion/memtable/TsFileProcessor.java | 4 +-
.../dataregion/wal/buffer/WALBuffer.java | 10 +-
.../wal/checkpoint/CheckpointManager.java | 4 +-
.../DataNodeInternalRPCServiceImplDiskTest.java | 43 +-
...eInternalRPCServiceImplSetSystemStatusTest.java | 91 +++
.../metadata/ShowClusterDetailsTaskTest.java | 172 ++++
.../config/metadata/ShowClusterTaskTest.java | 168 ++++
.../config/metadata/ShowConfigNodesTaskTest.java | 121 +++
.../config/metadata/ShowDataNodesTaskTest.java | 114 +++
.../wal/buffer/WALBufferRollRecoveryTest.java | 4 +-
.../rescon/disk/FolderManagerTest.java | 57 +-
.../apache/iotdb/commons/cluster/NodeStatus.java | 102 ++-
.../apache/iotdb/commons/cluster/RegionStatus.java | 5 +
.../apache/iotdb/commons/conf/CommonConfig.java | 57 +-
.../apache/iotdb/commons/disk/FolderManager.java | 14 +-
.../commons/disk/strategy/DirectoryStrategy.java | 4 +-
.../commons/enums/HandleSystemErrorStrategy.java | 6 +-
.../schema/column/ColumnHeaderConstant.java | 2 +
.../commons/schema/table/InformationSchema.java | 3 +
.../iotdb/commons/cluster/NodeStatusTest.java | 91 +++
.../iotdb/commons/conf/CommonConfigTest.java | 222 ++++-
.../src/main/thrift/confignode.thrift | 21 +-
111 files changed, 7705 insertions(+), 600 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java
index 63c2041199d..b69e86c12bd 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/EnvUtils.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.it.env.cluster;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.it.framework.IoTDBTestLogger;
import org.apache.tsfile.external.commons.lang3.SystemUtils;
@@ -69,6 +70,17 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR;
public class EnvUtils {
+ /**
+ * The status a node locally stopped via {@code AbstractNodeWrapper.stop()} is expected to be in.
+ * On Windows, {@code Process.destroy()} terminates the node process without running the JVM
+ * shutdown hooks, so the graceful-shutdown report is never sent and the ConfigNode marks the node
+ * Unknown by heartbeat timeout. On Unix, the shutdown hook reports the stop and the node becomes
+ * Stopped.
+ */
+ public static NodeStatus getNodeStatusAfterLocalStop() {
+ return SystemUtils.IS_OS_WINDOWS ? NodeStatus.Unknown : NodeStatus.Stopped;
+ }
+
public static int[] searchAvailablePorts() {
int length = 10;
while (true) {
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java
index e12ab129d21..3f477c1c95f 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeErrorStartUpIT.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TDataNodeRestartResp;
import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.config.MppBaseConfig;
import org.apache.iotdb.it.env.cluster.config.MppCommonConfig;
import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
@@ -212,15 +213,35 @@ public class IoTDBClusterNodeErrorStartUpIT {
dataNodeRestartResp.getStatus().getCode());
Assert.assertTrue(dataNodeRestartResp.getStatus().getMessage().contains("whose nodeId="));
- // Shutdown and check
+ // Local stop runs shutdown hooks on Unix, but terminates the process directly on Windows.
+ // A ConfigNode that was the leader at shutdown time can also remain Unknown if its report
+ // cannot reach another leader.
EnvFactory.getEnv().shutdownConfigNode(1);
EnvFactory.getEnv().shutdownDataNode(0);
EnvFactory.getEnv()
.ensureNodeStatus(
- Arrays.asList(
- EnvFactory.getEnv().getConfigNodeWrapper(1),
- EnvFactory.getEnv().getDataNodeWrapper(0)),
- Arrays.asList(NodeStatus.Unknown, NodeStatus.Unknown));
+ Arrays.asList(EnvFactory.getEnv().getDataNodeWrapper(0)),
+ Arrays.asList(EnvUtils.getNodeStatusAfterLocalStop()));
+ boolean isConfigNodeDown = false;
+ for (int retry = 0; retry < 30; retry++) {
+ TShowClusterResp showClusterResp = client.showCluster();
+ for (TConfigNodeLocation configNodeLocation : showClusterResp.getConfigNodeList()) {
+ if (configNodeLocation.getConsensusEndPoint().getPort()
+ == registeredConfigNodeWrapper.getConsensusPort()) {
+ String configNodeStatus =
+ showClusterResp.getNodeStatus().get(configNodeLocation.getConfigNodeId());
+ if (NodeStatus.Stopped.getStatus().equals(configNodeStatus)
+ || NodeStatus.Unknown.getStatus().equals(configNodeStatus)) {
+ isConfigNodeDown = true;
+ }
+ }
+ }
+ if (isConfigNodeDown) {
+ break;
+ }
+ Thread.sleep(1000);
+ }
+ Assert.assertTrue(isConfigNodeDown);
/* Restart and updatePeer */
// TODO: Delete this IT after enable modify internal TEndPoints
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java
index 8917f163557..e947a7c5273 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBClusterNodeShutdownHookIT.java
@@ -19,12 +19,15 @@
package org.apache.iotdb.confignode.it.cluster;
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -39,7 +42,6 @@ import org.junit.runner.RunWith;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
@RunWith(IoTDBTestRunner.class)
@Category({ClusterIT.class})
@@ -75,29 +77,57 @@ public class IoTDBClusterNodeShutdownHookIT {
try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
- // The unknown Nodes should be detected immediately with the help of shutdown hook
+ // Unix shutdown hooks report Stopped; Windows local stop is detected as Unknown.
+ // A ConfigNode whose report cannot reach the newly elected leader remains Unknown too.
+ TShowClusterResp showClusterResp = client.showCluster();
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), showClusterResp.getStatus().getCode());
+
+ int stoppedDataNodeId = -1;
+ for (TDataNodeLocation dataNodeLocation : showClusterResp.getDataNodeList()) {
+ if (dataNodeLocation.getInternalEndPoint().getPort()
+ == EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()) {
+ stoppedDataNodeId = dataNodeLocation.getDataNodeId();
+ }
+ }
+ Assert.assertNotEquals(-1, stoppedDataNodeId);
+
+ int stoppedConfigNodeId = -1;
+ for (TConfigNodeLocation configNodeLocation : showClusterResp.getConfigNodeList()) {
+ if (configNodeLocation.getConsensusEndPoint().getPort()
+ == EnvFactory.getEnv().getConfigNodeWrapper(1).getConsensusPort()) {
+ stoppedConfigNodeId = configNodeLocation.getConfigNodeId();
+ }
+ }
+ Assert.assertNotEquals(-1, stoppedConfigNodeId);
+
boolean isDetected = false;
- for (int retry = 0; retry < 5; retry++) {
- TShowClusterResp showClusterResp = client.showCluster();
+ for (int retry = 0; retry < 30; retry++) {
+ showClusterResp = client.showCluster();
Assert.assertEquals(
TSStatusCode.SUCCESS_STATUS.getStatusCode(), showClusterResp.getStatus().getCode());
- AtomicInteger unknownNum = new AtomicInteger(0);
- showClusterResp
- .getNodeStatus()
- .forEach(
- (nodeId, nodeStatus) -> {
- if (NodeStatus.Unknown.getStatus().equals(nodeStatus)) {
- unknownNum.getAndIncrement();
- }
- });
- if (unknownNum.get() == 2) {
+
+ // Windows does not run the shutdown hook, so allow heartbeat failure detection to finish.
+ final String dataNodeStatus = showClusterResp.getNodeStatus().get(stoppedDataNodeId);
+ final boolean isDataNodeStopped =
+ EnvUtils.getNodeStatusAfterLocalStop().getStatus().equals(dataNodeStatus);
+
+ // The stopped ConfigNode is Stopped when its report reached a leader, and may otherwise
+ // remain Unknown until heartbeat timeout
+ final String configNodeStatus = showClusterResp.getNodeStatus().get(stoppedConfigNodeId);
+ final boolean isConfigNodeDetected =
+ NodeStatus.Stopped.getStatus().equals(configNodeStatus)
+ || NodeStatus.Unknown.getStatus().equals(configNodeStatus);
+
+ if (isDataNodeStopped && isConfigNodeDetected) {
isDetected = true;
break;
}
TimeUnit.SECONDS.sleep(1);
}
- Assert.assertTrue(isDetected);
+ Assert.assertTrue(
+ "Timed out waiting for the stopped DataNode and ConfigNode to be detected", isDetected);
}
}
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java
new file mode 100644
index 00000000000..13dd39c7a31
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusFailoverIT.java
@@ -0,0 +1,743 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.it.cluster;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.ClientPoolFactory;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import com.sun.tools.attach.VirtualMachine;
+import org.apache.tsfile.utils.Pair;
+import org.junit.After;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.ErrorCollector;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collections;
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/** Process-level status persistence checks with three actual ConfigRegion voters. */
+@RunWith(IoTDBTestRunner.class)
+@Category(ClusterIT.class)
+public class IoTDBNodeStatusFailoverIT {
+ @Rule public TemporaryFolder temporary = new TemporaryFolder();
+ @Rule public ErrorCollector failures = new ErrorCollector();
+ private Path agentJar;
+ private int queryNode;
+
+ @After
+ public void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testGracefulLeaderShutdownPreservesStoppedAndRemoving() throws Exception {
+ init(3);
+ int stopped = stoppedFixture(0);
+ int removing = removingFixture(1);
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ Path marker = submit(leader, "shutdown");
+ awaitFile(marker);
+ assertTrue(
+ "System.exit must finish all JVM shutdown hooks",
+ EnvFactory.getEnv()
+ .getConfigNodeWrapper(leader)
+ .getInstance()
+ .waitFor(60, TimeUnit.SECONDS));
+ assertNotEquals(leader, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ assertStatus(0, stopped, NodeStatus.Stopped);
+ assertStatus(1, removing, NodeStatus.Removing);
+ EnvFactory.getEnv().startConfigNode(leader);
+ awaitConfigNodeRunning(leader);
+ awaitPersisted(leader, stopped, NodeStatus.Stopped);
+ transfer(leader);
+ assertStatus(0, stopped, NodeStatus.Stopped);
+ assertStatus(1, removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testGracefulSeedShutdownReportsStopped() throws Exception {
+ init(1);
+ transfer(1);
+ int seedId = configLocation(cluster(), 0).getConfigNodeId();
+ // Use System.exit so the actual shutdown hook runs on Windows as well as Unix.
+ awaitFile(submit(0, "shutdown"));
+ assertTrue(
+ EnvFactory.getEnv().getConfigNodeWrapper(0).getInstance().waitFor(60, TimeUnit.SECONDS));
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(0)),
+ Collections.singletonList(NodeStatus.Stopped));
+ awaitPersisted(1, seedId, NodeStatus.Stopped);
+ awaitPersisted(2, seedId, NodeStatus.Stopped);
+
+ EnvFactory.getEnv().startConfigNode(0);
+ awaitConfigNodeRunning(0);
+ awaitPersisted(0, seedId, null);
+ awaitPersisted(1, seedId, null);
+ awaitPersisted(2, seedId, null);
+ }
+
+ @Test
+ public void testForcedLeaderShutdownWithTwoDataNodes() throws Exception {
+ init(2);
+ int stopped = stoppedFixture(0);
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ EnvFactory.getEnv().getConfigNodeWrapper(leader).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leader).isAlive());
+ assertNotEquals(leader, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ assertStatus(0, stopped, NodeStatus.Stopped);
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(leader)),
+ Collections.singletonList(NodeStatus.Unknown));
+ EnvFactory.getEnv().startConfigNode(leader);
+ awaitConfigNodeRunning(leader);
+ awaitPersisted(leader, stopped, NodeStatus.Stopped);
+ transfer(leader);
+ assertStatus(0, stopped, NodeStatus.Stopped);
+ }
+
+ @Test
+ public void testNewLeaderChangesSurviveReturnToOriginalLeader() throws Exception {
+ init(3);
+ int stopped = stoppedFixture(0);
+ int removing = removingFixture(1);
+ int first = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ int second = (first + 1) % 3;
+ transfer(second);
+ assertStatus(0, stopped, NodeStatus.Stopped);
+ assertStatus(1, removing, NodeStatus.Removing);
+ EnvFactory.getEnv().startDataNode(0);
+ assertStatus(0, stopped, NodeStatus.Running);
+ assertSuccess(control(second, "update:" + removing + ":Running"));
+ awaitPersisted(first, stopped, null);
+ awaitPersisted(first, removing, null);
+ transfer((first + 2) % 3);
+ transfer(first);
+ EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly();
+ assertStatus(0, stopped, NodeStatus.Unknown);
+ transfer(second);
+ assertStatus(0, stopped, NodeStatus.Unknown);
+ assertStatus(1, removing, NodeStatus.Unknown);
+ }
+
+ @Test
+ public void testReadOnlyReasonPersistsAcrossLeaderTransferAndClearsAfterRestart()
+ throws Exception {
+ init(2);
+ TDataNodeLocation dataNode = dataLocation(cluster(), 0);
+ int dataNodeId = dataNode.getDataNodeId();
+ IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> pool =
+ new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
+ .createClientManager(
+ new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+ try (SyncDataNodeInternalServiceClient client =
+ pool.borrowClient(dataNode.getInternalEndPoint())) {
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ client.setSystemStatus(NodeStatus.ReadOnly.getStatus()).getCode());
+ } finally {
+ pool.close();
+ }
+ assertStatus(0, dataNodeId, NodeStatus.ReadOnly);
+ assertEquals(NodeStatus.MANUAL, cluster().getNodeStatusReason().get(dataNodeId));
+ // Inspect every voter's applied record before election; SHOW alone could pass on heartbeats.
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, dataNodeId, NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ }
+
+ int nextLeader = (EnvFactory.getEnv().getLeaderConfigNodeIndex() + 1) % 3;
+ transfer(nextLeader);
+ assertEquals(nextLeader, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, dataNodeId, NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ }
+ assertStatus(0, dataNodeId, NodeStatus.ReadOnly);
+ assertEquals(NodeStatus.MANUAL, cluster().getNodeStatusReason().get(dataNodeId));
+
+ // Explicitly report after stopping: Windows process termination does not run shutdown hooks.
+ assertEquals(dataNodeId, stoppedFixture(0));
+ assertNull(cluster().getNodeStatusReason().get(dataNodeId));
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, dataNodeId, NodeStatus.Stopped, null);
+ }
+
+ EnvFactory.getEnv().startDataNode(0);
+ assertStatus(0, dataNodeId, NodeStatus.Running);
+ assertNull(cluster().getNodeStatusReason().get(dataNodeId));
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, dataNodeId, null, null);
+ }
+ }
+
+ @Test
+ public void testLiveOldLeaderPartitionAndQuorumRecovery() throws Exception {
+ init(3);
+ int stopped = stoppedFixture(0);
+ int first = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ TConfigNodeLocation firstLocation = configLocation(cluster(), first);
+ for (int i = 0; i < 3; i++) {
+ control(i, "partition:" + firstLocation.getConfigNodeId());
+ }
+ int majorityLeader = awaitOtherLeader(first);
+ assertTrue(
+ "Partition must leave the old leader process alive",
+ EnvFactory.getEnv().getConfigNodeWrapper(first).isAlive());
+ assertSuccess(control(majorityLeader, "update:" + stopped + ":Removing"));
+ try {
+ assertStatus(0, stopped, NodeStatus.Removing);
+ } catch (SQLException e) {
+ // Preserve a failure for SQL routing without skipping consensus recovery assertions.
+ failures.addError(
+ new AssertionError(
+ "SHOW through the healthy DataNode cannot reach the elected majority", e));
+ }
+ heal();
+ awaitPersisted(first, stopped, NodeStatus.Removing);
+ transfer(first);
+ assertStatus(0, stopped, NodeStatus.Removing);
+
+ // Keep every process alive but prevent any pair of ConfigRegion voters communicating.
+ for (int i = 0; i < 3; i++) {
+ control(i, "partition:*");
+ }
+ Properties noQuorum = control(first, "update:" + stopped + ":Running");
+ assertNotEquals(
+ "No quorum cannot confirm clearing Removing", "200", noQuorum.getProperty("code"));
+ assertEquals(
+ "Removing", control(first, "inspect:" + stopped).getProperty("persisted." + stopped));
+ heal();
+ int recovered = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ assertSuccess(control(recovered, "update:" + stopped + ":Running"));
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, stopped, null);
+ }
+ transfer((recovered + 1) % 3);
+ assertStatus(0, stopped, NodeStatus.Unknown);
+ }
+
+ private void init(int dataNodes) throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setConfigNodeRatisSnapshotTriggerThreshold(Integer.MAX_VALUE)
+ .setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(4000);
+ EnvFactory.getEnv().initClusterEnvironment(3, dataNodes);
+ queryNode = dataNodes - 1;
+ agentJar = temporary.getRoot().toPath().resolve("node-status-agent.jar");
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ manifest.getMainAttributes().putValue("Agent-Class", NodeStatusTestAgent.class.getName());
+ String resource = NodeStatusTestAgent.class.getName().replace('.', '/') + ".class";
+ try (JarOutputStream jar = new JarOutputStream(Files.newOutputStream(agentJar), manifest);
+ InputStream bytecode = getClass().getClassLoader().getResourceAsStream(resource)) {
+ assertNotNull(bytecode);
+ jar.putNextEntry(new JarEntry(resource));
+ bytecode.transferTo(jar);
+ jar.closeEntry();
+ }
+ }
+
+ @Test
+ public void testNoQuorumTransitionMatrix() throws Exception {
+ init(2);
+ int nodeId = stoppedFixture(0);
+ NodeStatus[][] transitions = {
+ {NodeStatus.Running, NodeStatus.Stopped},
+ {NodeStatus.Stopped, NodeStatus.Running},
+ {NodeStatus.Removing, NodeStatus.Running},
+ {NodeStatus.Removing, NodeStatus.Stopped}
+ };
+ for (NodeStatus[] transition : transitions) {
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ assertSuccess(control(leader, "update:" + nodeId + ":" + transition[0]));
+ NodeStatus persisted = transition[0] == NodeStatus.Running ? null : transition[0];
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, nodeId, persisted);
+ }
+ try {
+ for (int i = 0; i < 3; i++) {
+ control(i, "partition:*");
+ }
+ Properties result = awaitFile(submit(leader, "update:" + nodeId + ":" + transition[1]));
+ assertNotEquals(
+ "No quorum: " + transition[0] + " -> " + transition[1] + " " + result,
+ "200",
+ result.getProperty("code"));
+ for (int i = 0; i < 3; i++) {
+ assertEquals(
+ String.valueOf(persisted),
+ control(i, "inspect:" + nodeId).getProperty("persisted." + nodeId));
+ }
+ // A read may reject or return a local cache before the old leader steps down. Record
+ // the actual direct RPC response rather than assert an undocumented linearizable read.
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getConfigNodeConnection(leader)) {
+ TShowClusterResp read = client.showCluster();
+ System.out.println(
+ "No-quorum SHOW RPC: " + transition[0] + " -> " + transition[1] + ", " + read);
+ }
+ try (Connection connection = queryConnection();
+ Statement statement = connection.createStatement()) {
+ try (ResultSet rows = statement.executeQuery("SHOW DATANODES")) {
+ while (rows.next()) {
+ if (rows.getInt(1) == nodeId) {
+ System.out.println("No-quorum SHOW SQL cached status: " + rows.getString("Status"));
+ }
+ }
+ }
+ } catch (SQLException unavailable) {
+ System.out.println("No-quorum SHOW SQL unavailable: " + unavailable.getMessage());
+ }
+ } finally {
+ heal();
+ }
+ int recovered = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ assertSuccess(control(recovered, "update:" + nodeId + ":" + transition[1]));
+ NodeStatus finalPersisted = transition[1] == NodeStatus.Running ? null : transition[1];
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, nodeId, finalPersisted);
+ }
+ }
+ }
+
+ @Test
+ public void testSnapshotAndFollowingClearLogRecoverTogether() throws Exception {
+ init(3);
+ int stopped = stoppedFixture(0);
+ int removing = removingFixture(1);
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, stopped, NodeStatus.Stopped);
+ awaitPersisted(i, removing, NodeStatus.Removing);
+ control(i, "snapshot");
+ assertSnapshot(i, stopped, NodeStatus.Stopped, removing, NodeStatus.Removing);
+ }
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ assertSuccess(control(leader, "update:" + stopped + ":Running"));
+ assertSuccess(control(leader, "update:" + removing + ":Stopped"));
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, stopped, null);
+ awaitPersisted(i, removing, NodeStatus.Stopped);
+ }
+ restartConfigNodes();
+ assertStatus(0, stopped, NodeStatus.Unknown);
+ assertStatus(1, removing, NodeStatus.Stopped);
+ for (int i = 0; i < 3; i++) {
+ control(i, "snapshot");
+ assertSnapshot(i, stopped, null, removing, NodeStatus.Stopped);
+ }
+ restartConfigNodes();
+ assertStatus(0, stopped, NodeStatus.Unknown);
+ assertStatus(1, removing, NodeStatus.Stopped);
+ }
+
+ private void assertSnapshot(
+ int configNode, int firstId, NodeStatus first, int secondId, NodeStatus second)
+ throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30);
+ do {
+ List<Path> snapshots;
+ // Stop at sm/<term>_<index>, without opening temporary snapshot directories on Windows.
+ try (Stream<Path> paths =
+ Files.walk(
+ Paths.get(
+ EnvFactory.getEnv().getConfigNodeWrapper(configNode).getNodePath(),
+ "data",
+ "confignode",
+ "consensus"),
+ 3)) {
+ snapshots =
+ paths
+ .filter(
+ p ->
+ p.getFileName().toString().matches("\\d+_\\d+")
+ && p.getParent().getFileName().toString().equals("sm"))
+ .collect(Collectors.toList());
+ } catch (IOException | UncheckedIOException e) {
+ // Ratis may delete an old snapshot while the directories are being enumerated.
+ TimeUnit.MILLISECONDS.sleep(200);
+ continue;
+ }
+ for (Path snapshot : snapshots) {
+ NodeInfo restored = new NodeInfo();
+ try {
+ restored.processLoadSnapshot(snapshot.toFile());
+ } catch (IOException e) {
+ // The snapshot may have been replaced since it was enumerated.
+ continue;
+ }
+ if (java.util.Objects.equals(
+ first == null ? null : new Pair<>(first, null),
+ restored.getPersistedNodeStatus(firstId))
+ && java.util.Objects.equals(
+ second == null ? null : new Pair<>(second, null),
+ restored.getPersistedNodeStatus(secondId))) {
+ return;
+ }
+ }
+ TimeUnit.MILLISECONDS.sleep(200);
+ } while (System.nanoTime() < deadline);
+ fail("No completed snapshot has the expected durable markers on ConfigNode " + configNode);
+ }
+
+ private void restartConfigNodes() throws Exception {
+ EnvFactory.getEnv().getConfigNodeWrapperList().forEach(n -> n.stopForcibly());
+ EnvFactory.getEnv().getConfigNodeWrapperList().forEach(n -> n.start());
+ EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ new java.util.ArrayList<>(EnvFactory.getEnv().getConfigNodeWrapperList()),
+ java.util.Arrays.asList(NodeStatus.Running, NodeStatus.Running, NodeStatus.Running));
+ }
+
+ private void awaitConfigNodeRunning(int index) {
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(index)),
+ Collections.singletonList(NodeStatus.Running));
+ }
+
+ @Test
+ public void testRealRemovalPersistsIntentAndDeletesRegistration() throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setSchemaRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setDataRegionConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setSchemaReplicationFactor(2)
+ .setDataReplicationFactor(2);
+ init(3);
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ int victim = -1;
+ int victimIndex = -1;
+ try (Connection connection = queryConnection();
+ Statement statement = connection.createStatement()) {
+ statement.execute("CREATE DATABASE root.status_remove");
+ statement.execute(
+ "CREATE TIMESERIES root.status_remove.d.s WITH DATATYPE=INT32, ENCODING=RLE");
+ statement.execute("INSERT INTO root.status_remove.d(time,s) VALUES (1,1)");
+ TShowClusterResp before = cluster();
+ int queryId = dataLocation(before, queryNode).getDataNodeId();
+ try (ResultSet regions = statement.executeQuery("SHOW REGIONS")) {
+ while (regions.next()) {
+ int candidate = regions.getInt("DataNodeId");
+ if (candidate != queryId) {
+ victim = candidate;
+ break;
+ }
+ }
+ }
+ assertTrue("The victim must host a real region replica", victim >= 0);
+ for (int i = 0; i < 3; i++) {
+ if (dataLocation(before, i).getDataNodeId() == victim) {
+ victimIndex = i;
+ }
+ }
+ assertTrue(victimIndex >= 0);
+ Path barrier = submit(leader, "barrier");
+ assertNull(awaitFile(barrier).getProperty("error"));
+ try {
+ statement.execute("REMOVE DATANODE " + victim);
+ awaitFile(Paths.get(barrier + ".reached"));
+ assertStatus(victimIndex, victim, NodeStatus.Removing);
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, victim, NodeStatus.Removing);
+ }
+ } finally {
+ Files.createFile(Paths.get(barrier + ".release"));
+ }
+ long deadline = System.nanoTime() + TimeUnit.MINUTES.toNanos(3);
+ boolean removed = false;
+ do {
+ final int victimId = victim;
+ removed =
+ cluster().getDataNodeList().stream().noneMatch(n -> n.getDataNodeId() == victimId);
+ if (!removed) {
+ TimeUnit.MILLISECONDS.sleep(200);
+ }
+ } while (!removed && System.nanoTime() < deadline);
+ assertTrue("RemoveDataNodesProcedure must unregister the node", removed);
+ for (int i = 0; i < 3; i++) {
+ awaitPersisted(i, victim, null);
+ }
+ transfer((leader + 1) % 3);
+ try (ResultSet rows = statement.executeQuery("SHOW DATANODES")) {
+ while (rows.next()) {
+ assertNotEquals(victim, rows.getInt(1));
+ }
+ }
+ try (ResultSet rows = statement.executeQuery("SELECT s FROM root.status_remove.d")) {
+ assertTrue(rows.next());
+ assertEquals(1, rows.getInt(2));
+ assertFalse(rows.next());
+ }
+ }
+ }
+
+ private int stoppedFixture(int index) throws Exception {
+ TDataNodeLocation node = dataLocation(cluster(), index);
+ EnvFactory.getEnv().getDataNodeWrapper(index).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getDataNodeWrapper(index).isAlive());
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ assertEquals(200, client.reportDataNodeShutdown(node).getCode());
+ }
+ assertStatus(index, node.getDataNodeId(), NodeStatus.Stopped);
+ return node.getDataNodeId();
+ }
+
+ private int removingFixture(int index) throws Exception {
+ TDataNodeLocation node = dataLocation(cluster(), index);
+ IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> pool =
+ new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
+ .createClientManager(
+ new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+ try (SyncDataNodeInternalServiceClient client = pool.borrowClient(node.getInternalEndPoint())) {
+ assertEquals(200, client.setSystemStatus("Removing").getCode());
+ } finally {
+ pool.close();
+ }
+ assertStatus(index, node.getDataNodeId(), NodeStatus.Removing);
+ EnvFactory.getEnv().getDataNodeWrapper(index).stopForcibly();
+ return node.getDataNodeId();
+ }
+
+ private void assertStatus(int index, int nodeId, NodeStatus expected) throws Exception {
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(index)),
+ Collections.singletonList(expected));
+ assertEquals(expected.getStatus(), cluster().getNodeStatus().get(nodeId));
+ // Pin the SQL connection to the healthy query node; do not fan out to stopped DataNodes.
+ try (Connection connection = queryConnection();
+ Statement statement = connection.createStatement()) {
+ for (String sql : new String[] {"SHOW CLUSTER", "SHOW DATANODES"}) {
+ boolean found = false;
+ try (ResultSet rows = statement.executeQuery(sql)) {
+ while (rows.next()) {
+ if (rows.getInt(1) == nodeId) {
+ assertEquals(sql, expected.getStatus(), rows.getString("Status"));
+ found = true;
+ }
+ }
+ }
+ assertTrue(sql + " did not contain DataNode " + nodeId, found);
+ }
+ }
+ }
+
+ private void transfer(int target) throws Exception {
+ int leader = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ if (leader == target) {
+ return;
+ }
+ long previousTerm = Long.parseLong(control(leader, "inspect").getProperty("term"));
+ TConfigNodeLocation location = configLocation(cluster(), target);
+ control(
+ leader,
+ "transfer:" + location.getConfigNodeId() + ":" + location.getConsensusEndPoint().getPort());
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
+ do {
+ if (EnvFactory.getEnv().getLeaderConfigNodeIndex() == target) {
+ assertTrue(Long.parseLong(control(target, "inspect").getProperty("term")) > previousTerm);
+ return;
+ }
+ TimeUnit.MILLISECONDS.sleep(200);
+ } while (System.nanoTime() < deadline);
+ fail("Leadership did not transfer to ConfigNode " + target);
+ }
+
+ private int awaitOtherLeader(int excluded) throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
+ do {
+ for (int i = 0; i < 3; i++) {
+ if (i != excluded) {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getConfigNodeConnection(i)) {
+ if (client.showCluster().getStatus().getCode() == 200) {
+ return i;
+ }
+ } catch (Exception ignored) {
+ /* Election can temporarily reject a read. */
+ }
+ }
+ }
+ TimeUnit.MILLISECONDS.sleep(200);
+ } while (System.nanoTime() < deadline);
+ throw new AssertionError("Majority did not elect a ready leader");
+ }
+
+ private void heal() throws Exception {
+ for (int i = 0; i < 3; i++) {
+ control(i, "heal");
+ }
+ }
+
+ private void awaitPersisted(int node, int dataNode, NodeStatus expected) throws Exception {
+ awaitPersisted(node, dataNode, expected, null);
+ }
+
+ private void awaitPersisted(int node, int dataNode, NodeStatus expected, String expectedReason)
+ throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60);
+ do {
+ Properties state = control(node, "inspect:" + dataNode);
+ if (String.valueOf(expected).equals(state.getProperty("persisted." + dataNode))
+ && String.valueOf(expectedReason)
+ .equals(state.getProperty("persistedReason." + dataNode))) {
+ return;
+ }
+ TimeUnit.MILLISECONDS.sleep(200);
+ } while (System.nanoTime() < deadline);
+ fail(
+ "ConfigNode "
+ + node
+ + " did not persist "
+ + expected
+ + " with reason "
+ + expectedReason
+ + " for "
+ + dataNode);
+ }
+
+ private Path submit(int node, String command) throws Exception {
+ Path output =
+ temporary.getRoot().toPath().resolve("control-" + System.nanoTime() + ".properties");
+ VirtualMachine vm =
+ VirtualMachine.attach(
+ Long.toString(EnvFactory.getEnv().getConfigNodeWrapper(node).getInstance().pid()));
+ try {
+ vm.loadAgent(agentJar.toString(), command + "|" + output);
+ } finally {
+ vm.detach();
+ }
+ return output;
+ }
+
+ private Properties control(int node, String command) throws Exception {
+ Properties result = awaitFile(submit(node, command));
+ assertNull(command + " failed: " + result, result.getProperty("error"));
+ return result;
+ }
+
+ private Properties awaitFile(Path output) throws Exception {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(90);
+ while (!Files.exists(output) && System.nanoTime() < deadline) {
+ TimeUnit.MILLISECONDS.sleep(100);
+ }
+ assertTrue("Test control timed out: " + output, Files.exists(output));
+ Properties result = new Properties();
+ try (InputStream stream = Files.newInputStream(output)) {
+ result.load(stream);
+ }
+ return result;
+ }
+
+ private TShowClusterResp cluster() throws Exception {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ TShowClusterResp result = client.showCluster();
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), result.getStatus().getCode());
+ return result;
+ }
+ }
+
+ private TDataNodeLocation dataLocation(TShowClusterResp cluster, int index) {
+ return cluster.getDataNodeList().stream()
+ .filter(
+ n ->
+ n.getInternalEndPoint().getPort()
+ == EnvFactory.getEnv().getDataNodeWrapper(index).getInternalPort())
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ }
+
+ private TConfigNodeLocation configLocation(TShowClusterResp cluster, int index) {
+ return cluster.getConfigNodeList().stream()
+ .filter(
+ n ->
+ n.getInternalEndPoint().getPort()
+ == EnvFactory.getEnv().getConfigNodeWrapper(index).getPort())
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ }
+
+ private void assertSuccess(Properties result) {
+ assertEquals(result.toString(), "200", result.getProperty("code"));
+ }
+
+ private Connection queryConnection() throws SQLException {
+ Properties properties = new Properties();
+ properties.setProperty("user", "root");
+ properties.setProperty("password", "root");
+ properties.setProperty("network_timeout", "15000");
+ properties.setProperty("sql_dialect", "tree");
+ return DriverManager.getConnection(
+ "jdbc:iotdb://"
+ + EnvFactory.getEnv().getDataNodeWrapper(queryNode).getIp()
+ + ":"
+ + EnvFactory.getEnv().getDataNodeWrapper(queryNode).getPort()
+ + "/",
+ properties);
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java
new file mode 100644
index 00000000000..6829780c2f3
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/IoTDBNodeStatusPersistenceIT.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.it.cluster;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.client.ClientPoolFactory;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.exception.ClientManagerException;
+import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
+import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
+import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
+import org.apache.iotdb.consensus.ConsensusFactory;
+import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.node.ConfigNodeWrapper;
+import org.apache.iotdb.it.framework.IoTDBTestRunner;
+import org.apache.iotdb.itbase.category.ClusterIT;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.TException;
+import org.apache.tsfile.utils.Pair;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.fail;
+
+@RunWith(IoTDBTestRunner.class)
+@Category({ClusterIT.class})
+public class IoTDBNodeStatusPersistenceIT {
+
+ @After
+ public void tearDown() {
+ EnvFactory.getEnv().cleanClusterEnvironment();
+ }
+
+ @Test
+ public void testStoppedNodesSurviveLeaderFailureAndLogRecovery() throws Exception {
+ initCluster(5, Integer.MAX_VALUE);
+ final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ final int stoppedConfigNodeIndex = (leaderIndex + 1) % 5;
+ stopAndReportNodes(stoppedConfigNodeIndex);
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped);
+
+ // Five ConfigNodes retain a quorum with both a stopped follower and a failed leader.
+ EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly();
+ assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped);
+
+ EnvFactory.getEnv().startConfigNode(leaderIndex);
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex)),
+ Collections.singletonList(NodeStatus.Running));
+
+ // Restart every surviving ConfigNode and restore the stopped states from consensus logs.
+ restartLiveConfigNodes();
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped);
+
+ EnvFactory.getEnv().startDataNode(0);
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Running);
+
+ // Keep the DataNode offline during the next election so fresh heartbeats cannot hide a
+ // resurrected Stopped status. A crash without a shutdown report must become Unknown.
+ EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly();
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Unknown);
+ final int nextLeaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ EnvFactory.getEnv().getConfigNodeWrapper(nextLeaderIndex).stopForcibly();
+ assertNotEquals(nextLeaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Unknown);
+ }
+
+ @Test
+ public void testStoppedNodesSurviveSnapshotRecovery() throws Exception {
+ initCluster(3, 1);
+ final int stoppedConfigNodeIndex = (EnvFactory.getEnv().getLeaderConfigNodeIndex() + 1) % 3;
+ stopAndReportNodes(stoppedConfigNodeIndex);
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped);
+
+ final List<ConfigNodeWrapper> liveConfigNodes = getLiveConfigNodes();
+ final TShowClusterResp cluster;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ cluster = client.showCluster();
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode());
+ }
+ final int stoppedConfigNodeId =
+ findConfigNodeLocation(cluster, stoppedConfigNodeIndex).getConfigNodeId();
+ final int stoppedDataNodeId = cluster.getDataNodeList().get(0).getDataNodeId();
+
+ // Verify actual completed snapshots, including their contents, before restarting the quorum.
+ // A low snapshot threshold alone would not prove snapshot recovery was exercised.
+ for (ConfigNodeWrapper configNode : liveConfigNodes) {
+ awaitNodeStatusSnapshot(
+ configNode,
+ Map.of(
+ stoppedConfigNodeId, new Pair<>(NodeStatus.Stopped, null),
+ stoppedDataNodeId, new Pair<>(NodeStatus.Stopped, null)));
+ }
+ restartLiveConfigNodes();
+ assertNodeStatuses(stoppedConfigNodeIndex, NodeStatus.Stopped);
+ }
+
+ @Test
+ public void testRemovingNodeSurvivesLeaderFailure() throws Exception {
+ initCluster(3, Integer.MAX_VALUE);
+ // Use the removal procedure's DataNode RPC. The leader learns Removing through heartbeats.
+ setDataNodeSystemStatus(NodeStatus.Removing);
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)),
+ Collections.singletonList(NodeStatus.Removing));
+
+ // The new leader must restore Removing without any help from fresh DataNode heartbeats.
+ EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive());
+ final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).isAlive());
+ assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)),
+ Collections.singletonList(NodeStatus.Removing));
+ }
+
+ @Test
+ public void testReadOnlyReasonRebuiltAfterLeaderFailureAndClearedAfterCrash() throws Exception {
+ initCluster(3, Integer.MAX_VALUE);
+ setDataNodeSystemStatus(NodeStatus.ReadOnly);
+ awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+
+ // The new leader restores ReadOnly and its reason; live heartbeats continue refreshing them.
+ final int leaderIndex = EnvFactory.getEnv().getLeaderConfigNodeIndex();
+ EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(leaderIndex).isAlive());
+ assertNotEquals(leaderIndex, EnvFactory.getEnv().getLeaderConfigNodeIndex());
+ awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+
+ // Without a shutdown report there is no durable Stopped marker. An unreachable ReadOnly
+ // DataNode must become Unknown, with its old reason cleared in both node display RPCs.
+ EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive());
+ awaitDataNodeStatusWithReason(NodeStatus.Unknown, null);
+ }
+
+ @Test
+ public void testReadOnlyReasonSurvivesSnapshotRecovery() throws Exception {
+ initCluster(3, 1);
+ setDataNodeSystemStatus(NodeStatus.ReadOnly);
+ awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+
+ final int dataNodeId;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ dataNodeId = client.showCluster().getDataNodeList().get(0).getDataNodeId();
+ }
+ // Inspect completed snapshots so live heartbeats cannot hide a missing durable reason.
+ for (ConfigNodeWrapper configNode : getLiveConfigNodes()) {
+ awaitNodeStatusSnapshot(
+ configNode,
+ Collections.singletonMap(dataNodeId, new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL)));
+ }
+ restartLiveConfigNodes();
+ awaitDataNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+
+ setDataNodeSystemStatus(NodeStatus.Running);
+ awaitDataNodeStatusWithReason(NodeStatus.Running, null);
+ }
+
+ private void setDataNodeSystemStatus(NodeStatus status) throws Exception {
+ final IClientManager<TEndPoint, SyncDataNodeInternalServiceClient> clientManager =
+ new IClientManager.Factory<TEndPoint, SyncDataNodeInternalServiceClient>()
+ .createClientManager(
+ new ClientPoolFactory.SyncDataNodeInternalServiceClientPoolFactory());
+ try (SyncDataNodeInternalServiceClient client =
+ clientManager.borrowClient(
+ new TEndPoint(
+ EnvFactory.getEnv().getDataNodeWrapper(0).getIp(),
+ EnvFactory.getEnv().getDataNodeWrapper(0).getInternalPort()))) {
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ client.setSystemStatus(status.getStatus()).getCode());
+ } finally {
+ clientManager.close();
+ }
+ }
+
+ private void awaitDataNodeStatusWithReason(NodeStatus expectedStatus, String expectedReason)
+ throws InterruptedException {
+ Throwable lastFailure = null;
+ for (int retry = 0; retry < 60; retry++) {
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final TShowClusterResp cluster = client.showCluster();
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode());
+ final int dataNodeId = cluster.getDataNodeList().get(0).getDataNodeId();
+ assertEquals(expectedStatus.getStatus(), cluster.getNodeStatus().get(dataNodeId));
+ assertEquals(expectedReason, cluster.getNodeStatusReason().get(dataNodeId));
+
+ final TShowDataNodesResp dataNodes = client.showDataNodes();
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), dataNodes.getStatus().getCode());
+ final TDataNodeInfo dataNode =
+ dataNodes.getDataNodesInfoList().stream()
+ .filter(node -> node.getDataNodeId() == dataNodeId)
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ assertEquals(expectedStatus.getStatus(), dataNode.getStatus());
+ assertEquals(expectedReason, dataNode.getStatusReason());
+ return;
+ } catch (IOException | ClientManagerException | TException | AssertionError e) {
+ lastFailure = e;
+ }
+ TimeUnit.SECONDS.sleep(1);
+ }
+ throw new AssertionError(
+ "DataNode did not reach " + expectedStatus + " with reason " + expectedReason, lastFailure);
+ }
+
+ private void initCluster(int configNodeCount, int snapshotThreshold) throws Exception {
+ EnvFactory.getEnv()
+ .getConfig()
+ .getCommonConfig()
+ .setConfigNodeConsensusProtocolClass(ConsensusFactory.RATIS_CONSENSUS)
+ .setConfigNodeRatisSnapshotTriggerThreshold(snapshotThreshold)
+ .setConfigRegionRatisRPCLeaderElectionTimeoutMaxMs(4000);
+ EnvFactory.getEnv().initClusterEnvironment(configNodeCount, 1);
+ }
+
+ private void stopAndReportNodes(int configNodeIndex) throws Exception {
+ final TConfigNodeLocation configNodeLocation;
+ final TDataNodeLocation dataNodeLocation;
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ final TShowClusterResp cluster = client.showCluster();
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), cluster.getStatus().getCode());
+ configNodeLocation = findConfigNodeLocation(cluster, configNodeIndex);
+ dataNodeLocation = cluster.getDataNodeList().get(0);
+ }
+
+ EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).stopForcibly();
+ EnvFactory.getEnv().getDataNodeWrapper(0).stopForcibly();
+ assertFalse(EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).isAlive());
+ assertFalse(EnvFactory.getEnv().getDataNodeWrapper(0).isAlive());
+
+ // Deliver the shutdown hooks' reports explicitly: Process.destroy() does not execute JVM
+ // shutdown hooks on every platform. These tests exercise the reports' durable handling.
+ try (SyncConfigNodeIServiceClient client =
+ (SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ client.reportConfigNodeShutdown(configNodeLocation).getCode());
+ assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ client.reportDataNodeShutdown(dataNodeLocation).getCode());
+ }
+ }
+
+ private TConfigNodeLocation findConfigNodeLocation(
+ TShowClusterResp cluster, int configNodeIndex) {
+ return cluster.getConfigNodeList().stream()
+ .filter(
+ node ->
+ node.getInternalEndPoint().getPort()
+ == EnvFactory.getEnv().getConfigNodeWrapper(configNodeIndex).getPort())
+ .findFirst()
+ .orElseThrow(AssertionError::new);
+ }
+
+ private void assertNodeStatuses(int stoppedConfigNodeIndex, NodeStatus dataNodeStatus) {
+ EnvFactory.getEnv()
+ .ensureNodeStatus(
+ Arrays.asList(
+ EnvFactory.getEnv().getConfigNodeWrapper(stoppedConfigNodeIndex),
+ EnvFactory.getEnv().getDataNodeWrapper(0)),
+ Arrays.asList(NodeStatus.Stopped, dataNodeStatus));
+ }
+
+ private List<ConfigNodeWrapper> getLiveConfigNodes() {
+ return EnvFactory.getEnv().getConfigNodeWrapperList().stream()
+ .filter(ConfigNodeWrapper::isAlive)
+ .collect(Collectors.toList());
+ }
+
+ private void restartLiveConfigNodes() {
+ final List<ConfigNodeWrapper> liveConfigNodes = getLiveConfigNodes();
+ liveConfigNodes.forEach(ConfigNodeWrapper::stopForcibly);
+ liveConfigNodes.forEach(node -> assertFalse(node.isAlive()));
+ liveConfigNodes.forEach(ConfigNodeWrapper::start);
+ }
+
+ private void awaitNodeStatusSnapshot(
+ ConfigNodeWrapper configNode, Map<Integer, Pair<NodeStatus, String>> expectedStatuses)
+ throws Exception {
+ for (int retry = 0; retry < 60; retry++) {
+ final List<Path> snapshotFiles = new ArrayList<>();
+ try {
+ final List<Path> groups;
+ try (Stream<Path> paths =
+ Files.list(Paths.get(configNode.getNodePath(), "data", "confignode", "consensus"))) {
+ groups = paths.filter(Files::isDirectory).collect(Collectors.toList());
+ }
+ // Never walk temporary snapshot directories: an open directory handle can prevent
+ // Ratis from atomically renaming them on Windows.
+ for (Path group : groups) {
+ try (Stream<Path> paths = Files.list(group.resolve("sm"))) {
+ snapshotFiles.addAll(
+ paths
+ .filter(path -> path.getFileName().toString().matches("\\d+_\\d+"))
+ .map(path -> path.resolve("node_info.bin"))
+ .collect(Collectors.toList()));
+ }
+ }
+ } catch (IOException | UncheckedIOException e) {
+ // Ratis can delete an older snapshot while this directory tree is being enumerated.
+ TimeUnit.SECONDS.sleep(1);
+ continue;
+ }
+ for (Path snapshotFile : snapshotFiles) {
+ final NodeInfo recovered = new NodeInfo();
+ try {
+ recovered.processLoadSnapshot(snapshotFile.getParent().toFile());
+ } catch (IOException e) {
+ // A newer snapshot may have replaced this one since the directory was enumerated.
+ continue;
+ }
+ if (expectedStatuses.entrySet().stream()
+ .allMatch(
+ entry ->
+ java.util.Objects.equals(
+ entry.getValue(), recovered.getPersistedNodeStatus(entry.getKey())))) {
+ return;
+ }
+ }
+ TimeUnit.SECONDS.sleep(1);
+ }
+ fail("No completed snapshot contains " + expectedStatuses + " on " + configNode.getId());
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java
new file mode 100644
index 00000000000..529afd9bccc
--- /dev/null
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/cluster/NodeStatusTestAgent.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.it.cluster;
+
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.request.AsyncRequestManager;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.utils.function.CheckedTriConsumer;
+import org.apache.iotdb.confignode.client.async.CnToDnAsyncRequestType;
+import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.service.ConfigNode;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.ratis.utils.Utils;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.ratis.util.CodeInjectionForTesting;
+import org.apache.thrift.TException;
+import org.apache.tsfile.utils.Pair;
+
+import java.io.OutputStream;
+import java.lang.instrument.Instrumentation;
+import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+/** Loaded only into test JVMs. No production endpoint or production bytecode is changed. */
+public final class NodeStatusTestAgent {
+ private NodeStatusTestAgent() {}
+
+ public static void agentmain(String arguments, Instrumentation ignored) {
+ String[] parts = arguments.split("\\|", 2);
+ String command = parts[0];
+ Path output = Paths.get(parts[1]);
+ // Return from attach immediately even if a consensus write needs to time out.
+ Thread worker = new Thread(() -> execute(command, output), "node-status-test-control");
+ worker.setDaemon(true);
+ worker.start();
+ }
+
+ private static void execute(String command, Path output) {
+ Properties result = new Properties();
+ try {
+ String[] args = command.split(":");
+ if ("shutdown".equals(args[0])) {
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> store(output, result)));
+ System.exit(0);
+ return;
+ }
+ ConfigManager manager = ConfigNode.getInstance().getConfigManager();
+ ConsensusManager consensus = manager.getConsensusManager();
+ switch (args[0]) {
+ case "snapshot":
+ consensus.getConsensusImpl().triggerSnapshot(consensus.getConsensusGroupId(), true);
+ break;
+ case "barrier":
+ installRemovalBarrier(output);
+ break;
+ case "partition":
+ String isolated =
+ "*".equals(args[1])
+ ? "*"
+ : Utils.fromNodeIdToRaftPeerId(Integer.parseInt(args[1])).toString();
+ for (String point :
+ new String[] {"requestVote", "appendEntries", "startLeaderElection"}) {
+ CodeInjectionForTesting.put(
+ "RaftServerImpl." + point,
+ (local, remote, values) -> {
+ if ("*".equals(isolated)
+ || isolated.equals(String.valueOf(local))
+ || isolated.equals(String.valueOf(remote))) {
+ // Fail at receipt, before term/log processing. This drops Raft messages in both
+ // directions while leaving the JVM and ConfigNode business RPC server alive.
+ throw new IllegalStateException("Injected Raft message partition");
+ }
+ return false;
+ });
+ }
+ break;
+ case "heal":
+ for (String point :
+ new String[] {"requestVote", "appendEntries", "startLeaderElection"}) {
+ CodeInjectionForTesting.remove("RaftServerImpl." + point);
+ }
+ break;
+ case "transfer":
+ consensus
+ .getConsensusImpl()
+ .transferLeader(
+ consensus.getConsensusGroupId(),
+ new Peer(
+ consensus.getConsensusGroupId(),
+ Integer.parseInt(args[1]),
+ new TEndPoint("127.0.0.1", Integer.parseInt(args[2]))));
+ break;
+ case "update":
+ int id = Integer.parseInt(args[1]);
+ NodeStatus status = NodeStatus.parse(args[2]);
+ TSStatus response = manager.getLoadManager().trySetNodeStatus(id, status, true);
+ result.setProperty("code", Integer.toString(response.getCode()));
+ break;
+ case "inspect":
+ NodeInfo nodeInfo = manager.getNodeManager().getNodeInfo();
+ for (int i = 1; i < args.length; i++) {
+ int nodeId = Integer.parseInt(args[i]);
+ Pair<NodeStatus, String> persistedStatus = nodeInfo.getPersistedNodeStatus(nodeId);
+ result.setProperty(
+ "persisted." + nodeId,
+ String.valueOf(persistedStatus == null ? null : persistedStatus.left));
+ result.setProperty(
+ "persistedReason." + nodeId,
+ String.valueOf(persistedStatus == null ? null : persistedStatus.right));
+ }
+ break;
+ default:
+ throw new IllegalArgumentException(command);
+ }
+ result.setProperty(
+ "term",
+ Long.toString(
+ consensus.getConsensusImpl().getLogicalClock(consensus.getConsensusGroupId())));
+ result.setProperty("leader", Boolean.toString(consensus.isLeader()));
+ } catch (Throwable e) {
+ result.setProperty("error", e.toString());
+ }
+ store(output, result);
+ }
+
+ private static void store(Path output, Properties result) {
+ try {
+ Path temporary = output.resolveSibling(output.getFileName() + ".tmp");
+ try (OutputStream stream = Files.newOutputStream(temporary)) {
+ result.store(stream, "Node status test control");
+ }
+ Files.move(temporary, output);
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private static void installRemovalBarrier(Path output) throws Exception {
+ CnToDnInternalServiceAsyncRequestManager requests =
+ CnToDnInternalServiceAsyncRequestManager.getInstance();
+ Field field = AsyncRequestManager.class.getDeclaredField("actionMap");
+ field.setAccessible(true);
+ Map actions = new HashMap((Map) field.get(requests));
+ CheckedTriConsumer<Object, Object, Object, TException> original =
+ (CheckedTriConsumer<Object, Object, Object, TException>)
+ actions.get(CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE);
+ Path reached = Paths.get(output + ".reached");
+ Path release = Paths.get(output + ".release");
+ CheckedTriConsumer<Object, Object, Object, TException> gated =
+ (request, client, callback) -> {
+ if (!Files.exists(reached)) {
+ store(reached, new Properties());
+ }
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(120);
+ while (!Files.exists(release)) {
+ if (System.nanoTime() >= deadline) {
+ throw new TException("Removal test barrier timed out");
+ }
+ try {
+ TimeUnit.MILLISECONDS.sleep(100);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new TException(e);
+ }
+ }
+ try {
+ original.accept(request, client, callback);
+ } catch (Exception e) {
+ throw new TException(e);
+ }
+ };
+ actions.put(CnToDnAsyncRequestType.CLEAN_DATA_NODE_CACHE, gated);
+ field.set(requests, ImmutableMap.copyOf(actions));
+ }
+}
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java
index fc0b73d6e1c..6fb971e94d7 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/load/IoTDBRegionGroupLeaderDistributionIT.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -208,7 +209,7 @@ public class IoTDBRegionGroupLeaderDistributionIT {
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(0)),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
// Check leader distribution
isDistributionBalanced = false;
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java
index 954f54e01bd..3e5195586be 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionCreationIT.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.cluster.RegionStatus;
import org.apache.iotdb.commons.pipe.config.constant.SystemConstant;
import org.apache.iotdb.confignode.it.utils.ConfigNodeTestUtils;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionReq;
import org.apache.iotdb.confignode.rpc.thrift.TDataPartitionTableResp;
import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema;
@@ -39,6 +40,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -318,36 +320,34 @@ public class IoTDBPartitionCreationIT {
// Shutdown 1 DataNode
// Current cluster: 1C5D
- // DataNode status: Running, Running, Removing, ReadOnly, Unknown
+ // DataNode status: Running, Running, Removing, ReadOnly, Stopped (Unknown on Windows).
// Region distribution: [0, 1, 2], [0, 1, 2], [0], [1], [2]
EnvFactory.getEnv().shutdownDataNode(4);
// Wait for shutdown check
- while (true) {
- AtomicBoolean containUnknown = new AtomicBoolean(false);
+ boolean isShutdownDetected = false;
+ for (int retry = 0; retry < 60; retry++) {
TShowDataNodesResp showDataNodesResp = client.showDataNodes();
- showDataNodesResp
- .getDataNodesInfoList()
- .forEach(
- dataNodeInfo -> {
- if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) {
- containUnknown.set(true);
- }
- });
-
- if (containUnknown.get()) {
+ for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
+ if (EnvUtils.getNodeStatusAfterLocalStop().getStatus().equals(dataNodeInfo.getStatus())) {
+ isShutdownDetected = true;
+ break;
+ }
+ }
+ if (isShutdownDetected) {
break;
}
TimeUnit.SECONDS.sleep(1);
}
+ Assert.assertTrue(isShutdownDetected);
// Register 1 DataNode and Create 1 DataPartition to extend 1 DataRegionGroup
// The new DataRegions wouldn't be allocated to the Removing and ReadOnly DataNode
- // But the new DataRegion can be allocated to the Unknown DataNode
+ // But the new DataRegion can be allocated to the offline DataNode.
// Current cluster: 1C6D
- // Status: Running, Running, Removing, ReadOnly, Unknown, Running
+ // Status: Running, Running, Removing, ReadOnly, Stopped (Unknown on Windows), Running.
// RegionGroup: [0, 1, 2, 3], [0, 1, 2], [0], [1], [2, 3], [3]
EnvFactory.getEnv().registerNewDataNode(false);
- // Use thread sleep to replace verifying because the Unknown DataNode can not pass the
+ // Use thread sleep to replace verifying because the offline DataNode can not pass the
// connection check
TimeUnit.SECONDS.sleep(25);
partitionSlotsMap =
@@ -446,23 +446,25 @@ public class IoTDBPartitionCreationIT {
// RegionGroup: [0, 1, 2, 3], [0, 1, 2], [0], [1], [2, 3], [3]
EnvFactory.getEnv().startDataNode(4);
// Wait for restart check
- while (true) {
- AtomicBoolean containUnknown = new AtomicBoolean(false);
+ boolean isRestartDetected = false;
+ for (int retry = 0; retry < 60; retry++) {
TShowDataNodesResp showDataNodesResp = client.showDataNodes();
- showDataNodesResp
- .getDataNodesInfoList()
- .forEach(
- dataNodeInfo -> {
- if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) {
- containUnknown.set(true);
- }
- });
-
- if (!containUnknown.get()) {
+ boolean containDown = false;
+ for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
+ // The restarted DataNode keeps Stopped until its first heartbeat revives it
+ if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())
+ || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) {
+ containDown = true;
+ break;
+ }
+ }
+ if (!containDown) {
+ isRestartDetected = true;
break;
}
TimeUnit.SECONDS.sleep(1);
}
+ Assert.assertTrue(isRestartDetected);
// Check Region count and status
for (int i = 0; i < 30; i++) {
runningCnt = 0;
diff --git a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java
index 8d6e4a5f831..a9262f69818 100644
--- a/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/confignode/it/partition/IoTDBPartitionDurableIT.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowRegionResp;
import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
@@ -285,49 +286,49 @@ public class IoTDBPartitionDurableIT {
}
@Test
- public void testUnknownDataNode() throws Exception {
+ public void testStoppedDataNode() throws Exception {
// Shutdown a DataNode, the ConfigNode should still be able to create RegionGroup
+ NodeStatus stoppedStatus = EnvUtils.getNodeStatusAfterLocalStop();
EnvFactory.getEnv().shutdownDataNode(testDataNodeId);
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(EnvFactory.getEnv().getDataNodeWrapper(testDataNodeId)),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(stoppedStatus));
try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
// Wait for shutdown check
TShowClusterResp showClusterResp;
- while (true) {
- AtomicBoolean containUnknown = new AtomicBoolean(false);
+ boolean isShutdownDetected = false;
+ for (int retry = 0; retry < 60; retry++) {
TShowDataNodesResp showDataNodesResp = client.showDataNodes();
- showDataNodesResp
- .getDataNodesInfoList()
- .forEach(
- dataNodeInfo -> {
- if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) {
- containUnknown.set(true);
- }
- });
-
- if (containUnknown.get()) {
+ for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
+ if (stoppedStatus.getStatus().equals(dataNodeInfo.getStatus())) {
+ isShutdownDetected = true;
+ break;
+ }
+ }
+ if (isShutdownDetected) {
break;
}
TimeUnit.SECONDS.sleep(1);
}
+ Assert.assertTrue(isShutdownDetected);
int runningCnt = 0;
- int unknownCnt = 0;
+ int stoppedCnt = 0;
showClusterResp = client.showCluster();
for (TDataNodeLocation dataNodeLocation : showClusterResp.getDataNodeList()) {
if (NodeStatus.Running.getStatus()
.equals(showClusterResp.getNodeStatus().get(dataNodeLocation.getDataNodeId()))) {
runningCnt += 1;
- } else if (NodeStatus.Unknown.getStatus()
+ } else if (stoppedStatus
+ .getStatus()
.equals(showClusterResp.getNodeStatus().get(dataNodeLocation.getDataNodeId()))) {
- unknownCnt += 1;
+ stoppedCnt += 1;
}
}
Assert.assertEquals(2, runningCnt);
- Assert.assertEquals(1, unknownCnt);
+ Assert.assertEquals(1, stoppedCnt);
// Test getOrCreateDataPartition, ConfigNode should create DataPartition and return
Map<String, Map<TSeriesPartitionSlot, TTimeSlotList>> partitionSlotsMap =
ConfigNodeTestUtils.constructPartitionSlotsMap(
@@ -369,7 +370,7 @@ public class IoTDBPartitionDurableIT {
// Check Region count
runningCnt = 0;
- unknownCnt = 0;
+ int regionUnknownCnt = 0;
TShowRegionResp showRegionResp = client.showRegion(new TShowRegionReq());
showRegionResp
.getRegionInfoList()
@@ -383,12 +384,12 @@ public class IoTDBPartitionDurableIT {
if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) {
runningCnt += 1;
} else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) {
- unknownCnt += 1;
+ regionUnknownCnt += 1;
}
}
// The runningCnt should be exactly twice as the unknownCnt
// since there exists one DataNode is shutdown
- Assert.assertEquals(unknownCnt * 2, runningCnt);
+ Assert.assertEquals(regionUnknownCnt * 2, runningCnt);
// Test getOrCreateDataPartition, ConfigNode should create DataPartition and return
partitionSlotsMap =
@@ -430,7 +431,7 @@ public class IoTDBPartitionDurableIT {
// Check Region count and status
runningCnt = 0;
- unknownCnt = 0;
+ regionUnknownCnt = 0;
showRegionResp = client.showRegion(new TShowRegionReq());
showRegionResp
.getRegionInfoList()
@@ -444,12 +445,12 @@ public class IoTDBPartitionDurableIT {
if (RegionStatus.Running.getStatus().equals(regionInfo.getStatus())) {
runningCnt += 1;
} else if (RegionStatus.Unknown.getStatus().equals(regionInfo.getStatus())) {
- unknownCnt += 1;
+ regionUnknownCnt += 1;
}
}
// The runningCnt should be exactly twice as the unknownCnt
// since there exists one DataNode is shutdown
- Assert.assertEquals(unknownCnt * 2, runningCnt);
+ Assert.assertEquals(regionUnknownCnt * 2, runningCnt);
EnvFactory.getEnv().startDataNode(testDataNodeId);
EnvFactory.getEnv()
diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java
index 9bbdffb9fa8..b43190e9645 100644
--- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/single/IoTDBShowReceiversIT.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.db.it.utils.TestUtils;
import org.apache.iotdb.db.pipe.sink.payload.evolvable.request.PipeTransferDataNodeHandshakeV1Req;
import org.apache.iotdb.isession.SessionConfig;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.MultiClusterIT1;
@@ -115,7 +116,8 @@ public class IoTDBShowReceiversIT extends AbstractPipeSingleIT {
env.shutdownDataNode(stoppedDataNodeIndex);
env.ensureNodeStatus(
- Collections.singletonList(stoppedDataNode), Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(stoppedDataNode),
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
assertShowReceiversWithoutDataNode(
"show receivers", BaseEnv.TREE_SQL_DIALECT, queryDataNode, stoppedDataNodeId);
@@ -245,7 +247,7 @@ public class IoTDBShowReceiversIT extends AbstractPipeSingleIT {
env.ensureNodeStatus(
Collections.singletonList(restartedDataNode),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
assertNoThriftReceiverRuntimeOnDataNode(
"show receivers", BaseEnv.TREE_SQL_DIALECT, restartedDataNodeId);
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java
index efeeb888e3f..4dac5e9feb4 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBSetSystemStatusTableIT.java
@@ -64,18 +64,45 @@ public class IoTDBSetSystemStatusTableIT {
() -> {
ResultSet resultSet = statement.executeQuery("SHOW DATANODES");
int num = 0;
+ int manualReasonNum = 0;
try {
while (resultSet.next()) {
String status = resultSet.getString("Status");
if (status.equals("ReadOnly")) {
num++;
}
+ // The ReadOnly requested through SQL is reported with the Manual reason.
+ // The StatusReason column only appears once at least one node has reported
+ // its reason, so a missing column means the condition is not satisfied yet.
+ if ("Manual".equals(resultSet.getString("StatusReason"))) {
+ manualReasonNum++;
+ }
}
- } catch (InconsistentDataException e) {
+ } catch (Exception e) {
+ // The StatusReason column (and the Manual reason behind it) may not be
+ // propagated yet, or the cluster may be transiently inconsistent while the
+ // status is changing: treat both as "not satisfied yet".
return false;
}
- return num == EnvFactory.getEnv().getDataNodeWrapperList().size();
+ return num == EnvFactory.getEnv().getDataNodeWrapperList().size()
+ && manualReasonNum == EnvFactory.getEnv().getDataNodeWrapperList().size();
});
+ assertShowNodeColumns(true);
+
+ // The table-model NODES view keeps the status and its reason in two separate columns
+ // instead of merging them into "ReadOnly(Manual)".
+ try (ResultSet nodesResultSet =
+ statement.executeQuery("select * from information_schema.nodes")) {
+ int dataNodeNum = 0;
+ while (nodesResultSet.next()) {
+ if ("DataNode".equals(nodesResultSet.getString("node_type"))) {
+ dataNodeNum++;
+ Assert.assertEquals("ReadOnly", nodesResultSet.getString("status"));
+ Assert.assertEquals("Manual", nodesResultSet.getString("status_reason"));
+ }
+ }
+ Assert.assertEquals(EnvFactory.getEnv().getDataNodeWrapperList().size(), dataNodeNum);
+ }
statement.execute("SET SYSTEM TO RUNNING ON CLUSTER");
Awaitility.await()
@@ -87,6 +114,9 @@ public class IoTDBSetSystemStatusTableIT {
int num = 0;
try {
+ if (resultSet.getMetaData().getColumnCount() != 6) {
+ return false;
+ }
while (resultSet.next()) {
String status = resultSet.getString("Status");
if (status.equals("Running")) {
@@ -98,8 +128,44 @@ public class IoTDBSetSystemStatusTableIT {
}
return num == EnvFactory.getEnv().getDataNodeWrapperList().size();
});
+ assertShowNodeColumns(false);
} catch (Exception e) {
Assert.fail(e.getMessage());
}
}
+
+ private static void assertShowNodeColumns(boolean hasReason) throws Exception {
+ for (String dialect : new String[] {BaseEnv.TREE_SQL_DIALECT, BaseEnv.TABLE_SQL_DIALECT}) {
+ try (Connection connection = EnvFactory.getEnv().getConnection(dialect);
+ Statement statement = connection.createStatement()) {
+ assertShowColumns(statement, "SHOW CLUSTER", 7, hasReason);
+ assertShowColumns(statement, "SHOW CLUSTER DETAILS", 13, hasReason);
+ assertShowColumns(statement, "SHOW DATANODES", 6, hasReason);
+ // SET SYSTEM changes DataNodes only; ConfigNodes keep the layout without a reason column.
+ assertShowColumns(statement, "SHOW CONFIGNODES", 5, false);
+ }
+ }
+ }
+
+ private static void assertShowColumns(
+ Statement statement, String sql, int baseColumnCount, boolean hasReason) throws Exception {
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
+ int columnCount = resultSet.getMetaData().getColumnCount();
+ Assert.assertEquals(sql, baseColumnCount + (hasReason ? 1 : 0), columnCount);
+ if (hasReason) {
+ Assert.assertEquals(
+ sql, "StatusReason", resultSet.getMetaData().getColumnName(columnCount));
+ }
+ Assert.assertTrue(sql, resultSet.next());
+ do {
+ if (hasReason) {
+ String expectedReason =
+ sql.startsWith("SHOW CLUSTER") && !"DataNode".equals(resultSet.getString("NodeType"))
+ ? null
+ : "Manual";
+ Assert.assertEquals(sql, expectedReason, resultSet.getString(columnCount));
+ }
+ } while (resultSet.next());
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java
index 3e8bc96f1db..0d62de9dd73 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/recent/IoTDBTableAggregationQueryWithNetworkPartitionIT.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.isession.ITableSession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.TableClusterIT;
@@ -167,7 +168,7 @@ public class IoTDBTableAggregationQueryWithNetworkPartitionIT {
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(dataNodeWrapper),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
}
List<String> otherNodes = new ArrayList<>();
@@ -209,7 +210,7 @@ public class IoTDBTableAggregationQueryWithNetworkPartitionIT {
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(dataNodeWrapper),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
}
List<String> otherNodes = new ArrayList<>();
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java
index 74e009afb17..b9f905c30ed 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/query/view/recent/IoTDBTableViewQueryWithCachedDeviceIT.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.isession.ISession;
import org.apache.iotdb.isession.ITableSession;
import org.apache.iotdb.isession.SessionDataSet;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
@@ -97,7 +98,7 @@ public class IoTDBTableViewQueryWithCachedDeviceIT {
EnvFactory.getEnv()
.ensureNodeStatus(
Collections.singletonList(dataNodeWrapper),
- Collections.singletonList(NodeStatus.Unknown));
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
}
EnvFactory.getEnv().startAllDataNodes();
for (DataNodeWrapper dataNodeWrapper : EnvFactory.getEnv().getDataNodeWrapperList()) {
diff --git a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
index c7f85ea500e..be42be99298 100644
--- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBDatabaseIT.java
@@ -698,6 +698,7 @@ public class IoTDBDatabaseIT {
"node_id,INT32,TAG,",
"node_type,STRING,ATTRIBUTE,",
"status,STRING,ATTRIBUTE,",
+ "status_reason,STRING,ATTRIBUTE,",
"internal_address,STRING,ATTRIBUTE,",
"internal_port,INT32,ATTRIBUTE,",
"version,STRING,ATTRIBUTE,",
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java
index 18b5f897e5c..90795d969ff 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBConnectionsIT.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.it.env.EnvFactory;
import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper;
@@ -49,7 +50,6 @@ import java.sql.Statement;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.iotdb.db.it.utils.TestUtils.createUser;
import static org.apache.iotdb.itbase.env.BaseEnv.TABLE_SQL_DIALECT;
@@ -278,23 +278,24 @@ public class IoTDBConnectionsIT {
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
// Wait for shutdown check
- while (true) {
- AtomicBoolean containUnknown = new AtomicBoolean(false);
+ boolean isShutdownDetected = false;
+ for (int retry = 0; retry < 60; retry++) {
TShowDataNodesResp showDataNodesResp = client.showDataNodes();
- showDataNodesResp
- .getDataNodesInfoList()
- .forEach(
- dataNodeInfo -> {
- if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) {
- containUnknown.set(true);
- }
- });
-
- if (containUnknown.get()) {
+ for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
+ // A gracefully stopped DataNode is reported as Stopped by its shutdown hook; if
+ // the report fails it becomes Unknown instead
+ if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())
+ || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) {
+ isShutdownDetected = true;
+ break;
+ }
+ }
+ if (isShutdownDetected) {
break;
}
TimeUnit.SECONDS.sleep(1);
}
+ Assert.assertTrue(isShutdownDetected);
}
int activeDataNodeId = (int) allDataNodeId.toArray()[1];
@@ -327,23 +328,25 @@ public class IoTDBConnectionsIT {
try (SyncConfigNodeIServiceClient client =
(SyncConfigNodeIServiceClient) EnvFactory.getEnv().getLeaderConfigNodeConnection()) {
// Wait for restart check
- while (true) {
- AtomicBoolean containUnknown = new AtomicBoolean(false);
+ boolean isRestartDetected = false;
+ for (int retry = 0; retry < 60; retry++) {
TShowDataNodesResp showDataNodesResp = client.showDataNodes();
- showDataNodesResp
- .getDataNodesInfoList()
- .forEach(
- dataNodeInfo -> {
- if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())) {
- containUnknown.set(true);
- }
- });
-
- if (!containUnknown.get()) {
+ boolean containDown = false;
+ for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
+ // The restarted DataNode keeps Stopped until its first heartbeat revives it
+ if (NodeStatus.Unknown.getStatus().equals(dataNodeInfo.getStatus())
+ || NodeStatus.Stopped.getStatus().equals(dataNodeInfo.getStatus())) {
+ containDown = true;
+ break;
+ }
+ }
+ if (!containDown) {
+ isRestartDetected = true;
break;
}
TimeUnit.SECONDS.sleep(1);
}
+ Assert.assertTrue(isRestartDetected);
}
// The ConfigNode may report the restarted DataNode as Running before its client RPC service is
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java
index 0a801477866..0ce3be416d4 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/pool/SessionPoolIT.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.isession.SessionConfig;
import org.apache.iotdb.isession.pool.ISessionPool;
import org.apache.iotdb.isession.pool.SessionDataSetWrapper;
import org.apache.iotdb.it.env.EnvFactory;
+import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.framework.IoTDBTestRunner;
import org.apache.iotdb.itbase.category.ClusterIT;
import org.apache.iotdb.itbase.category.LocalStandaloneIT;
@@ -252,7 +253,10 @@ public class SessionPoolIT {
node.stop();
EnvFactory.getEnv()
.ensureNodeStatus(
- Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
+ // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
+ // Windows the process is hard-killed, so it is marked Unknown instead)
+ Collections.singletonList(node),
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
// user does not know what happens.
while (wrapper.hasNext()) {
wrapper.next();
@@ -263,7 +267,10 @@ public class SessionPoolIT {
node.stop();
EnvFactory.getEnv()
.ensureNodeStatus(
- Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
+ // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
+ // Windows the process is hard-killed, so it is marked Unknown instead)
+ Collections.singletonList(node),
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
node.start();
EnvFactory.getEnv()
.ensureNodeStatus(
@@ -289,7 +296,10 @@ public class SessionPoolIT {
node.stop();
EnvFactory.getEnv()
.ensureNodeStatus(
- Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
+ // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
+ // Windows the process is hard-killed, so it is marked Unknown instead)
+ Collections.singletonList(node),
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
node.start();
EnvFactory.getEnv()
.ensureNodeStatus(
@@ -345,7 +355,10 @@ public class SessionPoolIT {
node.stop();
EnvFactory.getEnv()
.ensureNodeStatus(
- Collections.singletonList(node), Collections.singletonList(NodeStatus.Unknown));
+ // A gracefully stopped DataNode is reported as Stopped by its shutdown hook (on
+ // Windows the process is hard-killed, so it is marked Unknown instead)
+ Collections.singletonList(node),
+ Collections.singletonList(EnvUtils.getNodeStatusAfterLocalStop()));
pool = EnvFactory.getEnv().getSessionPool(1);
// all this ten data will fail.
write10Data(pool, false);
diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
index 800d59bf727..6fae63f6536 100644
--- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
@@ -687,4 +687,7 @@ public final class ConfigNodeMessages {
public static final String
EXCEPTION_IOTDB_EDGE_CONFIGNODE_INTERNAL_PORT_ARG_IS_NOT_READY_WITHIN_03697FF5 =
"IoTDB Edge: ConfigNode internal port %s is not ready within %s ms";
+ public static final String
+ LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5 =
+ "Failed to transfer ConfigNode leadership before shutdown; continuing shutdown.";
}
diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
index 0723aa05e2a..183ab36c478 100644
--- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ManagerMessages.java
@@ -483,8 +483,8 @@ public final class ManagerMessages {
"Successfully transferred config region snapshot {}.";
public static final String THERE_IS_NO_RUNNING_DATANODE_TO_EXECUTE_CQ =
"There is no RUNNING DataNode to execute CQ {}";
- public static final String THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN =
- "The ConfigNode-{} will be shutdown soon, mark it as Unknown";
+ public static final String LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665 =
+ "Node {} reported shutdown; current status is {}.";
public static final String THE_CONFIG_REGION_AIR_GAP_CONNECTOR_DOES_NOT_SUPPORT_TRANSFERRING =
"The config region air gap connector does not support transferring single file piece bytes.";
public static final String THE_CONFIG_REGION_SINK_DOES_NOT_SUPPORT_TRANSFERRING_SINGLE_FILE =
@@ -493,8 +493,6 @@ public final class ManagerMessages {
"The config region snapshots %s cannot be parsed.";
public static final String THE_DATABASE_DOESN_T_EXIST_MAYBE_IT_HAS_BEEN_PRE =
"The Database: {} doesn't exist. Maybe it has been pre-deleted.";
- public static final String THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN =
- "The DataNode-{} will be shutdown soon, mark it as Unknown";
public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_GREEDYREGIONGROUPALLOCATOR_IS_YET =
"The removeNodeReplicaSelect method of GreedyRegionGroupAllocator is yet to be implemented.";
public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_PARTITEGRAPHPLACEMENTREGIONGROUPALLOCATOR =
@@ -717,4 +715,6 @@ public final class ManagerMessages {
MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D =
"%s Please manually check later whether the procedure is executed successfully.";
+ public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 =
+ "ConfigNode leader is waiting for node status persistence.";
}
diff --git a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
index 94abb3faab4..18aca012439 100644
--- a/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/en/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
@@ -1220,6 +1220,9 @@ public final class ProcedureMessages {
+ " cache status is {}";
public static final String MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_UPDATE_CONSENSUSGROUP_PEER_INFORMATION_FAILED_FCE5302B = "Remove ConfigNode failed because update ConsensusGroup peer information failed.";
public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 = "Can't remove datanode due to the limit of replication factor, ";
+ public static final String
+ MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6 =
+ "Submit RemoveDataNodesProcedure failed, because when there are other unknown, stopped or readonly nodes in the consensus group that are not remove nodes, the remove operation cannot be performed for security reasons. Please check the status of the node %s and ensure it is running.";
public static final String MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C = "availableDataNodeSize: %s, maxReplicaFactor: %s, max allowed removed Data Node size is: %s";
public static final String EXCEPTION_NOT_SUPPORTED_0A83F963 = " is not supported";
public static final String LOG_START_ADD_TRIGGER_ARG_TRIGGERTABLE_CONFIG_NODES_NEEDTOSAVEJAR_ARG_0C23D81E = "Start to add trigger [{}] in TriggerTable on Config Nodes, needToSaveJar[{}]";
@@ -1423,4 +1426,8 @@ public final class ProcedureMessages {
"rolling back after failure: %s.";
public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 =
"rolling back after an earlier failure.";
+ public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F =
+ "DataNode setSystemStatus RPC failed: %s: %s: %s; ";
+ public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 =
+ "ConfigNode trySetNodeStatus failed: %s: %s: %s; ";
}
diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
index 14713cbb801..20787786a74 100644
--- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ConfigNodeMessages.java
@@ -730,4 +730,7 @@ public final class ConfigNodeMessages {
public static final String
EXCEPTION_IOTDB_EDGE_CONFIGNODE_INTERNAL_PORT_ARG_IS_NOT_READY_WITHIN_03697FF5 =
"IoTDB Edge：ConfigNode 内部端口 %s 在 %s ms 内未就绪";
+ public static final String
+ LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5 =
+ "关闭前转移 ConfigNode leader 失败，继续关闭。";
}
diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
index a140efc2f4d..fa3d63dddb3 100644
--- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ManagerMessages.java
@@ -478,8 +478,8 @@ public final class ManagerMessages {
"成功传输 config region 快照 {}。";
public static final String THERE_IS_NO_RUNNING_DATANODE_TO_EXECUTE_CQ =
"没有处于 RUNNING 状态的 DataNode 可用于执行 CQ {}";
- public static final String THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN =
- "ConfigNode-{} 即将关闭，将其标记为 Unknown";
+ public static final String LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665 =
+ "节点 {} 已报告停止；当前状态为 {}。";
public static final String THE_CONFIG_REGION_AIR_GAP_CONNECTOR_DOES_NOT_SUPPORT_TRANSFERRING =
"config region air gap connector 不支持传输单文件分片字节。";
public static final String THE_CONFIG_REGION_SINK_DOES_NOT_SUPPORT_TRANSFERRING_SINGLE_FILE =
@@ -488,8 +488,6 @@ public final class ManagerMessages {
"无法解析 config region 快照 %s。";
public static final String THE_DATABASE_DOESN_T_EXIST_MAYBE_IT_HAS_BEEN_PRE =
"Database: {} 不存在，可能已被预删除。";
- public static final String THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN =
- "DataNode-{} 即将关闭，将其标记为 Unknown";
public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_GREEDYREGIONGROUPALLOCATOR_IS_YET =
"GreedyRegionGroupAllocator 的 removeNodeReplicaSelect 方法尚未实现。";
public static final String THE_REMOVENODEREPLICASELECT_METHOD_OF_PARTITEGRAPHPLACEMENTREGIONGROUPALLOCATOR =
@@ -696,4 +694,6 @@ public final class ManagerMessages {
MESSAGE_ARG_PLEASE_MANUALLY_CHECK_LATER_WHETHER_THE_PROCEDURE_IS_EXECUTED_SUCCESSFULLY_A82B739D =
"%s 请稍后手动检查该 Procedure 是否执行成功。";
+ public static final String MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809 =
+ "ConfigNode leader 正在等待节点状态持久化。";
}
diff --git a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
index d4d5dc2770a..14e52d72b9e 100644
--- a/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
+++ b/iotdb-core/confignode/src/main/i18n/zh/org/apache/iotdb/confignode/i18n/ProcedureMessages.java
@@ -1170,6 +1170,9 @@ public final class ProcedureMessages {
"移除 ConfigNode 失败，原因：更新 ConsensusGroup peer 信息失败。";
public static final String MESSAGE_CAN_T_REMOVE_DATANODE_LIMIT_REPLICATION_FACTOR_D960E3A6 =
"无法移除 DataNode，原因：受副本因子限制，";
+ public static final String
+ MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6 =
+ "提交 RemoveDataNodesProcedure 失败：共识组内存在非移除目标且状态为 Unknown、Stopped 或 ReadOnly 的其他节点时，出于安全考虑无法执行移除操作。请检查节点 %s 的状态并确保其处于 Running 状态。";
public static final String
MESSAGE_AVAILABLEDATANODESIZE_ARG_MAXREPLICAFACTOR_ARG_MAX_ALLOWED_REMOVED_DATA_NODE_SIZE_FB8C382C =
"availableDataNodeSize：%s，maxReplicaFactor：%s，允许移除的最大 DataNode 数量为：%s";
@@ -1508,4 +1511,8 @@ public final class ProcedureMessages {
public static final String MESSAGE_ROLLING_BACK_AFTER_FAILURE_ARG_474DF456 = "正在回滚，失败原因：%s。";
public static final String MESSAGE_ROLLING_BACK_AFTER_AN_EARLIER_FAILURE_850D0AF5 =
"正在回滚此前发生的失败。";
+ public static final String EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F =
+ "DataNode setSystemStatus RPC 失败：%s：%s：%s； ";
+ public static final String EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4 =
+ "ConfigNode trySetNodeStatus 失败：%s：%s：%s； ";
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java
index 03e6c0bfe31..8e734ed635d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/AINodeHeartbeatHandler.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.confignode.client.async.handlers.heartbeat;
import org.apache.iotdb.ainode.rpc.thrift.TAIHeartbeatResp;
import org.apache.iotdb.commons.client.ThriftClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.confignode.manager.load.LoadManager;
import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
@@ -49,8 +48,7 @@ public class AINodeHeartbeatHandler implements AsyncMethodCallback<TAIHeartbeatR
@Override
public void onError(Exception e) {
if (ThriftClient.isConnectionBroken(e)) {
- loadManager.forceUpdateNodeCache(
- NodeType.AINode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown));
+ loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false);
}
loadManager.getLoadCache().resetHeartbeatProcessing(nodeId);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java
index 96225e4fb7b..df3dad71840 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/ConfigNodeHeartbeatHandler.java
@@ -21,7 +21,6 @@ package org.apache.iotdb.confignode.client.async.handlers.heartbeat;
import org.apache.iotdb.commons.client.ThriftClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.confignode.manager.load.LoadManager;
import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeHeartbeatResp;
@@ -48,8 +47,7 @@ public class ConfigNodeHeartbeatHandler implements AsyncMethodCallback<TConfigNo
@Override
public void onError(Exception e) {
if (ThriftClient.isConnectionBroken(e)) {
- loadManager.forceUpdateNodeCache(
- NodeType.ConfigNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown));
+ loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false);
}
loadManager.getLoadCache().resetHeartbeatProcessing(nodeId);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java
index 14e1dcbb86f..d2d84f90d32 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandler.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.commons.client.ThriftClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.cluster.RegionStatus;
import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
@@ -105,7 +104,8 @@ public class DataNodeHeartbeatHandler implements AsyncMethodCallback<TDataNodeHe
}
private void cacheRegionGroupHeartbeatSamples(TDataNodeHeartbeatResp heartbeatResp) {
- RegionStatus regionStatus = RegionStatus.valueOf(heartbeatResp.getStatus());
+ RegionStatus regionStatus =
+ RegionStatus.fromNodeStatus(NodeStatus.parse(heartbeatResp.getStatus()));
Map<TConsensusGroupId, Boolean> judgedLeaders =
heartbeatResp.isSetJudgedLeaders()
@@ -220,8 +220,7 @@ public class DataNodeHeartbeatHandler implements AsyncMethodCallback<TDataNodeHe
@Override
public void onError(Exception e) {
if (ThriftClient.isConnectionBroken(e)) {
- loadManager.forceUpdateNodeCache(
- NodeType.DataNode, nodeId, new NodeHeartbeatSample(NodeStatus.Unknown));
+ loadManager.trySetNodeStatus(nodeId, NodeStatus.Unknown, false);
}
loadManager.getLoadCache().resetHeartbeatProcessing(nodeId);
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
index c70b8c12fcf..9c01288d287 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlan.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan;
@@ -359,6 +360,9 @@ public abstract class ConfigPhysicalPlan implements IConsensusRequest {
case UpdateVersionInfo:
plan = new UpdateVersionInfoPlan();
break;
+ case UpdateNodeStatus:
+ plan = new UpdateNodeStatusPlan();
+ break;
case UpdateClusterId:
plan = new UpdateClusterIdPlan();
break;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
index 144cb8d897a..eed13c91715 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java
@@ -30,6 +30,8 @@ public enum ConfigPhysicalPlanType {
UpdateVersionInfo((short) 2),
UpdateClusterId((short) 3),
+ UpdateNodeStatus((short) 4),
+
/** DataNode. */
RegisterDataNode((short) 100),
GetDataNodeConfiguration((short) 101),
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java
new file mode 100644
index 00000000000..35140c61771
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/confignode/UpdateNodeStatusPlan.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.consensus.request.write.confignode;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan;
+import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlanType;
+
+import org.apache.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Objects;
+
+public class UpdateNodeStatusPlan extends ConfigPhysicalPlan {
+
+ private int nodeId;
+
+ // A null status removes the persisted record, including its reason.
+ private NodeStatus status;
+
+ private String statusReason;
+
+ public UpdateNodeStatusPlan() {
+ super(ConfigPhysicalPlanType.UpdateNodeStatus);
+ }
+
+ public UpdateNodeStatusPlan(int nodeId, NodeStatus status) {
+ this(nodeId, status, null);
+ }
+
+ public UpdateNodeStatusPlan(int nodeId, NodeStatus status, String statusReason) {
+ this();
+ this.nodeId = nodeId;
+ // Resolve clearing before serialization so replay does not depend on future persistence rules.
+ this.status = status != null && status.isPersistentStatus() ? status : null;
+ this.statusReason = this.status == null ? null : statusReason;
+ }
+
+ public int getNodeId() {
+ return nodeId;
+ }
+
+ public NodeStatus getStatus() {
+ return status;
+ }
+
+ public String getStatusReason() {
+ return statusReason;
+ }
+
+ @Override
+ protected void serializeImpl(DataOutputStream stream) throws IOException {
+ ReadWriteIOUtils.write(getType().getPlanType(), stream);
+ ReadWriteIOUtils.write(nodeId, stream);
+ ReadWriteIOUtils.write(status == null ? null : status.getStatus(), stream);
+ ReadWriteIOUtils.write(statusReason, stream);
+ }
+
+ @Override
+ protected void deserializeImpl(ByteBuffer buffer) {
+ nodeId = ReadWriteIOUtils.readInt(buffer);
+ String statusName = ReadWriteIOUtils.readString(buffer);
+ status = statusName == null ? null : NodeStatus.parse(statusName);
+ statusReason = ReadWriteIOUtils.readString(buffer);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ UpdateNodeStatusPlan that = (UpdateNodeStatusPlan) o;
+ return nodeId == that.nodeId
+ && status == that.status
+ && Objects.equals(statusReason, that.statusReason);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(nodeId, status, statusReason);
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 2a6b7e0f68c..98c0a1bb9fc 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -120,7 +120,7 @@ import org.apache.iotdb.confignode.manager.cq.CQManager;
import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceInfo;
import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager;
import org.apache.iotdb.confignode.manager.load.LoadManager;
-import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
import org.apache.iotdb.confignode.manager.node.ClusterNodeStartUtils;
import org.apache.iotdb.confignode.manager.node.NodeManager;
import org.apache.iotdb.confignode.manager.node.NodeMetrics;
@@ -575,17 +575,19 @@ public class ConfigManager implements IManager {
@Override
public TSStatus reportDataNodeShutdown(TDataNodeLocation dataNodeLocation) {
+ return reportNodeShutdown(dataNodeLocation.getDataNodeId());
+ }
+
+ private TSStatus reportNodeShutdown(int nodeId) {
TSStatus status = confirmLeader();
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- // Force updating the target DataNode's status to Unknown
- getLoadManager()
- .forceUpdateNodeCache(
- NodeType.DataNode,
- dataNodeLocation.getDataNodeId(),
- new NodeHeartbeatSample(NodeStatus.Unknown));
- LOGGER.info(
- ManagerMessages.THE_DATANODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN,
- dataNodeLocation.getDataNodeId());
+ status = getLoadManager().trySetNodeStatus(nodeId, NodeStatus.Stopped, false);
+ if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ LOGGER.info(
+ ManagerMessages.LOG_NODE_ARG_REPORTED_SHUTDOWN_CURRENT_STATUS_IS_ARG_A375D665,
+ nodeId,
+ getLoadManager().getNodeStatus(nodeId));
+ }
}
return status;
}
@@ -620,7 +622,17 @@ public class ConfigManager implements IManager {
.sorted(Comparator.comparingInt(TDataNodeLocation::getDataNodeId))
.collect(Collectors.toList());
Map<Integer, TNodeVersionInfo> nodeVersionInfo = getNodeManager().getNodeVersionInfo();
- Map<Integer, String> nodeStatus = getLoadManager().getNodeStatusWithReason();
+ Map<Integer, NodeStatistics> nodeStatisticsSnapshot =
+ getLoadManager().getNodeStatisticsSnapshot();
+ Map<Integer, String> nodeStatus = new HashMap<>();
+ Map<Integer, String> nodeStatusReason = new HashMap<>();
+ nodeStatisticsSnapshot.forEach(
+ (nodeId, statistics) -> {
+ nodeStatus.put(nodeId, statistics.getStatus().getStatus());
+ if (statistics.getStatusReason() != null) {
+ nodeStatusReason.put(nodeId, statistics.getStatusReason());
+ }
+ });
configNodeLocations.forEach(
configNodeLocation ->
nodeStatus.putIfAbsent(
@@ -635,11 +647,9 @@ public class ConfigManager implements IManager {
.map(TAINodeConfiguration::getLocation)
.sorted(Comparator.comparingInt(TAINodeLocation::getAiNodeId))
.collect(Collectors.toList());
- Map<Integer, String> nodeStatusMap = getLoadManager().getNodeStatusWithReason();
aiNodeLocations.forEach(
aiNodeLocation ->
- nodeStatusMap.putIfAbsent(
- aiNodeLocation.getAiNodeId(), NodeStatus.Unknown.toString()));
+ nodeStatus.putIfAbsent(aiNodeLocation.getAiNodeId(), NodeStatus.Unknown.toString()));
return new TShowClusterResp()
.setStatus(status)
@@ -647,6 +657,7 @@ public class ConfigManager implements IManager {
.setDataNodeList(dataNodeLocations)
.setAiNodeList(aiNodeLocations)
.setNodeStatus(nodeStatus)
+ .setNodeStatusReason(nodeStatusReason)
.setNodeVersionInfo(nodeVersionInfo);
} else {
return new TShowClusterResp()
@@ -655,6 +666,7 @@ public class ConfigManager implements IManager {
.setDataNodeList(Collections.emptyList())
.setAiNodeList(Collections.emptyList())
.setNodeStatus(Collections.emptyMap())
+ .setNodeStatusReason(Collections.emptyMap())
.setNodeVersionInfo(Collections.emptyMap());
}
}
@@ -1629,19 +1641,7 @@ public class ConfigManager implements IManager {
@Override
public TSStatus reportConfigNodeShutdown(TConfigNodeLocation configNodeLocation) {
- TSStatus status = confirmLeader();
- if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- // Force updating the target ConfigNode's status to Unknown
- getLoadManager()
- .forceUpdateNodeCache(
- NodeType.ConfigNode,
- configNodeLocation.getConfigNodeId(),
- new NodeHeartbeatSample(NodeStatus.Unknown));
- LOGGER.info(
- ManagerMessages.THE_CONFIGNODE_WILL_BE_SHUTDOWN_SOON_MARK_IT_AS_UNKNOWN,
- configNodeLocation.getConfigNodeId());
- }
- return status;
+ return reportNodeShutdown(configNodeLocation.getConfigNodeId());
}
@Override
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
index bae45596812..cc203caeeba 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java
@@ -345,9 +345,13 @@ public interface IManager {
/**
* Report that the specified DataNode will be shutdown.
*
- * <p>The ConfigNode-leader will mark it as {@link NodeStatus#Unknown}
+ * <p>The ConfigNode-leader requests {@link NodeStatus#Stopped}, retaining an existing {@link
+ * NodeStatus#Removing}. Success confirms that the resulting status is persisted and can be
+ * restored by a new leader. If the report is not received, heartbeat timeout produces {@link
+ * NodeStatus#Unknown} only when there is no Stopped or Removing status to retain, including one
+ * restored from persistence. A failed write may still update the current leader's memory.
*
- * @return {@link TSStatusCode#SUCCESS_STATUS} if reporting successfully
+ * @return {@link TSStatusCode#SUCCESS_STATUS} if the resulting status is persisted
*/
TSStatus reportDataNodeShutdown(TDataNodeLocation dataNodeLocation);
@@ -538,10 +542,14 @@ public interface IManager {
TSStatus removeConfigNode(RemoveConfigNodePlan removeConfigNodePlan);
/**
- * Report that the specified ConfigNode will be shutdown. The ConfigNode-leader will mark it as
- * Unknown.
+ * Report that the specified ConfigNode will be shutdown. The leader requests {@link
+ * NodeStatus#Stopped}, retaining an existing {@link NodeStatus#Removing}. Success confirms that
+ * the resulting status is persisted and can be restored by a new leader. If the report is not
+ * received, heartbeat timeout produces {@link NodeStatus#Unknown} only when there is no Stopped
+ * or Removing status to retain, including one restored from persistence. A failed write may still
+ * update the current leader's memory.
*
- * @return {@link TSStatusCode#SUCCESS_STATUS} if reporting successfully
+ * @return {@link TSStatusCode#SUCCESS_STATUS} if the resulting status is persisted
*/
TSStatus reportConfigNodeShutdown(TConfigNodeLocation configNodeLocation);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
index c97654b649e..f063a933ebc 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ProcedureManager.java
@@ -58,6 +58,7 @@ import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNo
import org.apache.iotdb.confignode.consensus.request.write.procedure.UpdateProcedurePlan;
import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGroupsPlan;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
+import org.apache.iotdb.confignode.i18n.ProcedureMessages;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator;
import org.apache.iotdb.confignode.persistence.ProcedureInfo;
@@ -776,8 +777,8 @@ public class ProcedureManager {
removedDataNodesRegionSet.add(regionMigrationPlan.getRegionId());
}
- // 4. Check if there are any other unknown or readonly DataNodes in the consensus group that are
- // not the remove DataNodes
+ // 4. Check if there are any other unknown, stopped or readonly DataNodes in the consensus
+ // group that are not the remove DataNodes
for (TDataNodeLocation removeDataNode : dataNodeLocations) {
Set<TDataNodeLocation> relatedDataNodes =
@@ -787,13 +788,11 @@ public class ProcedureManager {
for (TDataNodeLocation relatedDataNode : relatedDataNodes) {
NodeStatus nodeStatus =
getConfigManager().getLoadManager().getNodeStatus(relatedDataNode.getDataNodeId());
- if (nodeStatus == NodeStatus.Unknown || nodeStatus == NodeStatus.ReadOnly) {
+ if (nodeStatus.isOffline() || nodeStatus == NodeStatus.ReadOnly) {
failMessage =
String.format(
- "Submit RemoveDataNodesProcedure failed, "
- + "because when there are other unknown or readonly nodes in the consensus group that are not remove nodes, "
- + "the remove operation cannot be performed for security reasons. "
- + "Please check the status of the node %s and ensure it is running.",
+ ProcedureMessages
+ .MESSAGE_SUBMIT_REMOVEDATANODESPROCEDURE_FAILED_BECAUSE_WHEN_THERE_ARE_OTHER_UNKNOWN_STOPPED_OR_READONLY_NODES_IN_THE_CONSENSUS_GROUP_THAT_ARE_NOT_REMOVE_NODES_THE_REMOVE_OPERATION_CANNOT_BE_PERFORMED_FOR_SECURITY_REASONS_PLEASE_CHECK_THE_STATUS_OF_THE_NODE_ARG_AND_ENSURE_IT_IS_RUNNING_5063B3F6,
relatedDataNode.getDataNodeId());
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java
index 2f222a93ac6..f4fe41fe656 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThread.java
@@ -139,8 +139,7 @@ public class RetryFailedTasksThread {
NodeStatus nodeStatus = loadManager.getNodeStatus(dataNodeLocation.getDataNodeId());
if (nodeStatus == NodeStatus.Running) {
oldUnknownNodes.remove(dataNodeLocation);
- } else if (!oldUnknownNodes.contains(dataNodeLocation)
- && nodeStatus == NodeStatus.Unknown) {
+ } else if (!oldUnknownNodes.contains(dataNodeLocation) && nodeStatus.isOffline()) {
newUnknownNodes.add(dataNodeLocation);
}
});
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
index 5a424ea5a5e..d5d7085bfa8 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
@@ -22,9 +22,9 @@ package org.apache.iotdb.confignode.manager.load;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.common.rpc.thrift.TSeriesPartitionSlot;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.cluster.RegionStatus;
import org.apache.iotdb.commons.partition.DataPartitionTable;
import org.apache.iotdb.commons.partition.SchemaPartitionTable;
@@ -33,13 +33,14 @@ import org.apache.iotdb.confignode.consensus.request.write.region.CreateRegionGr
import org.apache.iotdb.confignode.exception.DatabaseNotExistsException;
import org.apache.iotdb.confignode.exception.NoAvailableRegionGroupException;
import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException;
+import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.IManager;
import org.apache.iotdb.confignode.manager.load.balancer.PartitionBalancer;
import org.apache.iotdb.confignode.manager.load.balancer.RegionBalancer;
import org.apache.iotdb.confignode.manager.load.balancer.RouteBalancer;
import org.apache.iotdb.confignode.manager.load.cache.LoadCache;
import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
-import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.service.EventService;
import org.apache.iotdb.confignode.manager.load.service.HeartbeatService;
@@ -92,6 +93,10 @@ public class LoadManager {
this.routeBalancer = new RouteBalancer(configManager);
this.loadCache = new LoadCache();
+ this.loadCache.setNodeStatusPersistence(
+ configManager.getNodeManager().getNodeInfo()::getPersistedNodeStatus,
+ // Consensus is initialized after construction; obtain the manager when submitting the plan.
+ plan -> configManager.getConsensusManager().write(plan));
setHeartbeatService(configManager, loadCache);
this.statisticsService = new StatisticsService(loadCache);
this.topologyService = new TopologyService(configManager, loadCache::updateTopology);
@@ -209,7 +214,13 @@ public class LoadManager {
return false;
}
- loadCache.updateNodeStatistics(false);
+ if (!loadCache.updateNodeStatistics()) {
+ // In particular, do not serve as leader until its own old Stopped marker is cleared.
+ loadReadyReason =
+ ManagerMessages.MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809;
+ return false;
+ }
+
eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary();
List<String> unreadyReasons = loadCache.getNodeHeartbeatUnreadyReasons();
@@ -259,22 +270,24 @@ public class LoadManager {
}
/**
- * Safely get the specified Node's current status with reason.
+ * Safely get the specified Node's current status reason.
*
* @param nodeId The specified NodeId
- * @return The specified Node's current status if the nodeCache contains it, Unknown otherwise
+ * @return The reason why the Node is in its current status, null if the node has no reason or the
+ * cache doesn't exist
*/
- public String getNodeStatusWithReason(int nodeId) {
- return loadCache.getNodeStatusWithReason(nodeId);
+ public String getNodeStatusReason(int nodeId) {
+ return loadCache.getNodeStatusReason(nodeId);
}
/**
- * Get all Node's current status with reason.
+ * Get all Nodes' current statistics in a single traversal of the node cache, so that each node's
+ * status and reason come from the same statistics snapshot.
*
- * @return Map<NodeId, NodeStatus with reason>
+ * @return Map<NodeId, NodeStatistics>
*/
- public Map<Integer, String> getNodeStatusWithReason() {
- return loadCache.getNodeStatusWithReason();
+ public Map<Integer, NodeStatistics> getNodeStatisticsSnapshot() {
+ return loadCache.getCurrentNodeStatisticsMap();
}
/**
@@ -336,42 +349,37 @@ public class LoadManager {
return loadCache.getLowestLoadDataNode(dataNodeIds);
}
+ /** Try to set the specified Node's status without a reason. */
+ public TSStatus trySetNodeStatus(int nodeId, NodeStatus nodeStatus, boolean force) {
+ return trySetNodeStatus(nodeId, nodeStatus, null, force);
+ }
+
/**
- * Force update the specified Node's cache, update statistics and broadcast statistics change
- * event if necessary.
+ * Try to set the specified Node's status according to transition rules and broadcast statistics
+ * changes, including when persistence fails. Does not send a status-change RPC to the node.
*
- * @param nodeType Specified NodeType
* @param nodeId Specified NodeId
- * @param heartbeatSample Specified NodeHeartbeatSample
+ * @param nodeStatus requested status
+ * @param statusReason reason supplied by the caller, or null when no reason is known
+ * @param force whether an administrative update may replace Stopped or Removing
+ * @return processing result; success does not imply that the requested status replaced the old
+ * one
*/
- public void forceUpdateNodeCache(
- NodeType nodeType, int nodeId, NodeHeartbeatSample heartbeatSample) {
- switch (nodeType) {
- case ConfigNode:
- loadCache.cacheConfigNodeHeartbeatSample(nodeId, heartbeatSample);
- break;
- case DataNode:
- loadCache.cacheDataNodeHeartbeatSample(nodeId, heartbeatSample);
- break;
- case AINode:
- loadCache.cacheAINodeHeartbeatSample(nodeId, heartbeatSample);
- break;
- default:
- break;
- }
- loadCache.updateNodeStatistics(true);
+ public TSStatus trySetNodeStatus(
+ int nodeId, NodeStatus nodeStatus, String statusReason, boolean force) {
+ TSStatus status = loadCache.trySetNodeStatus(nodeId, nodeStatus, statusReason, force);
eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary();
+ return status;
}
/**
- * Remove the NodeHeartbeatCache of the specified Node, update statistics and broadcast statistics
- * change event if necessary.
+ * Remove the NodeHeartbeatCache of the specified Node and broadcast statistics changes if
+ * necessary.
*
* @param nodeId the index of the specified Node
*/
public void removeNodeCache(int nodeId) {
loadCache.removeNodeCache(nodeId);
- loadCache.updateNodeStatistics(true);
eventService.checkAndBroadcastNodeStatisticsChangeEventIfNecessary();
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
index 73583151f98..44dbcc37912 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancer.java
@@ -84,17 +84,15 @@ public class RegionBalancer {
throws NotEnoughDataNodeException, DatabaseNotExistsException {
// Some new RegionGroups will have to occupy unknown DataNodes if the number of online
- // DataNodes is insufficient (Unknown DataNodes are intentionally kept as candidates).
- // However, DataNodes that an in-progress RemoveDataNodesProcedure is removing must be
- // excluded: placing a new replica on a node that is about to disappear would strand that
- // replica and stall the removal forever. A status filter is not enough here, because a
+ // DataNodes is insufficient (Unknown and Stopped DataNodes are intentionally kept as
+ // candidates). However, DataNodes that an in-progress RemoveDataNodesProcedure is removing
+ // must be excluded: placing a new replica on a node that is about to disappear would strand
+ // that replica and stall the removal forever. A status filter is not enough here, because a
// DataNode killed (e.g. kill -9) before removal is reported as Unknown (not Removing) by the
// failure detector, so we additionally drop every DataNode that is currently being removed.
final Set<Integer> removingDataNodeIds = getProcedureManager().getRemovingDataNodeIds();
final List<TDataNodeConfiguration> availableDataNodes =
- getNodeManager()
- .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.Unknown)
- .stream()
+ getNodeManager().filterDataNodeThroughStatus(NodeStatus::isRegionCandidate).stream()
.filter(
dataNode -> !removingDataNodeIds.contains(dataNode.getLocation().getDataNodeId()))
.collect(Collectors.toList());
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java
index 4c5ee322aa1..da22a140cd4 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/AbstractLoadCache.java
@@ -94,18 +94,15 @@ public abstract class AbstractLoadCache {
* @return The latest heartbeat sample.
*/
public AbstractHeartbeatSample getLastSample() {
- return slidingWindow.isEmpty() ? null : slidingWindow.get(slidingWindow.size() - 1);
+ synchronized (slidingWindow) {
+ return slidingWindow.isEmpty() ? null : slidingWindow.get(slidingWindow.size() - 1);
+ }
}
public boolean hasHeartbeatSample() {
return getLastSample() != null;
}
- /**
- * Update currentStatistics based on the latest heartbeat sample that cached in the slidingWindow.
- */
- public abstract void updateCurrentStatistics(boolean forceUpdate);
-
public AbstractStatistics getCurrentStatistics() {
return currentStatistics.get();
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java
index 3416246811e..b7cf0639f3d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/LoadCache.java
@@ -27,11 +27,14 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.cluster.RegionStatus;
import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.IManager;
import org.apache.iotdb.confignode.manager.ProcedureManager;
@@ -49,8 +52,12 @@ import org.apache.iotdb.confignode.manager.load.cache.region.RegionGroupStatisti
import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics;
import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.ratis.util.function.CheckedFunction;
import org.apache.thrift.annotation.Nullable;
+import org.apache.tsfile.utils.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,6 +77,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
+import java.util.function.IntFunction;
import java.util.stream.Collectors;
/** Maintain all kinds of heartbeat samples and statistics. */
@@ -104,6 +112,9 @@ public class LoadCache {
private Map<Integer, Set<Integer>> topologyGraph;
private final AtomicBoolean topologyUpdated;
+ private IntFunction<Pair<NodeStatus, String>> persistedNodeStatusReader;
+ private CheckedFunction<UpdateNodeStatusPlan, TSStatus, ConsensusException> nodeStatusWriter;
+
public LoadCache() {
this.nodeCacheMap = new ConcurrentHashMap<>();
this.heartbeatProcessingMap = new ConcurrentHashMap<>();
@@ -116,6 +127,19 @@ public class LoadCache {
this.topologyUpdated = new AtomicBoolean(false);
}
+ public void setNodeStatusPersistence(
+ IntFunction<Pair<NodeStatus, String>> persistedNodeStatusReader,
+ CheckedFunction<UpdateNodeStatusPlan, TSStatus, ConsensusException> nodeStatusWriter) {
+ Objects.requireNonNull(persistedNodeStatusReader);
+ Objects.requireNonNull(nodeStatusWriter);
+ this.persistedNodeStatusReader = persistedNodeStatusReader;
+ this.nodeStatusWriter = nodeStatusWriter;
+ }
+
+ private boolean hasPersistence() {
+ return persistedNodeStatusReader != null && nodeStatusWriter != null;
+ }
+
public void initHeartbeatCache(final IManager configManager) {
initNodeHeartbeatCache(
configManager.getNodeManager().getRegisteredConfigNodes(),
@@ -148,10 +172,13 @@ public class LoadCache {
}
});
// Force set itself and never update
- nodeCacheMap.put(
- ConfigNodeHeartbeatCache.CURRENT_NODE_ID,
+ ConfigNodeHeartbeatCache currentNodeCache =
new ConfigNodeHeartbeatCache(
- CURRENT_NODE_ID, ConfigNodeHeartbeatCache.CURRENT_NODE_STATISTICS));
+ CURRENT_NODE_ID, ConfigNodeHeartbeatCache.CURRENT_NODE_STATISTICS);
+ if (hasPersistence()) {
+ currentNodeCache.initPersistence(null, this::persistNodeStatus);
+ }
+ nodeCacheMap.put(CURRENT_NODE_ID, currentNodeCache);
// Init DataNodeHeartbeatCache
registeredDataNodes.forEach(
@@ -216,24 +243,58 @@ public class LoadCache {
}
/**
- * Create a new NodeHeartbeatCache for the specified Node.
+ * Create a NodeHeartbeatCache for the specified Node if it does not already exist.
*
* @param nodeType The specified NodeType
* @param nodeId The specified NodeId
*/
public void createNodeHeartbeatCache(NodeType nodeType, int nodeId) {
- switch (nodeType) {
- case ConfigNode:
- nodeCacheMap.put(nodeId, new ConfigNodeHeartbeatCache(nodeId));
- break;
- case DataNode:
- nodeCacheMap.put(nodeId, new DataNodeHeartbeatCache(nodeId));
- break;
- case AINode:
- nodeCacheMap.put(nodeId, new AINodeHeartbeatCache(nodeId));
- break;
+ // A recovered registration procedure can repeat this call while a status write is in progress.
+ // Keep the existing cache and its heartbeat processing flag so that pending updates stay
+ // visible.
+ nodeCacheMap.computeIfAbsent(
+ nodeId,
+ id -> {
+ BaseNodeCache cache =
+ switch (nodeType) {
+ case ConfigNode -> new ConfigNodeHeartbeatCache(id);
+ case DataNode -> new DataNodeHeartbeatCache(id);
+ case AINode -> new AINodeHeartbeatCache(id);
+ };
+ if (hasPersistence()) {
+ cache.initPersistence(persistedNodeStatusReader.apply(id), this::persistNodeStatus);
+ }
+ return cache;
+ });
+ heartbeatProcessingMap.putIfAbsent(nodeId, new AtomicBoolean(false));
+ }
+
+ private TSStatus persistNodeStatus(BaseNodeCache cache, NodeStatus status, String reason) {
+ int nodeId = cache.getNodeId();
+ if (nodeCacheMap.get(nodeId) != cache) {
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ }
+ UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(nodeId, status, reason);
+ Pair<NodeStatus, String> persistedStatus =
+ plan.getStatus() == null ? null : new Pair<>(plan.getStatus(), plan.getStatusReason());
+ if (Objects.equals(persistedNodeStatusReader.apply(nodeId), persistedStatus)) {
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+ try {
+ return nodeStatusWriter.apply(plan);
+ } catch (ConsensusException e) {
+ LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e);
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode())
+ .setMessage(e.getMessage());
}
- heartbeatProcessingMap.put(nodeId, new AtomicBoolean(false));
+ }
+
+ public TSStatus trySetNodeStatus(
+ int nodeId, NodeStatus status, String statusReason, boolean force) {
+ BaseNodeCache cache = nodeCacheMap.get(nodeId);
+ return cache == null
+ ? new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
+ : cache.trySetNodeStatus(status, statusReason, force);
}
/**
@@ -269,9 +330,9 @@ public class LoadCache {
* @param sample the latest heartbeat sample
*/
public void cacheAINodeHeartbeatSample(int nodeId, NodeHeartbeatSample sample) {
- nodeCacheMap
- .computeIfAbsent(nodeId, empty -> new AINodeHeartbeatCache(nodeId))
- .cacheHeartbeatSample(sample);
+ // Only cache sample when the corresponding loadCache exists
+ Optional.ofNullable(nodeCacheMap.get(nodeId))
+ .ifPresent(node -> node.cacheHeartbeatSample(sample));
Optional.ofNullable(heartbeatProcessingMap.get(nodeId)).ifPresent(node -> node.set(false));
}
@@ -366,10 +427,13 @@ public class LoadCache {
}
/** Update the NodeStatistics of all Nodes. */
- public void updateNodeStatistics(boolean forceUpdate) {
- nodeCacheMap
- .values()
- .forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(forceUpdate));
+ public boolean updateNodeStatistics() {
+ boolean success = true;
+ for (BaseNodeCache cache : nodeCacheMap.values()) {
+ success &=
+ cache.updateNodeStatistics().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode();
+ }
+ return success;
}
/** Update the RegionGroupStatistics of all RegionGroups. */
@@ -379,9 +443,7 @@ public class LoadCache {
/** Update the ConsensusGroupStatistics of all RegionGroups. */
public void updateConsensusGroupStatistics() {
- consensusGroupCacheMap
- .values()
- .forEach(consensusGroupCache -> consensusGroupCache.updateCurrentStatistics(false));
+ consensusGroupCacheMap.values().forEach(ConsensusGroupCache::updateCurrentStatistics);
}
/**
@@ -506,7 +568,11 @@ public class LoadCache {
}
if ((nodeCache instanceof ConfigNodeHeartbeatCache
|| nodeCache instanceof DataNodeHeartbeatCache)
- && !nodeCache.hasHeartbeatSample()) {
+ && !nodeCache.hasHeartbeatSample()
+ // Stopped/Removing can be restored from persisted records without waiting for a
+ // heartbeat. ReadOnly still needs a heartbeat to confirm that the node is reachable.
+ && nodeCache.getNodeStatus() != NodeStatus.Stopped
+ && nodeCache.getNodeStatus() != NodeStatus.Removing) {
unreadyNodes.add(nodeId);
}
});
@@ -536,25 +602,15 @@ public class LoadCache {
}
/**
- * Safely get the specified Node's current status with reason.
+ * Safely get the specified Node's current status reason.
*
* @param nodeId The specified NodeId
- * @return The specified Node's current status if the nodeCache contains it, Unknown otherwise
- */
- public String getNodeStatusWithReason(int nodeId) {
- return Optional.ofNullable(nodeCacheMap.get(nodeId))
- .map(BaseNodeCache::getNodeStatusWithReason)
- .orElseGet(() -> NodeStatus.Unknown.getStatus() + "(NoHeartbeat)");
- }
-
- /**
- * Get all Node's current status with reason.
- *
- * @return Map<NodeId, NodeStatus with reason>
+ * @return The reason why the Node is in its current status, null if the node has no reason or the
+ * cache doesn't exist
*/
- public Map<Integer, String> getNodeStatusWithReason() {
- return nodeCacheMap.entrySet().stream()
- .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getNodeStatusWithReason()));
+ public String getNodeStatusReason(int nodeId) {
+ BaseNodeCache nodeCache = nodeCacheMap.get(nodeId);
+ return nodeCache == null ? null : nodeCache.getNodeStatusReason();
}
/**
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java
index 7dcacc4fd80..2524673ff56 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/consensus/ConsensusGroupCache.java
@@ -35,8 +35,7 @@ public class ConsensusGroupCache extends AbstractLoadCache {
this.currentStatistics.set(ConsensusGroupStatistics.generateDefaultConsensusGroupStatistics());
}
- @Override
- public synchronized void updateCurrentStatistics(boolean forceUpdate) {
+ public synchronized void updateCurrentStatistics() {
ConsensusGroupHeartbeatSample lastSample;
synchronized (slidingWindow) {
lastSample = (ConsensusGroupHeartbeatSample) getLastSample();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java
index ca15f9cb515..8a8a765a18a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/AINodeHeartbeatCache.java
@@ -19,57 +19,9 @@
package org.apache.iotdb.confignode.manager.load.cache.node;
-import org.apache.iotdb.common.rpc.thrift.TLoadSample;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicReference;
-
public class AINodeHeartbeatCache extends BaseNodeCache {
- private final AtomicReference<TLoadSample> latestLoadSample;
-
public AINodeHeartbeatCache(int aiNodeId) {
super(aiNodeId);
- this.latestLoadSample = new AtomicReference<>(new TLoadSample());
- }
-
- @Override
- public void updateCurrentStatistics(boolean forceUpdate) {
- NodeHeartbeatSample lastSample;
- final List<AbstractHeartbeatSample> heartbeatHistory;
- /* Update Node status */
- NodeStatus status = null;
- String statusReason = null;
- long currentNanoTime = System.nanoTime();
- synchronized (slidingWindow) {
- lastSample = (NodeHeartbeatSample) getLastSample();
- heartbeatHistory = Collections.unmodifiableList(slidingWindow);
- /* Update load sample */
- if (lastSample != null && lastSample.isSetLoadSample()) {
- latestLoadSample.set((lastSample.getLoadSample()));
- }
-
- if (lastSample != null && NodeStatus.Removing.equals(lastSample.getStatus())) {
- status = NodeStatus.Removing;
- } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) {
- /* Failure detector decides that this AINode is UNKNOWN */
- status = NodeStatus.Unknown;
- } else if (lastSample != null) {
- status = lastSample.getStatus();
- statusReason = lastSample.getStatusReason();
- }
- }
-
- long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE;
-
- NodeStatistics newStatistics =
- new NodeStatistics(currentNanoTime, status, statusReason, loadScore);
- if (!currentStatistics.get().equals(newStatistics)) {
- // Update the current NodeStatistics if necessary
- currentStatistics.set(newStatistics);
- }
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java
index 4eeb96344cc..07a4a4877f0 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/BaseNodeCache.java
@@ -19,8 +19,17 @@
package org.apache.iotdb.confignode.manager.load.cache.node;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.AbstractLoadCache;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.external.commons.lang3.function.TriFunction;
+import org.apache.tsfile.utils.Pair;
+
+import java.util.Collections;
+import java.util.List;
/**
* NodeCache caches the NodeHeartbeatSamples of a Node. Update and cache the current statistics of
@@ -30,6 +39,8 @@ public abstract class BaseNodeCache extends AbstractLoadCache {
protected final int nodeId;
+ private TriFunction<BaseNodeCache, NodeStatus, String, TSStatus> nodeStatusPersister;
+
/** Constructor for NodeCache with default NodeStatistics. */
protected BaseNodeCache(int nodeId) {
super();
@@ -41,6 +52,73 @@ public abstract class BaseNodeCache extends AbstractLoadCache {
return nodeId;
}
+ /** Initialize a new cache before adding it to LoadCache's nodeCacheMap. */
+ public void initPersistence(
+ Pair<NodeStatus, String> persistedStatus,
+ TriFunction<BaseNodeCache, NodeStatus, String, TSStatus> nodeStatusPersister) {
+ this.nodeStatusPersister = nodeStatusPersister;
+ if (persistedStatus != null) {
+ currentStatistics.set(
+ new NodeStatistics(
+ System.nanoTime(), persistedStatus.left, persistedStatus.right, Long.MAX_VALUE));
+ }
+ }
+
+ public TSStatus updateNodeStatistics() {
+ synchronized (slidingWindow) {
+ return applyNodeStatistics(calculateCurrentStatistics(), false);
+ }
+ }
+
+ /**
+ * Try to set the requested status and record a sample for later statistics refreshes. Transition
+ * rules may retain the previous status even when this method returns success. A persistence
+ * failure is returned without discarding the new statistics.
+ */
+ public TSStatus trySetNodeStatus(NodeStatus status, String statusReason, boolean force) {
+ synchronized (slidingWindow) {
+ NodeHeartbeatSample sample = new NodeHeartbeatSample(status, statusReason);
+ cacheHeartbeatSample(sample);
+ return applyNodeStatistics(
+ new NodeStatistics(
+ sample.getSampleLogicalTimestamp(),
+ status,
+ sample.getStatusReason(),
+ NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE),
+ force);
+ }
+ }
+
+ private TSStatus applyNodeStatistics(NodeStatistics newStats, boolean force) {
+ newStats = NodeStatistics.transition((NodeStatistics) currentStatistics.get(), newStats, force);
+ TSStatus result =
+ nodeStatusPersister == null
+ ? new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode())
+ : nodeStatusPersister.apply(this, newStats.getStatus(), newStats.getStatusReason());
+ currentStatistics.set(newStats);
+ return result;
+ }
+
+ /** Called with the slidingWindow lock held through calculation, persistence and publication. */
+ protected NodeStatistics calculateCurrentStatistics() {
+ NodeStatus status;
+ String statusReason = null;
+ long currentNanoTime = System.nanoTime();
+ NodeHeartbeatSample lastSample = (NodeHeartbeatSample) getLastSample();
+ List<AbstractHeartbeatSample> heartbeatHistory = Collections.unmodifiableList(slidingWindow);
+ if (lastSample == null || !failureDetector.isAvailable(nodeId, heartbeatHistory)) {
+ status = NodeStatus.Unknown;
+ } else {
+ status = lastSample.getStatus();
+ statusReason = lastSample.getStatusReason();
+ }
+ return new NodeStatistics(
+ currentNanoTime,
+ status,
+ statusReason,
+ NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE);
+ }
+
/**
* TODO: The loadScore of each Node will be changed to Double
*
@@ -60,10 +138,8 @@ public abstract class BaseNodeCache extends AbstractLoadCache {
/**
* @return The reason why lead to current NodeStatus.
*/
- public String getNodeStatusWithReason() {
+ public String getNodeStatusReason() {
NodeStatistics statistics = (NodeStatistics) this.currentStatistics.get();
- return statistics.getStatusReason() == null
- ? statistics.getStatus().getStatus()
- : statistics.getStatus().getStatus() + "(" + statistics.getStatusReason() + ")";
+ return statistics.getStatusReason();
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java
index 4d675e1e8a4..ae0b20e3a62 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/ConfigNodeHeartbeatCache.java
@@ -21,10 +21,6 @@ package org.apache.iotdb.confignode.manager.load.cache.node;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
-import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample;
-
-import java.util.Collections;
-import java.util.List;
/** Heartbeat cache for cluster ConfigNodes. */
public class ConfigNodeHeartbeatCache extends BaseNodeCache {
@@ -48,37 +44,10 @@ public class ConfigNodeHeartbeatCache extends BaseNodeCache {
}
@Override
- public synchronized void updateCurrentStatistics(boolean forceUpdate) {
- // Skip itself and the Removing status can not be updated
- if (nodeId == CURRENT_NODE_ID || NodeStatus.Removing.equals(getNodeStatus())) {
- return;
- }
-
- NodeHeartbeatSample lastSample;
- // Update Node status
- NodeStatus status;
- long currentNanoTime = System.nanoTime();
- final List<AbstractHeartbeatSample> heartbeatHistory;
- synchronized (slidingWindow) {
- lastSample = (NodeHeartbeatSample) getLastSample();
- heartbeatHistory = Collections.unmodifiableList(slidingWindow);
-
- if (lastSample == null) {
- /* First heartbeat not received from this ConfigNode, status is UNKNOWN */
- status = NodeStatus.Unknown;
- } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) {
- /* Failure detector decides that this ConfigNode is UNKNOWN */
- status = NodeStatus.Unknown;
- } else {
- status = lastSample.getStatus();
- }
+ protected NodeStatistics calculateCurrentStatistics() {
+ if (nodeId == CURRENT_NODE_ID) {
+ return (NodeStatistics) currentStatistics.get();
}
-
- /* Update loadScore */
- // Only consider Running ConfigNode as available currently
- // TODO: Construct load score module
- long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE;
-
- currentStatistics.set(new NodeStatistics(currentNanoTime, status, null, loadScore));
+ return super.calculateCurrentStatistics();
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java
index e3d6d15c835..97451c36579 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/DataNodeHeartbeatCache.java
@@ -20,22 +20,12 @@
package org.apache.iotdb.confignode.manager.load.cache.node;
import org.apache.iotdb.common.rpc.thrift.TLoadSample;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.confignode.i18n.ManagerMessages;
-import org.apache.iotdb.confignode.manager.load.cache.AbstractHeartbeatSample;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
/** Heartbeat cache for cluster DataNodes. */
public class DataNodeHeartbeatCache extends BaseNodeCache {
- private static final Logger LOGGER = LoggerFactory.getLogger(DataNodeHeartbeatCache.class);
-
// TODO: The load sample may be moved into NodeStatistics in the future
private final AtomicReference<TLoadSample> latestLoadSample;
@@ -46,49 +36,12 @@ public class DataNodeHeartbeatCache extends BaseNodeCache {
}
@Override
- public synchronized void updateCurrentStatistics(boolean forceUpdate) {
- // The Removing status can not be updated
- if (!forceUpdate && NodeStatus.Removing.equals(getNodeStatus())) {
- return;
- }
-
- NodeHeartbeatSample lastSample;
- final List<AbstractHeartbeatSample> heartbeatHistory;
- /* Update Node status */
- NodeStatus status;
- String statusReason = null;
- long currentNanoTime = System.nanoTime();
- synchronized (slidingWindow) {
- lastSample = (NodeHeartbeatSample) getLastSample();
- heartbeatHistory = Collections.unmodifiableList(slidingWindow);
- /* Update load sample */
- if (lastSample != null && lastSample.isSetLoadSample()) {
- latestLoadSample.set(lastSample.getLoadSample());
- }
-
- if (lastSample == null) {
- /* First heartbeat not received from this DataNode, status is UNKNOWN */
- status = NodeStatus.Unknown;
- } else if (!failureDetector.isAvailable(nodeId, heartbeatHistory)) {
- /* Failure detector decides that this DataNode is UNKNOWN */
- status = NodeStatus.Unknown;
- } else {
- status = lastSample.getStatus();
- statusReason = lastSample.getStatusReason();
- }
- }
-
- /* Update loadScore */
- // Only consider Running DataNode as available currently
- // TODO: Construct load score module
- long loadScore = NodeStatus.isNormalStatus(status) ? 0 : Long.MAX_VALUE;
-
- currentStatistics.set(new NodeStatistics(currentNanoTime, status, statusReason, loadScore));
-
- if (forceUpdate) {
- LOGGER.debug(
- ManagerMessages.FORCE_UPDATE_NODECACHE_STATUS_CURRENTNANOTIME, status, currentNanoTime);
+ protected NodeStatistics calculateCurrentStatistics() {
+ NodeHeartbeatSample lastSample = (NodeHeartbeatSample) getLastSample();
+ if (lastSample != null && lastSample.isSetLoadSample()) {
+ latestLoadSample.set(lastSample.getLoadSample());
}
+ return super.calculateCurrentStatistics();
}
public double getFreeDiskSpace() {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java
index 8217593f5d6..6d9be6080af 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeHeartbeatSample.java
@@ -37,9 +37,14 @@ public class NodeHeartbeatSample extends AbstractHeartbeatSample {
/** Constructor for generating default sample with specified status */
public NodeHeartbeatSample(NodeStatus status) {
+ this(status, null);
+ }
+
+ /** Constructor for an explicit management status with its reason. */
+ public NodeHeartbeatSample(NodeStatus status, String statusReason) {
super(System.nanoTime());
this.status = status;
- this.statusReason = null;
+ this.statusReason = statusReason;
this.loadSample = null;
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java
index 12c554d51e8..04ed6c50289 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/node/NodeStatistics.java
@@ -55,6 +55,23 @@ public class NodeStatistics extends AbstractStatistics {
return new NodeStatistics(Long.MIN_VALUE, NodeStatus.Unknown, null, Long.MAX_VALUE);
}
+ /**
+ * Resolves the requested statistics through {@link NodeStatus#transition}. If the requested
+ * status is accepted, its reason and load score are kept. Otherwise, the final status must not
+ * carry the rejected status's reason or load score, e.g. Removing must not inherit ReadOnly's
+ * DiskFull reason. The requested statistics timestamp is retained in both cases.
+ */
+ public static NodeStatistics transition(
+ NodeStatistics previous, NodeStatistics requested, boolean force) {
+ NodeStatus status = NodeStatus.transition(previous.status, requested.status, force);
+ if (status == requested.status) {
+ return requested;
+ }
+ // if the transition output is not the same as requested, it could be Stopped or Removing status
+ // both do not have status reason, set reason to null.
+ return new NodeStatistics(requested.getStatisticsNanoTimestamp(), status, null, Long.MAX_VALUE);
+ }
+
public NodeStatus getStatus() {
return status;
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java
index a374c923f24..23a777b7fb6 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionCache.java
@@ -42,8 +42,7 @@ public class RegionCache extends AbstractLoadCache {
this.currentStatistics.set(RegionStatistics.generateDefaultRegionStatistics());
}
- @Override
- public synchronized void updateCurrentStatistics(boolean forceUpdate) {
+ public synchronized void updateCurrentStatistics() {
RegionHeartbeatSample lastSample;
List<AbstractHeartbeatSample> history;
synchronized (slidingWindow) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java
index 2cdecc757fd..e6d65bd5003 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/cache/region/RegionGroupCache.java
@@ -101,7 +101,7 @@ public class RegionGroupCache {
* slidingWindow.
*/
public synchronized void updateCurrentStatistics() {
- regionCacheMap.values().forEach(regionCache -> regionCache.updateCurrentStatistics(false));
+ regionCacheMap.values().forEach(RegionCache::updateCurrentStatistics);
Map<Integer, RegionStatistics> regionStatisticsMap =
regionCacheMap.entrySet().stream()
.collect(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java
index 620276767f0..8d61102ac59 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/service/StatisticsService.java
@@ -98,7 +98,7 @@ public class StatisticsService {
}
private void updateLoadStatistics() {
- loadCache.updateNodeStatistics(false);
+ loadCache.updateNodeStatistics();
loadCache.updateRegionGroupStatistics();
loadCache.updateConsensusGroupStatistics();
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
index 64d4cf741e1..c6790da284a 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
@@ -75,6 +75,7 @@ import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
import org.apache.iotdb.confignode.manager.externalservice.ExternalServiceManager;
import org.apache.iotdb.confignode.manager.load.LoadManager;
import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
import org.apache.iotdb.confignode.manager.partition.PartitionMetrics;
import org.apache.iotdb.confignode.manager.pipe.coordinator.PipeManager;
@@ -151,6 +152,10 @@ public class NodeManager {
this.removeConfigNodeLock = new ReentrantLock();
}
+ public NodeInfo getNodeInfo() {
+ return nodeInfo;
+ }
+
/**
* Get system configurations.
*
@@ -501,7 +506,7 @@ public class NodeManager {
for (TAINodeConfiguration aiNodeConfiguration : getRegisteredAINodes()) {
TAINodeInfo aiNodeInfo = new TAINodeInfo();
aiNodeInfo.setAiNodeId(aiNodeConfiguration.getLocation().getAiNodeId());
- aiNodeInfo.setStatus(getLoadManager().getNodeStatusWithReason(aiNodeInfo.getAiNodeId()));
+ aiNodeInfo.setStatus(getLoadManager().getNodeStatus(aiNodeInfo.getAiNodeId()).getStatus());
aiNodeInfo.setInternalAddress(aiNodeConfiguration.getLocation().getInternalEndPoint().ip);
aiNodeInfo.setInternalPort(aiNodeConfiguration.getLocation().getInternalEndPoint().port);
aiNodeInfoList.add(aiNodeInfo);
@@ -714,13 +719,20 @@ public class NodeManager {
public List<TDataNodeInfo> getRegisteredDataNodeInfoList() {
List<TDataNodeInfo> dataNodeInfoList = new ArrayList<>();
List<TDataNodeConfiguration> registeredDataNodes = this.getRegisteredDataNodes();
+ Map<Integer, NodeStatistics> nodeStatisticsSnapshot =
+ getLoadManager().getNodeStatisticsSnapshot();
if (registeredDataNodes != null) {
registeredDataNodes.forEach(
registeredDataNode -> {
TDataNodeInfo dataNodeInfo = new TDataNodeInfo();
int dataNodeId = registeredDataNode.getLocation().getDataNodeId();
dataNodeInfo.setDataNodeId(dataNodeId);
- dataNodeInfo.setStatus(getLoadManager().getNodeStatusWithReason(dataNodeId));
+ NodeStatistics statistics = nodeStatisticsSnapshot.get(dataNodeId);
+ dataNodeInfo.setStatus(
+ statistics == null
+ ? NodeStatus.Unknown.getStatus()
+ : statistics.getStatus().getStatus());
+ dataNodeInfo.setStatusReason(statistics == null ? null : statistics.getStatusReason());
dataNodeInfo.setRpcAddresss(
registeredDataNode.getLocation().getClientRpcEndPoint().getIp());
dataNodeInfo.setRpcPort(
@@ -850,13 +862,20 @@ public class NodeManager {
public List<TConfigNodeInfo> getRegisteredConfigNodeInfoList() {
List<TConfigNodeInfo> configNodeInfoList = new ArrayList<>();
List<TConfigNodeLocation> registeredConfigNodes = this.getRegisteredConfigNodes();
+ Map<Integer, NodeStatistics> nodeStatisticsSnapshot =
+ getLoadManager().getNodeStatisticsSnapshot();
if (registeredConfigNodes != null) {
registeredConfigNodes.forEach(
configNodeLocation -> {
TConfigNodeInfo info = new TConfigNodeInfo();
int configNodeId = configNodeLocation.getConfigNodeId();
info.setConfigNodeId(configNodeId);
- info.setStatus(getLoadManager().getNodeStatusWithReason(configNodeId));
+ NodeStatistics statistics = nodeStatisticsSnapshot.get(configNodeId);
+ info.setStatus(
+ statistics == null
+ ? NodeStatus.Unknown.getStatus()
+ : statistics.getStatus().getStatus());
+ info.setStatusReason(statistics == null ? null : statistics.getStatusReason());
info.setInternalAddress(configNodeLocation.getInternalEndPoint().getIp());
info.setInternalPort(configNodeLocation.getInternalEndPoint().getPort());
info.setRoleType(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java
index dd8fe2e70cf..cfe16f29268 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinator.java
@@ -84,8 +84,7 @@ public class SubscriptionRuntimeCoordinator {
final NodeStatus oldStatus = getNodeStatus(pair.getLeft());
final NodeStatus newStatus = getNodeStatus(pair.getRight());
return oldStatus != newStatus
- && (isRuntimeSensitiveStatus(oldStatus)
- || isRuntimeSensitiveStatus(newStatus));
+ && (oldStatus.mayBeOffline() || newStatus.mayBeOffline());
});
if (!shouldRefreshRuntime) {
return;
@@ -153,8 +152,4 @@ public class SubscriptionRuntimeCoordinator {
private static NodeStatus getNodeStatus(final NodeStatistics statistics) {
return statistics == null ? NodeStatus.Unknown : statistics.getStatus();
}
-
- private static boolean isRuntimeSensitiveStatus(final NodeStatus status) {
- return status == NodeStatus.Unknown || status == NodeStatus.Removing;
- }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
index a09c283a48d..8919308a159 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/executor/ConfigPlanExecutor.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorPlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan;
@@ -523,6 +524,8 @@ public class ConfigPlanExecutor {
return nodeInfo.removeConfigNode((RemoveConfigNodePlan) physicalPlan);
case UpdateVersionInfo:
return nodeInfo.updateVersionInfo((UpdateVersionInfoPlan) physicalPlan);
+ case UpdateNodeStatus:
+ return nodeInfo.applyNodeStatusPlan((UpdateNodeStatusPlan) physicalPlan);
case UpdateClusterId:
return clusterInfo.updateClusterId((UpdateClusterIdPlan) physicalPlan);
case CreateFunction:
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
index d98a05c8a8c..098a1545011 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
@@ -36,6 +36,7 @@ import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePl
import org.apache.iotdb.confignode.consensus.request.write.ainode.UpdateAINodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan;
import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
@@ -50,6 +51,7 @@ import org.apache.thrift.TException;
import org.apache.thrift.protocol.TBinaryProtocol;
import org.apache.thrift.protocol.TProtocol;
import org.apache.thrift.transport.TIOStreamTransport;
+import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,6 +111,10 @@ public class NodeInfo implements SnapshotProcessor {
private final Map<Integer, TNodeVersionInfo> nodeVersionInfo;
private final ReentrantReadWriteLock versionInfoReadWriteLock;
+ // Map<NodeId, Pair<status, reason>>. A null reason still belongs to a stored status.
+ private final Map<Integer, Pair<NodeStatus, String>> persistedNodeStatuses;
+ private final ReentrantReadWriteLock persistedNodeStatusReadWriteLock;
+
private static final String SNAPSHOT_FILENAME = "node_info.bin";
public NodeInfo() {
@@ -123,6 +129,9 @@ public class NodeInfo implements SnapshotProcessor {
this.nodeVersionInfo = new ConcurrentHashMap<>();
this.versionInfoReadWriteLock = new ReentrantReadWriteLock();
+
+ this.persistedNodeStatuses = new HashMap<>();
+ this.persistedNodeStatusReadWriteLock = new ReentrantReadWriteLock();
}
/**
@@ -177,15 +186,18 @@ public class NodeInfo implements SnapshotProcessor {
dataNodeInfoReadWriteLock.writeLock().lock();
versionInfoReadWriteLock.writeLock().lock();
+ persistedNodeStatusReadWriteLock.writeLock().lock();
try {
req.getDataNodeLocations()
.forEach(
removeDataNodes -> {
registeredDataNodes.remove(removeDataNodes.getDataNodeId());
nodeVersionInfo.remove(removeDataNodes.getDataNodeId());
+ persistedNodeStatuses.remove(removeDataNodes.getDataNodeId());
LOGGER.info(ConfigNodeMessages.REMOVED_THE_DATANODE_FROM_CLUSTER, removeDataNodes);
});
} finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
versionInfoReadWriteLock.writeLock().unlock();
dataNodeInfoReadWriteLock.writeLock().unlock();
}
@@ -461,9 +473,11 @@ public class NodeInfo implements SnapshotProcessor {
TSStatus status = new TSStatus();
configNodeInfoReadWriteLock.writeLock().lock();
versionInfoReadWriteLock.writeLock().lock();
+ persistedNodeStatusReadWriteLock.writeLock().lock();
try {
registeredConfigNodes.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId());
nodeVersionInfo.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId());
+ persistedNodeStatuses.remove(removeConfigNodePlan.getConfigNodeLocation().getConfigNodeId());
SystemPropertiesUtils.storeConfigNodeList(new ArrayList<>(registeredConfigNodes.values()));
LOGGER.info(
ConfigNodeMessages.SUCCESSFULLY_REMOVE_CONFIGNODE_CURRENT_CONFIGNODEGROUP,
@@ -477,6 +491,7 @@ public class NodeInfo implements SnapshotProcessor {
ConfigNodeMessages
.MESSAGE_REMOVE_CONFIGNODE_FAILED_BECAUSE_CURRENT_CONFIGNODE_CAN_T_STORE_CONFIGNODE_8AB3BCB4);
} finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
versionInfoReadWriteLock.writeLock().unlock();
configNodeInfoReadWriteLock.writeLock().unlock();
}
@@ -525,9 +540,9 @@ public class NodeInfo implements SnapshotProcessor {
}
/**
- * Persist Information about remove dataNode.
+ * Persist information about removing an AINode.
*
- * @param req RemoveDataNodePlan
+ * @param req RemoveAINodePlan
* @return {@link TSStatus}
*/
public TSStatus removeAINode(RemoveAINodePlan req) {
@@ -538,12 +553,15 @@ public class NodeInfo implements SnapshotProcessor {
aiNodeInfoReadWriteLock.writeLock().lock();
versionInfoReadWriteLock.writeLock().lock();
+ persistedNodeStatusReadWriteLock.writeLock().lock();
TAINodeLocation removedAINode = req.getAINodeLocation();
try {
registeredAINodes.remove(removedAINode.getAiNodeId());
nodeVersionInfo.remove(removedAINode.getAiNodeId());
+ persistedNodeStatuses.remove(removedAINode.getAiNodeId());
LOGGER.info(ConfigNodeMessages.REMOVED_THE_AINODE_FROM_CLUSTER, removedAINode);
} finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
versionInfoReadWriteLock.writeLock().unlock();
aiNodeInfoReadWriteLock.writeLock().unlock();
}
@@ -633,6 +651,52 @@ public class NodeInfo implements SnapshotProcessor {
return nextNodeId.incrementAndGet();
}
+ public TSStatus applyNodeStatusPlan(UpdateNodeStatusPlan plan) {
+ // Registration maps support concurrent reads. Node removal also holds this lock, so it cannot
+ // interleave with the existence check and status update.
+ persistedNodeStatusReadWriteLock.writeLock().lock();
+ try {
+ int nodeId = plan.getNodeId();
+
+ if (registeredConfigNodes.containsKey(nodeId)
+ || registeredDataNodes.containsKey(nodeId)
+ || registeredAINodes.containsKey(nodeId)) {
+ if (plan.getStatus() == null) {
+ persistedNodeStatuses.remove(nodeId);
+ } else {
+ persistedNodeStatuses.put(nodeId, new Pair<>(plan.getStatus(), plan.getStatusReason()));
+ }
+ }
+
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ } finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
+ }
+ }
+
+ public Pair<NodeStatus, String> getPersistedNodeStatus(int nodeId) {
+ persistedNodeStatusReadWriteLock.readLock().lock();
+ try {
+ Pair<NodeStatus, String> status = persistedNodeStatuses.get(nodeId);
+ // Pair is mutable, so callers must not receive the value stored in NodeInfo.
+ return status == null ? null : new Pair<>(status.left, status.right);
+ } finally {
+ persistedNodeStatusReadWriteLock.readLock().unlock();
+ }
+ }
+
+ public Map<Integer, Pair<NodeStatus, String>> getPersistedNodeStatuses() {
+ persistedNodeStatusReadWriteLock.readLock().lock();
+ try {
+ Map<Integer, Pair<NodeStatus, String>> result = new HashMap<>();
+ persistedNodeStatuses.forEach(
+ (nodeId, status) -> result.put(nodeId, new Pair<>(status.left, status.right)));
+ return result;
+ } finally {
+ persistedNodeStatusReadWriteLock.readLock().unlock();
+ }
+ }
+
@Override
public boolean processTakeSnapshot(File snapshotDir) throws IOException, TException {
File snapshotFile = new File(snapshotDir, SNAPSHOT_FILENAME);
@@ -648,6 +712,7 @@ public class NodeInfo implements SnapshotProcessor {
dataNodeInfoReadWriteLock.readLock().lock();
aiNodeInfoReadWriteLock.readLock().lock();
versionInfoReadWriteLock.readLock().lock();
+ persistedNodeStatusReadWriteLock.readLock().lock();
try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile);
TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileOutputStream)) {
@@ -663,6 +728,8 @@ public class NodeInfo implements SnapshotProcessor {
serializeVersionInfo(fileOutputStream);
+ serializePersistedNodeStatuses(fileOutputStream);
+
tioStreamTransport.flush();
fileOutputStream.getFD().sync();
@@ -671,6 +738,7 @@ public class NodeInfo implements SnapshotProcessor {
return tmpFile.renameTo(snapshotFile);
} finally {
+ persistedNodeStatusReadWriteLock.readLock().unlock();
versionInfoReadWriteLock.readLock().unlock();
aiNodeInfoReadWriteLock.readLock().unlock();
dataNodeInfoReadWriteLock.readLock().unlock();
@@ -724,6 +792,15 @@ public class NodeInfo implements SnapshotProcessor {
}
}
+ private void serializePersistedNodeStatuses(OutputStream outputStream) throws IOException {
+ ReadWriteIOUtils.write(persistedNodeStatuses.size(), outputStream);
+ for (Entry<Integer, Pair<NodeStatus, String>> entry : persistedNodeStatuses.entrySet()) {
+ ReadWriteIOUtils.write(entry.getKey(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue().left.getStatus(), outputStream);
+ ReadWriteIOUtils.write(entry.getValue().right, outputStream);
+ }
+ }
+
@Override
public void processLoadSnapshot(File snapshotDir) throws IOException, TException {
@@ -739,6 +816,7 @@ public class NodeInfo implements SnapshotProcessor {
dataNodeInfoReadWriteLock.writeLock().lock();
aiNodeInfoReadWriteLock.writeLock().lock();
versionInfoReadWriteLock.writeLock().lock();
+ persistedNodeStatusReadWriteLock.writeLock().lock();
try (ByteArrayInputStream inputStream =
new ByteArrayInputStream(Files.readAllBytes(snapshotFile.toPath()));
@@ -759,7 +837,9 @@ public class NodeInfo implements SnapshotProcessor {
deserializeBuildInfo(inputStream);
+ deserializePersistedNodeStatuses(inputStream);
} finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
versionInfoReadWriteLock.writeLock().unlock();
aiNodeInfoReadWriteLock.writeLock().unlock();
dataNodeInfoReadWriteLock.writeLock().unlock();
@@ -830,6 +910,20 @@ public class NodeInfo implements SnapshotProcessor {
}
}
+ private void deserializePersistedNodeStatuses(InputStream inputStream) throws IOException {
+ if (inputStream.available() == 0) {
+ return;
+ }
+
+ int size = ReadWriteIOUtils.readInt(inputStream);
+ for (int i = 0; i < size; i++) {
+ int nodeId = ReadWriteIOUtils.readInt(inputStream);
+ NodeStatus status = NodeStatus.parse(ReadWriteIOUtils.readString(inputStream));
+ String statusReason = ReadWriteIOUtils.readString(inputStream);
+ persistedNodeStatuses.put(nodeId, new Pair<>(status, statusReason));
+ }
+ }
+
public static int getMinimumDataNode() {
return MINIMUM_DATANODE;
}
@@ -840,6 +934,13 @@ public class NodeInfo implements SnapshotProcessor {
registeredConfigNodes.clear();
registeredAINodes.clear();
nodeVersionInfo.clear();
+
+ persistedNodeStatusReadWriteLock.writeLock().lock();
+ try {
+ persistedNodeStatuses.clear();
+ } finally {
+ persistedNodeStatusReadWriteLock.writeLock().unlock();
+ }
}
@Override
@@ -855,11 +956,17 @@ public class NodeInfo implements SnapshotProcessor {
&& nextNodeId.get() == nodeInfo.nextNodeId.get()
&& registeredDataNodes.equals(nodeInfo.registeredDataNodes)
&& registeredAINodes.equals(nodeInfo.registeredAINodes)
- && nodeVersionInfo.equals(nodeInfo.nodeVersionInfo);
+ && nodeVersionInfo.equals(nodeInfo.nodeVersionInfo)
+ && getPersistedNodeStatuses().equals(nodeInfo.getPersistedNodeStatuses());
}
@Override
public int hashCode() {
- return Objects.hash(registeredConfigNodes, nextNodeId, registeredDataNodes, nodeVersionInfo);
+ return Objects.hash(
+ registeredConfigNodes,
+ nextNodeId,
+ registeredDataNodes,
+ nodeVersionInfo,
+ getPersistedNodeStatuses());
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index 6f2f90b7392..d1cfe162467 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -1032,10 +1032,9 @@ public class ConfigNodeProcedureEnv {
return responseMap;
}
- private boolean isRuntimeActiveWriterNode(final int dataNodeId) {
- return dataNodeId >= 0
- && getLoadManager().getNodeStatus(dataNodeId) != NodeStatus.Unknown
- && getLoadManager().getNodeStatus(dataNodeId) != NodeStatus.Removing;
+ boolean isRuntimeActiveWriterNode(final int dataNodeId) {
+ final NodeStatus nodeStatus = getLoadManager().getNodeStatus(dataNodeId);
+ return dataNodeId >= 0 && !nodeStatus.mayBeOffline();
}
private static Map<Integer, TPushPipeMetaResp> sendPipeMetaRequest(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
index 22c65e1798b..f378ef13218 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandler.java
@@ -310,7 +310,7 @@ public class RegionMaintainHandler {
new TMaintainPeerReq(regionId, originalDataNode, procedureId);
final NodeStatus nodeStatus = getDataNodeStatus(originalDataNode.getDataNodeId());
- final boolean useFullRetry = !NodeStatus.Unknown.equals(nodeStatus);
+ final boolean useFullRetry = !nodeStatus.isOffline();
if (!useFullRetry) {
LOGGER.info(
ProcedureMessages.DATANODE_IS_SUBMIT_DELETE_OLD_REGION_PEER_WITH_A_SINGLE,
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java
index 9455d34a0dd..5404b92060d 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java
@@ -26,7 +26,6 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.cluster.RegionStatus;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.NodeUrlUtils;
@@ -43,7 +42,6 @@ import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.manager.lease.DataNodeContactTracker;
import org.apache.iotdb.confignode.manager.load.balancer.region.GreedyCopySetRegionGroupAllocator;
import org.apache.iotdb.confignode.manager.load.balancer.region.IRegionGroupAllocator;
-import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
import org.apache.iotdb.confignode.manager.partition.PartitionMetrics;
import org.apache.iotdb.confignode.persistence.node.NodeInfo;
@@ -55,6 +53,7 @@ import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -114,9 +113,11 @@ public class RemoveDataNodeHandler {
(int)
removedDataNodes.stream()
.filter(
- x ->
- configManager.getLoadManager().getNodeStatus(x.getDataNodeId())
- != NodeStatus.Unknown)
+ node ->
+ !configManager
+ .getLoadManager()
+ .getNodeStatus(node.getDataNodeId())
+ .isOffline())
.count();
return availableDatanodeSize - removedDataNodeSize >= NodeInfo.getMinimumDataNode();
@@ -132,7 +133,8 @@ public class RemoveDataNodeHandler {
* Removing, Running, etc.)
*/
public void changeDataNodeStatus(
- List<TDataNodeLocation> removedDataNodes, Map<Integer, NodeStatus> nodeStatusMap) {
+ List<TDataNodeLocation> removedDataNodes, Map<Integer, NodeStatus> nodeStatusMap)
+ throws IOException {
LOGGER.info(
ProcedureMessages.BEGIN_TO_CHANGE_DATANODE_STATUS_NODESTATUSMAP,
REMOVE_DATANODE_PROCESS,
@@ -150,27 +152,48 @@ public class RemoveDataNodeHandler {
CnToDnInternalServiceAsyncRequestManager.getInstance()
.sendAsyncRequestWithRetry(changeDataNodeStatusContext);
+ StringBuilder statusChangeFailures = new StringBuilder();
for (Map.Entry<Integer, TSStatus> entry :
changeDataNodeStatusContext.getResponseMap().entrySet()) {
int dataNodeId = entry.getKey();
NodeStatus nodeStatus = nodeStatusMap.get(dataNodeId);
- RegionStatus regionStatus = RegionStatus.valueOf(nodeStatus.getStatus());
- if (!isSucceed(entry.getValue())) {
+ // Removing or restoring an offline status does not require a DataNode acknowledgement.
+ // Persistence is still required below, even if the RPC failed.
+ if (!isSucceed(entry.getValue()) && !nodeStatus.mayBeOffline()) {
LOGGER.error(
ProcedureMessages.FAILED_TO_CHANGE_DATANODE_STATUS_DATANODEID_NODESTATUS,
REMOVE_DATANODE_PROCESS,
dataNodeId,
nodeStatus);
+ statusChangeFailures.append(
+ String.format(
+ ProcedureMessages
+ .EXCEPTION_DATANODE_SETSYSTEMSTATUS_RPC_FAILED_ARG_ARG_ARG_7E8E5F9F,
+ dataNodeId,
+ nodeStatus,
+ entry.getValue()));
continue;
}
// Force updating NodeStatus
long currentTime = System.nanoTime();
- configManager
- .getLoadManager()
- .forceUpdateNodeCache(
- NodeType.DataNode, dataNodeId, new NodeHeartbeatSample(currentTime, nodeStatus));
+ // Match the ReadOnly reason assigned by the DataNode's SET_SYSTEM_STATUS RPC.
+ String statusReason = nodeStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null;
+ TSStatus status =
+ configManager
+ .getLoadManager()
+ .trySetNodeStatus(dataNodeId, nodeStatus, statusReason, true);
+ if (!isSucceed(status)) {
+ // Do not advance the persisted procedure state before its node status is durable.
+ statusChangeFailures.append(
+ String.format(
+ ProcedureMessages.EXCEPTION_CONFIGNODE_TRYSETNODESTATUS_FAILED_ARG_ARG_ARG_C9BA78F4,
+ dataNodeId,
+ nodeStatus,
+ status));
+ continue;
+ }
LOGGER.info(
ProcedureMessages.FORCE_UPDATE_NODECACHE_DATANODEID_NODESTATUS_CURRENTTIME,
@@ -180,6 +203,7 @@ public class RemoveDataNodeHandler {
currentTime);
// Force update RegionStatus
+ RegionStatus regionStatus = RegionStatus.fromNodeStatus(nodeStatus);
if (regionStatus != RegionStatus.Removing) {
Map<TConsensusGroupId, Map<Integer, RegionHeartbeatSample>> heartbeatSampleMap =
new TreeMap<>();
@@ -195,6 +219,9 @@ public class RemoveDataNodeHandler {
configManager.getLoadManager().forceUpdateRegionGroupCache(heartbeatSampleMap);
}
}
+ if (statusChangeFailures.length() > 0) {
+ throw new IOException(statusChangeFailures.toString());
+ }
}
/**
@@ -234,7 +261,7 @@ public class RemoveDataNodeHandler {
final List<TDataNodeConfiguration> availableDataNodes =
configManager
.getNodeManager()
- .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.Unknown)
+ .filterDataNodeThroughStatus(NodeStatus::isRegionCandidate)
.stream()
.filter(node -> !removedDataNodesSet.contains(node.getLocation().getDataNodeId()))
.collect(Collectors.toList());
@@ -440,12 +467,18 @@ public class RemoveDataNodeHandler {
*
* @param removedDataNodes the list of DataNodeLocations to be removed
*/
- public void removeDataNodePersistence(List<TDataNodeLocation> removedDataNodes) {
+ public void removeDataNodePersistence(List<TDataNodeLocation> removedDataNodes)
+ throws IOException {
// Remove consensus record
try {
- configManager.getConsensusManager().write(new RemoveDataNodePlan(removedDataNodes));
+ TSStatus status =
+ configManager.getConsensusManager().write(new RemoveDataNodePlan(removedDataNodes));
+ if (!isSucceed(status)) {
+ throw new IOException(status.toString());
+ }
} catch (ConsensusException e) {
LOGGER.warn(ConfigNodeMessages.FAILED_IN_THE_WRITE_API_EXECUTING_THE_CONSENSUS_LAYER_DUE, e);
+ throw new IOException(e);
}
// Adjust maxRegionGroupNum
@@ -596,9 +629,11 @@ public class RemoveDataNodeHandler {
(int)
removeDataNodePlan.getDataNodeLocations().stream()
.filter(
- x ->
- configManager.getLoadManager().getNodeStatus(x.getDataNodeId())
- != NodeStatus.Unknown)
+ node ->
+ !configManager
+ .getLoadManager()
+ .getNodeStatus(node.getDataNodeId())
+ .isOffline())
.count();
if (availableDatanodeSize - removedDataNodeSize < NodeInfo.getMinimumDataNode()) {
status.setCode(TSStatusCode.NO_ENOUGH_DATANODE.getStatusCode());
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java
index 95c320ef878..23b041c2587 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveAINodeProcedure.java
@@ -98,6 +98,7 @@ public class RemoveAINodeProcedure extends AbstractNodeProcedure<RemoveAINodeSta
removedAINode,
response.getMessage()));
}
+ env.getConfigManager().getLoadManager().removeNodeCache(removedAINode.getAiNodeId());
return Flow.NO_MORE_STATE;
default:
throw new UnsupportedOperationException(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java
index 400d8f8facd..ff63a8531fb 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedure.java
@@ -120,6 +120,8 @@ public class RemoveDataNodesProcedure extends AbstractNodeProcedure<RemoveDataNo
removedDataNodes);
return Flow.NO_MORE_STATE;
}
+ // Retry preparation separately: changing nodes to Removing affects the capacity check.
+ break;
case REMOVE_DATA_NODE_PREPARE:
Map<Integer, NodeStatus> removedNodeStatusMap = new HashMap<>();
removedDataNodes.forEach(
@@ -159,6 +161,8 @@ public class RemoveDataNodesProcedure extends AbstractNodeProcedure<RemoveDataNo
e);
if (getCycles() > RETRY_THRESHOLD) {
setFailure(new ProcedureException(ProcedureMessages.STATE_STUCK_AT + state));
+ } else {
+ setNextState(state);
}
}
}
@@ -211,7 +215,7 @@ public class RemoveDataNodesProcedure extends AbstractNodeProcedure<RemoveDataNo
dataNodeLocation.getDataNodeId(), dataNodeLocation.getInternalEndPoint().getIp());
}
- private void checkRegionStatusAndStopDataNode(ConfigNodeProcedureEnv env) {
+ private void checkRegionStatusAndStopDataNode(ConfigNodeProcedureEnv env) throws IOException {
List<TRegionReplicaSet> replicaSets =
env.getConfigManager().getPartitionManager().getAllReplicaSets();
List<TDataNodeLocation> rollBackDataNodes = new ArrayList<>();
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
index ea91664834b..241eea8a702 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtils.java
@@ -282,9 +282,12 @@ public class SchemaUtils {
final ConfigManager configManager) {
return configManager.getNodeManager().getRegisteredDataNodeLocations().entrySet().stream()
.filter(
- entry ->
- configManager.getLoadManager().getNodeStatus(entry.getKey()) != NodeStatus.Unknown
- || !DataNodeContactTracker.getInstance().isDataNodeFenced(entry.getKey()))
+ entry -> {
+ final NodeStatus status =
+ configManager.getLoadManager().getNodeStatus(entry.getKey());
+ return !status.isOffline()
+ || !DataNodeContactTracker.getInstance().isDataNodeFenced(entry.getKey());
+ })
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
index 6b894804dc5..2296d54cd02 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHook.java
@@ -30,6 +30,10 @@ import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
import org.apache.iotdb.confignode.conf.ConfigNodeConstant;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.i18n.ConfigNodeMessages;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.exception.ConsensusException;
import org.apache.iotdb.db.utils.MemUtils;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -37,6 +41,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
public class ConfigNodeShutdownHook extends Thread {
@@ -45,49 +51,59 @@ public class ConfigNodeShutdownHook extends Thread {
private static final ConfigNodeConfig CONF = ConfigNodeDescriptor.getInstance().getConf();
private static final int SHUTDOWN_REPORT_RETRY_NUM = 2;
+ private final ConfigNode configNode;
+ private final SyncConfigNodeClientPool clientPool;
+
+ public ConfigNodeShutdownHook() {
+ this(ConfigNode.getInstance(), SyncConfigNodeClientPool.getInstance());
+ }
+
+ ConfigNodeShutdownHook(ConfigNode configNode, SyncConfigNodeClientPool clientPool) {
+ this.configNode = configNode;
+ this.clientPool = clientPool;
+ }
+
@Override
public void run() {
LOGGER.info(ConfigNodeMessages.CONFIGNODE_EXITING);
- boolean isLeader = getConfigNodeInstance().getConfigManager().getConsensusManager().isLeader();
+ TConfigNodeLocation localNode =
+ new TConfigNodeLocation(
+ CONF.getConfigNodeId(),
+ new TEndPoint(CONF.getInternalAddress(), CONF.getInternalPort()),
+ new TEndPoint(CONF.getInternalAddress(), CONF.getConsensusPort()));
+ // Save the same peer list for leadership transfer and the later shutdown report.
+ ConfigManager configManager = configNode.getConfigManager();
+ List<TConfigNodeLocation> otherNodes =
+ configManager == null
+ ? List.of()
+ : configManager.getNodeManager().getRegisteredConfigNodes().stream()
+ .filter(node -> !node.getInternalEndPoint().equals(localNode.getInternalEndPoint()))
+ .toList();
+ List<TEndPoint> reportTargets =
+ otherNodes.stream().map(TConfigNodeLocation::getInternalEndPoint).toList();
+ TEndPoint seed = CONF.getSeedConfigNode();
+ if (reportTargets.isEmpty() && seed != null && !seed.equals(localNode.getInternalEndPoint())) {
+ reportTargets = List.of(seed);
+ }
+ if (configManager != null) {
+ transferLeader(configManager, otherNodes);
+ }
try {
- ConfigNode.getInstance().deactivate();
+ configNode.deactivate();
} catch (IOException e) {
LOGGER.error(ConfigNodeMessages.MEET_ERROR_WHEN_DEACTIVATE_CONFIGNODE, e);
}
- if (!isLeader) {
- // Set and report shutdown to cluster ConfigNode-leader
- CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Unknown);
- boolean isReportSuccess = false;
- TEndPoint seedConfigNode = CONF.getSeedConfigNode();
- for (int retry = 0; retry < SHUTDOWN_REPORT_RETRY_NUM; retry++) {
- TSStatus result =
- (TSStatus)
- SyncConfigNodeClientPool.getInstance()
- .sendSyncRequestToConfigNodeWithRetry(
- seedConfigNode,
- new TConfigNodeLocation(
- CONF.getConfigNodeId(),
- new TEndPoint(CONF.getInternalAddress(), CONF.getInternalPort()),
- new TEndPoint(CONF.getInternalAddress(), CONF.getConsensusPort())),
- CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN);
-
- if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- // Report success
- isReportSuccess = true;
- break;
- } else if (result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()) {
- // Redirect
- seedConfigNode = result.getRedirectNode();
- }
- }
- if (!isReportSuccess) {
- LOGGER.error(
- ConfigNodeMessages
- .REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE);
- }
+ // Set and report shutdown to the cluster ConfigNode-leader best-effort, regardless of
+ // leadership: a leader that just stepped down may still reach the newly elected leader via
+ // redirect. If no leader is reachable, the new leader will mark this node Unknown by
+ // heartbeat timeout instead.
+ CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.Stopped);
+ if (!reportShutdown(localNode, reportTargets)) {
+ LOGGER.error(
+ ConfigNodeMessages.REPORTING_CONFIGNODE_SHUTDOWN_FAILED_THE_CLUSTER_WILL_STILL_TAKE_THE);
}
if (LOGGER.isInfoEnabled()) {
@@ -99,7 +115,83 @@ public class ConfigNodeShutdownHook extends Thread {
}
}
- protected ConfigNode getConfigNodeInstance() {
- return ConfigNode.getInstance();
+ private void transferLeader(ConfigManager configManager, List<TConfigNodeLocation> otherNodes) {
+ ConsensusManager consensusManager = configManager.getConsensusManager();
+ if (consensusManager == null || !consensusManager.isLeader()) {
+ return;
+ }
+ TConfigNodeLocation target =
+ otherNodes.stream()
+ .filter(
+ node ->
+ configManager.getLoadManager().getNodeStatus(node.getConfigNodeId())
+ == NodeStatus.Running)
+ .findFirst()
+ .orElse(null);
+ if (target == null) {
+ return;
+ }
+ try {
+ // The local consensus service must remain open until the transfer attempt finishes.
+ consensusManager
+ .getConsensusImpl()
+ .transferLeader(
+ consensusManager.getConsensusGroupId(),
+ new Peer(
+ consensusManager.getConsensusGroupId(),
+ target.getConfigNodeId(),
+ target.getConsensusEndPoint()));
+ } catch (ConsensusException e) {
+ LOGGER.warn(
+ ConfigNodeMessages
+ .LOG_FAILED_TO_TRANSFER_CONFIGNODE_LEADERSHIP_BEFORE_SHUTDOWN_CONTINUING_SHUTDOWN_2B9364D5,
+ e);
+ }
+ }
+
+ private boolean reportShutdown(TConfigNodeLocation localNode, List<TEndPoint> reportTargets) {
+ if (reportTargets.isEmpty()) {
+ return false;
+ }
+ long deadline =
+ System.nanoTime()
+ + TimeUnit.MILLISECONDS.toNanos(
+ CommonDescriptor.getInstance().getConfig().getCnConnectionTimeoutInMS());
+ while (System.nanoTime() < deadline && !Thread.currentThread().isInterrupted()) {
+ for (TEndPoint target : reportTargets) {
+ for (int retry = 0; retry < SHUTDOWN_REPORT_RETRY_NUM; retry++) {
+ if (System.nanoTime() >= deadline || Thread.currentThread().isInterrupted()) {
+ return false;
+ }
+ TSStatus result =
+ (TSStatus)
+ clientPool.sendSyncRequestToConfigNodeWithRetry(
+ target, localNode, CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN);
+ if (result.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ return true;
+ }
+ if (result.getCode() == TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()
+ && result.isSetRedirectNode()) {
+ target = result.getRedirectNode();
+ if (target.equals(localNode.getInternalEndPoint())) {
+ break;
+ }
+ }
+ }
+ }
+ // A newly elected leader may still be initializing its services after the transfer.
+ long remaining = deadline - System.nanoTime();
+ if (remaining <= 0) {
+ break;
+ }
+ try {
+ TimeUnit.NANOSECONDS.sleep(Math.min(remaining, TimeUnit.SECONDS.toNanos(1)));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOGGER.warn(ConfigNodeMessages.RETRY_WAIT_FAILED, e);
+ return false;
+ }
+ }
+ return false;
}
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java
new file mode 100644
index 00000000000..08e50ac9f76
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/client/async/handlers/heartbeat/DataNodeHeartbeatHandlerTest.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.client.async.handlers.heartbeat;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.NodeType;
+import org.apache.iotdb.commons.cluster.RegionStatus;
+import org.apache.iotdb.confignode.manager.lease.DataNodeContactTracker;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.cache.LoadCache;
+import org.apache.iotdb.confignode.manager.pipe.coordinator.runtime.PipeRuntimeCoordinator;
+import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
+
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DataNodeHeartbeatHandlerTest {
+
+ @Test
+ public void testStoppedHeartbeatUpdatesNodeAndRegions() {
+ int nodeId = 7;
+ TConsensusGroupId dataRegionId = new TConsensusGroupId(TConsensusGroupType.DataRegion, 10);
+ TConsensusGroupId schemaRegionId = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 20);
+ Map<TConsensusGroupId, Boolean> judgedLeaders =
+ Map.of(dataRegionId, false, schemaRegionId, false);
+ LoadCache loadCache = new LoadCache();
+ loadCache.createNodeHeartbeatCache(NodeType.DataNode, nodeId);
+ judgedLeaders
+ .keySet()
+ .forEach(id -> loadCache.createRegionGroupHeartbeatCache("root.db", id, Set.of(nodeId)));
+ LoadManager loadManager = mock(LoadManager.class);
+ when(loadManager.getLoadCache()).thenReturn(loadCache);
+ DataNodeHeartbeatHandler handler =
+ new DataNodeHeartbeatHandler(
+ nodeId,
+ loadManager,
+ new HashMap<>(),
+ new HashMap<>(),
+ new HashMap<>(),
+ ignored -> {},
+ ignored -> {},
+ mock(PipeRuntimeCoordinator.class));
+
+ try {
+ handler.onComplete(
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.Running.getStatus())
+ .setJudgedLeaders(judgedLeaders));
+ loadCache.updateNodeStatistics();
+ loadCache.updateRegionGroupStatistics();
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(nodeId));
+ for (TConsensusGroupId regionId : judgedLeaders.keySet()) {
+ assertEquals(RegionStatus.Running, loadCache.getRegionStatus(regionId, nodeId));
+ }
+
+ handler.onComplete(
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.Stopped.getStatus())
+ .setJudgedLeaders(judgedLeaders));
+ loadCache.updateNodeStatistics();
+ loadCache.updateRegionGroupStatistics();
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(nodeId));
+ for (TConsensusGroupId regionId : judgedLeaders.keySet()) {
+ assertEquals(
+ RegionStatus.Unknown, loadCache.getRegionCacheLastSampleStatus(regionId, nodeId));
+ assertEquals(RegionStatus.Unknown, loadCache.getRegionStatus(regionId, nodeId));
+ }
+ } finally {
+ DataNodeContactTracker.getInstance().removeDataNode(nodeId);
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
index 0e858589803..3ec2538c6d4 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanSerDeTest.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.common.rpc.thrift.TThrottleQuota;
import org.apache.iotdb.common.rpc.thrift.TTimePartitionSlot;
import org.apache.iotdb.common.rpc.thrift.TTimedQuota;
import org.apache.iotdb.common.rpc.thrift.ThrottleType;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.consensus.index.impl.IoTProgressIndex;
import org.apache.iotdb.commons.consensus.index.impl.MinimumProgressIndex;
import org.apache.iotdb.commons.exception.IllegalPathException;
@@ -73,6 +74,7 @@ import org.apache.iotdb.confignode.consensus.request.write.auth.AuthorTreePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateClusterIdPlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.ActiveCQPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.AddCQPlan;
import org.apache.iotdb.confignode.consensus.request.write.cq.DropCQPlan;
@@ -191,10 +193,12 @@ import org.apache.tsfile.file.metadata.enums.CompressionType;
import org.apache.tsfile.file.metadata.enums.TSEncoding;
import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.Pair;
+import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -214,6 +218,69 @@ import static org.junit.Assert.assertEquals;
public class ConfigPhysicalPlanSerDeTest {
+ @Test
+ public void testUpdateNodeStatusPlan() throws IOException {
+ for (NodeStatus status :
+ new NodeStatus[] {
+ NodeStatus.Stopped,
+ NodeStatus.Removing,
+ NodeStatus.ReadOnly,
+ NodeStatus.Running,
+ NodeStatus.Unknown,
+ null
+ }) {
+ for (String reason :
+ new String[] {
+ null,
+ NodeStatus.MANUAL,
+ NodeStatus.DISK_FULL,
+ NodeStatus.STOPPING,
+ NodeStatus.UNRECOVERABLE_ERROR + ": failed to recover root.test-0"
+ }) {
+ UpdateNodeStatusPlan original = new UpdateNodeStatusPlan(37, status, reason);
+ UpdateNodeStatusPlan restored =
+ (UpdateNodeStatusPlan)
+ ConfigPhysicalPlan.Factory.create(original.serializeToByteBuffer());
+ Assert.assertEquals(original, restored);
+ Assert.assertEquals(original.hashCode(), restored.hashCode());
+ Assert.assertEquals(37, restored.getNodeId());
+ NodeStatus persistedStatus = status != null && status.isPersistentStatus() ? status : null;
+ Assert.assertEquals(persistedStatus, restored.getStatus());
+ Assert.assertEquals(persistedStatus == null ? null : reason, restored.getStatusReason());
+ ByteBuffer serialized = original.serializeToByteBuffer();
+ Assert.assertEquals(
+ ConfigPhysicalPlanType.UpdateNodeStatus.getPlanType(), serialized.getShort());
+ Assert.assertEquals(37, ReadWriteIOUtils.readInt(serialized));
+ Assert.assertEquals(
+ persistedStatus == null ? null : persistedStatus.getStatus(),
+ ReadWriteIOUtils.readString(serialized));
+ Assert.assertEquals(restored.getStatusReason(), ReadWriteIOUtils.readString(serialized));
+ Assert.assertFalse(serialized.hasRemaining());
+ }
+ }
+ Assert.assertNotEquals(
+ new UpdateNodeStatusPlan(37, NodeStatus.ReadOnly, NodeStatus.MANUAL),
+ new UpdateNodeStatusPlan(37, NodeStatus.ReadOnly, NodeStatus.DISK_FULL));
+ }
+
+ @Test
+ public void testNodeStatusPlanNormalizesNonPersistentStatuses() {
+ for (String reason : new String[] {null, "test reason"}) {
+ for (NodeStatus status :
+ new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing, NodeStatus.ReadOnly}) {
+ UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(37, status, reason);
+ Assert.assertEquals(status, plan.getStatus());
+ Assert.assertEquals(reason, plan.getStatusReason());
+ }
+ for (NodeStatus status : new NodeStatus[] {NodeStatus.Running, NodeStatus.Unknown, null}) {
+ UpdateNodeStatusPlan plan = new UpdateNodeStatusPlan(37, status, reason);
+ Assert.assertNull(plan.getStatus());
+ Assert.assertNull(plan.getStatusReason());
+ Assert.assertEquals(new UpdateNodeStatusPlan(37, null), plan);
+ }
+ }
+ }
+
@Test
public void RegisterDataNodePlanTest() throws IOException {
TDataNodeLocation dataNodeLocation = new TDataNodeLocation();
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java
new file mode 100644
index 00000000000..4a8cf8e0552
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ConfigManagerShutdownReportTest.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class ConfigManagerShutdownReportTest {
+
+ private static final TSStatus SUCCESS_STATUS = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+
+ private static ConfigManager CONFIG_MANAGER_SPY;
+ private static LoadManager LOAD_MANAGER;
+
+ private static final int DATA_NODE_ID = 1;
+ private static final int CONFIG_NODE_ID = 2;
+
+ private static final TDataNodeLocation DATA_NODE_LOCATION =
+ new TDataNodeLocation(
+ DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 2000),
+ new TEndPoint("127.0.0.1", 2001),
+ new TEndPoint("127.0.0.1", 2002),
+ new TEndPoint("127.0.0.1", 2003),
+ new TEndPoint("127.0.0.1", 2004));
+
+ private static final TConfigNodeLocation CONFIG_NODE_LOCATION =
+ new TConfigNodeLocation(
+ CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001));
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ final ConfigManager configManager = new ConfigManager();
+ CONFIG_MANAGER_SPY = spy(configManager);
+ LOAD_MANAGER = mock(LoadManager.class);
+ when(CONFIG_MANAGER_SPY.getLoadManager()).thenReturn(LOAD_MANAGER);
+ doReturn(SUCCESS_STATUS).when(CONFIG_MANAGER_SPY).confirmLeader();
+ }
+
+ @Before
+ public void clearPreviousInteractions() {
+ // The shared LoadManager mock keeps interactions from previous tests; verification must only
+ // consider calls from the current test.
+ clearInvocations(LOAD_MANAGER);
+ when(LOAD_MANAGER.trySetNodeStatus(anyInt(), any(NodeStatus.class), eq(false)))
+ .thenReturn(SUCCESS_STATUS);
+ }
+
+ @Test
+ public void testReportDataNodeShutdownUpdatesRemovingDataNodeThroughCache() {
+ when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Removing);
+
+ Assert.assertEquals(
+ SUCCESS_STATUS.getCode(),
+ CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION).getCode());
+
+ verify(LOAD_MANAGER).trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+
+ @Test
+ public void testReportDataNodeShutdownRequestsStoppedStatus() {
+ when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Stopped);
+
+ Assert.assertEquals(
+ SUCCESS_STATUS.getCode(),
+ CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION).getCode());
+
+ verify(LOAD_MANAGER).trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+
+ @Test
+ public void testReportConfigNodeShutdownUpdatesRemovingConfigNodeThroughCache() {
+ when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(NodeStatus.Removing);
+
+ Assert.assertEquals(
+ SUCCESS_STATUS.getCode(),
+ CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION).getCode());
+
+ verify(LOAD_MANAGER).trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+
+ @Test
+ public void testReportConfigNodeShutdownRequestsStoppedStatus() {
+ when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(NodeStatus.Stopped);
+
+ Assert.assertEquals(
+ SUCCESS_STATUS.getCode(),
+ CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION).getCode());
+
+ verify(LOAD_MANAGER).trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+
+ @Test
+ public void testReportDataNodeShutdownPropagatesPersistenceFailure() {
+ TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR);
+ when(LOAD_MANAGER.trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false)))
+ .thenReturn(failure);
+
+ for (NodeStatus currentStatus : new NodeStatus[] {NodeStatus.Running, NodeStatus.Removing}) {
+ when(LOAD_MANAGER.getNodeStatus(DATA_NODE_ID)).thenReturn(currentStatus);
+ Assert.assertSame(failure, CONFIG_MANAGER_SPY.reportDataNodeShutdown(DATA_NODE_LOCATION));
+ }
+ verify(LOAD_MANAGER, times(2))
+ .trySetNodeStatus(eq(DATA_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+
+ @Test
+ public void testReportConfigNodeShutdownPropagatesPersistenceFailure() {
+ TSStatus failure = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR);
+ when(LOAD_MANAGER.trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false)))
+ .thenReturn(failure);
+
+ for (NodeStatus currentStatus : new NodeStatus[] {NodeStatus.Running, NodeStatus.Removing}) {
+ when(LOAD_MANAGER.getNodeStatus(CONFIG_NODE_ID)).thenReturn(currentStatus);
+ Assert.assertSame(failure, CONFIG_MANAGER_SPY.reportConfigNodeShutdown(CONFIG_NODE_LOCATION));
+ }
+ verify(LOAD_MANAGER, times(2))
+ .trySetNodeStatus(eq(CONFIG_NODE_ID), eq(NodeStatus.Stopped), eq(false));
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java
index 4de883b9c15..0289d0b6420 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/ProcedureManagerTest.java
@@ -111,6 +111,7 @@ public class ProcedureManagerTest {
@BeforeClass
public static void setUp() throws IOException {
IManager CONFIG_MANAGER = new ConfigManager();
+ IManager CONFIG_MANAGER_SPY = spy(CONFIG_MANAGER);
ProcedureManager procedureManager = CONFIG_MANAGER.getProcedureManager();
PROCEDURE_MANAGER = spy(procedureManager);
@@ -130,6 +131,10 @@ public class ProcedureManagerTest {
when(PROCEDURE_EXECUTOR.getProcedures()).thenReturn(procedureMap);
when(PROCEDURE_MANAGER.getEnv()).thenReturn(ENV);
when(ENV.getRemoveDataNodeHandler()).thenReturn(REMOVE_DATA_NODE_HANDLER);
+ // Inject the LoadManager spy into the call chain used by checkRemoveDataNodes, so that the
+ // node-status stubs below take effect on the production path
+ when(PROCEDURE_MANAGER.getConfigManager()).thenReturn(CONFIG_MANAGER_SPY);
+ when(CONFIG_MANAGER_SPY.getLoadManager()).thenReturn(LOAD_MANAGER);
}
@Test
@@ -197,6 +202,26 @@ public class ProcedureManagerTest {
TSStatus status = PROCEDURE_MANAGER.checkRemoveDataNodes(removedDataNodes);
Assert.assertTrue(isFailed(status));
+ Assert.assertTrue(status.getMessage().contains("unknown"));
+ }
+
+ @Test
+ public void testCheckRemoveDataNodeWithAnotherStoppedDataNode() {
+ Set<TDataNodeLocation> relatedDataNodes = new HashSet<>();
+ relatedDataNodes.add(removeDataNodeLocationA);
+ relatedDataNodes.add(coordinatorDataNodeLocation);
+
+ when(REMOVE_DATA_NODE_HANDLER.getRelatedDataNodeLocations(removeDataNodeLocationA))
+ .thenReturn(relatedDataNodes);
+
+ when(LOAD_MANAGER.getNodeStatus(removeDataNodeLocationA.getDataNodeId()))
+ .thenReturn(NodeStatus.Running);
+ when(LOAD_MANAGER.getNodeStatus(coordinatorDataNodeLocation.getDataNodeId()))
+ .thenReturn(NodeStatus.Stopped);
+
+ TSStatus status = PROCEDURE_MANAGER.checkRemoveDataNodes(removedDataNodes);
+ Assert.assertTrue(isFailed(status));
+ Assert.assertTrue(status.getMessage().contains("stopped"));
}
@Test
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java
new file mode 100644
index 00000000000..972421961ec
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/RetryFailedTasksThreadTest.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.rpc.RpcUtils;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.Test;
+
+import java.lang.reflect.Method;
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RetryFailedTasksThreadTest {
+
+ private static final int STOPPED_DATA_NODE_ID = 1;
+
+ private final IManager configManager = mock(IManager.class);
+ private final NodeManager nodeManager = mock(NodeManager.class);
+ private final LoadManager loadManager = mock(LoadManager.class);
+
+ private final TDataNodeConfiguration stoppedDataNode =
+ new TDataNodeConfiguration()
+ .setLocation(
+ new TDataNodeLocation(
+ STOPPED_DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 2000),
+ new TEndPoint("127.0.0.1", 2001),
+ new TEndPoint("127.0.0.1", 2002),
+ new TEndPoint("127.0.0.1", 2003),
+ new TEndPoint("127.0.0.1", 2004)));
+
+ private RetryFailedTasksThread retryFailedTasksThread;
+
+ // The thread captures its managers at construction time, so it must be created after the stubs.
+ private void createThreadWithStoppedDataNode() {
+ when(configManager.getNodeManager()).thenReturn(nodeManager);
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ when(nodeManager.getRegisteredDataNodes())
+ .thenReturn(Collections.singletonList(stoppedDataNode));
+ when(loadManager.getNodeStatus(STOPPED_DATA_NODE_ID)).thenReturn(NodeStatus.Stopped);
+ when(configManager.transfer(anyList()))
+ .thenReturn(RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS));
+ retryFailedTasksThread = new RetryFailedTasksThread(configManager);
+ }
+
+ private void invokeTriggerDetectTask() throws Exception {
+ final Method method = RetryFailedTasksThread.class.getDeclaredMethod("triggerDetectTask");
+ method.setAccessible(true);
+ method.invoke(retryFailedTasksThread);
+ }
+
+ @Test
+ public void testStoppedDataNodeTriggersRegionTransfer() throws Exception {
+ createThreadWithStoppedDataNode();
+
+ invokeTriggerDetectTask();
+
+ // A Stopped node is handled like Unknown: its regions are transferred.
+ verify(configManager)
+ .transfer(
+ argThat(
+ (List<TDataNodeLocation> nodes) ->
+ nodes.size() == 1 && nodes.get(0).getDataNodeId() == STOPPED_DATA_NODE_ID));
+ }
+
+ @Test
+ public void testContinuingStoppedDataNodeIsNotTransferredTwice() throws Exception {
+ createThreadWithStoppedDataNode();
+
+ invokeTriggerDetectTask();
+ invokeTriggerDetectTask();
+
+ // Like a continuing Unknown node, a continuing Stopped node only triggers one transfer.
+ verify(configManager, times(1)).transfer(anyList());
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java
index e2c51df9693..e5028f0bca9 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/LoadManagerTest.java
@@ -19,22 +19,29 @@
package org.apache.iotdb.confignode.manager.load;
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.cluster.RegionStatus;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.i18n.ManagerMessages;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.confignode.manager.IManager;
import org.apache.iotdb.confignode.manager.load.cache.LoadCache;
import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.consensus.ConsensusGroupStatistics;
+import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache;
import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionGroupStatistics;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.cache.region.RegionStatistics;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
import org.apache.iotdb.confignode.manager.partition.RegionGroupStatus;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.tsfile.utils.Pair;
import org.junit.Assert;
@@ -49,9 +56,17 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
public class LoadManagerTest {
private static LoadManager LOAD_MANAGER;
private static LoadCache LOAD_CACHE;
@@ -65,6 +80,9 @@ public class LoadManagerTest {
IManager CONFIG_MANAGER = new ConfigManager();
LOAD_MANAGER = CONFIG_MANAGER.getLoadManager();
LOAD_CACHE = LOAD_MANAGER.getLoadCache();
+ // These cache/event tests use successful stub writes without starting consensus.
+ LOAD_CACHE.setNodeStatusPersistence(
+ nodeId -> null, plan -> new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
}
@Before
@@ -90,7 +108,7 @@ public class LoadManagerTest {
// Simulate update to Running status
LOAD_CACHE.cacheConfigNodeHeartbeatSample(0, new NodeHeartbeatSample(NodeStatus.Running));
LOAD_CACHE.cacheDataNodeHeartbeatSample(1, new NodeHeartbeatSample(NodeStatus.Running));
- LOAD_CACHE.updateNodeStatistics(false);
+ LOAD_CACHE.updateNodeStatistics();
LOAD_MANAGER.getEventService().checkAndBroadcastNodeStatisticsChangeEventIfNecessary();
NODE_SEMAPHORE.acquire();
Assert.assertEquals(NodeStatus.Running, LOAD_CACHE.getNodeStatus(0));
@@ -105,8 +123,7 @@ public class LoadManagerTest {
differentNodeStatisticsMap.get(1));
// Force update to Removing status
- LOAD_MANAGER.forceUpdateNodeCache(
- NodeType.DataNode, 1, new NodeHeartbeatSample(NodeStatus.Removing));
+ LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.Removing, true);
NODE_SEMAPHORE.acquire();
Assert.assertEquals(NodeStatus.Removing, LOAD_CACHE.getNodeStatus(1));
differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap();
@@ -118,10 +135,43 @@ public class LoadManagerTest {
// Removing status can't be updated to any other status automatically
LOAD_CACHE.cacheDataNodeHeartbeatSample(1, new NodeHeartbeatSample(NodeStatus.ReadOnly));
- LOAD_CACHE.updateNodeStatistics(false);
+ LOAD_CACHE.updateNodeStatistics();
LOAD_MANAGER.getEventService().checkAndBroadcastNodeStatisticsChangeEventIfNecessary();
Assert.assertEquals(NodeStatus.Removing, LOAD_CACHE.getNodeStatus(1));
+ // An explicit rollback forwards its reason through LoadManager and the change event.
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.ReadOnly, NodeStatus.MANUAL, true).getCode());
+ Assert.assertTrue(NODE_SEMAPHORE.tryAcquire(5, TimeUnit.SECONDS));
+ Assert.assertEquals(NodeStatus.ReadOnly, LOAD_CACHE.getNodeStatus(1));
+ Assert.assertEquals(NodeStatus.MANUAL, LOAD_CACHE.getNodeStatusReason(1));
+ differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap();
+ Assert.assertEquals(1, differentNodeStatisticsMap.size());
+ Assert.assertEquals(
+ NodeStatus.Removing, differentNodeStatisticsMap.get(1).getLeft().getStatus());
+ Assert.assertEquals(
+ NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getRight().getStatus());
+ Assert.assertEquals(
+ NodeStatus.MANUAL, differentNodeStatisticsMap.get(1).getRight().getStatusReason());
+
+ // The overload without a reason clears it and broadcasts the reason-only change.
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ LOAD_MANAGER.trySetNodeStatus(1, NodeStatus.ReadOnly, false).getCode());
+ Assert.assertTrue(NODE_SEMAPHORE.tryAcquire(5, TimeUnit.SECONDS));
+ Assert.assertEquals(NodeStatus.ReadOnly, LOAD_CACHE.getNodeStatus(1));
+ Assert.assertNull(LOAD_CACHE.getNodeStatusReason(1));
+ differentNodeStatisticsMap = FAKE_SUBSCRIBER.getDifferentNodeStatisticsMap();
+ Assert.assertEquals(1, differentNodeStatisticsMap.size());
+ Assert.assertEquals(
+ NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getLeft().getStatus());
+ Assert.assertEquals(
+ NodeStatus.MANUAL, differentNodeStatisticsMap.get(1).getLeft().getStatusReason());
+ Assert.assertEquals(
+ NodeStatus.ReadOnly, differentNodeStatisticsMap.get(1).getRight().getStatus());
+ Assert.assertNull(differentNodeStatisticsMap.get(1).getRight().getStatusReason());
+
// Remove NodeCache
LOAD_MANAGER.removeNodeCache(1);
NODE_SEMAPHORE.acquire();
@@ -129,7 +179,7 @@ public class LoadManagerTest {
// Only DataNode 1 is updated
Assert.assertEquals(1, differentNodeStatisticsMap.size());
Assert.assertEquals(
- new Pair<>(new NodeStatistics(NodeStatus.Removing), null),
+ new Pair<>(new NodeStatistics(NodeStatus.ReadOnly), null),
differentNodeStatisticsMap.get(1));
}
@@ -316,7 +366,7 @@ public class LoadManagerTest {
dataNodeId ->
loadCache.cacheDataNodeHeartbeatSample(
dataNodeId, new NodeHeartbeatSample(NodeStatus.Running)));
- loadCache.updateNodeStatistics(false);
+ loadCache.updateNodeStatistics();
Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty());
}
@@ -335,13 +385,64 @@ public class LoadManagerTest {
loadCache.cacheDataNodeHeartbeatSample(
dataNodeId, new NodeHeartbeatSample(NodeStatus.Running));
});
- loadCache.updateNodeStatistics(false);
+ loadCache.updateNodeStatistics();
loadCache.updateRegionGroupStatistics();
loadCache.updateConsensusGroupStatistics();
Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty());
}
+ @Test
+ public void testLoadWarmUpWaitsForLeaderStoppedMarkerToBeCleared() throws Exception {
+ int leaderId = ConfigNodeHeartbeatCache.CURRENT_NODE_ID;
+ IManager configManager = mock(IManager.class, RETURNS_DEEP_STUBS);
+ NodeManager nodeManager = configManager.getNodeManager();
+ when(nodeManager.getRegisteredConfigNodes())
+ .thenReturn(Collections.singletonList(new TConfigNodeLocation().setConfigNodeId(leaderId)));
+ when(nodeManager.getRegisteredDataNodes()).thenReturn(Collections.emptyList());
+ when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.emptyList());
+ when(configManager.getClusterSchemaManager().getDatabaseNames(null))
+ .thenReturn(Collections.emptyList());
+ AtomicReference<Pair<NodeStatus, String>> persistedLeaderStatus =
+ new AtomicReference<>(new Pair<>(NodeStatus.Stopped, null));
+ Function<UpdateNodeStatusPlan, TSStatus> nodeStatusWriter = mock(Function.class);
+ when(nodeStatusWriter.apply(new UpdateNodeStatusPlan(leaderId, null)))
+ .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()))
+ .thenReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()))
+ .thenAnswer(
+ invocation -> {
+ persistedLeaderStatus.set(null);
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ });
+ LoadManager loadManager = new LoadManager(configManager);
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ LoadCache loadCache = loadManager.getLoadCache();
+ loadCache.setNodeStatusPersistence(id -> persistedLeaderStatus.get(), nodeStatusWriter::apply);
+ loadCache.initHeartbeatCache(configManager);
+
+ try {
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(leaderId));
+ Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty());
+ loadManager.markLoadServicesStartedForTest(
+ System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(31));
+
+ Assert.assertFalse(loadManager.isLoadReady());
+ Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), persistedLeaderStatus.get());
+ Assert.assertEquals(
+ ManagerMessages.MESSAGE_CONFIGNODE_LEADER_IS_WAITING_FOR_NODE_STATUS_PERSISTENCE_8CA96809,
+ loadManager.getLoadReadyReason());
+ // The first-heartbeat timeout must not bypass a repeated persistence failure.
+ Assert.assertFalse(loadManager.isLoadReady());
+ Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), persistedLeaderStatus.get());
+
+ Assert.assertTrue(loadManager.isLoadReady());
+ Assert.assertNull(persistedLeaderStatus.get());
+ verify(nodeStatusWriter, times(3)).apply(new UpdateNodeStatusPlan(leaderId, null));
+ } finally {
+ loadManager.stopLoadServices();
+ }
+ }
+
@Test
public void testLoadWarmUpToleratesMissingFirstHeartbeatAfterThirtySeconds() {
LOAD_CACHE.clearHeartbeatCache();
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java
new file mode 100644
index 00000000000..536379d79c5
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/RegionBalancerTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.load.balancer;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.exception.NotEnoughDataNodeException;
+import org.apache.iotdb.confignode.manager.IManager;
+import org.apache.iotdb.confignode.manager.ProcedureManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentMatchers;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.function.Function;
+
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RegionBalancerTest {
+
+ private static final int STOPPED_DATA_NODE_ID = 1;
+ private static final int REMOVING_DATA_NODE_ID = 2;
+
+ private final IManager configManager = mock(IManager.class);
+ private final NodeManager nodeManager = mock(NodeManager.class);
+ private final ProcedureManager procedureManager = mock(ProcedureManager.class);
+ private final ClusterSchemaManager clusterSchemaManager = mock(ClusterSchemaManager.class);
+ private final PartitionManager partitionManager = mock(PartitionManager.class);
+ private final LoadManager loadManager = mock(LoadManager.class);
+
+ private final TDataNodeConfiguration stoppedDataNode =
+ new TDataNodeConfiguration()
+ .setLocation(
+ new TDataNodeLocation(
+ STOPPED_DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 2000),
+ new TEndPoint("127.0.0.1", 2001),
+ new TEndPoint("127.0.0.1", 2002),
+ new TEndPoint("127.0.0.1", 2003),
+ new TEndPoint("127.0.0.1", 2004)));
+ private final TDataNodeConfiguration removingDataNode =
+ new TDataNodeConfiguration()
+ .setLocation(
+ new TDataNodeLocation(
+ REMOVING_DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 3000),
+ new TEndPoint("127.0.0.1", 3001),
+ new TEndPoint("127.0.0.1", 3002),
+ new TEndPoint("127.0.0.1", 3003),
+ new TEndPoint("127.0.0.1", 3004)));
+
+ private final RegionBalancer regionBalancer = new RegionBalancer(configManager);
+
+ @Before
+ public void setUp() {
+ when(configManager.getNodeManager()).thenReturn(nodeManager);
+ when(configManager.getProcedureManager()).thenReturn(procedureManager);
+ when(configManager.getClusterSchemaManager()).thenReturn(clusterSchemaManager);
+ when(configManager.getPartitionManager()).thenReturn(partitionManager);
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ }
+
+ @Test
+ public void testStoppedDataNodesAreKeptAsAllocationCandidates() throws Exception {
+ // With an empty allotment only the candidate filtering runs. Verify the supplied predicate
+ // accepts Running, Unknown and Stopped while excluding ReadOnly and Removing.
+ when(procedureManager.getRemovingDataNodeIds()).thenReturn(Collections.emptySet());
+ when(nodeManager.filterDataNodeThroughStatus(
+ ArgumentMatchers.<Function<NodeStatus, Boolean>>any()))
+ .thenReturn(Arrays.asList(stoppedDataNode, removingDataNode));
+
+ regionBalancer.genRegionGroupsAllocationPlan(
+ Collections.emptyMap(), TConsensusGroupType.DataRegion);
+
+ verify(nodeManager)
+ .filterDataNodeThroughStatus(
+ argThat(
+ (Function<NodeStatus, Boolean> predicate) ->
+ predicate != null
+ && predicate.apply(NodeStatus.Running)
+ && predicate.apply(NodeStatus.Unknown)
+ && predicate.apply(NodeStatus.Stopped)
+ && !predicate.apply(NodeStatus.ReadOnly)
+ && !predicate.apply(NodeStatus.Removing)));
+ }
+
+ @Test
+ public void testRemovingDataNodesAreExcludedFromCandidates() throws Exception {
+ when(procedureManager.getRemovingDataNodeIds())
+ .thenReturn(new HashSet<>(Collections.singletonList(REMOVING_DATA_NODE_ID)));
+ // The mocked status filter returns both nodes; the in-progress removal must drop the second.
+ when(nodeManager.filterDataNodeThroughStatus(
+ ArgumentMatchers.<Function<NodeStatus, Boolean>>any()))
+ .thenReturn(Arrays.asList(stoppedDataNode, removingDataNode));
+ when(clusterSchemaManager.getReplicationFactor(eq("db1"), eq(TConsensusGroupType.DataRegion)))
+ .thenReturn(3);
+
+ try {
+ regionBalancer.genRegionGroupsAllocationPlan(
+ Collections.singletonMap("db1", 1), TConsensusGroupType.DataRegion);
+ Assert.fail("Expected NotEnoughDataNodeException");
+ } catch (NotEnoughDataNodeException e) {
+ // The remaining candidates keep the Stopped node but not the node being removed.
+ Assert.assertTrue(
+ "Stopped node should be kept: " + e.getMessage(),
+ e.getMessage().contains("dataNodeId:" + STOPPED_DATA_NODE_ID));
+ Assert.assertFalse(
+ "Removing node should be excluded: " + e.getMessage(),
+ e.getMessage().contains("dataNodeId:" + REMOVING_DATA_NODE_ID));
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java
index 48104bf78a3..d601b645dbf 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/GreedyPriorityTest.java
@@ -65,7 +65,7 @@ public class GreedyPriorityTest {
nodeCacheMap.put(i, new DataNodeHeartbeatCache(i));
nodeCacheMap.get(i).cacheHeartbeatSample(new NodeHeartbeatSample(currentTimeNs, statuses[i]));
}
- nodeCacheMap.values().forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(false));
+ nodeCacheMap.values().forEach(BaseNodeCache::updateNodeStatistics);
/* Build TRegionReplicaSet */
TConsensusGroupId groupId1 = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1);
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java
index 5af3a677aab..90dbddc3f27 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/priority/LeaderPriorityBalancerTest.java
@@ -65,7 +65,7 @@ public class LeaderPriorityBalancerTest {
.cacheHeartbeatSample(new NodeHeartbeatSample(currentTimeNs, NodeStatus.Running));
}
}
- nodeCacheMap.values().forEach(baseNodeCache -> baseNodeCache.updateCurrentStatistics(false));
+ nodeCacheMap.values().forEach(BaseNodeCache::updateNodeStatistics);
// Build TRegionReplicaSet
TConsensusGroupId groupId1 = new TConsensusGroupId(TConsensusGroupType.SchemaRegion, 1);
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java
index caa3e839439..7bbd366be57 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/ConsensusGroupCacheTest.java
@@ -31,7 +31,7 @@ public class ConsensusGroupCacheTest {
ConsensusGroupCache consensusGroupCache = new ConsensusGroupCache();
ConsensusGroupHeartbeatSample sample = new ConsensusGroupHeartbeatSample(1L, 1);
consensusGroupCache.cacheHeartbeatSample(sample);
- consensusGroupCache.updateCurrentStatistics(false);
+ consensusGroupCache.updateCurrentStatistics();
Assert.assertEquals(1, consensusGroupCache.getLeaderId());
}
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java
new file mode 100644
index 00000000000..fa4bc5fb00f
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCacheNodeStatusTest.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.load.cache;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.IManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.utils.Pair;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.clearInvocations;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class LoadCacheNodeStatusTest {
+ private static final int NODE_ID = 1;
+
+ private NodeInfo nodeInfo;
+ private LoadCache loadCache;
+ private ConsensusManager consensusManager;
+
+ @Before
+ public void setUp() throws Exception {
+ nodeInfo = new NodeInfo();
+ nodeInfo.registerDataNode(
+ new RegisterDataNodePlan(
+ new TDataNodeConfiguration()
+ .setLocation(new TDataNodeLocation().setDataNodeId(NODE_ID))));
+ IManager manager = mock(IManager.class, RETURNS_DEEP_STUBS);
+ consensusManager = mock(ConsensusManager.class);
+ when(manager.getConsensusManager()).thenReturn(consensusManager);
+ when(consensusManager.write(any(UpdateNodeStatusPlan.class)))
+ .thenAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)));
+ when(manager.getNodeManager()).thenReturn(new NodeManager(manager, nodeInfo));
+ when(manager.getClusterSchemaManager().getDatabaseNames(null))
+ .thenReturn(Collections.emptyList());
+ loadCache = new LoadCache();
+ loadCache.setNodeStatusPersistence(nodeInfo::getPersistedNodeStatus, consensusManager::write);
+ loadCache.initHeartbeatCache(manager);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testLoadManagerSuppliesPersistenceBeforeConsensusInitialization() throws Exception {
+ ConfigManager manager = new ConfigManager();
+ try {
+ NodeInfo info = manager.getNodeManager().getNodeInfo();
+ info.registerDataNode(
+ new RegisterDataNodePlan(
+ new TDataNodeConfiguration()
+ .setLocation(new TDataNodeLocation().setDataNodeId(NODE_ID))));
+ LoadCache cache = manager.getLoadManager().getLoadCache();
+ cache.initHeartbeatCache(manager);
+
+ // Use the real methods supplied by LoadManager, replacing only consensus submission.
+ Field consensusField = ConfigManager.class.getDeclaredField("consensusManager");
+ consensusField.setAccessible(true);
+ ((AtomicReference<ConsensusManager>) consensusField.get(manager)).set(consensusManager);
+ doThrow(new ConsensusException("quorum unavailable"))
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ TSStatus result = cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false);
+ assertFailure(result);
+ assertEquals("quorum unavailable", result.getMessage());
+ assertEquals(NodeStatus.Stopped, cache.getNodeStatus(NODE_ID));
+ assertNull(info.getPersistedNodeStatus(NODE_ID));
+
+ doAnswer(invocation -> info.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertTrue(cache.updateNodeStatistics());
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(NODE_ID));
+ assertSuccess(cache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ verify(consensusManager, times(2))
+ .write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Stopped));
+ } finally {
+ manager.close();
+ }
+ }
+
+ @Test
+ public void testOnlyDurableTransitionsWriteConsensus() throws Exception {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false));
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false));
+ verify(consensusManager, never()).write(any());
+
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ verify(consensusManager, times(1)).write(any());
+
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true));
+ assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, times(3)).write(any());
+ }
+
+ @Test
+ public void testFailedReadOnlyReasonChangePublishesStatisticsAndRetriesPersistence()
+ throws Exception {
+ assertSuccess(
+ loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false));
+ UpdateNodeStatusPlan reasonChange =
+ new UpdateNodeStatusPlan(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ doReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()))
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(reasonChange);
+
+ assertFailure(
+ loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL, false));
+ assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(NODE_ID));
+ assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL),
+ nodeInfo.getPersistedNodeStatus(NODE_ID));
+
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL),
+ nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertTrue(loadCache.updateNodeStatistics());
+ verify(consensusManager, times(2)).write(reasonChange);
+ }
+
+ @Test
+ public void testReadOnlyReasonIsRemovedWhenPersistentStatusChanges() throws Exception {
+ for (NodeStatus requested : new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing}) {
+ assertSuccess(
+ loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL, true));
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, requested, null, false));
+ assertEquals(requested, loadCache.getNodeStatus(NODE_ID));
+ assertNull(loadCache.getNodeStatusReason(NODE_ID));
+ assertEquals(new Pair<>(requested, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ }
+ }
+
+ @Test
+ public void testReadOnlyIsClearedOnDisconnectAndPersistedAgainAfterHeartbeat() throws Exception {
+ Pair<NodeStatus, String> readOnly = new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ assertSuccess(
+ loadCache.trySetNodeStatus(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false));
+ assertEquals(readOnly, nodeInfo.getPersistedNodeStatus(NODE_ID));
+
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false));
+ assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID));
+ assertNull(loadCache.getNodeStatusReason(NODE_ID));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+
+ loadCache.cacheDataNodeHeartbeatSample(
+ NODE_ID,
+ new NodeHeartbeatSample(
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.ReadOnly.getStatus())
+ .setStatusReason(NodeStatus.DISK_FULL)));
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.DISK_FULL, loadCache.getNodeStatusReason(NODE_ID));
+ assertEquals(readOnly, nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, times(2))
+ .write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL));
+ verify(consensusManager).write(new UpdateNodeStatusPlan(NODE_ID, null));
+ }
+
+ @Test
+ public void testFailedWriteIsRetriedAndClearIsDurable() throws Exception {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false));
+ doReturn(new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode()))
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ // A later connection failure must retain the stop and retry its uncommitted record.
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false));
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Stopped));
+
+ doThrow(new ConsensusException("test consensus failure"))
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, false));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ }
+
+ private static void assertSuccess(TSStatus status) {
+ assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode());
+ }
+
+ @Test
+ public void testCommittedWriteWithLostResponseIsNotClearedByUnknown() throws Exception {
+ doAnswer(
+ i -> {
+ nodeInfo.applyNodeStatusPlan(i.getArgument(0));
+ throw new ConsensusException("response lost after commit");
+ })
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false));
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, times(1)).write(any());
+ }
+
+ @Test
+ public void testOrdinaryUpdatesKeepRemovingWithoutQuorumButRollbackRequiresIt() throws Exception {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true));
+ clearInvocations(consensusManager);
+ doThrow(new ConsensusException("quorum unavailable")).when(consensusManager).write(any());
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true));
+ for (NodeStatus observed :
+ new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, observed, null, false));
+ assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(NODE_ID));
+ }
+ verify(consensusManager, never()).write(any());
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, true));
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true));
+ assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertFalse(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+
+ doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Removing));
+ }
+
+ @Test
+ public void testConnectionFailureKeepsStoppedUntilExplicitReset() throws Exception {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Stopped, null, false));
+ clearInvocations(consensusManager);
+
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, false));
+ assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(NODE_ID));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, never()).write(any());
+
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Unknown, null, true));
+ assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ loadCache.updateNodeStatistics();
+ assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(NODE_ID));
+ }
+
+ private static void assertFailure(TSStatus status) {
+ assertEquals(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(), status.getCode());
+ }
+
+ @Test
+ public void testCommittedClearWithLostResponseIsNotRevertedByStatistics() throws Exception {
+ assertSuccess(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Removing, null, true));
+ clearInvocations(consensusManager);
+ doAnswer(
+ i -> {
+ nodeInfo.applyNodeStatusPlan(i.getArgument(0));
+ throw new ConsensusException("clear response lost");
+ })
+ .when(consensusManager)
+ .write(any(UpdateNodeStatusPlan.class));
+ assertFailure(loadCache.trySetNodeStatus(NODE_ID, NodeStatus.Running, null, true));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertTrue(loadCache.updateNodeStatistics());
+ assertEquals(NodeStatus.Running, loadCache.getNodeStatus(NODE_ID));
+ assertNull(nodeInfo.getPersistedNodeStatus(NODE_ID));
+ verify(consensusManager, times(1)).write(any());
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(NODE_ID, NodeStatus.Removing));
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java
new file mode 100644
index 00000000000..c8b9c84134f
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/LoadCachePersistedNodeStatusTest.java
@@ -0,0 +1,898 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.load.cache;
+
+import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TAINodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.NodeType;
+import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
+import org.apache.iotdb.confignode.manager.IManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.manager.load.cache.node.AINodeHeartbeatCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.BaseNodeCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.DataNodeHeartbeatCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.manager.schema.ClusterSchemaManager;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.tsfile.utils.Pair;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class LoadCachePersistedNodeStatusTest {
+
+ private static final int SELF_ID = ConfigNodeHeartbeatCache.CURRENT_NODE_ID;
+ private static final int CONFIG_NODE_ID = SELF_ID + 100;
+ private static final int DATA_NODE_ID = SELF_ID + 101;
+ private static final int REMOVING_DATA_NODE_ID = SELF_ID + 102;
+ private static final int AI_NODE_ID = SELF_ID + 103;
+
+ private NodeInfo nodeInfo;
+ private ConsensusManager consensusManager;
+ private IManager configManager;
+ private NodeManager nodeManager;
+ private LoadCache loadCache;
+
+ @Before
+ public void setUp() throws Exception {
+ nodeInfo = spy(new NodeInfo());
+ configManager = mock(IManager.class);
+ nodeManager = mock(NodeManager.class);
+ consensusManager = mock(ConsensusManager.class);
+ ClusterSchemaManager schemaManager = mock(ClusterSchemaManager.class);
+
+ when(configManager.getNodeManager()).thenReturn(nodeManager);
+ when(configManager.getConsensusManager()).thenReturn(consensusManager);
+ when(configManager.getClusterSchemaManager()).thenReturn(schemaManager);
+ when(schemaManager.getDatabaseNames(null)).thenReturn(Collections.emptyList());
+ when(nodeManager.getRegisteredConfigNodes())
+ .thenReturn(
+ Arrays.asList(
+ new TConfigNodeLocation(
+ SELF_ID, new TEndPoint("127.0.0.1", 11000), new TEndPoint("127.0.0.1", 12000)),
+ new TConfigNodeLocation(
+ CONFIG_NODE_ID,
+ new TEndPoint("127.0.0.1", 11001),
+ new TEndPoint("127.0.0.1", 12001))));
+ when(nodeManager.getRegisteredDataNodes())
+ .thenReturn(
+ Arrays.asList(
+ new TDataNodeConfiguration()
+ .setLocation(new TDataNodeLocation().setDataNodeId(DATA_NODE_ID)),
+ new TDataNodeConfiguration()
+ .setLocation(new TDataNodeLocation().setDataNodeId(REMOVING_DATA_NODE_ID))));
+ when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.emptyList());
+ for (TConfigNodeLocation node : nodeManager.getRegisteredConfigNodes()) {
+ nodeInfo.applyConfigNode(new ApplyConfigNodePlan(node));
+ }
+ for (TDataNodeConfiguration node : nodeManager.getRegisteredDataNodes()) {
+ nodeInfo.registerDataNode(new RegisterDataNodePlan(node));
+ }
+ when(consensusManager.write(any(UpdateNodeStatusPlan.class)))
+ .thenAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)));
+ loadCache = new LoadCache();
+ loadCache.setNodeStatusPersistence(nodeInfo::getPersistedNodeStatus, consensusManager::write);
+ }
+
+ @Test
+ public void testInitializationRestoresStickyStatusesAndKeepsLeaderRunning() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(SELF_ID, NodeStatus.Stopped));
+
+ loadCache.initHeartbeatCache(configManager);
+
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(SELF_ID));
+ Assert.assertTrue(loadCache.getNodeHeartbeatUnreadyReasons().isEmpty());
+ verify(nodeInfo).getPersistedNodeStatus(CONFIG_NODE_ID);
+ verify(nodeInfo).getPersistedNodeStatus(DATA_NODE_ID);
+ verify(nodeInfo).getPersistedNodeStatus(REMOVING_DATA_NODE_ID);
+ verify(nodeInfo, never()).getPersistedNodeStatus(SELF_ID);
+
+ loadCache.updateNodeStatistics();
+
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(SELF_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(SELF_ID));
+ }
+
+ @Test
+ public void testRestoredStatusDoesNotCreateHeartbeatSamples() throws Exception {
+ for (NodeStatus status : Arrays.asList(NodeStatus.Stopped, NodeStatus.Removing)) {
+ for (BaseNodeCache cache :
+ Arrays.asList(
+ new DataNodeHeartbeatCache(DATA_NODE_ID),
+ new ConfigNodeHeartbeatCache(CONFIG_NODE_ID),
+ new AINodeHeartbeatCache(AI_NODE_ID))) {
+ cache.initPersistence(
+ new Pair<>(status, null), (ignoredCache, ignoredStatus, ignoredReason) -> success());
+ Assert.assertEquals(status, cache.getNodeStatus());
+ Assert.assertFalse(cache.hasHeartbeatSample());
+ cache.updateNodeStatistics();
+ Assert.assertEquals(status, cache.getNodeStatus());
+ Assert.assertFalse(cache.hasHeartbeatSample());
+ }
+ }
+ }
+
+ @Test
+ public void testRestoredReadOnlyRequiresHeartbeatAndUnknownClearsPersistence() throws Exception {
+ Pair<NodeStatus, String> persisted = new Pair<>(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ for (BaseNodeCache cache :
+ Arrays.asList(
+ new DataNodeHeartbeatCache(DATA_NODE_ID),
+ new ConfigNodeHeartbeatCache(CONFIG_NODE_ID),
+ new AINodeHeartbeatCache(AI_NODE_ID))) {
+ cache.initPersistence(persisted, (ignoredCache, ignoredStatus, ignoredReason) -> success());
+ Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, cache.getNodeStatusReason());
+ Assert.assertFalse(cache.hasHeartbeatSample());
+ }
+
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.DISK_FULL));
+ loadCache.initHeartbeatCache(configManager);
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.DISK_FULL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(
+ Collections.singletonList("nodes=[" + DATA_NODE_ID + "]"),
+ loadCache.getNodeHeartbeatUnreadyReasons());
+
+ // Restoring durable metadata is not evidence that the node is still reachable.
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ Collections.singletonList("nodes=[" + DATA_NODE_ID + "]"),
+ loadCache.getNodeHeartbeatUnreadyReasons());
+ verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testReadOnlyReasonChangesArePersistedAndRunningClearsRecord() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ // Each heartbeat replaces the previous reason, including a lower-priority or absent reason.
+ for (String reason :
+ Arrays.asList(NodeStatus.STOPPING, NodeStatus.MANUAL, NodeStatus.DISK_FULL, null)) {
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, readOnlySample(System.nanoTime(), reason));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(reason, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, reason), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+
+ // Repeating the same status and reason must not append another consensus log.
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, readOnlySample(System.nanoTime(), reason));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ verify(consensusManager, times(1))
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, reason));
+ }
+
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testLiveHeartbeatClearsStoppedStatusForBothNodeTypes() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+
+ loadCache.cacheConfigNodeHeartbeatSample(
+ CONFIG_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ loadCache.updateNodeStatistics();
+
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(CONFIG_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testRevivalUpdatesStatisticsWhileClearIsRetried() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ doReturn(failure())
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+
+ Assert.assertFalse(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID));
+ verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testClearDoesNotRecheckThePreviousPersistedStatus() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ // CLEAR executes the caller's decision even if the applied value has changed.
+ doAnswer(
+ invocation -> {
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Removing));
+ return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0));
+ })
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testReadOnlyRevivalPublishesReasonWhilePersistenceIsRetried() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ doReturn(failure())
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL));
+
+ Assert.assertFalse(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL),
+ nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager, times(2))
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL));
+
+ // A new leader restores the committed reason before receiving its first heartbeat.
+ loadCache.initHeartbeatCache(configManager);
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, readOnlySample(System.nanoTime(), NodeStatus.MANUAL));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testReadOnlyHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception {
+ long previousLeaderTimestamp = System.nanoTime();
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, readOnlySample(previousLeaderTimestamp, NodeStatus.MANUAL));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, NodeStatus.MANUAL),
+ nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.ReadOnly, NodeStatus.MANUAL));
+ }
+
+ @Test
+ public void testStoppingReasonIsClearedByShutdownAndCannotOverrideRemoving() throws Exception {
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+ for (int nodeId : Arrays.asList(DATA_NODE_ID, REMOVING_DATA_NODE_ID)) {
+ loadCache.cacheDataNodeHeartbeatSample(
+ nodeId, readOnlySample(System.nanoTime(), NodeStatus.STOPPING));
+ }
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.STOPPING, loadCache.getNodeStatusReason(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertNull(loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID));
+
+ for (int nodeId : Arrays.asList(DATA_NODE_ID, REMOVING_DATA_NODE_ID)) {
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode());
+ Assert.assertNull(loadCache.getNodeStatusReason(nodeId));
+ }
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null),
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+ }
+
+ @Test
+ public void testStoppedStatisticsUpdateAfterFailedPersistenceReturns() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, null, false);
+ CountDownLatch persistenceStarted = new CountDownLatch(1);
+ CountDownLatch finishPersistence = new CountDownLatch(1);
+ doAnswer(
+ invocation -> {
+ persistenceStarted.countDown();
+ Assert.assertTrue(finishPersistence.await(5, TimeUnit.SECONDS));
+ return failure();
+ })
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<TSStatus> update =
+ executor.submit(
+ () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false));
+ Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID));
+ finishPersistence.countDown();
+ Assert.assertEquals(failure().getCode(), update.get(5, TimeUnit.SECONDS).getCode());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+
+ // Even when the latest sample becomes Unknown, statistics retain the stop for retry.
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager, times(2))
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ } finally {
+ finishPersistence.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testHeartbeatAndStatisticsWaitUntilShutdownPersistenceCompletes() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Running, null, false);
+ NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running);
+ CountDownLatch persistenceStarted = new CountDownLatch(1);
+ CountDownLatch finishPersistence = new CountDownLatch(1);
+ CountDownLatch updatesStarted = new CountDownLatch(2);
+ doAnswer(
+ invocation -> {
+ persistenceStarted.countDown();
+ Assert.assertTrue(finishPersistence.await(10, TimeUnit.SECONDS));
+ return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0));
+ })
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ ExecutorService executor = Executors.newFixedThreadPool(3);
+ try {
+ Future<TSStatus> stopped =
+ executor.submit(
+ () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false));
+ Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS));
+ Future<?> heartbeat =
+ executor.submit(
+ () -> {
+ updatesStarted.countDown();
+ loadCache.cacheDataNodeHeartbeatSample(DATA_NODE_ID, delayedRunning);
+ });
+ Future<Boolean> statistics =
+ executor.submit(
+ () -> {
+ updatesStarted.countDown();
+ return loadCache.updateNodeStatistics();
+ });
+ Assert.assertTrue(updatesStarted.await(5, TimeUnit.SECONDS));
+ for (Future<?> pending : Arrays.asList(heartbeat, statistics)) {
+ try {
+ pending.get(100, TimeUnit.MILLISECONDS);
+ Assert.fail("The node update must wait for shutdown persistence");
+ } catch (TimeoutException expected) {
+ // Both operations must wait, while the last published status remains readable.
+ }
+ }
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager, times(1))
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+
+ finishPersistence.countDown();
+ Assert.assertEquals(success().getCode(), stopped.get(5, TimeUnit.SECONDS).getCode());
+ heartbeat.get(5, TimeUnit.SECONDS);
+ Assert.assertTrue(statistics.get(5, TimeUnit.SECONDS));
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ } finally {
+ finishPersistence.countDown();
+ executor.shutdownNow();
+ Assert.assertTrue(executor.awaitTermination(5, TimeUnit.SECONDS));
+ }
+ }
+
+ @Test
+ public void testHeartbeatIsNotFilteredByCacheInitializationTime() throws Exception {
+ long oldSampleTimestamp = System.nanoTime() - 1;
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(oldSampleTimestamp, NodeStatus.Running));
+ loadCache.updateNodeStatistics();
+
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ verify(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testForcingAnotherNodeDoesNotClearRemovingStatus() throws Exception {
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode());
+
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null),
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+ verify(consensusManager, never())
+ .write(
+ argThat(
+ plan ->
+ plan instanceof UpdateNodeStatusPlan
+ && ((UpdateNodeStatusPlan) plan).getNodeId() == REMOVING_DATA_NODE_ID));
+
+ // An explicit rollback of that node still clears its durable Removing marker.
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache
+ .trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, null, true)
+ .getCode());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID));
+ }
+
+ @Test
+ public void testExplicitRollbackRestoresOfflineAndRunningStatuses() throws Exception {
+ for (NodeStatus rollbackStatus :
+ Arrays.asList(
+ NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running, NodeStatus.ReadOnly)) {
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache
+ .trySetNodeStatus(
+ REMOVING_DATA_NODE_ID,
+ rollbackStatus,
+ rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null,
+ true)
+ .getCode());
+ Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null,
+ loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ rollbackStatus.isPersistentStatus()
+ ? new Pair<>(
+ rollbackStatus, rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null)
+ : null,
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+
+ // Ordinary statistics updates must preserve the successfully committed rollback.
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(rollbackStatus, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ rollbackStatus == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null,
+ loadCache.getNodeStatusReason(REMOVING_DATA_NODE_ID));
+ }
+ }
+
+ @Test
+ public void testFailedExplicitRollbackUpdatesStatisticsAndRetriesPeriodically() throws Exception {
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+ doReturn(failure())
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null));
+
+ Assert.assertEquals(
+ failure().getCode(),
+ loadCache
+ .trySetNodeStatus(REMOVING_DATA_NODE_ID, NodeStatus.Running, null, true)
+ .getCode());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null),
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(REMOVING_DATA_NODE_ID));
+ verify(consensusManager, times(2)).write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null));
+ verify(consensusManager, never())
+ .write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ }
+
+ @Test
+ public void testShutdownReportCannotOverrideRemovingStatus() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Removing));
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+
+ for (int nodeId : Arrays.asList(CONFIG_NODE_ID, REMOVING_DATA_NODE_ID)) {
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(nodeId, NodeStatus.Stopped, null, false).getCode());
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(nodeId));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(nodeId));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(nodeId, NodeStatus.Stopped));
+ }
+ }
+
+ @Test
+ public void testStatisticsFailureDoesNotPreventUpdatingOtherNodes() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ doReturn(failure()).when(consensusManager).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ loadCache.cacheConfigNodeHeartbeatSample(
+ CONFIG_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+
+ Assert.assertFalse(loadCache.updateNodeStatistics());
+
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(CONFIG_NODE_ID));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(SELF_ID, null));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testRepeatedCacheCreationPreservesPendingShutdown() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Running, null, false).getCode());
+ Assert.assertFalse(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID));
+ CountDownLatch persistenceStarted = new CountDownLatch(1);
+ CountDownLatch finishPersistence = new CountDownLatch(1);
+ doAnswer(
+ invocation -> {
+ persistenceStarted.countDown();
+ Assert.assertTrue(finishPersistence.await(10, TimeUnit.SECONDS));
+ return nodeInfo.applyNodeStatusPlan(invocation.getArgument(0));
+ })
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, NodeStatus.Stopped));
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ Future<TSStatus> shutdown =
+ executor.submit(
+ () -> loadCache.trySetNodeStatus(CONFIG_NODE_ID, NodeStatus.Stopped, null, false));
+ Assert.assertTrue(persistenceStarted.await(5, TimeUnit.SECONDS));
+ // A recovered AddConfigNodeProcedure may repeat REGISTER_SUCCESS after registration applied.
+ loadCache.createNodeHeartbeatCache(NodeType.ConfigNode, CONFIG_NODE_ID);
+ finishPersistence.countDown();
+ Assert.assertEquals(success().getCode(), shutdown.get(5, TimeUnit.SECONDS).getCode());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID));
+ Assert.assertTrue(loadCache.checkAndSetHeartbeatProcessing(CONFIG_NODE_ID));
+
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(CONFIG_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(CONFIG_NODE_ID));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(CONFIG_NODE_ID, null));
+ } finally {
+ finishPersistence.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ @Test
+ public void testDiscardedCacheCannotClearNewRemovingMarker() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ loadCache.initHeartbeatCache(configManager);
+ Field field = LoadCache.class.getDeclaredField("nodeCacheMap");
+ field.setAccessible(true);
+ Map<Integer, BaseNodeCache> caches = (Map<Integer, BaseNodeCache>) field.get(loadCache);
+ BaseNodeCache discarded = caches.get(DATA_NODE_ID);
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+ Assert.assertEquals(
+ failure().getCode(), discarded.trySetNodeStatus(NodeStatus.Running, null, true).getCode());
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(DATA_NODE_ID));
+ verify(consensusManager, never()).write(new UpdateNodeStatusPlan(DATA_NODE_ID, null));
+ }
+
+ @Test
+ public void testReadOnlyHeartbeatReplacesStoppedButNotRemoving() throws Exception {
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(REMOVING_DATA_NODE_ID, NodeStatus.Removing));
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly));
+ loadCache.cacheDataNodeHeartbeatSample(
+ REMOVING_DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.ReadOnly));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.ReadOnly, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null),
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+ }
+
+ @Test
+ public void testAINodeRestoresStoppedAndRemovingAcrossLeaderChange() throws Exception {
+ registerAINode();
+ loadCache.initHeartbeatCache(configManager);
+ for (NodeStatus status : Arrays.asList(NodeStatus.Stopped, NodeStatus.Removing)) {
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(AI_NODE_ID, status, null, false).getCode());
+ Assert.assertEquals(new Pair<>(status, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+ loadCache.initHeartbeatCache(configManager);
+ Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(status, loadCache.getNodeStatus(AI_NODE_ID));
+ }
+ loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Removing, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+ }
+
+ @Test
+ public void testAINodeUpdatesStatisticsOnPersistenceFailureAndRetries() throws Exception {
+ registerAINode();
+ loadCache.initHeartbeatCache(configManager);
+ doReturn(failure())
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(AI_NODE_ID, NodeStatus.Stopped));
+ Assert.assertEquals(
+ failure().getCode(),
+ loadCache.trySetNodeStatus(AI_NODE_ID, NodeStatus.Stopped, null, false).getCode());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+ loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Unknown));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+
+ doReturn(failure())
+ .doAnswer(invocation -> nodeInfo.applyNodeStatusPlan(invocation.getArgument(0)))
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(AI_NODE_ID, null));
+ loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ Assert.assertFalse(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+ Assert.assertTrue(loadCache.updateNodeStatistics());
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(AI_NODE_ID));
+ Assert.assertNull(nodeInfo.getPersistedNodeStatus(AI_NODE_ID));
+ }
+
+ @Test
+ public void testDelayedAINodeHeartbeatDoesNotRecreateRemovedCache() throws Exception {
+ registerAINode();
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.removeNodeCache(AI_NODE_ID);
+ loadCache.cacheAINodeHeartbeatSample(AI_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ Assert.assertFalse(loadCache.getCurrentNodeStatisticsMap().containsKey(AI_NODE_ID));
+ }
+
+ private void registerAINode() {
+ TAINodeConfiguration node =
+ new TAINodeConfiguration().setLocation(new TAINodeLocation().setAiNodeId(AI_NODE_ID));
+ nodeInfo.registerAINode(new RegisterAINodePlan(node));
+ when(nodeManager.getRegisteredAINodes()).thenReturn(Collections.singletonList(node));
+ }
+
+ @Test
+ public void testOlderHeartbeatCannotUndoShutdownReport() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ NodeHeartbeatSample delayedRunning = new NodeHeartbeatSample(NodeStatus.Running);
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode());
+ for (NodeHeartbeatSample olderHeartbeat :
+ Arrays.asList(
+ delayedRunning,
+ new NodeHeartbeatSample(
+ System.nanoTime() - TimeUnit.DAYS.toNanos(1), NodeStatus.Running))) {
+ loadCache.cacheDataNodeHeartbeatSample(DATA_NODE_ID, olderHeartbeat);
+ loadCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Stopped, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ }
+ }
+
+ @Test
+ public void testBlockedNodeWriteDoesNotHoldOtherNodeCacheLock() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ doAnswer(
+ i -> {
+ entered.countDown();
+ Assert.assertTrue(release.await(10, TimeUnit.SECONDS));
+ return nodeInfo.applyNodeStatusPlan(i.getArgument(0));
+ })
+ .when(consensusManager)
+ .write(new UpdateNodeStatusPlan(DATA_NODE_ID, NodeStatus.Stopped));
+ ExecutorService executor = Executors.newFixedThreadPool(2);
+ try {
+ Future<TSStatus> blocked =
+ executor.submit(
+ () -> loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, true));
+ Assert.assertTrue(entered.await(5, TimeUnit.SECONDS));
+ Future<TSStatus> independent =
+ executor.submit(
+ () ->
+ loadCache.trySetNodeStatus(
+ REMOVING_DATA_NODE_ID, NodeStatus.Removing, null, true));
+ Assert.assertEquals(success().getCode(), independent.get(5, TimeUnit.SECONDS).getCode());
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null),
+ nodeInfo.getPersistedNodeStatus(REMOVING_DATA_NODE_ID));
+ Assert.assertFalse(blocked.isDone());
+ release.countDown();
+ Assert.assertEquals(success().getCode(), blocked.get(5, TimeUnit.SECONDS).getCode());
+ } finally {
+ release.countDown();
+ executor.shutdownNow();
+ }
+ }
+
+ private static NodeHeartbeatSample readOnlySample(long timestamp, String reason) {
+ return new NodeHeartbeatSample(
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(timestamp)
+ .setStatus(NodeStatus.ReadOnly.getStatus())
+ .setStatusReason(reason));
+ }
+
+ private static TSStatus success() {
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ @Test
+ public void testLateShutdownReportConvergesAfterRestartedNodeHeartbeat() throws Exception {
+ loadCache.initHeartbeatCache(configManager);
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ loadCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ // Shutdown reports carry no DataNode incarnation. Document the current temporary Stopped
+ // result, then require a fresh heartbeat from the new process to durably clear that marker.
+ Assert.assertEquals(
+ success().getCode(),
+ loadCache.trySetNodeStatus(DATA_NODE_ID, NodeStatus.Stopped, null, false).getCode());
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(DATA_NODE_ID));
+ loadCache.cacheDataNodeHeartbeatSample(
+ DATA_NODE_ID, new NodeHeartbeatSample(NodeStatus.Running));
+ loadCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Running, loadCache.getNodeStatus(DATA_NODE_ID));
+ Assert.assertFalse(nodeInfo.getPersistedNodeStatuses().containsKey(DATA_NODE_ID));
+ }
+
+ private static TSStatus failure() {
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java
index a4006929566..05262d3955d 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/cache/NodeCacheTest.java
@@ -19,13 +19,22 @@
package org.apache.iotdb.confignode.manager.load.cache;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.cluster.NodeType;
+import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.manager.load.cache.node.AINodeHeartbeatCache;
+import org.apache.iotdb.confignode.manager.load.cache.node.BaseNodeCache;
import org.apache.iotdb.confignode.manager.load.cache.node.ConfigNodeHeartbeatCache;
import org.apache.iotdb.confignode.manager.load.cache.node.DataNodeHeartbeatCache;
import org.apache.iotdb.confignode.manager.load.cache.node.NodeHeartbeatSample;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
+import org.apache.iotdb.mpp.rpc.thrift.TDataNodeHeartbeatResp;
import org.junit.Assert;
import org.junit.Test;
+import java.util.Map;
+
public class NodeCacheTest {
@Test
@@ -35,7 +44,7 @@ public class NodeCacheTest {
long currentTime = System.nanoTime();
dataNodeHeartbeatCache.cacheHeartbeatSample(
new NodeHeartbeatSample(currentTime, NodeStatus.Running));
- dataNodeHeartbeatCache.updateCurrentStatistics(false);
+ dataNodeHeartbeatCache.updateNodeStatistics();
Assert.assertEquals(NodeStatus.Running, dataNodeHeartbeatCache.getNodeStatus());
Assert.assertEquals(0, dataNodeHeartbeatCache.getLoadScore());
@@ -44,8 +53,223 @@ public class NodeCacheTest {
currentTime = System.nanoTime();
configNodeHeartbeatCache.cacheHeartbeatSample(
new NodeHeartbeatSample(currentTime, NodeStatus.Running));
- configNodeHeartbeatCache.updateCurrentStatistics(false);
+ configNodeHeartbeatCache.updateNodeStatistics();
Assert.assertEquals(NodeStatus.Running, configNodeHeartbeatCache.getNodeStatus());
Assert.assertEquals(0, configNodeHeartbeatCache.getLoadScore());
}
+
+ @Test
+ public void testRemovingAINodeStatisticsSurviveHeartbeatTimeout() {
+ ConfigNodeConfig config = ConfigNodeDescriptor.getInstance().getConf();
+ String detector = config.getFailureDetector();
+ long timeout = config.getFailureDetectorFixedThresholdInMs();
+ config.setFailureDetector(IFailureDetector.FIXED_DETECTOR);
+ config.setFailureDetectorFixedThresholdInMs(0);
+ try {
+ AINodeHeartbeatCache cache = new AINodeHeartbeatCache(3);
+ cache.trySetNodeStatus(NodeStatus.Removing, null, false);
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus());
+ Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore());
+ } finally {
+ config.setFailureDetector(detector);
+ config.setFailureDetectorFixedThresholdInMs(timeout);
+ }
+ }
+
+ @Test
+ public void stoppedStatusStickyAndRevivalTest() {
+ for (BaseNodeCache cache :
+ new BaseNodeCache[] {
+ new DataNodeHeartbeatCache(1),
+ new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1),
+ new AINodeHeartbeatCache(3)
+ }) {
+ String cacheType = cache.getClass().getSimpleName();
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Stopped));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus());
+ Assert.assertEquals(cacheType, Long.MAX_VALUE, cache.getLoadScore());
+
+ // Unknown observations cannot undo a reported stop.
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Unknown));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus());
+
+ // A live heartbeat revives a stopped node.
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Running));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Running, cache.getNodeStatus());
+ Assert.assertEquals(cacheType, 0, cache.getLoadScore());
+
+ // A stopped node can still enter removal.
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Stopped));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Stopped, cache.getNodeStatus());
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Removing));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Removing, cache.getNodeStatus());
+
+ // Ordinary updates, including fresh Running, must not cancel removal.
+ for (NodeStatus observed :
+ new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Unknown, NodeStatus.Running}) {
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(observed));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, NodeStatus.Removing, cache.getNodeStatus());
+ Assert.assertEquals(cacheType, Long.MAX_VALUE, cache.getLoadScore());
+ }
+
+ // Management restores the requested state explicitly, including offline states.
+ for (NodeStatus restored :
+ new NodeStatus[] {NodeStatus.Unknown, NodeStatus.Stopped, NodeStatus.Running}) {
+ cache.trySetNodeStatus(NodeStatus.Removing, null, true);
+ cache.trySetNodeStatus(restored, null, true);
+ Assert.assertEquals(cacheType, restored, cache.getNodeStatus());
+ cache.updateNodeStatistics();
+ Assert.assertEquals(cacheType, restored, cache.getNodeStatus());
+ }
+ }
+ }
+
+ @Test
+ public void testConnectionFailureIsNotUndoneByPreviousRunningHeartbeat() {
+ for (BaseNodeCache cache :
+ new BaseNodeCache[] {
+ new DataNodeHeartbeatCache(1),
+ new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1),
+ new AINodeHeartbeatCache(3)
+ }) {
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(NodeStatus.Running));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Running, cache.getNodeStatus());
+
+ cache.trySetNodeStatus(NodeStatus.Unknown, null, false);
+ Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus());
+ Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore());
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus());
+ }
+ }
+
+ @Test
+ public void testExplicitStatusUpdateDoesNotRequireLiveHeartbeat() {
+ ConfigNodeConfig config = ConfigNodeDescriptor.getInstance().getConf();
+ String detector = config.getFailureDetector();
+ long timeout = config.getFailureDetectorFixedThresholdInMs();
+ config.setFailureDetector(IFailureDetector.FIXED_DETECTOR);
+ config.setFailureDetectorFixedThresholdInMs(0);
+ try {
+ for (BaseNodeCache cache :
+ new BaseNodeCache[] {
+ new DataNodeHeartbeatCache(1),
+ new ConfigNodeHeartbeatCache(ConfigNodeHeartbeatCache.CURRENT_NODE_ID + 1),
+ new AINodeHeartbeatCache(3)
+ }) {
+ cache.cacheHeartbeatSample(new NodeHeartbeatSample(0, NodeStatus.Running));
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus());
+
+ // A shutdown report is known information even when heartbeat detection says unavailable.
+ cache.trySetNodeStatus(NodeStatus.Stopped, null, false);
+ Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus());
+ Assert.assertEquals(Long.MAX_VALUE, cache.getLoadScore());
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus());
+
+ cache.trySetNodeStatus(NodeStatus.Unknown, null, false);
+ Assert.assertEquals(NodeStatus.Stopped, cache.getNodeStatus());
+ cache.trySetNodeStatus(NodeStatus.Unknown, null, true);
+ Assert.assertEquals(NodeStatus.Unknown, cache.getNodeStatus());
+ }
+ } finally {
+ config.setFailureDetector(detector);
+ config.setFailureDetectorFixedThresholdInMs(timeout);
+ }
+ }
+
+ @Test
+ public void statusReasonPropagationTest() {
+ DataNodeHeartbeatCache dataNodeHeartbeatCache = new DataNodeHeartbeatCache(1);
+
+ // A heartbeat response carrying a status reason (e.g. ReadOnly + DiskFull from the DataNode)
+ // publishes the reason into the node statistics.
+ TDataNodeHeartbeatResp heartbeatResp =
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.ReadOnly.getStatus())
+ .setStatusReason(NodeStatus.DISK_FULL);
+ dataNodeHeartbeatCache.cacheHeartbeatSample(new NodeHeartbeatSample(heartbeatResp));
+ dataNodeHeartbeatCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.ReadOnly, dataNodeHeartbeatCache.getNodeStatus());
+ Assert.assertEquals(NodeStatus.DISK_FULL, dataNodeHeartbeatCache.getNodeStatusReason());
+
+ // A heartbeat response without a status reason clears the previous reason.
+ heartbeatResp =
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.Running.getStatus());
+ dataNodeHeartbeatCache.cacheHeartbeatSample(new NodeHeartbeatSample(heartbeatResp));
+ dataNodeHeartbeatCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Running, dataNodeHeartbeatCache.getNodeStatus());
+ Assert.assertNull(dataNodeHeartbeatCache.getNodeStatusReason());
+
+ // An Unknown decided by the failure detector (a stale heartbeat) never carries a reason. The
+ // stale sample is accepted because the fresh cache has an empty sliding window.
+ DataNodeHeartbeatCache staleCache = new DataNodeHeartbeatCache(3);
+ staleCache.cacheHeartbeatSample(
+ new NodeHeartbeatSample(System.nanoTime() - 60_000_000_000L, NodeStatus.ReadOnly));
+ staleCache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Unknown, staleCache.getNodeStatus());
+ Assert.assertNull(staleCache.getNodeStatusReason());
+ }
+
+ @Test
+ public void explicitStatusUpdatesKeepTheSuppliedReason() {
+ DataNodeHeartbeatCache cache = new DataNodeHeartbeatCache(1);
+ // ConfigNode must use the supplied reason, without inventing Manual or applying local priority.
+ for (String reason : new String[] {NodeStatus.MANUAL, NodeStatus.DISK_FULL, null}) {
+ cache.trySetNodeStatus(NodeStatus.ReadOnly, reason, false);
+ Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus());
+ Assert.assertEquals(reason, cache.getNodeStatusReason());
+ Assert.assertEquals(reason, ((NodeHeartbeatSample) cache.getLastSample()).getStatusReason());
+
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.ReadOnly, cache.getNodeStatus());
+ Assert.assertEquals(reason, cache.getNodeStatusReason());
+ }
+
+ // A rejected ReadOnly update cannot attach its reason to a node that remains Removing.
+ cache.trySetNodeStatus(NodeStatus.Removing, null, true);
+ cache.trySetNodeStatus(NodeStatus.ReadOnly, NodeStatus.DISK_FULL, false);
+ Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus());
+ Assert.assertNull(cache.getNodeStatusReason());
+ cache.updateNodeStatistics();
+ Assert.assertEquals(NodeStatus.Removing, cache.getNodeStatus());
+ Assert.assertNull(cache.getNodeStatusReason());
+ }
+
+ @Test
+ public void loadCacheSnapshotKeepsStatusAndReasonFromSameStatistics() {
+ LoadCache loadCache = new LoadCache();
+ loadCache.createNodeHeartbeatCache(NodeType.DataNode, 1);
+ loadCache.cacheDataNodeHeartbeatSample(
+ 1,
+ new NodeHeartbeatSample(
+ new TDataNodeHeartbeatResp()
+ .setHeartbeatTimestamp(System.nanoTime())
+ .setStatus(NodeStatus.ReadOnly.getStatus())
+ .setStatusReason(NodeStatus.MANUAL)));
+ loadCache.updateNodeStatistics();
+
+ // The snapshot used by SHOW CLUSTER reads status and reason from one statistics object.
+ Map<Integer, NodeStatistics> snapshot = loadCache.getCurrentNodeStatisticsMap();
+ Assert.assertEquals(1, snapshot.size());
+ Assert.assertEquals(NodeStatus.ReadOnly, snapshot.get(1).getStatus());
+ Assert.assertEquals(NodeStatus.MANUAL, snapshot.get(1).getStatusReason());
+ Assert.assertEquals(NodeStatus.MANUAL, loadCache.getNodeStatusReason(1));
+
+ // A missing cache yields no reason and Unknown status, without a fake merged string.
+ Assert.assertNull(loadCache.getNodeStatusReason(2));
+ Assert.assertEquals(NodeStatus.Unknown, loadCache.getNodeStatus(2));
+ }
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java
new file mode 100644
index 00000000000..f9e086eb85c
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/node/NodeManagerStatusReasonTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.node;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TNodeResource;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.manager.IManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
+import org.apache.iotdb.confignode.manager.partition.PartitionManager;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class NodeManagerStatusReasonTest {
+ private static final int NODE_ID = 1;
+ private static final int UNREPORTED_NODE_ID = 2;
+
+ private NodeInfo nodeInfo;
+ private NodeManager nodeManager;
+ private AtomicReference<NodeStatistics> currentStatistics;
+
+ @Before
+ public void setUp() {
+ nodeInfo = mock(NodeInfo.class);
+ IManager manager = mock(IManager.class);
+ LoadManager loadManager = mock(LoadManager.class);
+ PartitionManager partitionManager = mock(PartitionManager.class);
+ when(manager.getLoadManager()).thenReturn(loadManager);
+ when(manager.getPartitionManager()).thenReturn(partitionManager);
+ when(partitionManager.getAllReplicaSets()).thenReturn(Collections.emptyList());
+ nodeManager = new NodeManager(manager, nodeInfo);
+
+ currentStatistics =
+ new AtomicReference<>(
+ new NodeStatistics(1, NodeStatus.ReadOnly, NodeStatus.MANUAL, Long.MAX_VALUE));
+ NodeStatistics running = new NodeStatistics(2, NodeStatus.Running, null, 0);
+ // Simulate a heartbeat arriving immediately after the first statistics read. Reading
+ // status and reason separately would mix ReadOnly with the new heartbeat's null reason.
+ when(loadManager.getNodeStatisticsSnapshot())
+ .thenAnswer(
+ invocation -> Collections.singletonMap(NODE_ID, currentStatistics.getAndSet(running)));
+ when(loadManager.getNodeStatus(NODE_ID))
+ .thenAnswer(invocation -> currentStatistics.getAndSet(running).getStatus());
+ when(loadManager.getNodeStatusReason(NODE_ID))
+ .thenAnswer(invocation -> currentStatistics.get().getStatusReason());
+ when(loadManager.getNodeStatus(UNREPORTED_NODE_ID)).thenReturn(NodeStatus.Unknown);
+ }
+
+ @Test
+ public void testDataNodeStatusAndReasonRemainPairedDuringHeartbeatUpdate() {
+ when(nodeInfo.getRegisteredDataNodes())
+ .thenReturn(
+ Arrays.asList(
+ dataNodeConfiguration(NODE_ID), dataNodeConfiguration(UNREPORTED_NODE_ID)));
+
+ List<TDataNodeInfo> result = nodeManager.getRegisteredDataNodeInfoList();
+
+ assertEquals(2, result.size());
+ assertEquals(NodeStatus.ReadOnly.getStatus(), result.get(0).getStatus());
+ assertEquals(NodeStatus.MANUAL, result.get(0).getStatusReason());
+ assertEquals(NodeStatus.Unknown.getStatus(), result.get(1).getStatus());
+ assertNull(result.get(1).getStatusReason());
+ assertEquals(NodeStatus.Running, currentStatistics.get().getStatus());
+ assertNull(currentStatistics.get().getStatusReason());
+ }
+
+ @Test
+ public void testConfigNodeStatusAndReasonRemainPairedDuringHeartbeatUpdate() {
+ when(nodeInfo.getRegisteredConfigNodes())
+ .thenReturn(
+ Arrays.asList(configNodeLocation(NODE_ID), configNodeLocation(UNREPORTED_NODE_ID)));
+
+ List<TConfigNodeInfo> result = nodeManager.getRegisteredConfigNodeInfoList();
+
+ assertEquals(2, result.size());
+ assertEquals(NodeStatus.ReadOnly.getStatus(), result.get(0).getStatus());
+ assertEquals(NodeStatus.MANUAL, result.get(0).getStatusReason());
+ assertEquals(NodeStatus.Unknown.getStatus(), result.get(1).getStatus());
+ assertNull(result.get(1).getStatusReason());
+ assertEquals(NodeStatus.Running, currentStatistics.get().getStatus());
+ assertNull(currentStatistics.get().getStatusReason());
+ }
+
+ private static TDataNodeConfiguration dataNodeConfiguration(int nodeId) {
+ return new TDataNodeConfiguration()
+ .setLocation(
+ new TDataNodeLocation()
+ .setDataNodeId(nodeId)
+ .setClientRpcEndPoint(new TEndPoint("127.0.0.1", 6667 + nodeId)))
+ .setResource(new TNodeResource(1, 1024));
+ }
+
+ private static TConfigNodeLocation configNodeLocation(int nodeId) {
+ return new TConfigNodeLocation()
+ .setConfigNodeId(nodeId)
+ .setInternalEndPoint(new TEndPoint("127.0.0.1", 10710 + nodeId));
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java
new file mode 100644
index 00000000000..f3c0aab1a76
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/subscription/runtime/SubscriptionRuntimeCoordinatorTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.manager.subscription.runtime;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.subscription.meta.topic.TopicMeta;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.ProcedureManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.cache.node.NodeStatistics;
+import org.apache.iotdb.confignode.manager.load.subscriber.NodeStatisticsChangeEvent;
+import org.apache.iotdb.confignode.manager.subscription.SubscriptionCoordinator;
+import org.apache.iotdb.confignode.manager.subscription.SubscriptionManager;
+import org.apache.iotdb.confignode.persistence.subscription.SubscriptionInfo;
+import org.apache.iotdb.rpc.subscription.config.TopicConfig;
+
+import org.apache.tsfile.utils.Pair;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class SubscriptionRuntimeCoordinatorTest {
+
+ private static final int DATA_NODE_ID = 1;
+
+ private final ConfigManager configManager = mock(ConfigManager.class);
+ private final SubscriptionManager subscriptionManager = mock(SubscriptionManager.class);
+ private final SubscriptionCoordinator subscriptionCoordinator =
+ mock(SubscriptionCoordinator.class);
+ private final SubscriptionInfo subscriptionInfo = mock(SubscriptionInfo.class);
+ private final TopicMeta topicMeta = mock(TopicMeta.class);
+ private final TopicConfig topicConfig = mock(TopicConfig.class);
+ private final LoadManager loadManager = mock(LoadManager.class);
+ private final ProcedureManager procedureManager = mock(ProcedureManager.class);
+
+ private final SubscriptionRuntimeCoordinator coordinator =
+ new SubscriptionRuntimeCoordinator(configManager);
+
+ private void stubConsensusBasedTopic() {
+ when(configManager.getSubscriptionManager()).thenReturn(subscriptionManager);
+ when(subscriptionManager.getSubscriptionCoordinator()).thenReturn(subscriptionCoordinator);
+ when(subscriptionCoordinator.getSubscriptionInfo()).thenReturn(subscriptionInfo);
+ when(subscriptionInfo.getAllTopicMeta()).thenReturn(Collections.singletonList(topicMeta));
+ when(topicMeta.getConfig()).thenReturn(topicConfig);
+ when(topicConfig.isIncrementalMode()).thenReturn(true);
+
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ // A seeded DataRegion leader pair makes the refresh map non-empty.
+ when(loadManager.getRegionLeaderMap())
+ .thenReturn(
+ Collections.singletonMap(
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 1), DATA_NODE_ID));
+ when(configManager.getProcedureManager()).thenReturn(procedureManager);
+ }
+
+ private NodeStatisticsChangeEvent eventOf(
+ final NodeStatus oldStatus, final NodeStatus newStatus) {
+ final Map<Integer, Pair<NodeStatistics, NodeStatistics>> map = new HashMap<>();
+ map.put(DATA_NODE_ID, new Pair<>(new NodeStatistics(oldStatus), new NodeStatistics(newStatus)));
+ return new NodeStatisticsChangeEvent(map);
+ }
+
+ @Test
+ public void testStoppedStatusTriggersRuntimeRefresh() {
+ stubConsensusBasedTopic();
+
+ coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.Stopped));
+
+ // A Stopped node is handled like Unknown/Removing: its runtime leader pairs are refreshed.
+ verify(procedureManager).subscriptionHandleLeaderChange(any(Map.class), anyLong());
+ }
+
+ @Test
+ public void testUnknownStatusTriggersRuntimeRefresh() {
+ stubConsensusBasedTopic();
+
+ coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.Unknown));
+
+ verify(procedureManager).subscriptionHandleLeaderChange(any(Map.class), anyLong());
+ }
+
+ @Test
+ public void testReadOnlyStatusDoesNotTriggerRuntimeRefresh() {
+ stubConsensusBasedTopic();
+
+ coordinator.handleNodeStatisticsChange(eventOf(NodeStatus.Running, NodeStatus.ReadOnly));
+
+ // ReadOnly is not runtime-sensitive: the runtime leader pairs stay untouched.
+ verify(procedureManager, never()).subscriptionHandleLeaderChange(any(Map.class), anyLong());
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
index e3ced5b0694..667cbba357e 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
@@ -19,18 +19,30 @@
package org.apache.iotdb.confignode.persistence;
+import org.apache.iotdb.common.rpc.thrift.TAINodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TAINodeLocation;
import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TNodeResource;
+import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.exception.StartupException;
+import org.apache.iotdb.confignode.consensus.request.write.ainode.RegisterAINodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.ainode.RemoveAINodePlan;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.RemoveConfigNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateVersionInfoPlan;
import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.thrift.TException;
import org.apache.tsfile.external.commons.io.FileUtils;
+import org.apache.tsfile.utils.Pair;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -38,6 +50,9 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.util.Collections;
+import java.util.Map;
import static org.apache.iotdb.db.utils.constant.TestConstant.BASE_OUTPUT_PATH;
@@ -66,11 +81,207 @@ public class NodeInfoTest {
public void testSnapshot() throws TException, IOException {
registerConfigNodes();
registerDataNodes();
+ nodeInfo.registerAINode(new RegisterAINodePlan(generateTAINodeConfiguration()));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Stopped));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Removing));
Assert.assertTrue(nodeInfo.processTakeSnapshot(snapshotDir));
NodeInfo nodeInfo1 = new NodeInfo();
nodeInfo1.processLoadSnapshot(snapshotDir);
Assert.assertEquals(nodeInfo, nodeInfo1);
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo1.getPersistedNodeStatus(10001));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), nodeInfo1.getPersistedNodeStatus(10003));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, null), nodeInfo1.getPersistedNodeStatus(10004));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), nodeInfo1.getPersistedNodeStatus(10006));
+ }
+
+ @Test
+ public void testLoadSnapshotWithoutNodeStatuses() throws TException, IOException {
+ NodeInfo oldNodeInfo = new NodeInfo();
+ oldNodeInfo.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ oldNodeInfo.updateVersionInfo(
+ new UpdateVersionInfoPlan(new TNodeVersionInfo("2.0", "build"), 10003));
+ File oldSnapshotDir = new File(snapshotDir, "without-node-statuses");
+ Assert.assertTrue(oldSnapshotDir.mkdirs());
+ Assert.assertTrue(oldNodeInfo.processTakeSnapshot(oldSnapshotDir));
+ // The previous format ends immediately before the new, empty node-status map size.
+ try (RandomAccessFile snapshot =
+ new RandomAccessFile(new File(oldSnapshotDir, "node_info.bin"), "rw")) {
+ snapshot.setLength(snapshot.length() - Integer.BYTES);
+ }
+
+ NodeInfo restored = new NodeInfo();
+ restored.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing));
+ restored.processLoadSnapshot(oldSnapshotDir);
+ Assert.assertEquals(oldNodeInfo, restored);
+ Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty());
+ Assert.assertEquals(new TNodeVersionInfo("2.0", "build"), restored.getVersionInfo(10003));
+ }
+
+ @Test
+ public void testSetOperationsReplacePersistedStatus() {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing));
+ Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003));
+ Pair<NodeStatus, String> status = info.getPersistedNodeStatus(10003);
+ status.left = NodeStatus.ReadOnly;
+ status.right = NodeStatus.MANUAL;
+ Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003));
+ Map<Integer, Pair<NodeStatus, String>> snapshot = info.getPersistedNodeStatuses();
+ snapshot.get(10003).left = NodeStatus.ReadOnly;
+ snapshot.get(10003).right = NodeStatus.MANUAL;
+ Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003));
+ snapshot.clear();
+ Assert.assertEquals(new Pair<>(NodeStatus.Removing, null), info.getPersistedNodeStatus(10003));
+
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003));
+ info.clear();
+ Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty());
+ }
+
+ @Test
+ public void testReadOnlyReasonsSurviveSnapshot() throws TException, IOException {
+ NodeInfo info = new NodeInfo();
+ String[] reasons = {
+ null,
+ NodeStatus.MANUAL,
+ NodeStatus.DISK_FULL,
+ NodeStatus.STOPPING,
+ NodeStatus.UNRECOVERABLE_ERROR + ": failed to recover root.test-0"
+ };
+ for (int i = 0; i < reasons.length; i++) {
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3 + i)));
+ info.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(10003 + i, NodeStatus.ReadOnly, reasons[i]));
+ }
+ File directory = new File(snapshotDir, "readonly-reasons");
+ Assert.assertTrue(directory.mkdirs());
+ Assert.assertTrue(info.processTakeSnapshot(directory));
+
+ NodeInfo restored = new NodeInfo();
+ restored.processLoadSnapshot(directory);
+ Assert.assertEquals(info, restored);
+ for (int i = 0; i < reasons.length; i++) {
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, reasons[i]), restored.getPersistedNodeStatus(10003 + i));
+ }
+ }
+
+ @Test
+ public void testStoppedAndRemovingReasonsSurviveSnapshot() throws TException, IOException {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(4)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped, "stop reason"));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing, "remove reason"));
+ File directory = new File(snapshotDir, "stopped-removing-reasons");
+ Assert.assertTrue(directory.mkdirs());
+ Assert.assertTrue(info.processTakeSnapshot(directory));
+
+ NodeInfo restored = new NodeInfo();
+ restored.processLoadSnapshot(directory);
+ Assert.assertEquals(info, restored);
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Stopped, "stop reason"), restored.getPersistedNodeStatus(10003));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, "remove reason"), restored.getPersistedNodeStatus(10004));
+ }
+
+ @Test
+ public void testReadOnlyReasonReplacementAndStatusChange() {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ // Plans contain the caller's final decision, including lower-priority or absent reasons.
+ for (String reason : new String[] {NodeStatus.MANUAL, NodeStatus.DISK_FULL, null}) {
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, reason));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.ReadOnly, reason), info.getPersistedNodeStatus(10003));
+ }
+ info.applyNodeStatusPlan(
+ new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, NodeStatus.MANUAL));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ Assert.assertEquals(new Pair<>(NodeStatus.Stopped, null), info.getPersistedNodeStatus(10003));
+ }
+
+ @Test
+ public void testNodeRemovalDiscardsReadOnlyReason() {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ UpdateNodeStatusPlan readOnly =
+ new UpdateNodeStatusPlan(10003, NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ info.applyNodeStatusPlan(readOnly);
+ info.removeDataNode(
+ new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3))));
+ Assert.assertNull(info.getPersistedNodeStatus(10003));
+ info.applyNodeStatusPlan(readOnly);
+ Assert.assertNull(info.getPersistedNodeStatus(10003));
+ }
+
+ @Test
+ public void testClearRemovesAnyPersistedStatusAndAllowsRetry() {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ for (NodeStatus status :
+ new NodeStatus[] {NodeStatus.Stopped, NodeStatus.Removing, NodeStatus.ReadOnly}) {
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, status, NodeStatus.MANUAL));
+ Assert.assertNotNull(info.getPersistedNodeStatus(10003));
+
+ UpdateNodeStatusPlan clear = new UpdateNodeStatusPlan(10003, null);
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), info.applyNodeStatusPlan(clear).getCode());
+ Assert.assertNull(info.getPersistedNodeStatus(10003));
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), info.applyNodeStatusPlan(clear).getCode());
+ Assert.assertNull(info.getPersistedNodeStatus(10003));
+ }
+ }
+
+ @Test
+ public void testNodeRemovalDiscardsStatusAndIgnoresDelayedUpdates() {
+ NodeInfo info = new NodeInfo();
+ TConfigNodeLocation configNode =
+ new TConfigNodeLocation(
+ 10001, new TEndPoint("127.0.0.1", 22201), new TEndPoint("127.0.0.1", 22301));
+ info.applyConfigNode(new ApplyConfigNodePlan(configNode));
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing));
+ TAINodeConfiguration aiNode = generateTAINodeConfiguration();
+ info.registerAINode(new RegisterAINodePlan(aiNode));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Removing));
+
+ info.removeConfigNode(new RemoveConfigNodePlan(configNode));
+ info.removeDataNode(
+ new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3))));
+ info.removeAINode(new RemoveAINodePlan(aiNode.getLocation()));
+ Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty());
+
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10001, NodeStatus.Stopped)).getCode());
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing)).getCode());
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(),
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10006, NodeStatus.Stopped)).getCode());
+ Assert.assertTrue(info.getPersistedNodeStatuses().isEmpty());
+ }
+
+ private static TAINodeConfiguration generateTAINodeConfiguration() {
+ return new TAINodeConfiguration(
+ new TAINodeLocation(10006, new TEndPoint("127.0.0.1", 10810)), new TNodeResource(8, 1024L));
}
private void registerConfigNodes() {
@@ -85,6 +296,64 @@ public class NodeInfoTest {
}
}
+ @Test
+ public void testTruncatedStatusPayloadMustFailSnapshotLoad() throws Exception {
+ for (NodeStatus status : new NodeStatus[] {NodeStatus.Removing, NodeStatus.ReadOnly}) {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, status, NodeStatus.MANUAL));
+ File directory = new File(snapshotDir, "truncated-status-payload-" + status);
+ Assert.assertTrue(directory.mkdirs());
+ Assert.assertTrue(info.processTakeSnapshot(directory));
+ try (RandomAccessFile file =
+ new RandomAccessFile(new File(directory, "node_info.bin"), "rw")) {
+ file.setLength(file.length() - 2);
+ }
+ Assert.assertThrows(IOException.class, () -> new NodeInfo().processLoadSnapshot(directory));
+ }
+ }
+
+ @Test
+ public void testClearedStatusesStayClearedAfterSnapshotAndLaterPlans() throws Exception {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(4)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, null));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, null));
+ File directory = new File(snapshotDir, "cleared-statuses");
+ Assert.assertTrue(directory.mkdirs());
+ Assert.assertTrue(info.processTakeSnapshot(directory));
+ NodeInfo restored = new NodeInfo();
+ restored.processLoadSnapshot(directory);
+ Assert.assertTrue(restored.getPersistedNodeStatuses().isEmpty());
+ // Replay the entries after that snapshot in their original order.
+ restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10004, NodeStatus.Removing));
+ restored.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, null));
+ Assert.assertNull(restored.getPersistedNodeStatus(10003));
+ Assert.assertEquals(
+ new Pair<>(NodeStatus.Removing, null), restored.getPersistedNodeStatus(10004));
+ }
+
+ @Test
+ public void testRejoinedNodeDoesNotInheritRemovedNodeStatus() {
+ NodeInfo info = new NodeInfo();
+ info.registerDataNode(new RegisterDataNodePlan(generateTDataNodeConfiguration(3)));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Removing));
+ info.removeDataNode(
+ new RemoveDataNodePlan(Collections.singletonList(generateTDataNodeLocation(3))));
+ int newId = info.generateNextNodeId();
+ TDataNodeConfiguration rejoined = generateTDataNodeConfiguration(3);
+ rejoined.getLocation().setDataNodeId(newId);
+ info.registerDataNode(new RegisterDataNodePlan(rejoined));
+ info.applyNodeStatusPlan(new UpdateNodeStatusPlan(10003, NodeStatus.Stopped));
+ Assert.assertNotEquals(10003, newId);
+ Assert.assertNull(info.getPersistedNodeStatus(newId));
+ Assert.assertNull(info.getPersistedNodeStatus(10003));
+ }
+
private void registerDataNodes() {
for (int i = 3; i < 6; i++) {
RegisterDataNodePlan registerDataNodePlan =
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java
new file mode 100644
index 00000000000..7691eb45848
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnvTest.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.env;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.procedure.scheduler.ProcedureScheduler;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ConfigNodeProcedureEnvTest {
+
+ private static final int DATA_NODE_ID = 1;
+
+ private final ConfigManager configManager = mock(ConfigManager.class);
+ private final LoadManager loadManager = mock(LoadManager.class);
+ private final ConfigNodeProcedureEnv env =
+ new ConfigNodeProcedureEnv(configManager, mock(ProcedureScheduler.class));
+
+ @Before
+ public void setUp() {
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ }
+
+ @Test
+ public void testStoppedNodeIsNotRuntimeActiveWriter() {
+ when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Stopped);
+
+ // A Stopped node is handled like Unknown: it can not serve as an active runtime writer.
+ Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testUnknownNodeIsNotRuntimeActiveWriter() {
+ when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Unknown);
+
+ Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testRemovingNodeIsNotRuntimeActiveWriter() {
+ when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Removing);
+
+ Assert.assertFalse(env.isRuntimeActiveWriterNode(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testRunningNodeIsRuntimeActiveWriter() {
+ when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.Running);
+
+ Assert.assertTrue(env.isRuntimeActiveWriterNode(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testReadOnlyNodeIsRuntimeActiveWriter() {
+ when(loadManager.getNodeStatus(DATA_NODE_ID)).thenReturn(NodeStatus.ReadOnly);
+
+ // ReadOnly still responds and can serve as a runtime writer, unlike Unknown/Stopped/Removing.
+ Assert.assertTrue(env.isRuntimeActiveWriterNode(DATA_NODE_ID));
+ }
+
+ @Test
+ public void testNegativeNodeIdIsNotRuntimeActiveWriter() {
+ // The node id guard dominates the status check: even a Running status can not turn an
+ // invalid node id into an active writer.
+ when(loadManager.getNodeStatus(-1)).thenReturn(NodeStatus.Running);
+
+ Assert.assertFalse(env.isRuntimeActiveWriterNode(-1));
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java
index 8fd32922398..b94c9625968 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/env/RegionMaintainHandlerTest.java
@@ -29,6 +29,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.client.sync.CnToDnSyncRequestType;
import org.apache.iotdb.confignode.client.sync.SyncDataNodeClientPool;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.manager.ConfigManager;
@@ -39,6 +40,7 @@ import org.apache.iotdb.confignode.manager.node.NodeManager;
import org.apache.iotdb.confignode.manager.partition.PartitionManager;
import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeReq;
import org.apache.iotdb.mpp.rpc.thrift.TRegionLeaderChangeResp;
+import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.junit.After;
@@ -58,8 +60,12 @@ import java.util.Optional;
import static org.apache.iotdb.consensus.ConsensusFactory.RATIS_CONSENSUS;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -232,4 +238,66 @@ public class RegionMaintainHandlerTest {
clientManagerField.set(clientPool, originalClientManager);
}
}
+
+ private RegionMaintainHandler stubStatusAndSubmit(
+ final NodeStatus status, final TSStatus submitResult) {
+ RegionMaintainHandler retryHandler = spy(handler);
+ doReturn(status).when(retryHandler).getDataNodeStatus(original.getDataNodeId());
+ doReturn(submitResult)
+ .when(retryHandler)
+ .submitDataNodeSyncRequest(
+ any(TEndPoint.class),
+ any(Object.class),
+ any(CnToDnSyncRequestType.class),
+ any(Boolean.class));
+ return retryHandler;
+ }
+
+ @Test
+ public void testDeleteOldRegionPeerUsesSingleRetryForStoppedDataNode() {
+ final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.Stopped, success);
+
+ final TSStatus status = retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId);
+
+ assertEquals(success.getCode(), status.getCode());
+ // A Stopped node does not respond to requests either: like Unknown, it gets a single retry.
+ verify(retryHandler)
+ .submitDataNodeSyncRequest(
+ any(TEndPoint.class),
+ any(Object.class),
+ eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER),
+ eq(false));
+ }
+
+ @Test
+ public void testDeleteOldRegionPeerUsesFullRetryForRunningDataNode() {
+ final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.Running, success);
+
+ retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId);
+
+ verify(retryHandler)
+ .submitDataNodeSyncRequest(
+ any(TEndPoint.class),
+ any(Object.class),
+ eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER),
+ eq(true));
+ }
+
+ @Test
+ public void testDeleteOldRegionPeerUsesFullRetryForReadOnlyDataNode() {
+ final TSStatus success = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
+ final RegionMaintainHandler retryHandler = stubStatusAndSubmit(NodeStatus.ReadOnly, success);
+
+ retryHandler.submitDeleteOldRegionPeerTask(1L, original, regionId);
+
+ // Only Unknown and Stopped are treated as down; ReadOnly still responds.
+ verify(retryHandler)
+ .submitDataNodeSyncRequest(
+ any(TEndPoint.class),
+ any(Object.class),
+ eq(CnToDnSyncRequestType.DELETE_OLD_REGION_PEER),
+ eq(true));
+ }
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java
new file mode 100644
index 00000000000..6ec10161917
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodePersistenceTest.java
@@ -0,0 +1,581 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.confignode.procedure.impl.node;
+
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
+import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.client.IClientManager;
+import org.apache.iotdb.commons.client.async.AsyncDataNodeInternalServiceClient;
+import org.apache.iotdb.commons.client.request.AsyncRequestManager;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.client.async.CnToDnInternalServiceAsyncRequestManager;
+import org.apache.iotdb.confignode.consensus.request.write.confignode.UpdateNodeStatusPlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
+import org.apache.iotdb.confignode.consensus.request.write.datanode.RemoveDataNodePlan;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.cache.LoadCache;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.confignode.persistence.node.NodeInfo;
+import org.apache.iotdb.confignode.procedure.env.ConfigNodeProcedureEnv;
+import org.apache.iotdb.confignode.procedure.env.RemoveDataNodeHandler;
+import org.apache.iotdb.confignode.procedure.state.RemoveDataNodeState;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.apache.thrift.async.AsyncMethodCallback;
+import org.apache.tsfile.utils.Pair;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyList;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+/** Real procedure/handler/cache/NodeInfo, with faults only at the RPC and consensus boundaries. */
+public class RemoveDataNodePersistenceTest {
+ private static final int FIRST = 100;
+ private static final int SECOND = 101;
+ private final Map<Integer, NodeStatus> remoteStatuses = new HashMap<>();
+ private final Map<Integer, Boolean> failRpc = new HashMap<>();
+ private final Map<Integer, Boolean> failStop = new HashMap<>();
+ private NodeInfo nodeInfo;
+ private LoadCache cache;
+ private ConfigManager manager;
+ private ConsensusManager consensus;
+ private RemoveDataNodeHandler handler;
+ private ConfigNodeProcedureEnv env;
+ private Field clientManagerField;
+ private Object originalClientManager;
+ private List<TDataNodeLocation> nodes;
+ private boolean failStatusWrite;
+ private boolean applyRpcBeforeFailure;
+ private boolean failDeleteWrite;
+ private boolean throwDeleteWrite;
+ private int stopRequests;
+
+ @Before
+ public void setUp() throws Exception {
+ nodes = Arrays.asList(location(FIRST), location(SECOND));
+ nodeInfo = new NodeInfo();
+ manager = mock(ConfigManager.class, RETURNS_DEEP_STUBS);
+ consensus = mock(ConsensusManager.class);
+ when(manager.getConsensusManager()).thenReturn(consensus);
+ when(consensus.isLeader()).thenReturn(true);
+ when(consensus.write(any(RemoveDataNodePlan.class)))
+ .thenAnswer(
+ i -> {
+ if (throwDeleteWrite) {
+ throw new ConsensusException("injected delete failure");
+ }
+ return failDeleteWrite ? failure() : nodeInfo.removeDataNode(i.getArgument(0));
+ });
+ for (TDataNodeLocation node : nodes) {
+ nodeInfo.registerDataNode(
+ new RegisterDataNodePlan(new TDataNodeConfiguration().setLocation(node)));
+ remoteStatuses.put(node.getDataNodeId(), NodeStatus.Running);
+ }
+ when(manager.getNodeManager()).thenReturn(new NodeManager(manager, nodeInfo));
+ when(manager.getClusterSchemaManager().getDatabaseNames(null))
+ .thenReturn(Collections.emptyList());
+ when(manager.getPartitionManager().getAllReplicaSets()).thenReturn(Collections.emptyList());
+ when(manager.getPartitionManager().getAllReplicaSets(anyInt()))
+ .thenReturn(Collections.emptyList());
+ cache = new LoadCache();
+ cache.setNodeStatusPersistence(
+ nodeInfo::getPersistedNodeStatus,
+ plan -> failStatusWrite ? failure() : nodeInfo.applyNodeStatusPlan(plan));
+ cache.initHeartbeatCache(manager);
+ for (TDataNodeLocation node : nodes) {
+ assertSuccess(cache.trySetNodeStatus(node.getDataNodeId(), NodeStatus.Running, null, true));
+ }
+ LoadManager loadManager = mock(LoadManager.class);
+ when(manager.getLoadManager()).thenReturn(loadManager);
+ when(loadManager.trySetNodeStatus(anyInt(), any(), any(), anyBoolean()))
+ .thenAnswer(
+ i ->
+ cache.trySetNodeStatus(
+ i.getArgument(0), i.getArgument(1), i.getArgument(2), i.getArgument(3)));
+ when(loadManager.getNodeStatus(anyInt()))
+ .thenAnswer(i -> cache.getNodeStatus(i.getArgument(0)));
+ doAnswer(
+ i -> {
+ cache.removeNodeCache(i.getArgument(0));
+ return null;
+ })
+ .when(loadManager)
+ .removeNodeCache(anyInt());
+
+ // Keep the real request context, handler and retry loop. Restore the singleton after each test.
+ IClientManager<TEndPoint, AsyncDataNodeInternalServiceClient> transport =
+ mock(IClientManager.class);
+ for (TDataNodeLocation node : nodes) {
+ int id = node.getDataNodeId();
+ AsyncDataNodeInternalServiceClient client = mock(AsyncDataNodeInternalServiceClient.class);
+ when(transport.borrowClient(node.getInternalEndPoint())).thenReturn(client);
+ doAnswer(
+ i -> {
+ AsyncMethodCallback<TSStatus> callback = i.getArgument(1);
+ if (failRpc.getOrDefault(id, false)) {
+ if (applyRpcBeforeFailure) {
+ remoteStatuses.put(id, NodeStatus.parse(i.getArgument(0)));
+ }
+ callback.onComplete(failure());
+ } else {
+ remoteStatuses.put(id, NodeStatus.parse(i.getArgument(0)));
+ callback.onComplete(success());
+ }
+ return null;
+ })
+ .when(client)
+ .setSystemStatus(anyString(), any());
+ doAnswer(
+ i -> {
+ stopRequests++;
+ AsyncMethodCallback<TSStatus> callback = i.getArgument(0);
+ callback.onComplete(failStop.getOrDefault(id, false) ? failure() : success());
+ return null;
+ })
+ .when(client)
+ .stopAndClearDataNode(any());
+ }
+ clientManagerField = AsyncRequestManager.class.getDeclaredField("clientManager");
+ clientManagerField.setAccessible(true);
+ originalClientManager =
+ clientManagerField.get(CnToDnInternalServiceAsyncRequestManager.getInstance());
+ clientManagerField.set(CnToDnInternalServiceAsyncRequestManager.getInstance(), transport);
+ handler = spy(new RemoveDataNodeHandler(manager));
+ // Region migration and broadcast are independent of the status persistence under test.
+ doReturn(Collections.emptyList()).when(handler).selectedRegionMigrationPlans(anyList());
+ doNothing().when(handler).broadcastDataNodeStatusChange(anyList());
+ env = mock(ConfigNodeProcedureEnv.class);
+ when(env.getConfigManager()).thenReturn(manager);
+ when(env.getRemoveDataNodeHandler()).thenReturn(handler);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (clientManagerField != null && originalClientManager != null) {
+ clientManagerField.set(
+ CnToDnInternalServiceAsyncRequestManager.getInstance(), originalClientManager);
+ }
+ }
+
+ @Test
+ public void testPreparePersistsRemoving() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testPrepareRpcFailureStillPersistsRemoving() throws Exception {
+ failRpc.put(FIRST, true);
+ TestProcedure procedure =
+ procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE);
+ procedure.step(env);
+ assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state());
+ assertStates(FIRST, NodeStatus.Running, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testOfflineRemovalAndRetryDoNotRequireRpcSuccess() throws Exception {
+ for (NodeStatus offline : Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped)) {
+ assertSuccess(cache.trySetNodeStatus(FIRST, offline, null, true));
+ remoteStatuses.put(FIRST, offline);
+ failRpc.put(FIRST, true);
+ change(FIRST, NodeStatus.Removing);
+ assertStates(FIRST, offline, NodeStatus.Removing, NodeStatus.Removing);
+ change(FIRST, NodeStatus.Removing);
+ assertStates(FIRST, offline, NodeStatus.Removing, NodeStatus.Removing);
+ }
+ }
+
+ @Test
+ public void testOfflineRemovalStillRequiresPersistence() throws Exception {
+ assertSuccess(cache.trySetNodeStatus(FIRST, NodeStatus.Stopped, null, true));
+ failRpc.put(FIRST, true);
+ failStatusWrite = true;
+ expectChangeFailure(NodeStatus.Removing);
+ assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST));
+ assertEquals(new Pair<>(NodeStatus.Stopped, null), nodeInfo.getPersistedNodeStatus(FIRST));
+ failStatusWrite = false;
+ change(FIRST, NodeStatus.Removing);
+ assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(FIRST));
+ }
+
+ @Test
+ public void testOfflineCompensationDoesNotRequireRpcSuccess() throws Exception {
+ retainRegionOn(FIRST);
+ for (NodeStatus offline : Arrays.asList(NodeStatus.Unknown, NodeStatus.Stopped)) {
+ failRpc.put(FIRST, false);
+ change(FIRST, NodeStatus.Removing);
+ failRpc.put(FIRST, true);
+ assertTrue(procedure(offline, RemoveDataNodeState.STOP_DATA_NODE).finish(env));
+ assertStates(
+ FIRST, NodeStatus.Removing, offline, offline == NodeStatus.Stopped ? offline : null);
+ }
+ }
+
+ @Test
+ public void testInitialPrepareCommitFailureRetriesWithoutRecheckingCapacity() throws Exception {
+ for (int index = nodes.size(); index <= NodeInfo.getMinimumDataNode(); index++) {
+ nodeInfo.registerDataNode(
+ new RegisterDataNodePlan(
+ new TDataNodeConfiguration().setLocation(location(FIRST + index))));
+ }
+ cache.initHeartbeatCache(manager);
+ for (TDataNodeConfiguration node : nodeInfo.getRegisteredDataNodes()) {
+ assertSuccess(
+ cache.trySetNodeStatus(
+ node.getLocation().getDataNodeId(), NodeStatus.Running, null, true));
+ }
+ when(manager
+ .getLoadManager()
+ .filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.ReadOnly))
+ .thenAnswer(
+ i -> cache.filterDataNodeThroughStatus(NodeStatus.Running, NodeStatus.ReadOnly));
+
+ TestProcedure procedure =
+ procedure(NodeStatus.Running, RemoveDataNodeState.REGION_REPLICA_CHECK);
+ failStatusWrite = true;
+ procedure.step(env);
+ procedure.step(env);
+ assertFalse(procedure.isFailed());
+ assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state());
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null);
+
+ failStatusWrite = false;
+ procedure.step(env);
+ assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state());
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testPrepareCommitFailureUpdatesCacheAndRetries() throws Exception {
+ failStatusWrite = true;
+ TestProcedure procedure =
+ procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE);
+ procedure.step(env);
+ assertFalse(procedure.isFailed());
+ assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state());
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null);
+ failStatusWrite = false;
+ procedure.step(env);
+ assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state());
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testPrepareCommitFailureStopsAfterRetryThreshold() throws Exception {
+ failStatusWrite = true;
+ TestProcedure procedure =
+ procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE);
+ for (int attempt = 0; attempt < 5; attempt++) {
+ procedure.step(env);
+ assertFalse(procedure.isFailed());
+ assertEquals(RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE, procedure.state());
+ }
+ procedure.step(env);
+ assertTrue(procedure.isFailed());
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null);
+ }
+
+ @Test
+ public void testAppliedRpcWithLostResponseMustStillPersistIntent() throws Exception {
+ applyRpcBeforeFailure = true;
+ failRpc.put(FIRST, true);
+ TestProcedure procedure =
+ procedure(NodeStatus.Running, RemoveDataNodeState.REMOVE_DATA_NODE_PREPARE);
+ procedure.step(env);
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ assertEquals(RemoveDataNodeState.BROADCAST_DISABLE_DATA_NODE, procedure.state());
+ }
+
+ @Test
+ public void testPreparePartialRpcFailurePreservesPerNodeResults() throws Exception {
+ failRpc.put(SECOND, true);
+ Map<Integer, NodeStatus> desired = new HashMap<>();
+ desired.put(FIRST, NodeStatus.Removing);
+ desired.put(SECOND, NodeStatus.Removing);
+ handler.changeDataNodeStatus(nodes, desired);
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ assertStates(SECOND, NodeStatus.Running, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testRpcFailureOnRetryStillRequiresPersistence() throws Exception {
+ failStatusWrite = true;
+ expectChangeFailure(NodeStatus.Removing);
+ failRpc.put(FIRST, true);
+ expectChangeFailure(NodeStatus.Removing);
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, null);
+ failStatusWrite = false;
+ change(FIRST, NodeStatus.Removing);
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ }
+
+ @Test
+ public void testCompensationRestoresRunning() throws Exception {
+ assertCompensation(NodeStatus.Running);
+ }
+
+ @Test
+ public void testCompensationRestoresReadOnly() throws Exception {
+ assertCompensation(NodeStatus.ReadOnly);
+ assertEquals(NodeStatus.MANUAL, cache.getNodeStatusReason(FIRST));
+ }
+
+ @Test
+ public void testCompensationRestoresUnknown() throws Exception {
+ assertCompensation(NodeStatus.Unknown);
+ }
+
+ @Test
+ public void testCompensationRestoresStopped() throws Exception {
+ assertCompensation(NodeStatus.Stopped);
+ }
+
+ @Test
+ public void testCompensationRpcFailureMustNotFinishProcedure() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ retainRegionOn(FIRST);
+ failRpc.put(FIRST, true);
+ for (NodeStatus original : Arrays.asList(NodeStatus.Running, NodeStatus.ReadOnly)) {
+ assertFalse(
+ "Failed compensation must not finish",
+ procedure(original, RemoveDataNodeState.STOP_DATA_NODE).finish(env));
+ assertStates(FIRST, NodeStatus.Removing, NodeStatus.Removing, NodeStatus.Removing);
+ }
+ }
+
+ @Test
+ public void testCompensationCommitFailureRemainsRetryable() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ retainRegionOn(FIRST);
+ failStatusWrite = true;
+ TestProcedure procedure = procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE);
+ assertFalse(procedure.finish(env));
+ assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, NodeStatus.Removing);
+ failStatusWrite = false;
+ assertTrue(procedure.finish(env));
+ assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, null);
+ }
+
+ @Test
+ public void testDeleteErrorMustNotStopOrForgetRegisteredNode() throws Exception {
+ failDeleteWrite = true;
+ assertDeleteFailure();
+ }
+
+ @Test
+ public void testProcedureExecutorCanActuallyRetryCompensationCommitFailure() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ retainRegionOn(FIRST);
+ failStatusWrite = true;
+ TestProcedure procedure = procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE);
+ procedure.step(env);
+ assertEquals(
+ "The state-machine executor must retain the failed compensation step",
+ RemoveDataNodeState.STOP_DATA_NODE,
+ procedure.state());
+ failStatusWrite = false;
+ procedure.step(env);
+ assertStates(FIRST, NodeStatus.Running, NodeStatus.Running, null);
+ }
+
+ @Test
+ public void testDeleteExceptionMustNotStopOrForgetRegisteredNode() throws Exception {
+ throwDeleteWrite = true;
+ assertDeleteFailure();
+ }
+
+ @Test
+ public void testSuccessfulDeleteCannotBeResurrectedByLateStatus() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ assertTrue(procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE).finish(env));
+ assertFalse(
+ nodeInfo.getRegisteredDataNodes().stream()
+ .anyMatch(n -> n.getLocation().getDataNodeId() == FIRST));
+ assertNull(nodeInfo.getPersistedNodeStatus(FIRST));
+ nodeInfo.applyNodeStatusPlan(new UpdateNodeStatusPlan(FIRST, NodeStatus.Removing));
+ assertNull(nodeInfo.getPersistedNodeStatus(FIRST));
+ assertTrue(stopRequests > 0);
+ }
+
+ @Test
+ public void testFailedRemoteStopCannotResurrectDeletedNode() throws Exception {
+ failStop.put(FIRST, true);
+ testSuccessfulDeleteCannotBeResurrectedByLateStatus();
+ }
+
+ @Test
+ public void testPartialRemovalKeepsSuccessfulAndCompensatedResultsSeparate() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ change(SECOND, NodeStatus.Removing);
+ retainRegionOn(SECOND);
+ Map<Integer, NodeStatus> originals = new HashMap<>();
+ originals.put(FIRST, NodeStatus.Running);
+ originals.put(SECOND, NodeStatus.ReadOnly);
+ TestProcedure procedure =
+ new TestProcedure(nodes, originals, RemoveDataNodeState.STOP_DATA_NODE);
+ assertTrue(procedure.finish(env));
+ assertNull(nodeInfo.getPersistedNodeStatus(FIRST));
+ assertEquals(1, nodeInfo.getRegisteredDataNodes().size());
+ assertStates(SECOND, NodeStatus.ReadOnly, NodeStatus.ReadOnly, NodeStatus.ReadOnly);
+ }
+
+ private void assertDeleteFailure() throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ boolean finished =
+ procedure(NodeStatus.Running, RemoveDataNodeState.STOP_DATA_NODE).finish(env);
+ assertEquals("No remote stop before durable unregister", 0, stopRequests);
+ assertFalse("Failed unregister must remain retryable", finished);
+ assertEquals(NodeStatus.Removing, cache.getNodeStatus(FIRST));
+ assertEquals(new Pair<>(NodeStatus.Removing, null), nodeInfo.getPersistedNodeStatus(FIRST));
+ assertEquals(2, nodeInfo.getRegisteredDataNodes().size());
+ }
+
+ private void assertCompensation(NodeStatus original) throws Exception {
+ change(FIRST, NodeStatus.Removing);
+ retainRegionOn(FIRST);
+ assertTrue(
+ "Compensation should complete for " + original,
+ procedure(original, RemoveDataNodeState.STOP_DATA_NODE).finish(env));
+ assertStates(FIRST, original, original, original.isPersistentStatus() ? original : null);
+ }
+
+ private void retainRegionOn(int id) {
+ when(manager.getPartitionManager().getAllReplicaSets())
+ .thenReturn(
+ Collections.singletonList(
+ new TRegionReplicaSet(
+ new TConsensusGroupId(TConsensusGroupType.DataRegion, 1),
+ Collections.singletonList(location(id)))));
+ }
+
+ private void change(int id, NodeStatus status) throws Exception {
+ handler.changeDataNodeStatus(
+ Collections.singletonList(location(id)), Collections.singletonMap(id, status));
+ }
+
+ private void expectChangeFailure(NodeStatus status) throws Exception {
+ try {
+ change(FIRST, status);
+ fail("Unconfirmed status transition must fail");
+ } catch (IOException expected) {
+ assertNotNull(expected.getMessage());
+ }
+ }
+
+ private void assertStates(int id, NodeStatus remote, NodeStatus published, NodeStatus persisted) {
+ assertEquals("DataNode", remote, remoteStatuses.get(id));
+ assertEquals("LoadCache", published, cache.getNodeStatus(id));
+ assertEquals(
+ "NodeInfo",
+ persisted == null
+ ? null
+ : new Pair<>(persisted, persisted == NodeStatus.ReadOnly ? NodeStatus.MANUAL : null),
+ nodeInfo.getPersistedNodeStatus(id));
+ }
+
+ private TestProcedure procedure(NodeStatus original, RemoveDataNodeState initial) {
+ return new TestProcedure(
+ Collections.singletonList(location(FIRST)),
+ Collections.singletonMap(FIRST, original),
+ initial);
+ }
+
+ private static TDataNodeLocation location(int id) {
+ return new TDataNodeLocation()
+ .setDataNodeId(id)
+ .setInternalEndPoint(new TEndPoint("127.0.0.1", 10000 + id))
+ .setClientRpcEndPoint(new TEndPoint("127.0.0.1", 11000 + id));
+ }
+
+ private static TSStatus success() {
+ return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode());
+ }
+
+ private static TSStatus failure() {
+ return new TSStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode());
+ }
+
+ private static void assertSuccess(TSStatus status) {
+ assertEquals(success().getCode(), status.getCode());
+ }
+
+ private static class TestProcedure extends RemoveDataNodesProcedure {
+ private final RemoveDataNodeState initial;
+
+ private TestProcedure(
+ List<TDataNodeLocation> nodes,
+ Map<Integer, NodeStatus> originals,
+ RemoveDataNodeState initial) {
+ super(nodes, originals);
+ this.initial = initial;
+ }
+
+ @Override
+ protected RemoveDataNodeState getInitialState() {
+ return initial;
+ }
+
+ private void step(ConfigNodeProcedureEnv env) throws InterruptedException {
+ execute(env);
+ }
+
+ private RemoveDataNodeState state() {
+ return getCurrentState();
+ }
+
+ private boolean finish(ConfigNodeProcedureEnv env) {
+ return executeFromState(env, RemoveDataNodeState.STOP_DATA_NODE) == Flow.NO_MORE_STATE;
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java
new file mode 100644
index 00000000000..005cd93211f
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/schema/SchemaUtilsTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.procedure.impl.schema;
+
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class SchemaUtilsTest {
+
+ private static final int STOPPED_DATA_NODE_ID = 1;
+ private static final int UNKNOWN_DATA_NODE_ID = 2;
+ private static final int RUNNING_DATA_NODE_ID = 3;
+
+ private static TDataNodeLocation locationOf(final int dataNodeId) {
+ return new TDataNodeLocation(
+ dataNodeId,
+ new TEndPoint("127.0.0.1", dataNodeId * 1000),
+ new TEndPoint("127.0.0.1", dataNodeId * 1000 + 1),
+ new TEndPoint("127.0.0.1", dataNodeId * 1000 + 2),
+ new TEndPoint("127.0.0.1", dataNodeId * 1000 + 3),
+ new TEndPoint("127.0.0.1", dataNodeId * 1000 + 4));
+ }
+
+ @Test
+ public void testFilterFencedDataNodeTreatsStoppedLikeUnknown() {
+ final ConfigManager configManager = mock(ConfigManager.class);
+ final NodeManager nodeManager = mock(NodeManager.class);
+ final LoadManager loadManager = mock(LoadManager.class);
+ when(configManager.getNodeManager()).thenReturn(nodeManager);
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+
+ final Map<Integer, TDataNodeLocation> registered = new HashMap<>();
+ registered.put(STOPPED_DATA_NODE_ID, locationOf(STOPPED_DATA_NODE_ID));
+ registered.put(UNKNOWN_DATA_NODE_ID, locationOf(UNKNOWN_DATA_NODE_ID));
+ registered.put(RUNNING_DATA_NODE_ID, locationOf(RUNNING_DATA_NODE_ID));
+ when(nodeManager.getRegisteredDataNodeLocations()).thenReturn(registered);
+ when(loadManager.getNodeStatus(STOPPED_DATA_NODE_ID)).thenReturn(NodeStatus.Stopped);
+ when(loadManager.getNodeStatus(UNKNOWN_DATA_NODE_ID)).thenReturn(NodeStatus.Unknown);
+ when(loadManager.getNodeStatus(RUNNING_DATA_NODE_ID)).thenReturn(NodeStatus.Running);
+
+ // A negative fence threshold makes every never-contacted DataNode read as fenced, so the
+ // status condition decides alone.
+ final long originalFenceMs =
+ ConfigNodeDescriptor.getInstance().getConf().getMetadataLeaseFenceMs();
+ try {
+ ConfigNodeDescriptor.getInstance().getConf().setMetadataLeaseFenceMs(-100_000L);
+
+ final Map<Integer, TDataNodeLocation> filtered =
+ SchemaUtils.filterFencedDataNode(configManager);
+
+ // An additionally fenced Stopped node is skipped, exactly like an additionally fenced
+ // Unknown node. A Running node is kept even when fenced.
+ Assert.assertFalse(filtered.containsKey(STOPPED_DATA_NODE_ID));
+ Assert.assertFalse(filtered.containsKey(UNKNOWN_DATA_NODE_ID));
+ Assert.assertTrue(filtered.containsKey(RUNNING_DATA_NODE_ID));
+ } finally {
+ ConfigNodeDescriptor.getInstance().getConf().setMetadataLeaseFenceMs(originalFenceMs);
+ }
+ }
+}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java
new file mode 100644
index 00000000000..8ea80cff177
--- /dev/null
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/service/ConfigNodeShutdownHookTest.java
@@ -0,0 +1,346 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.service;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.confignode.client.CnToCnNodeRequestType;
+import org.apache.iotdb.confignode.client.sync.SyncConfigNodeClientPool;
+import org.apache.iotdb.confignode.conf.ConfigNodeConfig;
+import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
+import org.apache.iotdb.confignode.manager.ConfigManager;
+import org.apache.iotdb.confignode.manager.consensus.ConsensusManager;
+import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.node.NodeManager;
+import org.apache.iotdb.consensus.IConsensus;
+import org.apache.iotdb.consensus.common.Peer;
+import org.apache.iotdb.consensus.exception.ConsensusException;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
+import static org.mockito.Mockito.when;
+
+public class ConfigNodeShutdownHookTest {
+
+ private static final CnToCnNodeRequestType REPORT =
+ CnToCnNodeRequestType.REPORT_CONFIG_NODE_SHUTDOWN;
+
+ private final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
+ private final CommonConfig commonConf = CommonDescriptor.getInstance().getConfig();
+ private final ConfigNode configNode = mock(ConfigNode.class);
+ private final NodeManager nodeManager = mock(NodeManager.class);
+ private final LoadManager loadManager = mock(LoadManager.class);
+ private final ConsensusManager consensusManager = mock(ConsensusManager.class);
+ private final IConsensus consensus = mock(IConsensus.class);
+ private final SyncConfigNodeClientPool clientPool = mock(SyncConfigNodeClientPool.class);
+ private final TEndPoint seed = new TEndPoint("127.0.0.2", 10710);
+ private final TEndPoint peer = new TEndPoint("127.0.0.3", 10710);
+ private final List<TConfigNodeLocation> registeredNodes = new ArrayList<>();
+
+ private TConfigNodeLocation localNode;
+ private TEndPoint previousSeed;
+ private NodeStatus previousStatus;
+ private String previousReason;
+ private int previousConnectionTimeout;
+
+ @Before
+ public void setUp() {
+ previousSeed = conf.getSeedConfigNode();
+ previousStatus = commonConf.getNodeStatus();
+ previousReason = commonConf.getStatusReason();
+ previousConnectionTimeout = commonConf.getCnConnectionTimeoutInMS();
+ commonConf.setCnConnectionTimeoutInMS(5000);
+ localNode =
+ new TConfigNodeLocation(
+ conf.getConfigNodeId(),
+ new TEndPoint(conf.getInternalAddress(), conf.getInternalPort()),
+ new TEndPoint(conf.getInternalAddress(), conf.getConsensusPort()));
+ conf.setSeedConfigNode(seed);
+ registeredNodes.add(localNode);
+ registeredNodes.add(new TConfigNodeLocation(100, peer, new TEndPoint("127.0.0.3", 10720)));
+ ConfigManager configManager = mock(ConfigManager.class);
+ when(configNode.getConfigManager()).thenReturn(configManager);
+ when(configManager.getNodeManager()).thenReturn(nodeManager);
+ when(configManager.getLoadManager()).thenReturn(loadManager);
+ when(configManager.getConsensusManager()).thenReturn(consensusManager);
+ when(consensusManager.getConsensusImpl()).thenReturn(consensus);
+ when(consensusManager.getConsensusGroupId())
+ .thenReturn(ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID);
+ when(nodeManager.getRegisteredConfigNodes()).thenReturn(registeredNodes);
+ when(loadManager.getNodeStatus(100)).thenReturn(NodeStatus.Running);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(any(), any(), eq(REPORT)))
+ .thenReturn(new TSStatus(TSStatusCode.INTERNAL_REQUEST_RETRY_ERROR.getStatusCode()));
+ }
+
+ @After
+ public void tearDown() {
+ conf.setSeedConfigNode(previousSeed);
+ commonConf.setNodeStatusWithReason(previousStatus, previousReason);
+ commonConf.setCnConnectionTimeoutInMS(previousConnectionTimeout);
+ }
+
+ @Test
+ public void testStoppingSeedReportsToPeerAfterDeactivation() throws Exception {
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+ // Deactivation may make local metadata unavailable; the report must use its saved endpoints.
+ doAnswer(
+ invocation -> {
+ registeredNodes.clear();
+ return null;
+ })
+ .when(configNode)
+ .deactivate();
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ InOrder order = inOrder(nodeManager, configNode, clientPool);
+ order.verify(nodeManager).getRegisteredConfigNodes();
+ order.verify(configNode).deactivate();
+ order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verify(consensus, never()).transferLeader(any(), any());
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testLeaderTransfersBeforeDeactivation() throws Exception {
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+ when(consensusManager.isLeader()).thenReturn(true);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ InOrder order = inOrder(consensus, configNode, clientPool);
+ order
+ .verify(consensus)
+ .transferLeader(
+ ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID,
+ new Peer(
+ ConsensusManager.DEFAULT_CONSENSUS_GROUP_ID,
+ 100,
+ registeredNodes.get(1).getConsensusEndPoint()));
+ order.verify(configNode).deactivate();
+ order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verify(nodeManager).getRegisteredConfigNodes();
+ verifyNoMoreInteractions(consensus, clientPool);
+ }
+
+ @Test
+ public void testTransferFailureStillDeactivatesAndReports() throws Exception {
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+ when(consensusManager.isLeader()).thenReturn(true);
+ doThrow(new ConsensusException("transfer timed out"))
+ .when(consensus)
+ .transferLeader(any(), any());
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ InOrder order = inOrder(consensus, configNode, clientPool);
+ order.verify(consensus).transferLeader(any(), any());
+ order.verify(configNode).deactivate();
+ order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ }
+
+ @Test
+ public void testLeaderWithoutRunningPeerStillShutsDown() throws Exception {
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+ when(consensusManager.isLeader()).thenReturn(true);
+ when(loadManager.getNodeStatus(100)).thenReturn(NodeStatus.Unknown);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(consensus, never()).transferLeader(any(), any());
+ verify(configNode).deactivate();
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ }
+
+ @Test
+ public void testReportRetriesUntilNewLeaderIsReady() {
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(
+ new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()),
+ new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode()),
+ new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool, times(3)).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testUnavailableRegisteredNodeFallsBackToPeer() {
+ registeredNodes.add(1, new TConfigNodeLocation(101, seed, new TEndPoint("127.0.0.2", 10720)));
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool, times(2)).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT);
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testFollowerReportsWithoutStatusCacheOrSeed() {
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verifyNoMoreInteractions(loadManager, consensus);
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testSeedIsUsedOnlyWhenNoRegisteredPeersExist() {
+ registeredNodes.remove(1);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testSeedIsUsedBeforeConfigManagerInitialization() {
+ when(configNode.getConfigManager()).thenReturn(null);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testLocalSeedWithoutPeersDoesNotReportToItself() {
+ registeredNodes.remove(1);
+ conf.setSeedConfigNode(localNode.getInternalEndPoint());
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testReportFollowsRedirect() {
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(
+ new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode()).setRedirectNode(seed));
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ InOrder order = inOrder(clientPool);
+ order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ order.verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testRedirectToStoppingNodeFallsBackToPeer() {
+ registeredNodes.add(1, new TConfigNodeLocation(101, seed, new TEndPoint("127.0.0.2", 10720)));
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT))
+ .thenReturn(
+ new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode())
+ .setRedirectNode(localNode.getInternalEndPoint()));
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenReturn(new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()));
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(seed, localNode, REPORT);
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verify(clientPool, never())
+ .sendSyncRequestToConfigNodeWithRetry(localNode.getInternalEndPoint(), localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testReportStopsRetryingAfterDeadline() {
+ commonConf.setCnConnectionTimeoutInMS(100);
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenAnswer(
+ invocation -> {
+ TimeUnit.MILLISECONDS.sleep(150);
+ return new TSStatus(TSStatusCode.REDIRECTION_RECOMMEND.getStatusCode());
+ });
+
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ }
+
+ @Test
+ public void testReportStopsRetryingWhenInterrupted() {
+ when(clientPool.sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT))
+ .thenAnswer(
+ invocation -> {
+ Thread.currentThread().interrupt();
+ return new TSStatus(TSStatusCode.CONFIG_NODE_LEADER_WARMING_UP.getStatusCode());
+ });
+ try {
+ new ConfigNodeShutdownHook(configNode, clientPool).run();
+
+ assertTrue(Thread.currentThread().isInterrupted());
+ verify(clientPool).sendSyncRequestToConfigNodeWithRetry(peer, localNode, REPORT);
+ verifyNoMoreInteractions(clientPool);
+ } finally {
+ Thread.interrupted();
+ }
+ }
+}
diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index 405a5139e34..fc241312e51 100644
--- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -1081,8 +1081,7 @@ public final class DataNodeMiscMessages {
public static final String MISC_LOG_DATANODE_EXITS_JVM_MEMORY_USAGE_BE69D1F5 =
"DataNode exits. Jvm memory usage: {}";
public static final String MISC_LOG_FAILED_TO_REPORT_DATANODE_S_SHUTDOWN_TO_CONFIGNODE_THE_CLUSTER_E6727497 =
- "Failed to report DataNode's shutdown to ConfigNode. The cluster will still take the current "
- + "DataNode as Running for a few seconds.";
+ "Failed to confirm the DataNode shutdown report with the ConfigNode leader.";
public static final String MISC_LOG_SOMETHING_WRONG_HAPPENED_WHILE_CALLING_CONSENSUS_LAYER_S_8B8FBB16 =
"Something wrong happened while calling consensus layer's triggerSnapshot API.";
public static final String MISC_LOG_THE_ADDREGIONPEERTASK_HAS_ALREADY_BEEN_SUBMITTED_AND_WILL_4D398F73 =
diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
index a87a49eaaf5..b0266763747 100644
--- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
+++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodeMiscMessages.java
@@ -1080,7 +1080,7 @@ public final class DataNodeMiscMessages {
public static final String MISC_LOG_DATANODE_EXITS_JVM_MEMORY_USAGE_BE69D1F5 =
"DataNode 退出。JVM 内存使用情况：{}";
public static final String MISC_LOG_FAILED_TO_REPORT_DATANODE_S_SHUTDOWN_TO_CONFIGNODE_THE_CLUSTER_E6727497 =
- "无法向 ConfigNode 上报 DataNode 关闭状态。集群在接下来的几秒内仍会将当前 DataNode 视为 Running。";
+ "未能获得 ConfigNode leader 对 DataNode 关闭报告的确认。";
public static final String MISC_LOG_SOMETHING_WRONG_HAPPENED_WHILE_CALLING_CONSENSUS_LAYER_S_8B8FBB16 =
"调用共识层的 triggerSnapshot API 时发生错误。";
public static final String MISC_LOG_THE_ADDREGIONPEERTASK_HAS_ALREADY_BEEN_SUBMITTED_AND_WILL_4D398F73 =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
index 8f2c1080b3f..adce7c280fc 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImpl.java
@@ -2435,12 +2435,13 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
resp.setDataRegionRawDataSize(regionRawDataSize);
}
resp.setHeartbeatTimestamp(req.getHeartbeatTimestamp());
- resp.setStatus(commonConfig.getNodeStatus().getStatus());
- // Advertise that this DataNode supports metadata-lease self-fencing, so the ConfigNode may
- // treat
- // it as safely fenced when unreachable (older DataNodes that omit this are handled strictly).
- if (commonConfig.getStatusReason() != null) {
- resp.setStatusReason(commonConfig.getStatusReason());
+ // Read status and reason under the same monitor used by their setters.
+ synchronized (commonConfig) {
+ resp.setStatus(commonConfig.getNodeStatus().getStatus());
+ final String statusReason = commonConfig.getStatusReason();
+ if (statusReason != null) {
+ resp.setStatusReason(statusReason);
+ }
}
MetadataLeaseFencedException schemaUsageCollectionException = null;
if (req.getSchemaRegionIds() != null) {
@@ -2620,12 +2621,15 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
RamUsageEstimator.humanReadableUnits((long) totalDisk),
freeDiskRatio,
commonConfig.getDiskSpaceWarningThreshold());
- commonConfig.setNodeStatus(NodeStatus.ReadOnly);
- commonConfig.setStatusReason(NodeStatus.DISK_FULL);
- } else if (NodeStatus.ReadOnly.equals(commonConfig.getNodeStatus())
- && NodeStatus.DISK_FULL.equals(commonConfig.getStatusReason())) {
- commonConfig.setNodeStatus(NodeStatus.Running);
- commonConfig.setStatusReason(null);
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
+ } else {
+ synchronized (commonConfig) {
+ // A concurrent Manual/Stopping/error transition must not be cleared by disk recovery.
+ if (NodeStatus.ReadOnly.equals(commonConfig.getNodeStatus())
+ && NodeStatus.DISK_FULL.equals(commonConfig.getStatusReason())) {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ }
+ }
}
}
}
@@ -2800,7 +2804,12 @@ public class DataNodeInternalRPCServiceImpl implements IDataNodeRPCService.Iface
@Override
public TSStatus setSystemStatus(String status) throws TException {
try {
- commonConfig.setNodeStatus(NodeStatus.parse(status));
+ NodeStatus nodeStatus = NodeStatus.parse(status);
+ if (nodeStatus == NodeStatus.ReadOnly) {
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ } else {
+ commonConfig.setNodeStatus(nodeStatus);
+ }
} catch (Exception e) {
return RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage());
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
index f4fb59f7ca7..f57fc00154e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/relational/InformationSchemaContentSupplierFactory.java
@@ -1221,6 +1221,7 @@ public class InformationSchemaContentSupplierFactory {
location.getConfigNodeId(),
NODE_TYPE_CONFIG_NODE,
showClusterResp.getNodeStatus().get(location.getConfigNodeId()),
+ getNodeStatusReason(location.getConfigNodeId()),
location.getInternalEndPoint().getIp(),
location.getInternalEndPoint().getPort(),
showClusterResp.getNodeVersionInfo().get(location.getConfigNodeId()));
@@ -1232,6 +1233,7 @@ public class InformationSchemaContentSupplierFactory {
location.getDataNodeId(),
NODE_TYPE_DATA_NODE,
showClusterResp.getNodeStatus().get(location.getDataNodeId()),
+ getNodeStatusReason(location.getDataNodeId()),
location.getInternalEndPoint().getIp(),
location.getInternalEndPoint().getPort(),
showClusterResp.getNodeVersionInfo().get(location.getDataNodeId()));
@@ -1243,16 +1245,26 @@ public class InformationSchemaContentSupplierFactory {
location.getAiNodeId(),
NODE_TYPE_AI_NODE,
showClusterResp.getNodeStatus().get(location.getAiNodeId()),
+ getNodeStatusReason(location.getAiNodeId()),
location.getInternalEndPoint().getIp(),
location.getInternalEndPoint().getPort(),
showClusterResp.getNodeVersionInfo().get(location.getAiNodeId()));
}
}
+ private String getNodeStatusReason(int nodeId) {
+ // The optional field is absent when the ConfigNode is an old version.
+ if (!showClusterResp.isSetNodeStatusReason()) {
+ return null;
+ }
+ return showClusterResp.getNodeStatusReason().get(nodeId);
+ }
+
private void buildNodeTsBlock(
int nodeId,
String nodeType,
String nodeStatus,
+ String nodeStatusReason,
String internalAddress,
int internalPort,
TNodeVersionInfo versionInfo) {
@@ -1263,23 +1275,28 @@ public class InformationSchemaContentSupplierFactory {
} else {
columnBuilders[2].writeBinary(new Binary(nodeStatus, TSFileConfig.STRING_CHARSET));
}
+ if (nodeStatusReason == null) {
+ columnBuilders[3].appendNull();
+ } else {
+ columnBuilders[3].writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET));
+ }
if (internalAddress == null) {
- columnBuilders[3].appendNull();
+ columnBuilders[4].appendNull();
} else {
- columnBuilders[3].writeBinary(new Binary(internalAddress, TSFileConfig.STRING_CHARSET));
+ columnBuilders[4].writeBinary(new Binary(internalAddress, TSFileConfig.STRING_CHARSET));
}
- columnBuilders[4].writeInt(internalPort);
+ columnBuilders[5].writeInt(internalPort);
if (versionInfo == null || versionInfo.getVersion() == null) {
- columnBuilders[5].appendNull();
+ columnBuilders[6].appendNull();
} else {
- columnBuilders[5].writeBinary(
+ columnBuilders[6].writeBinary(
new Binary(versionInfo.getVersion(), TSFileConfig.STRING_CHARSET));
}
if (versionInfo == null || versionInfo.getBuildInfo() == null) {
- columnBuilders[6].appendNull();
+ columnBuilders[7].appendNull();
} else {
- columnBuilders[6].writeBinary(
+ columnBuilders[7].writeBinary(
new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET));
}
resultBuilder.declarePosition();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
index 46c4217f17c..a995d643970 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java
@@ -1590,7 +1590,13 @@ public class ClusterConfigTaskExecutor implements IConfigTaskExecutor {
}
} else {
try {
- CommonDescriptor.getInstance().getConfig().setNodeStatus(status);
+ if (status == NodeStatus.ReadOnly) {
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ } else {
+ CommonDescriptor.getInstance().getConfig().setNodeStatus(status);
+ }
tsStatus = RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
} catch (Exception e) {
tsStatus = RpcUtils.getStatus(TSStatusCode.EXECUTE_STATEMENT_ERROR, e.getMessage());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java
index 17778269b7e..2140bd53fff 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTask.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
@@ -38,6 +37,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.utils.Binary;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -57,6 +57,8 @@ public class ShowClusterDetailsTask implements IConfigTask {
TsBlockBuilder builder,
int nodeId,
String nodeStatus,
+ String nodeStatusReason,
+ boolean hasStatusReason,
String internalAddress,
int internalPort,
int configConsensusPort,
@@ -102,6 +104,15 @@ public class ShowClusterDetailsTask implements IConfigTask {
.getColumnBuilder(12)
.writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET));
}
+ if (hasStatusReason) {
+ if (nodeStatusReason == null) {
+ builder.getColumnBuilder(13).appendNull();
+ } else {
+ builder
+ .getColumnBuilder(13)
+ .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET));
+ }
+ }
builder.declarePosition();
}
@@ -109,6 +120,8 @@ public class ShowClusterDetailsTask implements IConfigTask {
TsBlockBuilder builder,
int nodeId,
String nodeStatus,
+ String nodeStatusReason,
+ boolean hasStatusReason,
String internalAddress,
int internalPort,
TNodeVersionInfo versionInfo) {
@@ -123,7 +136,6 @@ public class ShowClusterDetailsTask implements IConfigTask {
} else {
builder.getColumnBuilder(2).writeBinary(new Binary(nodeStatus, TSFileConfig.STRING_CHARSET));
}
-
if (internalAddress == null) {
builder.getColumnBuilder(3).appendNull();
} else {
@@ -152,6 +164,15 @@ public class ShowClusterDetailsTask implements IConfigTask {
.getColumnBuilder(12)
.writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET));
}
+ if (hasStatusReason) {
+ if (nodeStatusReason == null) {
+ builder.getColumnBuilder(13).appendNull();
+ } else {
+ builder
+ .getColumnBuilder(13)
+ .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET));
+ }
+ }
builder.declarePosition();
}
@@ -160,6 +181,8 @@ public class ShowClusterDetailsTask implements IConfigTask {
TsBlockBuilder builder,
int nodeId,
String nodeStatus,
+ String nodeStatusReason,
+ boolean hasStatusReason,
String internalAddress,
int internalPort,
String rpcAddress,
@@ -219,15 +242,31 @@ public class ShowClusterDetailsTask implements IConfigTask {
.getColumnBuilder(12)
.writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET));
}
+ if (hasStatusReason) {
+ if (nodeStatusReason == null) {
+ builder.getColumnBuilder(13).appendNull();
+ } else {
+ builder
+ .getColumnBuilder(13)
+ .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET));
+ }
+ }
builder.declarePosition();
}
public static void buildTSBlock(
TShowClusterResp clusterNodeInfos, SettableFuture<ConfigTaskResult> future) {
+ boolean hasStatusReason =
+ clusterNodeInfos.getNodeStatusReason() != null
+ && clusterNodeInfos.getNodeStatusReason().values().stream()
+ .anyMatch(reason -> reason != null && !reason.isEmpty());
+ List<ColumnHeader> columnHeaders =
+ new ArrayList<>(ColumnHeaderConstant.showClusterDetailsColumnHeaders);
+ if (hasStatusReason) {
+ columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT));
+ }
List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showClusterDetailsColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
+ columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList());
TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
clusterNodeInfos
@@ -238,6 +277,10 @@ public class ShowClusterDetailsTask implements IConfigTask {
builder,
e.getConfigNodeId(),
clusterNodeInfos.getNodeStatus().get(e.getConfigNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getConfigNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
e.getConsensusEndPoint().getPort(),
@@ -251,6 +294,10 @@ public class ShowClusterDetailsTask implements IConfigTask {
builder,
e.getDataNodeId(),
clusterNodeInfos.getNodeStatus().get(e.getDataNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getDataNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
e.getClientRpcEndPoint().getIp(),
@@ -267,11 +314,15 @@ public class ShowClusterDetailsTask implements IConfigTask {
builder,
e.getAiNodeId(),
clusterNodeInfos.getNodeStatus().get(e.getAiNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getAiNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
clusterNodeInfos.getNodeVersionInfo().get(e.getAiNodeId())));
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowClusterDetailsHeader();
+ DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true);
future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java
index 8837864ab13..f1ad0ffe8f5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTask.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
@@ -38,6 +37,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.utils.Binary;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -64,6 +64,8 @@ public class ShowClusterTask implements IConfigTask {
int nodeId,
String nodeType,
String nodeStatus,
+ String nodeStatusReason,
+ boolean hasStatusReason,
String hostAddress,
int port,
TNodeVersionInfo versionInfo) {
@@ -100,15 +102,31 @@ public class ShowClusterTask implements IConfigTask {
.writeBinary(new Binary(versionInfo.getBuildInfo(), TSFileConfig.STRING_CHARSET));
}
+ if (hasStatusReason) {
+ if (nodeStatusReason == null) {
+ builder.getColumnBuilder(7).appendNull();
+ } else {
+ builder
+ .getColumnBuilder(7)
+ .writeBinary(new Binary(nodeStatusReason, TSFileConfig.STRING_CHARSET));
+ }
+ }
builder.declarePosition();
}
public static void buildTsBlock(
TShowClusterResp clusterNodeInfos, SettableFuture<ConfigTaskResult> future) {
+ boolean hasStatusReason =
+ clusterNodeInfos.getNodeStatusReason() != null
+ && clusterNodeInfos.getNodeStatusReason().values().stream()
+ .anyMatch(reason -> reason != null && !reason.isEmpty());
+ List<ColumnHeader> columnHeaders =
+ new ArrayList<>(ColumnHeaderConstant.showClusterColumnHeaders);
+ if (hasStatusReason) {
+ columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT));
+ }
List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showClusterColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
+ columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList());
TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
clusterNodeInfos
@@ -120,6 +138,10 @@ public class ShowClusterTask implements IConfigTask {
e.getConfigNodeId(),
NODE_TYPE_CONFIG_NODE,
clusterNodeInfos.getNodeStatus().get(e.getConfigNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getConfigNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
clusterNodeInfos.getNodeVersionInfo().get(e.getConfigNodeId())));
@@ -133,6 +155,10 @@ public class ShowClusterTask implements IConfigTask {
e.getDataNodeId(),
NODE_TYPE_DATA_NODE,
clusterNodeInfos.getNodeStatus().get(e.getDataNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getDataNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
clusterNodeInfos.getNodeVersionInfo().get(e.getDataNodeId())));
@@ -147,11 +173,15 @@ public class ShowClusterTask implements IConfigTask {
e.getAiNodeId(),
NODE_TYPE_AI_NODE,
clusterNodeInfos.getNodeStatus().get(e.getAiNodeId()),
+ hasStatusReason
+ ? clusterNodeInfos.getNodeStatusReason().get(e.getAiNodeId())
+ : null,
+ hasStatusReason,
e.getInternalEndPoint().getIp(),
e.getInternalEndPoint().getPort(),
clusterNodeInfos.getNodeVersionInfo().get(e.getAiNodeId())));
}
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowClusterHeader();
+ DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true);
future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java
index a666a72d815..4bcbb9a5b3e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowConfigNodesTask.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowConfigNodesResp;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
@@ -36,6 +35,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.utils.BytesUtils;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -53,10 +53,20 @@ public class ShowConfigNodesTask implements IConfigTask {
public static void buildTSBlock(
TShowConfigNodesResp showConfigNodesResp, SettableFuture<ConfigTaskResult> future) {
+ boolean hasStatusReason =
+ showConfigNodesResp.getConfigNodesInfoList() != null
+ && showConfigNodesResp.getConfigNodesInfoList().stream()
+ .anyMatch(
+ configNodeInfo ->
+ configNodeInfo.getStatusReason() != null
+ && !configNodeInfo.getStatusReason().isEmpty());
+ List<ColumnHeader> columnHeaders =
+ new ArrayList<>(ColumnHeaderConstant.showConfigNodesColumnHeaders);
+ if (hasStatusReason) {
+ columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT));
+ }
List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showConfigNodesColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
+ columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList());
TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
if (showConfigNodesResp.getConfigNodesInfoList() != null) {
for (TConfigNodeInfo configNodeInfo : showConfigNodesResp.getConfigNodesInfoList()) {
@@ -68,10 +78,18 @@ public class ShowConfigNodesTask implements IConfigTask {
.writeBinary(BytesUtils.valueOf(configNodeInfo.getInternalAddress()));
builder.getColumnBuilder(3).writeInt(configNodeInfo.getInternalPort());
builder.getColumnBuilder(4).writeBinary(BytesUtils.valueOf(configNodeInfo.getRoleType()));
+ if (hasStatusReason) {
+ String statusReason = configNodeInfo.getStatusReason();
+ if (statusReason == null) {
+ builder.getColumnBuilder(5).appendNull();
+ } else {
+ builder.getColumnBuilder(5).writeBinary(BytesUtils.valueOf(statusReason));
+ }
+ }
builder.declarePosition();
}
}
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowConfigNodesHeader();
+ DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true);
future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java
index 1adb6e68cae..5ffdd6743ed 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowDataNodesTask.java
@@ -24,7 +24,6 @@ import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
import org.apache.iotdb.confignode.rpc.thrift.TDataNodeInfo;
import org.apache.iotdb.confignode.rpc.thrift.TShowDataNodesResp;
import org.apache.iotdb.db.queryengine.common.header.DatasetHeader;
-import org.apache.iotdb.db.queryengine.common.header.DatasetHeaderFactory;
import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
import org.apache.iotdb.db.queryengine.plan.execution.config.IConfigTask;
import org.apache.iotdb.db.queryengine.plan.execution.config.executor.IConfigTaskExecutor;
@@ -37,6 +36,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.read.common.block.TsBlockBuilder;
import org.apache.tsfile.utils.BytesUtils;
+import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
@@ -58,10 +58,20 @@ public class ShowDataNodesTask implements IConfigTask {
public static void buildTSBlock(
TShowDataNodesResp showDataNodesResp, SettableFuture<ConfigTaskResult> future) {
+ boolean hasStatusReason =
+ showDataNodesResp.getDataNodesInfoList() != null
+ && showDataNodesResp.getDataNodesInfoList().stream()
+ .anyMatch(
+ dataNodeInfo ->
+ dataNodeInfo.getStatusReason() != null
+ && !dataNodeInfo.getStatusReason().isEmpty());
+ List<ColumnHeader> columnHeaders =
+ new ArrayList<>(ColumnHeaderConstant.showDataNodesColumnHeaders);
+ if (hasStatusReason) {
+ columnHeaders.add(new ColumnHeader(ColumnHeaderConstant.STATUS_REASON, TSDataType.TEXT));
+ }
List<TSDataType> outputDataTypes =
- ColumnHeaderConstant.showDataNodesColumnHeaders.stream()
- .map(ColumnHeader::getColumnType)
- .collect(Collectors.toList());
+ columnHeaders.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList());
TsBlockBuilder builder = new TsBlockBuilder(outputDataTypes);
if (showDataNodesResp.getDataNodesInfoList() != null) {
for (TDataNodeInfo dataNodeInfo : showDataNodesResp.getDataNodesInfoList()) {
@@ -72,16 +82,22 @@ public class ShowDataNodesTask implements IConfigTask {
.writeBinary(
BytesUtils.valueOf(
dataNodeInfo.getStatus() == null ? "" : dataNodeInfo.getStatus()));
-
builder.getColumnBuilder(2).writeBinary(BytesUtils.valueOf(dataNodeInfo.getRpcAddresss()));
builder.getColumnBuilder(3).writeInt(dataNodeInfo.getRpcPort());
builder.getColumnBuilder(4).writeInt(dataNodeInfo.getDataRegionNum());
-
builder.getColumnBuilder(5).writeInt(dataNodeInfo.getSchemaRegionNum());
+ if (hasStatusReason) {
+ String statusReason = dataNodeInfo.getStatusReason();
+ if (statusReason == null) {
+ builder.getColumnBuilder(6).appendNull();
+ } else {
+ builder.getColumnBuilder(6).writeBinary(BytesUtils.valueOf(statusReason));
+ }
+ }
builder.declarePosition();
}
}
- DatasetHeader datasetHeader = DatasetHeaderFactory.getShowDataNodesHeader();
+ DatasetHeader datasetHeader = new DatasetHeader(columnHeaders, true);
future.set(new ConfigTaskResult(TSStatusCode.SUCCESS_STATUS, builder.build(), datasetHeader));
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
index 6dd082c0efa..16f17916eb0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNodeShutdownHook.java
@@ -117,7 +117,9 @@ public class DataNodeShutdownHook extends Thread {
// Reject write operations to make sure all tsfiles will be sealed
CommonDescriptor.getInstance().getConfig().setStopping(true);
- CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING);
// Wait all wal are flushed
WALManager.getInstance().waitAllWALFlushed();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 802d440cc44..ead75db6bac 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -2146,7 +2146,9 @@ public class DataRegion implements IDataRegionForQuery {
StorageEngineMessages
.STORAGE_LOG_DISK_SPACE_IS_INSUFFICIENT_WHEN_CREATING_TSFILE_PROCESSOR_4032BAF0,
e);
- CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+ CommonDescriptor.getInstance()
+ .getConfig()
+ .setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
throw new WriteProcessException(e.getMessage(), e.getErrorCode(), true);
} catch (IOException e) {
if (retryCnt < 3) {
@@ -2158,7 +2160,7 @@ public class DataRegion implements IDataRegionForQuery {
.STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729,
e);
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
throw new WriteProcessException(
String.format(
StorageEngineMessages
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 78c48a2702b..9589dcf9a40 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -1932,7 +1932,7 @@ public class TsFileProcessor {
tsFileResource.getTsFile().getAbsolutePath(),
e);
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
try {
logger.error(
StorageEngineMessages
@@ -2080,7 +2080,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getAbsolutePath(),
e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
break;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 13930f936d5..45432e7ab3d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -613,7 +613,7 @@ public class WALBuffer extends AbstractWALBuffer {
failListeners(syncFailure);
// SET SYSTEM TO RUNNING does not repair a failed batch or an unknown record boundary.
if (CommonDescriptor.getInstance().getConfig().isRunning()) {
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(syncFailure);
}
if (forceFlag && retryAfterFailedBatch) {
syncFailure = null;
@@ -643,7 +643,7 @@ public class WALBuffer extends AbstractWALBuffer {
}
failListeners(e);
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
switchSyncingBufferToIdle();
return;
}
@@ -676,7 +676,7 @@ public class WALBuffer extends AbstractWALBuffer {
syncFailure = e instanceof Exception exception ? exception : new IOException(e);
retryAfterFailedBatch = false;
failListeners(syncFailure);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
return;
} finally {
switchSyncingBufferToIdle();
@@ -710,7 +710,7 @@ public class WALBuffer extends AbstractWALBuffer {
retryAfterFailedBatch = false;
}
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
}
} else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging
// after rolling file
@@ -727,7 +727,7 @@ public class WALBuffer extends AbstractWALBuffer {
failListeners(e);
syncFailure = e;
retryAfterFailedBatch = false;
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index a1bba3bad40..ec96f43c7d4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -222,7 +222,7 @@ public class CheckpointManager implements AutoCloseable {
identifier,
e);
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
}
try {
@@ -243,7 +243,7 @@ public class CheckpointManager implements AutoCloseable {
identifier,
e);
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
- CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
+ CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(e);
}
} finally {
infoLock.unlock();
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
index 7dce3cc3c2d..e397baabb16 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplDiskTest.java
@@ -53,17 +53,17 @@ public class DataNodeInternalRPCServiceImplDiskTest {
originalDataNodeId = dataNodeConfig.getDataNodeId();
dataNodeConfig.setDataNodeId(0);
+ // Go through Running first so that a ReadOnly with a higher-priority reason left behind by
+ // another test can always be overridden.
commonConfig.setNodeStatus(NodeStatus.Running);
- commonConfig.setStatusReason(null);
commonConfig.setDiskSpaceWarningThreshold(0.05);
- commonConfig.setNodeStatus(NodeStatus.ReadOnly);
- commonConfig.setStatusReason(NodeStatus.DISK_FULL);
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.DISK_FULL);
}
@After
public void tearDown() {
- commonConfig.setNodeStatus(originalStatus);
- commonConfig.setStatusReason(originalStatusReason);
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason);
commonConfig.setDiskSpaceWarningThreshold(originalDiskSpaceWarningThreshold);
dataNodeConfig.setDataNodeId(originalDataNodeId);
}
@@ -95,7 +95,6 @@ public class DataNodeInternalRPCServiceImplDiskTest {
when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(100L);
commonConfig.setNodeStatus(NodeStatus.Running);
- commonConfig.setStatusReason(null);
DataNodeContext dataNodeContext = mock(DataNodeContext.class);
DataNodeInternalRPCServiceImpl service =
new DataNodeInternalRPCServiceImpl(dataNodeContext, systemMetrics);
@@ -105,4 +104,36 @@ public class DataNodeInternalRPCServiceImplDiskTest {
Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
Assert.assertEquals(NodeStatus.DISK_FULL, commonConfig.getStatusReason());
}
+
+ @Test
+ public void testRecoveryRequiresExactReadOnlyWithDiskFullReason() {
+ SystemMetrics systemMetrics = mock(SystemMetrics.class);
+ // The aggregate free ratio is 52%, so a ReadOnly + DiskFull state would recover.
+ when(systemMetrics.getSystemDiskAvailableSpace()).thenReturn(104L);
+ when(systemMetrics.getSystemDiskTotalSpace()).thenReturn(200L);
+
+ DataNodeContext dataNodeContext = mock(DataNodeContext.class);
+ DataNodeInternalRPCServiceImpl service =
+ new DataNodeInternalRPCServiceImpl(dataNodeContext, systemMetrics);
+
+ // A manually set ReadOnly must not be auto-recovered by a disk ratio that recovered.
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.MANUAL);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.MANUAL, commonConfig.getStatusReason());
+
+ // Neither can a ReadOnly entered through an unrecoverable error.
+ // (Manual outranks UnrecoverableError, so go through Running to enter it.)
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(
+ NodeStatus.ReadOnly, NodeStatus.UNRECOVERABLE_ERROR + ", 2026-09-02 10:00:00.000, broken");
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertTrue(commonConfig.getStatusReason().startsWith(NodeStatus.UNRECOVERABLE_ERROR));
+
+ commonConfig.setNodeStatusWithReason(NodeStatus.ReadOnly, NodeStatus.STOPPING);
+ service.sampleDiskLoad(new TLoadSample());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.STOPPING, commonConfig.getStatusReason());
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java
new file mode 100644
index 00000000000..c999737dae6
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/protocol/thrift/impl/DataNodeInternalRPCServiceImplSetSystemStatusTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.protocol.thrift.impl;
+
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.service.DataNode.DataNodeContext;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+
+public class DataNodeInternalRPCServiceImplSetSystemStatusTest {
+
+ private final CommonConfig commonConfig = CommonDescriptor.getInstance().getConfig();
+ private NodeStatus originalStatus;
+ private String originalStatusReason;
+
+ @BeforeClass
+ public static void setUpClass() {
+ // The static initializer of DataNodeInternalRPCServiceImpl (Coordinator) requires it.
+ IoTDBDescriptor.getInstance().getConfig().setDataNodeId(0);
+ }
+
+ @Before
+ public void setUp() {
+ originalStatus = commonConfig.getNodeStatus();
+ originalStatusReason = commonConfig.getStatusReason();
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ }
+
+ @After
+ public void tearDown() {
+ commonConfig.setNodeStatus(NodeStatus.Running);
+ commonConfig.setNodeStatusWithReason(originalStatus, originalStatusReason);
+ }
+
+ @Test
+ public void testSetSystemStatusReadOnlyCarriesManualReason() throws Exception {
+ final DataNodeInternalRPCServiceImpl service =
+ new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class));
+
+ // A ReadOnly requested through the RPC (i.e. by the ConfigNode on SET SYSTEM STATUS) is
+ // recorded with the Manual reason.
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), service.setSystemStatus("ReadOnly").getCode());
+ Assert.assertEquals(NodeStatus.ReadOnly, commonConfig.getNodeStatus());
+ Assert.assertEquals(NodeStatus.MANUAL, commonConfig.getStatusReason());
+
+ // Switching back to Running clears the reason.
+ Assert.assertEquals(
+ TSStatusCode.SUCCESS_STATUS.getStatusCode(), service.setSystemStatus("Running").getCode());
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ Assert.assertNull(commonConfig.getStatusReason());
+ }
+
+ @Test
+ public void testSetSystemStatusInvalidStatusReturnsError() throws Exception {
+ final DataNodeInternalRPCServiceImpl service =
+ new DataNodeInternalRPCServiceImpl(mock(DataNodeContext.class));
+
+ Assert.assertEquals(
+ TSStatusCode.EXECUTE_STATEMENT_ERROR.getStatusCode(),
+ service.setSystemStatus("NotAStatus").getCode());
+ Assert.assertEquals(NodeStatus.Running, commonConfig.getNodeStatus());
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java
new file mode 100644
index 00000000000..abf570d58fc
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterDetailsTaskTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.execution.config.metadata;
+
+import org.apache.iotdb.common.rpc.thrift.TAINodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.schema.column.ColumnHeader;
+import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
+import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.enums.TSDataType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ShowClusterDetailsTaskTest {
+
+ private static final int CONFIG_NODE_ID = 0;
+ private static final int DATA_NODE_ID = 1;
+ private static final int AI_NODE_ID = 2;
+
+ private static TShowClusterResp buildClusterResp(final Map<Integer, String> statusReasonMap) {
+ final TConfigNodeLocation configNode =
+ new TConfigNodeLocation(
+ CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001));
+ final TDataNodeLocation dataNode =
+ new TDataNodeLocation(
+ DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 2000),
+ new TEndPoint("127.0.0.1", 2001),
+ new TEndPoint("127.0.0.1", 2002),
+ new TEndPoint("127.0.0.1", 2003),
+ new TEndPoint("127.0.0.1", 2004));
+ final TAINodeLocation aiNode =
+ new TAINodeLocation(AI_NODE_ID, new TEndPoint("127.0.0.1", 3000));
+
+ final Map<Integer, String> nodeStatus = new HashMap<>();
+ nodeStatus.put(CONFIG_NODE_ID, NodeStatus.Running.getStatus());
+ nodeStatus.put(DATA_NODE_ID, NodeStatus.ReadOnly.getStatus());
+ nodeStatus.put(AI_NODE_ID, NodeStatus.Running.getStatus());
+
+ final Map<Integer, TNodeVersionInfo> nodeVersionInfo = new HashMap<>();
+ final TNodeVersionInfo versionInfo = new TNodeVersionInfo("2.0.11", "build");
+ nodeVersionInfo.put(CONFIG_NODE_ID, versionInfo);
+ nodeVersionInfo.put(DATA_NODE_ID, versionInfo);
+ nodeVersionInfo.put(AI_NODE_ID, versionInfo);
+
+ final TShowClusterResp resp =
+ new TShowClusterResp(
+ new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()),
+ Collections.singletonList(configNode),
+ Collections.singletonList(dataNode),
+ Collections.singletonList(aiNode),
+ nodeStatus,
+ nodeVersionInfo);
+ if (statusReasonMap != null) {
+ resp.setNodeStatusReason(statusReasonMap);
+ }
+ return resp;
+ }
+
+ private static ConfigTaskResult execute(final TShowClusterResp resp) throws Exception {
+ final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ ShowClusterDetailsTask.buildTSBlock(resp, future);
+ return future.get();
+ }
+
+ private static List<String> columnNames(final List<ColumnHeader> headers) {
+ return headers.stream().map(ColumnHeader::getColumnName).collect(Collectors.toList());
+ }
+
+ @Test
+ public void testOldLayoutWhenNoReasonFieldPresent() throws Exception {
+ final ConfigTaskResult result = execute(buildClusterResp(null));
+ final List<ColumnHeader> headers = result.getResultSetHeader().getColumnHeaders();
+
+ Assert.assertEquals(
+ columnNames(ColumnHeaderConstant.showClusterDetailsColumnHeaders), columnNames(headers));
+ Assert.assertEquals(
+ ColumnHeaderConstant.showClusterDetailsColumnHeaders.stream()
+ .map(ColumnHeader::getColumnType)
+ .collect(Collectors.toList()),
+ result.getResultSetHeader().getRespDataTypes());
+ final Column[] columns = result.getResultSet().getValueColumns();
+ Assert.assertEquals(headers.size(), columns.length);
+
+ Assert.assertEquals(NodeStatus.Running.getStatus(), columns[2].getBinary(0).toString());
+ Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString());
+ Assert.assertEquals("2002", columns[8].getBinary(1).toString());
+ Assert.assertEquals("2004", columns[9].getBinary(1).toString());
+ Assert.assertEquals("2003", columns[10].getBinary(1).toString());
+ Assert.assertEquals(AI_NODE_ID, columns[0].getInt(2));
+ Assert.assertEquals(3000, columns[4].getInt(2));
+ Assert.assertEquals("build", columns[12].getBinary(2).toString());
+ }
+
+ @Test
+ public void testReasonColumnAppendedWithNullForMissingReasons() throws Exception {
+ final Map<Integer, String> statusReasonMap = new HashMap<>();
+ statusReasonMap.put(DATA_NODE_ID, NodeStatus.DISK_FULL);
+ final ConfigTaskResult result = execute(buildClusterResp(statusReasonMap));
+ final List<ColumnHeader> headers = result.getResultSetHeader().getColumnHeaders();
+
+ Assert.assertEquals(
+ ColumnHeaderConstant.showClusterDetailsColumnHeaders.size() + 1, headers.size());
+ Assert.assertEquals(
+ columnNames(ColumnHeaderConstant.showClusterDetailsColumnHeaders),
+ columnNames(headers.subList(0, 13)));
+ Assert.assertEquals(ColumnHeaderConstant.STATUS_REASON, headers.get(13).getColumnName());
+ Assert.assertEquals(TSDataType.TEXT, headers.get(13).getColumnType());
+ Assert.assertEquals(
+ headers.stream().map(ColumnHeader::getColumnType).collect(Collectors.toList()),
+ result.getResultSetHeader().getRespDataTypes());
+ final Column[] columns = result.getResultSet().getValueColumns();
+ Assert.assertEquals(headers.size(), columns.length);
+
+ Assert.assertTrue(columns[13].isNull(0));
+ Assert.assertFalse(columns[13].isNull(1));
+ Assert.assertEquals(NodeStatus.DISK_FULL, columns[13].getBinary(1).toString());
+ Assert.assertEquals(NodeStatus.ReadOnly.getStatus(), columns[2].getBinary(1).toString());
+ Assert.assertEquals("127.0.0.1", columns[3].getBinary(1).toString());
+ Assert.assertEquals(2001, columns[4].getInt(1));
+ Assert.assertEquals("1001", columns[5].getBinary(0).toString());
+ Assert.assertEquals("", columns[6].getBinary(0).toString());
+ Assert.assertEquals("127.0.0.1", columns[6].getBinary(1).toString());
+ Assert.assertEquals("2000", columns[7].getBinary(1).toString());
+ Assert.assertEquals("2002", columns[8].getBinary(1).toString());
+ Assert.assertEquals("2004", columns[9].getBinary(1).toString());
+ Assert.assertEquals("2003", columns[10].getBinary(1).toString());
+ Assert.assertEquals("2.0.11", columns[11].getBinary(1).toString());
+ Assert.assertEquals("build", columns[12].getBinary(1).toString());
+ Assert.assertEquals(AI_NODE_ID, columns[0].getInt(2));
+ Assert.assertEquals("AINode", columns[1].getBinary(2).toString());
+ Assert.assertEquals("127.0.0.1", columns[3].getBinary(2).toString());
+ Assert.assertEquals(3000, columns[4].getInt(2));
+ Assert.assertEquals("", columns[5].getBinary(2).toString());
+ Assert.assertEquals("2.0.11", columns[11].getBinary(2).toString());
+ Assert.assertEquals("build", columns[12].getBinary(2).toString());
+ Assert.assertTrue(columns[13].isNull(2));
+ }
+}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java
new file mode 100644
index 00000000000..90d90d49de8
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/execution/config/metadata/ShowClusterTaskTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.queryengine.plan.execution.config.metadata;
+
+import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
+import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.cluster.NodeStatus;
+import org.apache.iotdb.commons.schema.column.ColumnHeader;
+import org.apache.iotdb.commons.schema.column.ColumnHeaderConstant;
+import org.apache.iotdb.confignode.rpc.thrift.TNodeVersionInfo;
+import org.apache.iotdb.confignode.rpc.thrift.TShowClusterResp;
+import org.apache.iotdb.db.queryengine.plan.execution.config.ConfigTaskResult;
+import org.apache.iotdb.rpc.TSStatusCode;
+
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.tsfile.block.column.Column;
+import org.apache.tsfile.enums.TSDataType;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class ShowClusterTaskTest {
+
+ private static final int CONFIG_NODE_ID = 0;
+ private static final int DATA_NODE_ID = 1;
+
+ private static TShowClusterResp buildClusterResp(final Map<Integer, String> statusReasonMap) {
+ final TConfigNodeLocation configNode =
+ new TConfigNodeLocation(
+ CONFIG_NODE_ID, new TEndPoint("127.0.0.1", 1000), new TEndPoint("127.0.0.1", 1001));
+ final TDataNodeLocation dataNode =
+ new TDataNodeLocation(
+ DATA_NODE_ID,
+ new TEndPoint("127.0.0.1", 2000),
+ new TEndPoint("127.0.0.1", 2001),
+ new TEndPoint("127.0.0.1", 2002),
+ new TEndPoint("127.0.0.1", 2003),
+ new TEndPoint("127.0.0.1", 2004));
+
+ final Map<Integer, String> nodeStatus = new HashMap<>();
+ nodeStatus.put(CONFIG_NODE_ID, NodeStatus.Running.getStatus());
+ nodeStatus.put(DATA_NODE_ID, NodeStatus.ReadOnly.getStatus());
+
+ final Map<Integer, TNodeVersionInfo> nodeVersionInfo = new HashMap<>();
+ final TNodeVersionInfo versionInfo = new TNodeVersionInfo("2.0.11", "build");
+ nodeVersionInfo.put(CONFIG_NODE_ID, versionInfo);
+ nodeVersionInfo.put(DATA_NODE_ID, versionInfo);
+
+ final TShowClusterResp resp =
+ new TShowClusterResp(
+ new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()),
+ Collections.singletonList(configNode),
+ Collections.singletonList(dataNode),
+ Collections.emptyList(),
+ nodeStatus,
+ nodeVersionInfo);
+ // An old ConfigNode response leaves the optional nodeStatusReason field unset.
+ if (statusReasonMap != null) {
+ resp.setNodeStatusReason(statusReasonMap);
+ }
+ return resp;
+ }
+
+ private static ConfigTaskResult execute(final TShowClusterResp resp) throws Exception {
+ final SettableFuture<ConfigTaskResult> future = SettableFuture.create();
+ ShowClusterTask.buildTsBlock(resp, future);
+ return future.get();
+ }
+
+ private static List<String> columnNames(final List<ColumnHeader> headers) {
... 1155 lines suppressed ...