You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/08/19 01:09:49 UTC

[iotdb] branch master updated: [IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/master by this push:
     new a68377e455 [IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)
a68377e455 is described below

commit a68377e45565070fa7ac2698c780bcf5947d6ea2
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Aug 19 09:09:44 2022 +0800

    [IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)
---
 .../iotdb/confignode/conf/ConfigNodeConfig.java    |  2 +-
 .../iotdb/confignode/conf/ConfigNodeConstant.java  |  7 ++++
 .../confignode/conf/ConfigNodeDescriptor.java      |  4 +--
 .../confignode/conf/ConfigNodeStartupCheck.java    |  4 +--
 .../iotdb/confignode/manager/NodeManager.java      | 31 ++++++++++++++----
 .../iotdb/confignode/manager/PartitionManager.java | 20 +++++++++---
 .../iotdb/confignode/manager/load/LoadManager.java |  2 +-
 .../manager/load/LoadManagerMetrics.java           | 36 +++++++++++----------
 .../manager/load/balancer/RouteBalancer.java       | 11 ++++---
 .../load/balancer/router/LazyGreedyRouter.java     |  3 +-
 .../{INodeCache.java => BaseNodeCache.java}        | 35 +++++++++++++++++---
 .../load/heartbeat/ConfigNodeHeartbeatCache.java   | 17 ++--------
 .../load/heartbeat/DataNodeHeartbeatCache.java     | 37 ++++++++++++----------
 .../manager/load/heartbeat/RegionGroupCache.java   |  2 +-
 .../iotdb/confignode/persistence/NodeInfo.java     | 12 ++++---
 .../procedure/env/ConfigNodeProcedureEnv.java      | 15 ++++++++-
 .../procedure/impl/AddConfigNodeProcedure.java     |  4 +--
 .../procedure/impl/RemoveDataNodeProcedure.java    |  6 ++--
 .../load/balancer/router/LeaderRouterTest.java     |  6 ++--
 .../balancer/router/LoadScoreGreedyRouterTest.java |  6 ++--
 .../apache/iotdb/commons/cluster/NodeStatus.java   | 14 ++++----
 21 files changed, 176 insertions(+), 98 deletions(-)

diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 42f929b997..faef338857 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -127,7 +127,7 @@ public class ConfigNodeConfig {
   private long heartbeatInterval = 1000;
 
   /** The routing policy of read/write requests */
-  private String routingPolicy = RouteBalancer.leaderPolicy;
+  private String routingPolicy = RouteBalancer.LEADER_POLICY;
 
   private String readConsistencyLevel = "strong";
 
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
index 425becaad4..4f58c58331 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
@@ -38,6 +38,13 @@ public class ConfigNodeConstant {
 
   public static final int MIN_SUPPORTED_JDK_VERSION = 8;
 
+  /** These variables are only used for cluster gauge metrics */
+  public static final String METRIC_TAG_TOTAL = "total";
+
+  public static final String METRIC_STATUS_REGISTER = "Registered";
+  public static final String METRIC_STATUS_ONLINE = "Online";
+  public static final String METRIC_STATUS_UNKNOWN = "Unknown";
+
   private ConfigNodeConstant() {
     // empty constructor
   }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index d7c6e47244..624e8c98c1 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -223,8 +223,8 @@ public class ConfigNodeDescriptor {
                   "heartbeat_interval", String.valueOf(conf.getHeartbeatInterval()))));
 
       String routingPolicy = properties.getProperty("routing_policy", conf.getRoutingPolicy());
-      if (routingPolicy.equals(RouteBalancer.greedyPolicy)
-          || routingPolicy.equals(RouteBalancer.leaderPolicy)) {
+      if (routingPolicy.equals(RouteBalancer.GREEDY_POLICY)
+          || routingPolicy.equals(RouteBalancer.LEADER_POLICY)) {
         conf.setRoutingPolicy(routingPolicy);
       } else {
         throw new IOException(
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
index e145795add..73c1f5ab20 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
@@ -92,8 +92,8 @@ public class ConfigNodeStartupCheck {
               "%s or %s", ConsensusFactory.StandAloneConsensus, ConsensusFactory.RatisConsensus));
     }
 
-    if (!conf.getRoutingPolicy().equals(RouteBalancer.leaderPolicy)
-        && !conf.getRoutingPolicy().equals(RouteBalancer.greedyPolicy)) {
+    if (!conf.getRoutingPolicy().equals(RouteBalancer.LEADER_POLICY)
+        && !conf.getRoutingPolicy().equals(RouteBalancer.GREEDY_POLICY)) {
       throw new ConfigurationException(
           "routing_policy", conf.getRoutingPolicy(), "leader or greedy");
     }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
index d602b0d5dc..d12da2ef63 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
@@ -45,9 +45,9 @@ import org.apache.iotdb.confignode.consensus.response.DataNodeConfigurationResp;
 import org.apache.iotdb.confignode.consensus.response.DataNodeRegisterResp;
 import org.apache.iotdb.confignode.consensus.response.DataNodeToStatusResp;
 import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.ConfigNodeHeartbeatCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
 import org.apache.iotdb.confignode.persistence.NodeInfo;
 import org.apache.iotdb.confignode.procedure.env.DataNodeRemoveHandler;
 import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo;
@@ -63,6 +63,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -95,7 +96,7 @@ public class NodeManager {
   // Monitor for leadership change
   private final Object scheduleMonitor = new Object();
   // Map<NodeId, INodeCache>
-  private final Map<Integer, INodeCache> nodeCacheMap;
+  private final Map<Integer, BaseNodeCache> nodeCacheMap;
   private final AtomicInteger heartbeatCounter = new AtomicInteger(0);
   private Future<?> currentHeartbeatFuture;
   private final ScheduledExecutorService heartBeatExecutor =
@@ -520,7 +521,7 @@ public class NodeManager {
     }
   }
 
-  public Map<Integer, INodeCache> getNodeCacheMap() {
+  public Map<Integer, BaseNodeCache> getNodeCacheMap() {
     return nodeCacheMap;
   }
 
@@ -531,7 +532,7 @@ public class NodeManager {
    * @return The specific Node's current status if the nodeCache contains it, Unknown otherwise
    */
   private String getNodeStatus(int nodeId) {
-    INodeCache nodeCache = nodeCacheMap.get(nodeId);
+    BaseNodeCache nodeCache = nodeCacheMap.get(nodeId);
     return nodeCache == null ? "Unknown" : nodeCache.getNodeStatus().getStatus();
   }
 
@@ -558,13 +559,14 @@ public class NodeManager {
    * @param status The specific NodeStatus
    * @return Filtered DataNodes with the specific NodeStatus
    */
-  public List<TDataNodeConfiguration> filterDataNodeThroughStatus(NodeStatus status) {
+  public List<TDataNodeConfiguration> filterDataNodeThroughStatus(NodeStatus... status) {
     return getRegisteredDataNodes().stream()
         .filter(
             registeredDataNode -> {
               int id = registeredDataNode.getLocation().getDataNodeId();
               return nodeCacheMap.containsKey(id)
-                  && status.equals(nodeCacheMap.get(id).getNodeStatus());
+                  && Arrays.stream(status)
+                      .anyMatch(s -> s.equals(nodeCacheMap.get(id).getNodeStatus()));
             })
         .collect(Collectors.toList());
   }
@@ -583,6 +585,23 @@ public class NodeManager {
     return result;
   }
 
+  public boolean isNodeRemoving(int dataNodeId) {
+    DataNodeHeartbeatCache cache =
+        (DataNodeHeartbeatCache) configManager.getNodeManager().getNodeCacheMap().get(dataNodeId);
+    if (cache != null) {
+      return cache.isRemoving();
+    }
+    return false;
+  }
+
+  public void setNodeRemovingStatus(int dataNodeId, boolean isRemoving) {
+    DataNodeHeartbeatCache cache =
+        (DataNodeHeartbeatCache) configManager.getNodeManager().getNodeCacheMap().get(dataNodeId);
+    if (cache != null) {
+      cache.setRemoving(isRemoving);
+    }
+  }
+
   public List<TConfigNodeLocation> getRegisteredConfigNodes() {
     return nodeInfo.getRegisteredConfigNodes();
   }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
index 6eba977594..a2b9941b52 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
@@ -535,7 +535,8 @@ public class PartitionManager {
   }
 
   /**
-   * Get the leadership of each RegionGroup
+   * Get the leadership of each RegionGroup If a node is in unknown or removing status, this node
+   * can't be leader
    *
    * @return Map<RegionGroupId, leader location>
    */
@@ -548,7 +549,12 @@ public class PartitionManager {
       regionGroupCacheMap.forEach(
           (consensusGroupId, regionGroupCache) -> {
             if (consensusGroupId.getType().equals(TConsensusGroupType.SchemaRegion)) {
-              result.put(consensusGroupId, regionGroupCache.getLeaderDataNodeId());
+              int leaderDataNodeId = regionGroupCache.getLeaderDataNodeId();
+              if (configManager.getNodeManager().isNodeRemoving(leaderDataNodeId)) {
+                result.put(consensusGroupId, -1);
+              } else {
+                result.put(consensusGroupId, leaderDataNodeId);
+              }
             }
           });
       getLoadManager()
@@ -561,8 +567,14 @@ public class PartitionManager {
                       regionReplicaSet.getDataNodeLocations().get(0).getDataNodeId()));
     } else {
       regionGroupCacheMap.forEach(
-          (consensusGroupId, regionGroupCache) ->
-              result.put(consensusGroupId, regionGroupCache.getLeaderDataNodeId()));
+          (consensusGroupId, regionGroupCache) -> {
+            int leaderDataNodeId = regionGroupCache.getLeaderDataNodeId();
+            if (configManager.getNodeManager().isNodeRemoving(leaderDataNodeId)) {
+              result.put(consensusGroupId, -1);
+            } else {
+              result.put(consensusGroupId, leaderDataNodeId);
+            }
+          });
     }
     return result;
   }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
index 27b0a14ef5..8ce812ade4 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
@@ -215,7 +215,7 @@ public class LoadManager {
       isNeedBroadcast = true;
     }
 
-    if (RouteBalancer.leaderPolicy.equals(CONF.getRoutingPolicy())) {
+    if (RouteBalancer.LEADER_POLICY.equals(CONF.getRoutingPolicy())) {
       // Check the condition of leader routing policy
       if (existChangeLeaderSchemaRegionGroup.get()) {
         // Broadcast the RegionRouteMap if some SchemaRegionGroups change their leader
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
index d1547f8e76..f359435fb2 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
@@ -36,6 +36,10 @@ import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_ONLINE;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_UNKNOWN;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_TAG_TOTAL;
+
 /** This class collates metrics about loadManager */
 public class LoadManagerMetrics {
 
@@ -148,9 +152,9 @@ public class LoadManagerMetrics {
             this,
             o -> getRunningConfigNodesNum(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Online.toString());
+            METRIC_STATUS_ONLINE);
 
     MetricService.getInstance()
         .getOrCreateAutoGauge(
@@ -159,9 +163,9 @@ public class LoadManagerMetrics {
             this,
             o -> getRunningDataNodesNum(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Online.toString());
+            METRIC_STATUS_ONLINE);
 
     MetricService.getInstance()
         .getOrCreateAutoGauge(
@@ -170,9 +174,9 @@ public class LoadManagerMetrics {
             this,
             o -> getUnknownConfigNodesNum(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Unknown.toString());
+            METRIC_STATUS_UNKNOWN);
 
     MetricService.getInstance()
         .getOrCreateAutoGauge(
@@ -181,9 +185,9 @@ public class LoadManagerMetrics {
             this,
             o -> getUnknownDataNodesNum(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Unknown.toString());
+            METRIC_STATUS_UNKNOWN);
   }
 
   /**
@@ -227,33 +231,33 @@ public class LoadManagerMetrics {
             MetricType.GAUGE,
             Metric.CONFIG_NODE.toString(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Online.toString());
+            METRIC_STATUS_ONLINE);
     MetricService.getInstance()
         .remove(
             MetricType.GAUGE,
             Metric.DATA_NODE.toString(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Online.toString());
+            METRIC_STATUS_ONLINE);
     MetricService.getInstance()
         .remove(
             MetricType.GAUGE,
             Metric.CONFIG_NODE.toString(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Unknown.toString());
+            METRIC_STATUS_UNKNOWN);
     MetricService.getInstance()
         .remove(
             MetricType.GAUGE,
             Metric.DATA_NODE.toString(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Unknown.toString());
+            METRIC_STATUS_UNKNOWN);
   }
 
   private NodeManager getNodeManager() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
index 1ef562245b..48db8ee587 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
@@ -42,8 +42,8 @@ import java.util.Map;
  */
 public class RouteBalancer {
 
-  public static final String leaderPolicy = "leader";
-  public static final String greedyPolicy = "greedy";
+  public static final String LEADER_POLICY = "leader";
+  public static final String GREEDY_POLICY = "greedy";
 
   private final IManager configManager;
 
@@ -84,7 +84,7 @@ public class RouteBalancer {
     String policy = ConfigNodeDescriptor.getInstance().getConf().getRoutingPolicy();
     switch (groupType) {
       case SchemaRegion:
-        if (policy.equals(leaderPolicy)) {
+        if (LEADER_POLICY.equals(policy)) {
           return new LeaderRouter(
               getPartitionManager().getAllLeadership(), getNodeManager().getAllLoadScores());
         } else {
@@ -98,9 +98,10 @@ public class RouteBalancer {
             .equals(ConsensusFactory.MultiLeaderConsensus)) {
           // Latent router for MultiLeader consensus protocol
           lazyGreedyRouter.updateUnknownDataNodes(
-              getNodeManager().filterDataNodeThroughStatus(NodeStatus.Unknown));
+              getNodeManager()
+                  .filterDataNodeThroughStatus(NodeStatus.Unknown, NodeStatus.Removing));
           return lazyGreedyRouter;
-        } else if (policy.equals(leaderPolicy)) {
+        } else if (LEADER_POLICY.equals(policy)) {
           return new LeaderRouter(
               getPartitionManager().getAllLeadership(), getNodeManager().getAllLoadScores());
         } else {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
index 83a4732381..549e8dc6b2 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
@@ -40,8 +40,9 @@ import java.util.stream.Collectors;
  */
 public class LazyGreedyRouter implements IRouter {
 
-  // Set<DataNodeId>
+  /** Set<DataNodeId> which stores the unknown and removing datanodes */
   private final Set<Integer> unknownDataNodes;
+
   private final Map<TConsensusGroupId, TRegionReplicaSet> routeMap;
 
   public LazyGreedyRouter() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
similarity index 59%
rename from confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java
rename to confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
index cec510f8ac..c78392544c 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
@@ -20,30 +20,55 @@ package org.apache.iotdb.confignode.manager.load.heartbeat;
 
 import org.apache.iotdb.commons.cluster.NodeStatus;
 
+import java.util.LinkedList;
+
 /** All the statistic interfaces that provided by HeartbeatCache */
-public interface INodeCache {
+public abstract class BaseNodeCache {
+
+  /** When the response time of heartbeat is more than 20s, the node is considered as down */
+  static final int HEARTBEAT_TIMEOUT_TIME = 20_000;
+
+  /** Max heartbeat cache samples store size */
+  static final int MAXIMUM_WINDOW_SIZE = 100;
+
+  /** SlidingWindow stores the heartbeat sample data */
+  final LinkedList<NodeHeartbeatSample> slidingWindow = new LinkedList<>();
+
+  /** Node status, using for `show cluster` command */
+  volatile NodeStatus status = NodeStatus.Unknown;
+
+  /** Represent that if this node is in removing status */
+  volatile boolean removing;
 
   /**
    * Cache the newest HeartbeatSample
    *
    * @param newHeartbeatSample The newest HeartbeatSample
    */
-  void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample);
+  public abstract void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample);
 
   /**
    * Invoking periodically to update Nodes' load statistics
    *
    * @return true if some load statistic changed
    */
-  boolean updateLoadStatistic();
+  public abstract boolean updateLoadStatistic();
 
   /**
    * TODO: The loadScore of each Node will be changed to Double
    *
    * @return The latest load score of a node, the higher the score the higher the load
    */
-  long getLoadScore();
+  public abstract long getLoadScore();
 
   /** @return The latest status of a node for showing cluster */
-  NodeStatus getNodeStatus();
+  public abstract NodeStatus getNodeStatus();
+
+  public void setRemoving(boolean removing) {
+    this.removing = removing;
+  }
+
+  public boolean isRemoving() {
+    return this.removing;
+  }
 }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
index 53a5dff62d..00e4057f4b 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
@@ -22,23 +22,12 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
 import org.apache.iotdb.commons.cluster.NodeStatus;
 import org.apache.iotdb.confignode.manager.NodeManager;
 
-import java.util.LinkedList;
-
-public class ConfigNodeHeartbeatCache implements INodeCache {
-
-  // Cache heartbeat samples
-  private static final int maximumWindowSize = 100;
-  private final LinkedList<NodeHeartbeatSample> slidingWindow;
+public class ConfigNodeHeartbeatCache extends BaseNodeCache {
 
   private final TConfigNodeLocation configNodeLocation;
 
-  // For showing cluster
-  private volatile NodeStatus status;
-
   public ConfigNodeHeartbeatCache(TConfigNodeLocation configNodeLocation) {
     this.configNodeLocation = configNodeLocation;
-    this.slidingWindow = new LinkedList<>();
-    this.status = NodeStatus.Unknown;
   }
 
   @Override
@@ -51,7 +40,7 @@ public class ConfigNodeHeartbeatCache implements INodeCache {
         slidingWindow.add(newHeartbeatSample);
       }
 
-      if (slidingWindow.size() > maximumWindowSize) {
+      if (slidingWindow.size() > MAXIMUM_WINDOW_SIZE) {
         slidingWindow.removeFirst();
       }
     }
@@ -82,7 +71,7 @@ public class ConfigNodeHeartbeatCache implements INodeCache {
     }
 
     // TODO: Optimize judge logic
-    if (System.currentTimeMillis() - lastSendTime > 20_000) {
+    if (System.currentTimeMillis() - lastSendTime > HEARTBEAT_TIMEOUT_TIME) {
       status = NodeStatus.Unknown;
     } else {
       status = NodeStatus.Running;
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
index 6e83e7a27d..ba56d867e7 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
@@ -20,31 +20,22 @@ package org.apache.iotdb.confignode.manager.load.heartbeat;
 
 import org.apache.iotdb.commons.cluster.NodeStatus;
 
-import java.util.LinkedList;
-
 /** DataNodeHeartbeatCache caches and maintains all the heartbeat data */
-public class DataNodeHeartbeatCache implements INodeCache {
-
-  // TODO: This class might be split into DataNodeCache and ConfigNodeCache
-
-  // Cache heartbeat samples
-  private static final int maximumWindowSize = 100;
-  private final LinkedList<NodeHeartbeatSample> slidingWindow;
+public class DataNodeHeartbeatCache extends BaseNodeCache {
 
-  // For guiding queries, the higher the score the higher the load
+  /** For guiding queries, the higher the score the higher the load */
   private volatile long loadScore;
-  // For showing cluster
-  private volatile NodeStatus status;
 
   public DataNodeHeartbeatCache() {
-    this.slidingWindow = new LinkedList<>();
-
     this.loadScore = 0;
-    this.status = NodeStatus.Unknown;
   }
 
   @Override
   public void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample) {
+    if (isRemoving()) {
+      return;
+    }
+
     synchronized (slidingWindow) {
       // Only sequential HeartbeatSamples are accepted.
       // And un-sequential HeartbeatSamples will be discarded.
@@ -53,7 +44,7 @@ public class DataNodeHeartbeatCache implements INodeCache {
         slidingWindow.add(newHeartbeatSample);
       }
 
-      if (slidingWindow.size() > maximumWindowSize) {
+      if (slidingWindow.size() > MAXIMUM_WINDOW_SIZE) {
         slidingWindow.removeFirst();
       }
     }
@@ -61,6 +52,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
 
   @Override
   public boolean updateLoadStatistic() {
+    if (isRemoving()) {
+      return false;
+    }
+
     long lastSendTime = 0;
     synchronized (slidingWindow) {
       if (slidingWindow.size() > 0) {
@@ -85,7 +80,7 @@ public class DataNodeHeartbeatCache implements INodeCache {
     }
 
     // TODO: Optimize judge logic
-    if (System.currentTimeMillis() - lastSendTime > 20_000) {
+    if (System.currentTimeMillis() - lastSendTime > HEARTBEAT_TIMEOUT_TIME) {
       status = NodeStatus.Unknown;
     } else {
       status = NodeStatus.Running;
@@ -95,6 +90,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
 
   @Override
   public long getLoadScore() {
+    if (isRemoving()) {
+      return Long.MAX_VALUE;
+    }
+
     // Return a copy of loadScore
     switch (status) {
       case Running:
@@ -108,6 +107,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
 
   @Override
   public NodeStatus getNodeStatus() {
+    if (isRemoving()) {
+      return NodeStatus.Removing;
+    }
+
     // Return a copy of status
     switch (status) {
       case Running:
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
index 0d6d937ad8..96b77e68c4 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
@@ -100,7 +100,7 @@ public class RegionGroupCache implements IRegionGroupCache {
       leaderDataNodeId.set(updateLeaderDataNodeId);
     }
 
-    return !(originLeaderDataNodeId == leaderDataNodeId.get());
+    return originLeaderDataNodeId != leaderDataNodeId.get();
   }
 
   @Override
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
index c749f2d5b2..7a0bfa72ca 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.cluster.NodeStatus;
 import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
 import org.apache.iotdb.commons.utils.TestOnly;
 import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
@@ -67,6 +66,9 @@ import java.util.concurrent.ConcurrentSkipListMap;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_REGISTER;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_TAG_TOTAL;
+
 /**
  * The NodeInfo stores cluster node information. The cluster node information including: 1. DataNode
  * information 2. ConfigNode information
@@ -109,9 +111,9 @@ public class NodeInfo implements SnapshotProcessor {
             registeredConfigNodes,
             o -> getRegisteredConfigNodeCount(),
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Registered.toString());
+            METRIC_STATUS_REGISTER);
     MetricService.getInstance()
         .getOrCreateAutoGauge(
             Metric.DATA_NODE.toString(),
@@ -119,9 +121,9 @@ public class NodeInfo implements SnapshotProcessor {
             registeredDataNodes,
             Map::size,
             Tag.NAME.toString(),
-            "total",
+            METRIC_TAG_TOTAL,
             Tag.STATUS.toString(),
-            NodeStatus.Registered.toString());
+            METRIC_STATUS_REGISTER);
   }
 
   /**
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index b54f9132e6..c5be031d59 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.confignode.procedure.env;
 import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
 import org.apache.iotdb.confignode.client.ConfigNodeRequestType;
@@ -298,7 +299,19 @@ public class ConfigNodeProcedureEnv {
   }
 
   /**
-   * Broadcast the CreateRegionGroupsPlan
+   * Mark the given datanode as removing status, and broadcast the region map, to avoid read or
+   * write request routing to this node.
+   *
+   * @param dataNodeLocation the datanode to be marked as removing status
+   */
+  public void markDataNodeAsRemovingAndBroadCast(TDataNodeLocation dataNodeLocation) {
+    int dataNodeId = dataNodeLocation.getDataNodeId();
+    configManager.getNodeManager().setNodeRemovingStatus(dataNodeId, true);
+    configManager.getLoadManager().broadcastLatestRegionRouteMap();
+  }
+
+  /**
+   * Do region creations and broadcast the CreateRegionGroupsPlan
    *
    * @return Those RegionGroups that failed to create
    */
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
index cf244fb52d..82131683f6 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
@@ -37,7 +37,7 @@ import java.nio.ByteBuffer;
 /** add config node procedure */
 public class AddConfigNodeProcedure extends AbstractNodeProcedure<AddConfigNodeState> {
   private static final Logger LOG = LoggerFactory.getLogger(AddConfigNodeProcedure.class);
-  private static final int retryThreshold = 5;
+  private static final int RETRY_THRESHOLD = 5;
 
   private TConfigNodeLocation tConfigNodeLocation;
 
@@ -85,7 +85,7 @@ public class AddConfigNodeProcedure extends AbstractNodeProcedure<AddConfigNodeS
             tConfigNodeLocation,
             state,
             e);
-        if (getCycles() > retryThreshold) {
+        if (getCycles() > RETRY_THRESHOLD) {
           setFailure(new ProcedureException("State stuck at " + state));
         }
       }
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
index cf327ec8f8..1743e1f111 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
@@ -40,7 +40,7 @@ import java.util.List;
 /** remove data node procedure */
 public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNodeState> {
   private static final Logger LOG = LoggerFactory.getLogger(RemoveDataNodeProcedure.class);
-  private static final int retryThreshold = 5;
+  private static final int RETRY_THRESHOLD = 5;
 
   private TDataNodeLocation disableDataNodeLocation;
 
@@ -63,6 +63,8 @@ public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNod
     try {
       switch (state) {
         case REMOVE_DATA_NODE_PREPARE:
+          // mark the datanode as removing status and broadcast region route map
+          env.markDataNodeAsRemovingAndBroadCast(disableDataNodeLocation);
           execDataNodeRegionIds =
               env.getDataNodeRemoveHandler().getDataNodeRegionIds(disableDataNodeLocation);
           LOG.info("DataNode region id is {}", execDataNodeRegionIds);
@@ -90,7 +92,7 @@ public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNod
             disableDataNodeLocation,
             state,
             e);
-        if (getCycles() > retryThreshold) {
+        if (getCycles() > RETRY_THRESHOLD) {
           setFailure(new ProcedureException("State stuck at " + state));
         }
       }
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
index db134efd25..de0fb369c5 100644
--- a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
@@ -23,8 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
 import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.IRegionGroupCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.NodeHeartbeatSample;
 import org.apache.iotdb.confignode.manager.load.heartbeat.RegionGroupCache;
@@ -59,7 +59,7 @@ public class LeaderRouterTest {
 
     // Build nodeCacheMap
     long currentTimeMillis = System.currentTimeMillis();
-    Map<Integer, INodeCache> nodeCacheMap = new HashMap<>();
+    Map<Integer, BaseNodeCache> nodeCacheMap = new HashMap<>();
     for (int i = 0; i < 6; i++) {
       nodeCacheMap.put(i, new DataNodeHeartbeatCache());
       // Simulate that the DataNode-i returned a heartbeat at (currentTime - i * 1000) ms
@@ -68,7 +68,7 @@ public class LeaderRouterTest {
           .cacheHeartbeatSample(
               new NodeHeartbeatSample(currentTimeMillis - i * 1000, currentTimeMillis - i * 1000));
     }
-    nodeCacheMap.values().forEach(INodeCache::updateLoadStatistic);
+    nodeCacheMap.values().forEach(BaseNodeCache::updateLoadStatistic);
 
     // Get the loadScoreMap
     Map<Integer, Long> loadScoreMap = new ConcurrentHashMap<>();
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
index faff66243e..ae13d16650 100644
--- a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
@@ -23,8 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
 import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
 import org.apache.iotdb.common.rpc.thrift.TEndPoint;
 import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
 import org.apache.iotdb.confignode.manager.load.heartbeat.NodeHeartbeatSample;
 
 import org.junit.Assert;
@@ -56,7 +56,7 @@ public class LoadScoreGreedyRouterTest {
 
     /* Build nodeCacheMap */
     long currentTimeMillis = System.currentTimeMillis();
-    Map<Integer, INodeCache> nodeCacheMap = new HashMap<>();
+    Map<Integer, BaseNodeCache> nodeCacheMap = new HashMap<>();
     for (int i = 0; i < 6; i++) {
       nodeCacheMap.put(i, new DataNodeHeartbeatCache());
       // Simulate that the DataNode-i returned a heartbeat at (currentTime - i * 1000) ms
@@ -65,7 +65,7 @@ public class LoadScoreGreedyRouterTest {
           .cacheHeartbeatSample(
               new NodeHeartbeatSample(currentTimeMillis - i * 1000, currentTimeMillis - i * 1000));
     }
-    nodeCacheMap.values().forEach(INodeCache::updateLoadStatistic);
+    nodeCacheMap.values().forEach(BaseNodeCache::updateLoadStatistic);
 
     /* Get the loadScoreMap */
     Map<Integer, Long> loadScoreMap = new ConcurrentHashMap<>();
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
index 8d2a3577a2..82cc285cf3 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
@@ -20,14 +20,14 @@ package org.apache.iotdb.commons.cluster;
 
 /** Node status for showing cluster */
 public enum NodeStatus {
-  // Node registered
-  Registered("Registered"),
-  // Node online ,right now Online is Running
-  Online("Online"),
-  // Node running properly
+  /** Node running properly */
   Running("Running"),
-  // Node connection failure
-  Unknown("Unknown");
+
+  /** Node connection failure */
+  Unknown("Unknown"),
+
+  /** Node is in removing */
+  Removing("Removing");
 
   private final String status;