You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/08/19 01:09:49 UTC
[iotdb] branch master updated: [IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new a68377e455 [IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)
a68377e455 is described below
commit a68377e45565070fa7ac2698c780bcf5947d6ea2
Author: Beyyes <cg...@foxmail.com>
AuthorDate: Fri Aug 19 09:09:44 2022 +0800
[IOTDB-4140] Mark the datanode as removing status when execute remove-datanode.sh (#7008)
---
.../iotdb/confignode/conf/ConfigNodeConfig.java | 2 +-
.../iotdb/confignode/conf/ConfigNodeConstant.java | 7 ++++
.../confignode/conf/ConfigNodeDescriptor.java | 4 +--
.../confignode/conf/ConfigNodeStartupCheck.java | 4 +--
.../iotdb/confignode/manager/NodeManager.java | 31 ++++++++++++++----
.../iotdb/confignode/manager/PartitionManager.java | 20 +++++++++---
.../iotdb/confignode/manager/load/LoadManager.java | 2 +-
.../manager/load/LoadManagerMetrics.java | 36 +++++++++++----------
.../manager/load/balancer/RouteBalancer.java | 11 ++++---
.../load/balancer/router/LazyGreedyRouter.java | 3 +-
.../{INodeCache.java => BaseNodeCache.java} | 35 +++++++++++++++++---
.../load/heartbeat/ConfigNodeHeartbeatCache.java | 17 ++--------
.../load/heartbeat/DataNodeHeartbeatCache.java | 37 ++++++++++++----------
.../manager/load/heartbeat/RegionGroupCache.java | 2 +-
.../iotdb/confignode/persistence/NodeInfo.java | 12 ++++---
.../procedure/env/ConfigNodeProcedureEnv.java | 15 ++++++++-
.../procedure/impl/AddConfigNodeProcedure.java | 4 +--
.../procedure/impl/RemoveDataNodeProcedure.java | 6 ++--
.../load/balancer/router/LeaderRouterTest.java | 6 ++--
.../balancer/router/LoadScoreGreedyRouterTest.java | 6 ++--
.../apache/iotdb/commons/cluster/NodeStatus.java | 14 ++++----
21 files changed, 176 insertions(+), 98 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
index 42f929b997..faef338857 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java
@@ -127,7 +127,7 @@ public class ConfigNodeConfig {
private long heartbeatInterval = 1000;
/** The routing policy of read/write requests */
- private String routingPolicy = RouteBalancer.leaderPolicy;
+ private String routingPolicy = RouteBalancer.LEADER_POLICY;
private String readConsistencyLevel = "strong";
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
index 425becaad4..4f58c58331 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConstant.java
@@ -38,6 +38,13 @@ public class ConfigNodeConstant {
public static final int MIN_SUPPORTED_JDK_VERSION = 8;
+ /** These variables are only used for cluster gauge metrics */
+ public static final String METRIC_TAG_TOTAL = "total";
+
+ public static final String METRIC_STATUS_REGISTER = "Registered";
+ public static final String METRIC_STATUS_ONLINE = "Online";
+ public static final String METRIC_STATUS_UNKNOWN = "Unknown";
+
private ConfigNodeConstant() {
// empty constructor
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
index d7c6e47244..624e8c98c1 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeDescriptor.java
@@ -223,8 +223,8 @@ public class ConfigNodeDescriptor {
"heartbeat_interval", String.valueOf(conf.getHeartbeatInterval()))));
String routingPolicy = properties.getProperty("routing_policy", conf.getRoutingPolicy());
- if (routingPolicy.equals(RouteBalancer.greedyPolicy)
- || routingPolicy.equals(RouteBalancer.leaderPolicy)) {
+ if (routingPolicy.equals(RouteBalancer.GREEDY_POLICY)
+ || routingPolicy.equals(RouteBalancer.LEADER_POLICY)) {
conf.setRoutingPolicy(routingPolicy);
} else {
throw new IOException(
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
index e145795add..73c1f5ab20 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeStartupCheck.java
@@ -92,8 +92,8 @@ public class ConfigNodeStartupCheck {
"%s or %s", ConsensusFactory.StandAloneConsensus, ConsensusFactory.RatisConsensus));
}
- if (!conf.getRoutingPolicy().equals(RouteBalancer.leaderPolicy)
- && !conf.getRoutingPolicy().equals(RouteBalancer.greedyPolicy)) {
+ if (!conf.getRoutingPolicy().equals(RouteBalancer.LEADER_POLICY)
+ && !conf.getRoutingPolicy().equals(RouteBalancer.GREEDY_POLICY)) {
throw new ConfigurationException(
"routing_policy", conf.getRoutingPolicy(), "leader or greedy");
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
index d602b0d5dc..d12da2ef63 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/NodeManager.java
@@ -45,9 +45,9 @@ import org.apache.iotdb.confignode.consensus.response.DataNodeConfigurationResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeRegisterResp;
import org.apache.iotdb.confignode.consensus.response.DataNodeToStatusResp;
import org.apache.iotdb.confignode.manager.load.LoadManager;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.ConfigNodeHeartbeatCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
import org.apache.iotdb.confignode.persistence.NodeInfo;
import org.apache.iotdb.confignode.procedure.env.DataNodeRemoveHandler;
import org.apache.iotdb.confignode.rpc.thrift.TConfigNodeInfo;
@@ -63,6 +63,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
@@ -95,7 +96,7 @@ public class NodeManager {
// Monitor for leadership change
private final Object scheduleMonitor = new Object();
// Map<NodeId, INodeCache>
- private final Map<Integer, INodeCache> nodeCacheMap;
+ private final Map<Integer, BaseNodeCache> nodeCacheMap;
private final AtomicInteger heartbeatCounter = new AtomicInteger(0);
private Future<?> currentHeartbeatFuture;
private final ScheduledExecutorService heartBeatExecutor =
@@ -520,7 +521,7 @@ public class NodeManager {
}
}
- public Map<Integer, INodeCache> getNodeCacheMap() {
+ public Map<Integer, BaseNodeCache> getNodeCacheMap() {
return nodeCacheMap;
}
@@ -531,7 +532,7 @@ public class NodeManager {
* @return The specific Node's current status if the nodeCache contains it, Unknown otherwise
*/
private String getNodeStatus(int nodeId) {
- INodeCache nodeCache = nodeCacheMap.get(nodeId);
+ BaseNodeCache nodeCache = nodeCacheMap.get(nodeId);
return nodeCache == null ? "Unknown" : nodeCache.getNodeStatus().getStatus();
}
@@ -558,13 +559,14 @@ public class NodeManager {
* @param status The specific NodeStatus
* @return Filtered DataNodes with the specific NodeStatus
*/
- public List<TDataNodeConfiguration> filterDataNodeThroughStatus(NodeStatus status) {
+ public List<TDataNodeConfiguration> filterDataNodeThroughStatus(NodeStatus... status) {
return getRegisteredDataNodes().stream()
.filter(
registeredDataNode -> {
int id = registeredDataNode.getLocation().getDataNodeId();
return nodeCacheMap.containsKey(id)
- && status.equals(nodeCacheMap.get(id).getNodeStatus());
+ && Arrays.stream(status)
+ .anyMatch(s -> s.equals(nodeCacheMap.get(id).getNodeStatus()));
})
.collect(Collectors.toList());
}
@@ -583,6 +585,23 @@ public class NodeManager {
return result;
}
+ public boolean isNodeRemoving(int dataNodeId) {
+ DataNodeHeartbeatCache cache =
+ (DataNodeHeartbeatCache) configManager.getNodeManager().getNodeCacheMap().get(dataNodeId);
+ if (cache != null) {
+ return cache.isRemoving();
+ }
+ return false;
+ }
+
+ public void setNodeRemovingStatus(int dataNodeId, boolean isRemoving) {
+ DataNodeHeartbeatCache cache =
+ (DataNodeHeartbeatCache) configManager.getNodeManager().getNodeCacheMap().get(dataNodeId);
+ if (cache != null) {
+ cache.setRemoving(isRemoving);
+ }
+ }
+
public List<TConfigNodeLocation> getRegisteredConfigNodes() {
return nodeInfo.getRegisteredConfigNodes();
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
index 6eba977594..a2b9941b52 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/PartitionManager.java
@@ -535,7 +535,8 @@ public class PartitionManager {
}
/**
- * Get the leadership of each RegionGroup
+ * Get the leadership of each RegionGroup If a node is in unknown or removing status, this node
+ * can't be leader
*
* @return Map<RegionGroupId, leader location>
*/
@@ -548,7 +549,12 @@ public class PartitionManager {
regionGroupCacheMap.forEach(
(consensusGroupId, regionGroupCache) -> {
if (consensusGroupId.getType().equals(TConsensusGroupType.SchemaRegion)) {
- result.put(consensusGroupId, regionGroupCache.getLeaderDataNodeId());
+ int leaderDataNodeId = regionGroupCache.getLeaderDataNodeId();
+ if (configManager.getNodeManager().isNodeRemoving(leaderDataNodeId)) {
+ result.put(consensusGroupId, -1);
+ } else {
+ result.put(consensusGroupId, leaderDataNodeId);
+ }
}
});
getLoadManager()
@@ -561,8 +567,14 @@ public class PartitionManager {
regionReplicaSet.getDataNodeLocations().get(0).getDataNodeId()));
} else {
regionGroupCacheMap.forEach(
- (consensusGroupId, regionGroupCache) ->
- result.put(consensusGroupId, regionGroupCache.getLeaderDataNodeId()));
+ (consensusGroupId, regionGroupCache) -> {
+ int leaderDataNodeId = regionGroupCache.getLeaderDataNodeId();
+ if (configManager.getNodeManager().isNodeRemoving(leaderDataNodeId)) {
+ result.put(consensusGroupId, -1);
+ } else {
+ result.put(consensusGroupId, leaderDataNodeId);
+ }
+ });
}
return result;
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
index 27b0a14ef5..8ce812ade4 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManager.java
@@ -215,7 +215,7 @@ public class LoadManager {
isNeedBroadcast = true;
}
- if (RouteBalancer.leaderPolicy.equals(CONF.getRoutingPolicy())) {
+ if (RouteBalancer.LEADER_POLICY.equals(CONF.getRoutingPolicy())) {
// Check the condition of leader routing policy
if (existChangeLeaderSchemaRegionGroup.get()) {
// Broadcast the RegionRouteMap if some SchemaRegionGroups change their leader
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
index d1547f8e76..f359435fb2 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/LoadManagerMetrics.java
@@ -36,6 +36,10 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_ONLINE;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_UNKNOWN;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_TAG_TOTAL;
+
/** This class collates metrics about loadManager */
public class LoadManagerMetrics {
@@ -148,9 +152,9 @@ public class LoadManagerMetrics {
this,
o -> getRunningConfigNodesNum(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Online.toString());
+ METRIC_STATUS_ONLINE);
MetricService.getInstance()
.getOrCreateAutoGauge(
@@ -159,9 +163,9 @@ public class LoadManagerMetrics {
this,
o -> getRunningDataNodesNum(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Online.toString());
+ METRIC_STATUS_ONLINE);
MetricService.getInstance()
.getOrCreateAutoGauge(
@@ -170,9 +174,9 @@ public class LoadManagerMetrics {
this,
o -> getUnknownConfigNodesNum(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Unknown.toString());
+ METRIC_STATUS_UNKNOWN);
MetricService.getInstance()
.getOrCreateAutoGauge(
@@ -181,9 +185,9 @@ public class LoadManagerMetrics {
this,
o -> getUnknownDataNodesNum(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Unknown.toString());
+ METRIC_STATUS_UNKNOWN);
}
/**
@@ -227,33 +231,33 @@ public class LoadManagerMetrics {
MetricType.GAUGE,
Metric.CONFIG_NODE.toString(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Online.toString());
+ METRIC_STATUS_ONLINE);
MetricService.getInstance()
.remove(
MetricType.GAUGE,
Metric.DATA_NODE.toString(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Online.toString());
+ METRIC_STATUS_ONLINE);
MetricService.getInstance()
.remove(
MetricType.GAUGE,
Metric.CONFIG_NODE.toString(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Unknown.toString());
+ METRIC_STATUS_UNKNOWN);
MetricService.getInstance()
.remove(
MetricType.GAUGE,
Metric.DATA_NODE.toString(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Unknown.toString());
+ METRIC_STATUS_UNKNOWN);
}
private NodeManager getNodeManager() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
index 1ef562245b..48db8ee587 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/RouteBalancer.java
@@ -42,8 +42,8 @@ import java.util.Map;
*/
public class RouteBalancer {
- public static final String leaderPolicy = "leader";
- public static final String greedyPolicy = "greedy";
+ public static final String LEADER_POLICY = "leader";
+ public static final String GREEDY_POLICY = "greedy";
private final IManager configManager;
@@ -84,7 +84,7 @@ public class RouteBalancer {
String policy = ConfigNodeDescriptor.getInstance().getConf().getRoutingPolicy();
switch (groupType) {
case SchemaRegion:
- if (policy.equals(leaderPolicy)) {
+ if (LEADER_POLICY.equals(policy)) {
return new LeaderRouter(
getPartitionManager().getAllLeadership(), getNodeManager().getAllLoadScores());
} else {
@@ -98,9 +98,10 @@ public class RouteBalancer {
.equals(ConsensusFactory.MultiLeaderConsensus)) {
// Latent router for MultiLeader consensus protocol
lazyGreedyRouter.updateUnknownDataNodes(
- getNodeManager().filterDataNodeThroughStatus(NodeStatus.Unknown));
+ getNodeManager()
+ .filterDataNodeThroughStatus(NodeStatus.Unknown, NodeStatus.Removing));
return lazyGreedyRouter;
- } else if (policy.equals(leaderPolicy)) {
+ } else if (LEADER_POLICY.equals(policy)) {
return new LeaderRouter(
getPartitionManager().getAllLeadership(), getNodeManager().getAllLoadScores());
} else {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
index 83a4732381..549e8dc6b2 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/LazyGreedyRouter.java
@@ -40,8 +40,9 @@ import java.util.stream.Collectors;
*/
public class LazyGreedyRouter implements IRouter {
- // Set<DataNodeId>
+ /** Set<DataNodeId> which stores the unknown and removing datanodes */
private final Set<Integer> unknownDataNodes;
+
private final Map<TConsensusGroupId, TRegionReplicaSet> routeMap;
public LazyGreedyRouter() {
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
similarity index 59%
rename from confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java
rename to confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
index cec510f8ac..c78392544c 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/INodeCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/BaseNodeCache.java
@@ -20,30 +20,55 @@ package org.apache.iotdb.confignode.manager.load.heartbeat;
import org.apache.iotdb.commons.cluster.NodeStatus;
+import java.util.LinkedList;
+
/** All the statistic interfaces that provided by HeartbeatCache */
-public interface INodeCache {
+public abstract class BaseNodeCache {
+
+ /** When the response time of heartbeat is more than 20s, the node is considered as down */
+ static final int HEARTBEAT_TIMEOUT_TIME = 20_000;
+
+ /** Max heartbeat cache samples store size */
+ static final int MAXIMUM_WINDOW_SIZE = 100;
+
+ /** SlidingWindow stores the heartbeat sample data */
+ final LinkedList<NodeHeartbeatSample> slidingWindow = new LinkedList<>();
+
+ /** Node status, using for `show cluster` command */
+ volatile NodeStatus status = NodeStatus.Unknown;
+
+ /** Represent that if this node is in removing status */
+ volatile boolean removing;
/**
* Cache the newest HeartbeatSample
*
* @param newHeartbeatSample The newest HeartbeatSample
*/
- void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample);
+ public abstract void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample);
/**
* Invoking periodically to update Nodes' load statistics
*
* @return true if some load statistic changed
*/
- boolean updateLoadStatistic();
+ public abstract boolean updateLoadStatistic();
/**
* TODO: The loadScore of each Node will be changed to Double
*
* @return The latest load score of a node, the higher the score the higher the load
*/
- long getLoadScore();
+ public abstract long getLoadScore();
/** @return The latest status of a node for showing cluster */
- NodeStatus getNodeStatus();
+ public abstract NodeStatus getNodeStatus();
+
+ public void setRemoving(boolean removing) {
+ this.removing = removing;
+ }
+
+ public boolean isRemoving() {
+ return this.removing;
+ }
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
index 53a5dff62d..00e4057f4b 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/ConfigNodeHeartbeatCache.java
@@ -22,23 +22,12 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.confignode.manager.NodeManager;
-import java.util.LinkedList;
-
-public class ConfigNodeHeartbeatCache implements INodeCache {
-
- // Cache heartbeat samples
- private static final int maximumWindowSize = 100;
- private final LinkedList<NodeHeartbeatSample> slidingWindow;
+public class ConfigNodeHeartbeatCache extends BaseNodeCache {
private final TConfigNodeLocation configNodeLocation;
- // For showing cluster
- private volatile NodeStatus status;
-
public ConfigNodeHeartbeatCache(TConfigNodeLocation configNodeLocation) {
this.configNodeLocation = configNodeLocation;
- this.slidingWindow = new LinkedList<>();
- this.status = NodeStatus.Unknown;
}
@Override
@@ -51,7 +40,7 @@ public class ConfigNodeHeartbeatCache implements INodeCache {
slidingWindow.add(newHeartbeatSample);
}
- if (slidingWindow.size() > maximumWindowSize) {
+ if (slidingWindow.size() > MAXIMUM_WINDOW_SIZE) {
slidingWindow.removeFirst();
}
}
@@ -82,7 +71,7 @@ public class ConfigNodeHeartbeatCache implements INodeCache {
}
// TODO: Optimize judge logic
- if (System.currentTimeMillis() - lastSendTime > 20_000) {
+ if (System.currentTimeMillis() - lastSendTime > HEARTBEAT_TIMEOUT_TIME) {
status = NodeStatus.Unknown;
} else {
status = NodeStatus.Running;
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
index 6e83e7a27d..ba56d867e7 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/DataNodeHeartbeatCache.java
@@ -20,31 +20,22 @@ package org.apache.iotdb.confignode.manager.load.heartbeat;
import org.apache.iotdb.commons.cluster.NodeStatus;
-import java.util.LinkedList;
-
/** DataNodeHeartbeatCache caches and maintains all the heartbeat data */
-public class DataNodeHeartbeatCache implements INodeCache {
-
- // TODO: This class might be split into DataNodeCache and ConfigNodeCache
-
- // Cache heartbeat samples
- private static final int maximumWindowSize = 100;
- private final LinkedList<NodeHeartbeatSample> slidingWindow;
+public class DataNodeHeartbeatCache extends BaseNodeCache {
- // For guiding queries, the higher the score the higher the load
+ /** For guiding queries, the higher the score the higher the load */
private volatile long loadScore;
- // For showing cluster
- private volatile NodeStatus status;
public DataNodeHeartbeatCache() {
- this.slidingWindow = new LinkedList<>();
-
this.loadScore = 0;
- this.status = NodeStatus.Unknown;
}
@Override
public void cacheHeartbeatSample(NodeHeartbeatSample newHeartbeatSample) {
+ if (isRemoving()) {
+ return;
+ }
+
synchronized (slidingWindow) {
// Only sequential HeartbeatSamples are accepted.
// And un-sequential HeartbeatSamples will be discarded.
@@ -53,7 +44,7 @@ public class DataNodeHeartbeatCache implements INodeCache {
slidingWindow.add(newHeartbeatSample);
}
- if (slidingWindow.size() > maximumWindowSize) {
+ if (slidingWindow.size() > MAXIMUM_WINDOW_SIZE) {
slidingWindow.removeFirst();
}
}
@@ -61,6 +52,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
@Override
public boolean updateLoadStatistic() {
+ if (isRemoving()) {
+ return false;
+ }
+
long lastSendTime = 0;
synchronized (slidingWindow) {
if (slidingWindow.size() > 0) {
@@ -85,7 +80,7 @@ public class DataNodeHeartbeatCache implements INodeCache {
}
// TODO: Optimize judge logic
- if (System.currentTimeMillis() - lastSendTime > 20_000) {
+ if (System.currentTimeMillis() - lastSendTime > HEARTBEAT_TIMEOUT_TIME) {
status = NodeStatus.Unknown;
} else {
status = NodeStatus.Running;
@@ -95,6 +90,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
@Override
public long getLoadScore() {
+ if (isRemoving()) {
+ return Long.MAX_VALUE;
+ }
+
// Return a copy of loadScore
switch (status) {
case Running:
@@ -108,6 +107,10 @@ public class DataNodeHeartbeatCache implements INodeCache {
@Override
public NodeStatus getNodeStatus() {
+ if (isRemoving()) {
+ return NodeStatus.Removing;
+ }
+
// Return a copy of status
switch (status) {
case Running:
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
index 0d6d937ad8..96b77e68c4 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/heartbeat/RegionGroupCache.java
@@ -100,7 +100,7 @@ public class RegionGroupCache implements IRegionGroupCache {
leaderDataNodeId.set(updateLeaderDataNodeId);
}
- return !(originLeaderDataNodeId == leaderDataNodeId.get());
+ return originLeaderDataNodeId != leaderDataNodeId.get();
}
@Override
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
index c749f2d5b2..7a0bfa72ca 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/persistence/NodeInfo.java
@@ -22,7 +22,6 @@ import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
-import org.apache.iotdb.commons.cluster.NodeStatus;
import org.apache.iotdb.commons.snapshot.SnapshotProcessor;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
@@ -67,6 +66,9 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_STATUS_REGISTER;
+import static org.apache.iotdb.confignode.conf.ConfigNodeConstant.METRIC_TAG_TOTAL;
+
/**
* The NodeInfo stores cluster node information. The cluster node information including: 1. DataNode
* information 2. ConfigNode information
@@ -109,9 +111,9 @@ public class NodeInfo implements SnapshotProcessor {
registeredConfigNodes,
o -> getRegisteredConfigNodeCount(),
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Registered.toString());
+ METRIC_STATUS_REGISTER);
MetricService.getInstance()
.getOrCreateAutoGauge(
Metric.DATA_NODE.toString(),
@@ -119,9 +121,9 @@ public class NodeInfo implements SnapshotProcessor {
registeredDataNodes,
Map::size,
Tag.NAME.toString(),
- "total",
+ METRIC_TAG_TOTAL,
Tag.STATUS.toString(),
- NodeStatus.Registered.toString());
+ METRIC_STATUS_REGISTER);
}
/**
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
index b54f9132e6..c5be031d59 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/ConfigNodeProcedureEnv.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.confignode.procedure.env;
import org.apache.iotdb.common.rpc.thrift.TConfigNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupId;
import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
+import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.confignode.client.ConfigNodeRequestType;
@@ -298,7 +299,19 @@ public class ConfigNodeProcedureEnv {
}
/**
- * Broadcast the CreateRegionGroupsPlan
+ * Mark the given datanode as removing status, and broadcast the region map, to avoid read or
+ * write request routing to this node.
+ *
+ * @param dataNodeLocation the datanode to be marked as removing status
+ */
+ public void markDataNodeAsRemovingAndBroadCast(TDataNodeLocation dataNodeLocation) {
+ int dataNodeId = dataNodeLocation.getDataNodeId();
+ configManager.getNodeManager().setNodeRemovingStatus(dataNodeId, true);
+ configManager.getLoadManager().broadcastLatestRegionRouteMap();
+ }
+
+ /**
+ * Do region creations and broadcast the CreateRegionGroupsPlan
*
* @return Those RegionGroups that failed to create
*/
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
index cf244fb52d..82131683f6 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/AddConfigNodeProcedure.java
@@ -37,7 +37,7 @@ import java.nio.ByteBuffer;
/** add config node procedure */
public class AddConfigNodeProcedure extends AbstractNodeProcedure<AddConfigNodeState> {
private static final Logger LOG = LoggerFactory.getLogger(AddConfigNodeProcedure.class);
- private static final int retryThreshold = 5;
+ private static final int RETRY_THRESHOLD = 5;
private TConfigNodeLocation tConfigNodeLocation;
@@ -85,7 +85,7 @@ public class AddConfigNodeProcedure extends AbstractNodeProcedure<AddConfigNodeS
tConfigNodeLocation,
state,
e);
- if (getCycles() > retryThreshold) {
+ if (getCycles() > RETRY_THRESHOLD) {
setFailure(new ProcedureException("State stuck at " + state));
}
}
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
index cf327ec8f8..1743e1f111 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/RemoveDataNodeProcedure.java
@@ -40,7 +40,7 @@ import java.util.List;
/** remove data node procedure */
public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNodeState> {
private static final Logger LOG = LoggerFactory.getLogger(RemoveDataNodeProcedure.class);
- private static final int retryThreshold = 5;
+ private static final int RETRY_THRESHOLD = 5;
private TDataNodeLocation disableDataNodeLocation;
@@ -63,6 +63,8 @@ public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNod
try {
switch (state) {
case REMOVE_DATA_NODE_PREPARE:
+ // mark the datanode as removing status and broadcast region route map
+ env.markDataNodeAsRemovingAndBroadCast(disableDataNodeLocation);
execDataNodeRegionIds =
env.getDataNodeRemoveHandler().getDataNodeRegionIds(disableDataNodeLocation);
LOG.info("DataNode region id is {}", execDataNodeRegionIds);
@@ -90,7 +92,7 @@ public class RemoveDataNodeProcedure extends AbstractNodeProcedure<RemoveDataNod
disableDataNodeLocation,
state,
e);
- if (getCycles() > retryThreshold) {
+ if (getCycles() > RETRY_THRESHOLD) {
setFailure(new ProcedureException("State stuck at " + state));
}
}
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
index db134efd25..de0fb369c5 100644
--- a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LeaderRouterTest.java
@@ -23,8 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.IRegionGroupCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.NodeHeartbeatSample;
import org.apache.iotdb.confignode.manager.load.heartbeat.RegionGroupCache;
@@ -59,7 +59,7 @@ public class LeaderRouterTest {
// Build nodeCacheMap
long currentTimeMillis = System.currentTimeMillis();
- Map<Integer, INodeCache> nodeCacheMap = new HashMap<>();
+ Map<Integer, BaseNodeCache> nodeCacheMap = new HashMap<>();
for (int i = 0; i < 6; i++) {
nodeCacheMap.put(i, new DataNodeHeartbeatCache());
// Simulate that the DataNode-i returned a heartbeat at (currentTime - i * 1000) ms
@@ -68,7 +68,7 @@ public class LeaderRouterTest {
.cacheHeartbeatSample(
new NodeHeartbeatSample(currentTimeMillis - i * 1000, currentTimeMillis - i * 1000));
}
- nodeCacheMap.values().forEach(INodeCache::updateLoadStatistic);
+ nodeCacheMap.values().forEach(BaseNodeCache::updateLoadStatistic);
// Get the loadScoreMap
Map<Integer, Long> loadScoreMap = new ConcurrentHashMap<>();
diff --git a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
index faff66243e..ae13d16650 100644
--- a/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
+++ b/confignode/src/test/java/org/apache/iotdb/confignode/manager/load/balancer/router/LoadScoreGreedyRouterTest.java
@@ -23,8 +23,8 @@ import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TRegionReplicaSet;
+import org.apache.iotdb.confignode.manager.load.heartbeat.BaseNodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.DataNodeHeartbeatCache;
-import org.apache.iotdb.confignode.manager.load.heartbeat.INodeCache;
import org.apache.iotdb.confignode.manager.load.heartbeat.NodeHeartbeatSample;
import org.junit.Assert;
@@ -56,7 +56,7 @@ public class LoadScoreGreedyRouterTest {
/* Build nodeCacheMap */
long currentTimeMillis = System.currentTimeMillis();
- Map<Integer, INodeCache> nodeCacheMap = new HashMap<>();
+ Map<Integer, BaseNodeCache> nodeCacheMap = new HashMap<>();
for (int i = 0; i < 6; i++) {
nodeCacheMap.put(i, new DataNodeHeartbeatCache());
// Simulate that the DataNode-i returned a heartbeat at (currentTime - i * 1000) ms
@@ -65,7 +65,7 @@ public class LoadScoreGreedyRouterTest {
.cacheHeartbeatSample(
new NodeHeartbeatSample(currentTimeMillis - i * 1000, currentTimeMillis - i * 1000));
}
- nodeCacheMap.values().forEach(INodeCache::updateLoadStatistic);
+ nodeCacheMap.values().forEach(BaseNodeCache::updateLoadStatistic);
/* Get the loadScoreMap */
Map<Integer, Long> loadScoreMap = new ConcurrentHashMap<>();
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
index 8d2a3577a2..82cc285cf3 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/cluster/NodeStatus.java
@@ -20,14 +20,14 @@ package org.apache.iotdb.commons.cluster;
/** Node status for showing cluster */
public enum NodeStatus {
- // Node registered
- Registered("Registered"),
- // Node online ,right now Online is Running
- Online("Online"),
- // Node running properly
+ /** Node running properly */
Running("Running"),
- // Node connection failure
- Unknown("Unknown");
+
+ /** Node connection failure */
+ Unknown("Unknown"),
+
+ /** Node is in removing */
+ Removing("Removing");
private final String status;