You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2023/05/11 06:28:41 UTC
[iotdb] branch master updated: [IOTDB-5843] Stall write requests when system shutting down (#9819)
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 115fc115f94 [IOTDB-5843] Stall write requests when system shutting down (#9819)
115fc115f94 is described below
commit 115fc115f945a2bebccc7dbef91479941a246e2a
Author: William Song <48...@users.noreply.github.com>
AuthorDate: Thu May 11 14:28:33 2023 +0800
[IOTDB-5843] Stall write requests when system shutting down (#9819)
---
.../consensus/ratis/ApplicationStateMachineProxy.java | 10 +++++-----
.../apache/iotdb/consensus/ratis/RatisConsensus.java | 3 +--
.../org/apache/iotdb/consensus/ratis/utils/Utils.java | 17 +++++++++++++++++
.../consensus/statemachine/DataRegionStateMachine.java | 3 +--
4 files changed, 24 insertions(+), 9 deletions(-)
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
index 06d955a573c..be1ebc43a5f 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
@@ -134,7 +134,7 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
}
Message ret = null;
- waitUntilSystemNotReadOnly();
+ waitUntilSystemAllowApply();
TSStatus finalStatus = null;
boolean shouldRetry = false;
boolean firstTry = true;
@@ -169,8 +169,8 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
new ResponseMessage(
new TSStatus(TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode())
.setMessage("internal error. statemachine throws a runtime exception: " + rte));
- if (applicationStateMachine.isReadOnly()) {
- waitUntilSystemNotReadOnly();
+ if (Utils.stallApply()) {
+ waitUntilSystemAllowApply();
shouldRetry = true;
} else {
break;
@@ -191,8 +191,8 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
return CompletableFuture.completedFuture(ret);
}
- private void waitUntilSystemNotReadOnly() {
- while (applicationStateMachine.isReadOnly()) {
+ private void waitUntilSystemAllowApply() {
+ while (Utils.stallApply()) {
try {
TimeUnit.SECONDS.sleep(60);
} catch (InterruptedException e) {
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
index 5d2b362acd9..d8fb18e1aae 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/RatisConsensus.java
@@ -29,7 +29,6 @@ import org.apache.iotdb.commons.client.exception.ClientManagerException;
import org.apache.iotdb.commons.client.property.ClientPoolProperty;
import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.threadpool.ScheduledExecutorUtil;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.utils.TestOnly;
@@ -248,7 +247,7 @@ class RatisConsensus implements IConsensus {
}
// current Peer is group leader and in ReadOnly State
- if (isLeader(consensusGroupId) && CommonDescriptor.getInstance().getConfig().isReadOnly()) {
+ if (isLeader(consensusGroupId) && Utils.rejectWrite()) {
try {
forceStepDownLeader(raftGroup);
} catch (Exception e) {
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Utils.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Utils.java
index 0a76426400c..507824a8e59 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Utils.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/utils/Utils.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.consensus.ratis.utils;
import org.apache.iotdb.common.rpc.thrift.TConsensusGroupType;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.consensus.common.Peer;
import org.apache.iotdb.consensus.config.RatisConfig;
@@ -50,6 +52,7 @@ public class Utils {
private static final byte PADDING_MAGIC = 0x47;
private static final String DATA_REGION_GROUP = "group-0001";
private static final String SCHEMA_REGION_GROUP = "group-0002";
+ private static volatile CommonConfig config = CommonDescriptor.getInstance().getConfig();
private Utils() {}
@@ -189,6 +192,20 @@ public class Utils {
return consensusGroupType;
}
+ public static boolean rejectWrite() {
+ return config.isReadOnly();
+ }
+
+ /**
+ * Normally, the RatisConsensus should reject write when system is read-only, i.e, {@link
+ * #rejectWrite()}. However, Ratis RaftServer close() will wait for applyIndex advancing to
+ * commitIndex. So when the system is shutting down, RatisConsensus should still allow
+ * statemachine to apply while rejecting new client write requests.
+ */
+ public static boolean stallApply() {
+ return config.isReadOnly() && !config.isStopping();
+ }
+
public static void initRatisConfig(RaftProperties properties, RatisConfig config) {
GrpcConfigKeys.setMessageSizeMax(properties, config.getGrpc().getMessageSizeMax());
GrpcConfigKeys.setFlowControlWindow(properties, config.getGrpc().getFlowControlWindow());
diff --git a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
index 41b54c50df8..a28892466ee 100644
--- a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
+++ b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
@@ -82,8 +82,7 @@ public class DataRegionStateMachine extends BaseStateMachine {
@Override
public boolean isReadOnly() {
- return CommonDescriptor.getInstance().getConfig().isReadOnly()
- && !CommonDescriptor.getInstance().getConfig().isStopping();
+ return CommonDescriptor.getInstance().getConfig().isReadOnly();
}
@Override