You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/08/16 07:57:31 UTC
[iotdb] branch master updated: [IOTDB-3843] Refine the using of setting read-only (#6968)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new b04489ebdd [IOTDB-3843] Refine the using of setting read-only (#6968)
b04489ebdd is described below
commit b04489ebdd3457a7e3fdb2bae11cf08b23167e59
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Tue Aug 16 15:57:27 2022 +0800
[IOTDB-3843] Refine the using of setting read-only (#6968)
---
.../statemachine/PartitionRegionStateMachine.java | 6 +++
.../org/apache/iotdb/consensus/IStateMachine.java | 4 ++
.../multileader/MultiLeaderConsensus.java | 12 ++++-
.../multileader/MultiLeaderServerImpl.java | 4 ++
.../service/MultiLeaderRPCServiceProcessor.java | 8 +++
.../ratis/ApplicationStateMachineProxy.java | 34 +++++++++---
.../consensus/standalone/StandAloneConsensus.java | 12 ++++-
.../consensus/standalone/StandAloneServerImpl.java | 5 ++
.../resources/conf/iotdb-datanode.properties | 4 ++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 44 ++++++++++++++--
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 +++
.../org/apache/iotdb/db/conf/SystemStatus.java | 32 ++++++++++++
.../db/conf/directories/DirectoryManager.java | 60 +++++++---------------
.../iotdb/db/conf/directories/FolderManager.java | 18 ++++++-
.../directories/strategy/DirectoryStrategy.java | 4 +-
.../statemachine/DataRegionStateMachine.java | 6 +++
.../statemachine/SchemaRegionStateMachine.java | 6 +++
.../compaction/task/CompactionRecoverTask.java | 3 +-
.../iotdb/db/engine/storagegroup/DataRegion.java | 46 ++++++++++-------
.../db/engine/storagegroup/TsFileProcessor.java | 42 ++++++++++-----
.../iotdb/db/metadata/logfile/MLogWriter.java | 25 ++++++---
.../db/mpp/plan/scheduler/StandaloneScheduler.java | 8 +++
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 5 +-
.../allocation/AbstractNodeAllocationStrategy.java | 8 +--
.../org/apache/iotdb/db/wal/buffer/WALBuffer.java | 15 +++---
.../iotdb/db/wal/checkpoint/CheckpointManager.java | 9 ++--
.../engine/modification/DeletionFileNodeTest.java | 2 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 3 +-
.../apache/iotdb/spark/db/EnvironmentUtils.java | 3 +-
29 files changed, 319 insertions(+), 115 deletions(-)
diff --git a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
index 7af80b66f8..947b31eb56 100644
--- a/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
+++ b/confignode/src/main/java/org/apache/iotdb/confignode/consensus/statemachine/PartitionRegionStateMachine.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.consensus.IStateMachine;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest;
import org.apache.iotdb.consensus.common.request.IConsensusRequest;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
@@ -163,4 +164,9 @@ public class PartitionRegionStateMachine implements IStateMachine, IStateMachine
public void stop() {
// do nothing
}
+
+ @Override
+ public boolean isReadOnly() {
+ return IoTDBDescriptor.getInstance().getConfig().isReadOnly();
+ }
}
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java b/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
index 18004f084c..3989272dee 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/IStateMachine.java
@@ -41,6 +41,10 @@ public interface IStateMachine {
void stop();
+ default boolean isReadOnly() {
+ return false;
+ }
+
/**
* apply a write-request from user
*
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderConsensus.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderConsensus.java
index 2ef4562f90..8a7582b403 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderConsensus.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderConsensus.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.consensus.multileader;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.client.IClientManager;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.exception.StartupException;
@@ -43,6 +44,7 @@ import org.apache.iotdb.consensus.multileader.client.AsyncMultiLeaderServiceClie
import org.apache.iotdb.consensus.multileader.client.MultiLeaderConsensusClientPool.AsyncMultiLeaderServiceClientPoolFactory;
import org.apache.iotdb.consensus.multileader.service.MultiLeaderRPCService;
import org.apache.iotdb.consensus.multileader.service.MultiLeaderRPCServiceProcessor;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -137,7 +139,15 @@ public class MultiLeaderConsensus implements IConsensus {
.setException(new ConsensusGroupNotExistException(groupId))
.build();
}
- return ConsensusWriteResponse.newBuilder().setStatus(impl.write(request)).build();
+
+ TSStatus status;
+ if (impl.isReadOnly()) {
+ status = new TSStatus(TSStatusCode.READ_ONLY_SYSTEM_ERROR.getStatusCode());
+ status.setMessage("Fail to do non-query operations because system is read-only.");
+ } else {
+ status = impl.write(request);
+ }
+ return ConsensusWriteResponse.newBuilder().setStatus(status).build();
}
@Override
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
index 27ccf2bcca..b106bca0c6 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/MultiLeaderServerImpl.java
@@ -250,4 +250,8 @@ public class MultiLeaderServerImpl {
public AtomicLong getIndexObject() {
return index;
}
+
+ public boolean isReadOnly() {
+ return stateMachine.isReadOnly();
+ }
}
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
index 70a0387c40..6337e4db02 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/multileader/service/MultiLeaderRPCServiceProcessor.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.consensus.multileader.service;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest;
import org.apache.iotdb.consensus.common.request.ByteBufferConsensusRequest;
import org.apache.iotdb.consensus.common.request.IConsensusRequest;
@@ -68,6 +69,13 @@ public class MultiLeaderRPCServiceProcessor implements MultiLeaderConsensusIServ
resultHandler.onComplete(new TSyncLogRes(Collections.singletonList(status)));
return;
}
+ if (impl.isReadOnly()) {
+ String message = "Fail to sync log because system is read-only.";
+ logger.error(message);
+ resultHandler.onError(
+ new IoTDBException(message, TSStatusCode.READ_ONLY_SYSTEM_ERROR.getStatusCode()));
+ return;
+ }
BatchIndexedConsensusRequest requestsInThisBatch = new BatchIndexedConsensusRequest();
// We use synchronized to ensure atomicity of executing multiple logs
if (!req.getBatches().isEmpty()) {
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
index 6588d5dbb4..601791f257 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/ratis/ApplicationStateMachineProxy.java
@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
public class ApplicationStateMachineProxy extends BaseStateMachine {
private final Logger logger = LoggerFactory.getLogger(ApplicationStateMachineProxy.class);
@@ -117,17 +118,36 @@ public class ApplicationStateMachineProxy extends BaseStateMachine {
}
Message ret;
- try {
- TSStatus result = applicationStateMachine.write(applicationRequest);
- ret = new ResponseMessage(result);
- } catch (Exception rte) {
- logger.error("application statemachine throws a runtime exception: ", rte);
- ret = Message.valueOf("internal error. statemachine throws a runtime exception: " + rte);
- }
+ waitUntilSystemNotReadOnly();
+ do {
+ try {
+ TSStatus result = applicationStateMachine.write(applicationRequest);
+ ret = new ResponseMessage(result);
+ break;
+ } catch (Exception rte) {
+ logger.error("application statemachine throws a runtime exception: ", rte);
+ ret = Message.valueOf("internal error. statemachine throws a runtime exception: " + rte);
+ if (applicationStateMachine.isReadOnly()) {
+ waitUntilSystemNotReadOnly();
+ } else {
+ break;
+ }
+ }
+ } while (!applicationStateMachine.isReadOnly());
return CompletableFuture.completedFuture(ret);
}
+ private void waitUntilSystemNotReadOnly() {
+ while (applicationStateMachine.isReadOnly()) {
+ try {
+ TimeUnit.SECONDS.sleep(60);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
@Override
public CompletableFuture<Message> query(Message request) {
if (!(request instanceof RequestMessage)) {
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneConsensus.java b/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneConsensus.java
index d87e27837f..fbacf70268 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneConsensus.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneConsensus.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.consensus.standalone;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
+import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.consensus.IConsensus;
@@ -35,6 +36,7 @@ import org.apache.iotdb.consensus.exception.ConsensusGroupAlreadyExistException;
import org.apache.iotdb.consensus.exception.ConsensusGroupNotExistException;
import org.apache.iotdb.consensus.exception.IllegalPeerEndpointException;
import org.apache.iotdb.consensus.exception.IllegalPeerNumException;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -111,7 +113,15 @@ class StandAloneConsensus implements IConsensus {
.setException(new ConsensusGroupNotExistException(groupId))
.build();
}
- return ConsensusWriteResponse.newBuilder().setStatus(impl.write(request)).build();
+
+ TSStatus status;
+ if (impl.isReadOnly()) {
+ status = new TSStatus(TSStatusCode.READ_ONLY_SYSTEM_ERROR.getStatusCode());
+ status.setMessage("Fail to do non-query operations because system is read-only.");
+ } else {
+ status = impl.write(request);
+ }
+ return ConsensusWriteResponse.newBuilder().setStatus(status).build();
}
@Override
diff --git a/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneServerImpl.java b/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneServerImpl.java
index 8438a5fbe4..494a12ee1e 100644
--- a/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneServerImpl.java
+++ b/consensus/src/main/java/org/apache/iotdb/consensus/standalone/StandAloneServerImpl.java
@@ -57,6 +57,11 @@ public class StandAloneServerImpl implements IStateMachine {
stateMachine.stop();
}
+ @Override
+ public boolean isReadOnly() {
+ return stateMachine.isReadOnly();
+ }
+
@Override
public TSStatus write(IConsensusRequest request) {
return stateMachine.write(request);
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 95cffb349d..9a70ba7680 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -307,6 +307,10 @@ timestamp_precision=ms
# Unit: ms
# default_ttl=36000000
+# Shutdown system or set it to read-only mode when unrecoverable error occurs.
+# Datatype: bool
+# allow_read_only_when_errors_occur=true
+
# Size of log buffer in each metadata operation plan(in byte).
# If the size of a metadata operation plan is larger than this parameter, then it will be rejected by SchemaRegion
# If it sets a value smaller than 0, use the default value 1024*1024
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 0929459035..a5de2f940f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -79,7 +79,11 @@ public class IoTDBConfig {
public static final Pattern NODE_PATTERN = Pattern.compile(NODE_MATCHER);
- private volatile boolean readOnly = false;
+ /** Shutdown system or set it to read-only mode when unrecoverable error occurs. */
+ private boolean allowReadOnlyWhenErrorsOccur = true;
+
+ /** Status of current system. */
+ private volatile SystemStatus status = SystemStatus.NORMAL;
/** whether to enable the mqtt service. */
private boolean enableMQTTService = false;
@@ -1493,12 +1497,44 @@ public class IoTDBConfig {
this.sessionTimeoutThreshold = sessionTimeoutThreshold;
}
+ boolean isAllowReadOnlyWhenErrorsOccur() {
+ return allowReadOnlyWhenErrorsOccur;
+ }
+
+ void setAllowReadOnlyWhenErrorsOccur(boolean allowReadOnlyWhenErrorsOccur) {
+ this.allowReadOnlyWhenErrorsOccur = allowReadOnlyWhenErrorsOccur;
+ }
+
public boolean isReadOnly() {
- return readOnly;
+ return status == SystemStatus.READ_ONLY
+ || (status == SystemStatus.ERROR && allowReadOnlyWhenErrorsOccur);
+ }
+
+ public SystemStatus getSystemStatus() {
+ return status;
}
- public void setReadOnly(boolean readOnly) {
- this.readOnly = readOnly;
+ public void setSystemStatus(SystemStatus newStatus) {
+ if (newStatus == SystemStatus.READ_ONLY) {
+ logger.error(
+ "Change system mode to read-only! Only query statements are permitted!",
+ new RuntimeException("System mode is set to READ_ONLY"));
+ } else if (newStatus == SystemStatus.ERROR) {
+ if (allowReadOnlyWhenErrorsOccur) {
+ logger.error(
+ "Unrecoverable error occurs! Make system read-only when allow_read_only_when_errors_occur is true.",
+ new RuntimeException("System mode is set to READ_ONLY"));
+ newStatus = SystemStatus.READ_ONLY;
+ } else {
+ logger.error(
+ "Unrecoverable error occurs! Shutdown system directly when allow_read_only_when_errors_occur is false.",
+ new RuntimeException("System mode is set to ERROR"));
+ System.exit(-1);
+ }
+ } else {
+ logger.info("Set system mode from {} to NORMAL.", status);
+ }
+ this.status = newStatus;
}
public String getRpcImplClassName() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index aecb8a54f8..80f906470f 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -670,6 +670,12 @@ public class IoTDBDescriptor {
conf.setKerberosPrincipal(
properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
+ conf.setAllowReadOnlyWhenErrorsOccur(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "allow_read_only_when_errors_occur",
+ String.valueOf(conf.isAllowReadOnlyWhenErrorsOccur()))));
+
// the num of memtables in each storage group
conf.setConcurrentWritingTimePartition(
Integer.parseInt(
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/SystemStatus.java b/server/src/main/java/org/apache/iotdb/db/conf/SystemStatus.java
new file mode 100644
index 0000000000..745fbb1529
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/conf/SystemStatus.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.conf;
+
+/** Status of current system */
+public enum SystemStatus {
+ /** System can read and write normally */
+ NORMAL,
+ /** Only query statements are permitted */
+ READ_ONLY,
+ /**
+ * Unrecoverable errors occur, system will be read-only or exit according to the param
+ * allow_read_only_when_errors_occur
+ */
+ ERROR,
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
index 273f071008..d9f50444bf 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/DirectoryManager.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.conf.directories;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.exception.LoadConfigurationException;
@@ -132,54 +133,27 @@ public class DirectoryManager {
}
public String getNextFolderForSequenceFile() throws DiskSpaceInsufficientException {
- return getSequenceFileFolder(getNextFolderIndexForSequenceFile());
- }
-
- /**
- * get next folder index for TsFile.
- *
- * @return next folder index
- */
- public int getNextFolderIndexForSequenceFile() throws DiskSpaceInsufficientException {
- return sequenceStrategy.nextFolderIndex();
- }
-
- public String getSequenceFileFolder(int index) {
- return sequenceFileFolders.get(index);
+ try {
+ return sequenceFileFolders.get(sequenceStrategy.nextFolderIndex());
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All disks of wal folders are full, change system mode to read-only.", e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
+ throw e;
+ }
}
public List<String> getAllSequenceFileFolders() {
return new ArrayList<>(sequenceFileFolders);
}
- private static class DirectoriesHolder {
-
- private static final DirectoryManager INSTANCE = new DirectoryManager();
- }
-
- public String getIndexRootFolder() {
- return IoTDBDescriptor.getInstance().getConfig().getIndexRootFolder();
- }
-
public String getNextFolderForUnSequenceFile() throws DiskSpaceInsufficientException {
- return getUnSequenceFileFolder(getNextFolderIndexForUnSequenceFile());
- }
-
- /**
- * get next folder index for OverflowFile.
- *
- * @return next folder index
- */
- public int getNextFolderIndexForUnSequenceFile() throws DiskSpaceInsufficientException {
- return unsequenceStrategy.nextFolderIndex();
- }
-
- public String getUnSequenceFileFolder(int index) {
- return unsequenceFileFolders.get(index);
- }
-
- public int getUnSequenceFileFolderIndex(String folder) {
- return unsequenceFileFolders.indexOf(folder);
+ try {
+ return unsequenceFileFolders.get(unsequenceStrategy.nextFolderIndex());
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All disks of wal folders are full, change system mode to read-only.", e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
+ throw e;
+ }
}
public List<String> getAllUnSequenceFileFolders() {
@@ -191,4 +165,8 @@ public class DirectoryManager {
folders.addAll(unsequenceFileFolders);
return folders;
}
+
+ private static class DirectoriesHolder {
+ private static final DirectoryManager INSTANCE = new DirectoryManager();
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/FolderManager.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/FolderManager.java
index bbe912877f..23e61a2663 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/FolderManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/FolderManager.java
@@ -18,6 +18,8 @@
*/
package org.apache.iotdb.db.conf.directories;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategy;
import org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategyType;
import org.apache.iotdb.db.conf.directories.strategy.MaxDiskUsableSpaceFirstStrategy;
@@ -56,10 +58,22 @@ public class FolderManager {
default:
throw new RuntimeException();
}
- this.selectStrategy.setFolders(folders);
+ try {
+ this.selectStrategy.setFolders(folders);
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All disks of wal folders are full, change system mode to read-only.", e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
+ throw e;
+ }
}
public String getNextFolder() throws DiskSpaceInsufficientException {
- return folders.get(selectStrategy.nextFolderIndex());
+ try {
+ return folders.get(selectStrategy.nextFolderIndex());
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error("All disks of wal folders are full, change system mode to read-only.", e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
+ throw e;
+ }
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java b/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
index 158b9be308..2758231515 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/directories/strategy/DirectoryStrategy.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.conf.directories.strategy;
import org.apache.iotdb.commons.utils.JVMCommonUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
import org.slf4j.Logger;
@@ -54,7 +55,8 @@ public abstract class DirectoryStrategy {
}
}
if (!hasSpace) {
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ logger.error("Disk space is insufficient, change system mode to read-only");
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
throw new DiskSpaceInsufficientException(folders);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
index fa7a4aa502..721f3c33af 100644
--- a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
+++ b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/DataRegionStateMachine.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.consensus.common.request.BatchIndexedConsensusRequest;
import org.apache.iotdb.consensus.common.request.IConsensusRequest;
import org.apache.iotdb.consensus.common.request.IndexedConsensusRequest;
import org.apache.iotdb.consensus.multileader.wal.GetConsensusReqReaderPlan;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.consensus.statemachine.visitor.DataExecutionVisitor;
import org.apache.iotdb.db.engine.StorageEngineV2;
import org.apache.iotdb.db.engine.snapshot.SnapshotLoader;
@@ -84,6 +85,11 @@ public class DataRegionStateMachine extends BaseStateMachine {
@Override
public void stop() {}
+ @Override
+ public boolean isReadOnly() {
+ return IoTDBDescriptor.getInstance().getConfig().isReadOnly();
+ }
+
@Override
public boolean takeSnapshot(File snapshotDir) {
try {
diff --git a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/SchemaRegionStateMachine.java b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/SchemaRegionStateMachine.java
index 93b2a1d06a..a0248fbaa4 100644
--- a/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/SchemaRegionStateMachine.java
+++ b/server/src/main/java/org/apache/iotdb/db/consensus/statemachine/SchemaRegionStateMachine.java
@@ -22,6 +22,7 @@ package org.apache.iotdb.db.consensus.statemachine;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.consensus.common.DataSet;
import org.apache.iotdb.consensus.common.request.IConsensusRequest;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.metadata.schemaregion.ISchemaRegion;
import org.apache.iotdb.db.metadata.visitor.SchemaExecutionVisitor;
import org.apache.iotdb.db.mpp.execution.fragment.FragmentInstanceManager;
@@ -51,6 +52,11 @@ public class SchemaRegionStateMachine extends BaseStateMachine {
@Override
public void stop() {}
+ @Override
+ public boolean isReadOnly() {
+ return IoTDBDescriptor.getInstance().getConfig().isReadOnly();
+ }
+
@Override
public boolean takeSnapshot(File snapshotDir) {
return schemaRegion.createSnapshot(snapshotDir);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java
index cdb107ee5c..9d8807e897 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/CompactionRecoverTask.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.compaction.task;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.engine.compaction.CompactionUtils;
import org.apache.iotdb.db.engine.compaction.log.CompactionLogAnalyzer;
import org.apache.iotdb.db.engine.compaction.log.CompactionLogger;
@@ -301,7 +302,7 @@ public class CompactionRecoverTask {
"{} [Compaction][ExceptionHandler] target file {} is not complete, and some source files is lost, do nothing. Set allowCompaction to false",
fullStorageGroupName,
targetFileIdentifier.getFilePath());
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
return false;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 4e8ece7bb4..e98ce60de4 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.StorageEngineV2;
@@ -1462,24 +1463,34 @@ public class DataRegion {
private TsFileProcessor getOrCreateTsFileProcessor(long timeRangeId, boolean sequence) {
TsFileProcessor tsFileProcessor = null;
- try {
- if (sequence) {
- tsFileProcessor =
- getOrCreateTsFileProcessorIntern(timeRangeId, workSequenceTsFileProcessors, true);
- } else {
- tsFileProcessor =
- getOrCreateTsFileProcessorIntern(timeRangeId, workUnsequenceTsFileProcessors, false);
+ int retryCnt = 0;
+ do {
+ try {
+ if (sequence) {
+ tsFileProcessor =
+ getOrCreateTsFileProcessorIntern(timeRangeId, workSequenceTsFileProcessors, true);
+ } else {
+ tsFileProcessor =
+ getOrCreateTsFileProcessorIntern(timeRangeId, workUnsequenceTsFileProcessors, false);
+ }
+ } catch (DiskSpaceInsufficientException e) {
+ logger.error(
+ "disk space is insufficient when creating TsFile processor, change system mode to read-only",
+ e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.READ_ONLY);
+ break;
+ } catch (IOException e) {
+ if (retryCnt < 3) {
+ logger.warn("meet IOException when creating TsFileProcessor, retry it again", e);
+ retryCnt++;
+ } else {
+ logger.error(
+ "meet IOException when creating TsFileProcessor, change system mode to error", e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.ERROR);
+ break;
+ }
}
- } catch (DiskSpaceInsufficientException e) {
- logger.error(
- "disk space is insufficient when creating TsFile processor, change system mode to read-only",
- e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
- } catch (IOException e) {
- logger.error(
- "meet IOException when creating TsFileProcessor, change system mode to read-only", e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
- }
+ } while (tsFileProcessor == null);
return tsFileProcessor;
}
@@ -2691,7 +2702,6 @@ public class DataRegion {
"Failed to append the tsfile {} to storage group processor {} because the disk space is insufficient.",
tsfileToBeInserted.getAbsolutePath(),
tsfileToBeInserted.getParentFile().getName());
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
throw new LoadFileException(e);
} catch (IllegalPathException e) {
logger.error(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 07b73c2d3b..9d1bac5d99 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.adapter.CompressionRatio;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.flush.CloseFileListener;
@@ -1233,11 +1234,11 @@ public class TsFileProcessor {
}
} else {
logger.error(
- "{}: {} meet error when flushing a memtable, change system mode to read-only",
+ "{}: {} meet error when flushing a memtable, change system mode to error",
storageGroupName,
tsFileResource.getTsFile().getName(),
e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.ERROR);
try {
logger.error(
"{}: {} IOTask meets error, truncate the corrupted data",
@@ -1320,7 +1321,9 @@ public class TsFileProcessor {
// for sync flush
syncReleaseFlushedMemTable(memTableToFlush);
- if (shouldClose && flushingMemTables.isEmpty() && writer != null) {
+ // retry to avoid unnecessary read-only mode
+ int retryCnt = 0;
+ while (shouldClose && flushingMemTables.isEmpty() && writer != null) {
try {
writer.mark();
updateCompressionRatio(memTableToFlush);
@@ -1328,7 +1331,7 @@ public class TsFileProcessor {
logger.debug(
"{}: {} flushingMemtables is empty and will close the file",
storageGroupName,
- tsFileResource.getTsFile().getName());
+ tsFileResource.getTsFile().getAbsolutePath());
}
endFile();
if (logger.isDebugEnabled()) {
@@ -1336,32 +1339,45 @@ public class TsFileProcessor {
}
} catch (Exception e) {
logger.error(
- "{} meet error when flush FileMetadata to {}, change system mode to read-only",
+ "{}: {} marking or ending file meet error",
storageGroupName,
tsFileResource.getTsFile().getAbsolutePath(),
e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ // truncate broken metadata
try {
writer.reset();
} catch (IOException e1) {
logger.error(
"{}: {} truncate corrupted data meets error",
storageGroupName,
- tsFileResource.getTsFile().getName(),
+ tsFileResource.getTsFile().getAbsolutePath(),
e1);
}
- logger.error(
- "{}: {} marking or ending file meet error",
- storageGroupName,
- tsFileResource.getTsFile().getName(),
- e);
+ // retry or set read-only
+ if (retryCnt < 3) {
+ logger.warn(
+ "{} meet error when flush FileMetadata to {}, retry it again",
+ storageGroupName,
+ tsFileResource.getTsFile().getAbsolutePath(),
+ e);
+ retryCnt++;
+ continue;
+ } else {
+ logger.error(
+ "{} meet error when flush FileMetadata to {}, change system mode to error",
+ storageGroupName,
+ tsFileResource.getTsFile().getAbsolutePath(),
+ e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.ERROR);
+ break;
+ }
}
// for sync close
if (logger.isDebugEnabled()) {
logger.debug(
"{}: {} try to get flushingMemtables lock.",
storageGroupName,
- tsFileResource.getTsFile().getName());
+ tsFileResource.getTsFile().getAbsolutePath());
}
synchronized (flushingMemTables) {
flushingMemTables.notifyAll();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java b/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
index cdb9031bcd..86d64ddf03 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/logfile/MLogWriter.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.metadata.mnode.IMNode;
import org.apache.iotdb.db.metadata.mnode.IMeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.IStorageGroupMNode;
@@ -109,12 +110,24 @@ public class MLogWriter implements AutoCloseable {
}
private void sync() {
- try {
- logWriter.write(mlogBuffer);
- } catch (IOException e) {
- logger.error(
- "MLog {} sync failed, change system mode to read-only", logFile.getAbsoluteFile(), e);
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
+ int retryCnt = 0;
+ mlogBuffer.mark();
+ while (true) {
+ try {
+ logWriter.write(mlogBuffer);
+ break;
+ } catch (IOException e) {
+ if (retryCnt < 3) {
+ logger.warn("MLog {} sync failed, retry it again", logFile.getAbsoluteFile(), e);
+ mlogBuffer.reset();
+ retryCnt++;
+ } else {
+ logger.error(
+ "MLog {} sync failed, change system mode to error", logFile.getAbsoluteFile(), e);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.ERROR);
+ break;
+ }
+ }
}
mlogBuffer.clear();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
index 09c7da7b43..5a1f6ee329 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/scheduler/StandaloneScheduler.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.client.sync.SyncDataNodeInternalServiceClient;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.consensus.SchemaRegionId;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.StorageEngineV2;
import org.apache.iotdb.db.engine.storagegroup.DataRegion;
import org.apache.iotdb.db.exception.WriteProcessException;
@@ -123,6 +124,13 @@ public class StandaloneScheduler implements IScheduler {
LOGGER.info("{} state tracker starts", getLogHeader());
break;
case WRITE:
+ // reject non-query operations when system is read-only
+ if (IoTDBDescriptor.getInstance().getConfig().isReadOnly()) {
+ TSStatus failedStatus = new TSStatus(TSStatusCode.READ_ONLY_SYSTEM_ERROR.getStatusCode());
+ failedStatus.setMessage("Fail to do non-query operations because system is read-only.");
+ stateMachine.transitionToFailed(failedStatus);
+ return;
+ }
try {
for (FragmentInstance fragmentInstance : instances) {
PlanNode planNode = fragmentInstance.getFragment().getRoot();
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 70c4883dcf..04f57311fb 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -39,6 +39,7 @@ import org.apache.iotdb.commons.utils.AuthUtils;
import org.apache.iotdb.db.auth.AuthorityChecker;
import org.apache.iotdb.db.auth.AuthorizerManager;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.BloomFilterCache;
import org.apache.iotdb.db.engine.cache.ChunkCache;
@@ -600,7 +601,9 @@ public class PlanExecutor implements IPlanExecutor {
}
private void operateSetSystemMode(SetSystemModePlan plan) {
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(plan.isReadOnly());
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setSystemStatus(plan.isReadOnly() ? SystemStatus.READ_ONLY : SystemStatus.NORMAL);
}
private void operateFlush(FlushPlan plan) throws StorageGroupNotSetException {
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/allocation/AbstractNodeAllocationStrategy.java b/server/src/main/java/org/apache/iotdb/db/wal/allocation/AbstractNodeAllocationStrategy.java
index 21c244511f..24d8ad0863 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/allocation/AbstractNodeAllocationStrategy.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/allocation/AbstractNodeAllocationStrategy.java
@@ -48,8 +48,9 @@ public abstract class AbstractNodeAllocationStrategy implements NodeAllocationSt
new FolderManager(
Arrays.asList(config.getWalDirs()), DirectoryStrategyType.SEQUENCE_STRATEGY);
} catch (DiskSpaceInsufficientException e) {
- logger.error("All disks of wal folders are full, change system mode to read-only.", e);
- config.setReadOnly(true);
+ logger.error(
+ "Fail to create wal node allocation strategy because all disks of wal folders are full.",
+ e);
}
}
@@ -59,8 +60,7 @@ public abstract class AbstractNodeAllocationStrategy implements NodeAllocationSt
try {
folder = folderManager.getNextFolder();
} catch (DiskSpaceInsufficientException e) {
- logger.error("All disks of wal folders are full, change system mode to read-only.", e);
- config.setReadOnly(true);
+ logger.error("Fail to create wal node because all disks of wal folders are full.", e);
return WALFakeNode.getFailureInstance(e);
}
folder = folder + File.separator + identifier;
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java b/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
index df76bec878..820766a468 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/buffer/WALBuffer.java
@@ -22,6 +22,7 @@ import org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.mpp.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.utils.MmapUtil;
import org.apache.iotdb.db.wal.exception.WALNodeClosedException;
@@ -418,8 +419,8 @@ public class WALBuffer extends AbstractWALBuffer {
currentWALFileWriter.write(syncingBuffer, info.metaData);
} catch (Throwable e) {
logger.error(
- "Fail to sync wal node-{}'s buffer, change system mode to read-only.", identifier, e);
- config.setReadOnly(true);
+ "Fail to sync wal node-{}'s buffer, change system mode to error.", identifier, e);
+ config.setSystemStatus(SystemStatus.ERROR);
} finally {
switchSyncingBufferToIdle();
}
@@ -436,13 +437,11 @@ public class WALBuffer extends AbstractWALBuffer {
}
} catch (IOException e) {
logger.error(
- "Fail to roll wal node-{}'s log writer, change system mode to read-only.",
- identifier,
- e);
+ "Fail to roll wal node-{}'s log writer, change system mode to error.", identifier, e);
if (info.rollWALFileWriterListener != null) {
info.rollWALFileWriterListener.fail(e);
}
- config.setReadOnly(true);
+ config.setSystemStatus(SystemStatus.ERROR);
}
} else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging
// after rolling file
@@ -451,13 +450,13 @@ public class WALBuffer extends AbstractWALBuffer {
forceSuccess = true;
} catch (IOException e) {
logger.error(
- "Fail to fsync wal node-{}'s log writer, change system mode to read-only.",
+ "Fail to fsync wal node-{}'s log writer, change system mode to error.",
identifier,
e);
for (WALFlushListener fsyncListener : info.fsyncListeners) {
fsyncListener.fail(e);
}
- config.setReadOnly(true);
+ config.setSystemStatus(SystemStatus.ERROR);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/checkpoint/CheckpointManager.java b/server/src/main/java/org/apache/iotdb/db/wal/checkpoint/CheckpointManager.java
index 9158ad246c..5d7bb51a42 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/checkpoint/CheckpointManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/checkpoint/CheckpointManager.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.wal.checkpoint;
import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.wal.io.CheckpointWriter;
import org.apache.iotdb.db.wal.io.ILogWriter;
import org.apache.iotdb.db.wal.utils.CheckpointFileUtils;
@@ -170,10 +171,10 @@ public class CheckpointManager implements AutoCloseable {
currentLogWriter.force();
} catch (IOException e) {
logger.error(
- "Fail to fsync wal node-{}'s checkpoint writer, change system mode to read-only.",
+ "Fail to fsync wal node-{}'s checkpoint writer, change system mode to error.",
identifier,
e);
- config.setReadOnly(true);
+ config.setSystemStatus(SystemStatus.ERROR);
}
try {
@@ -189,10 +190,10 @@ public class CheckpointManager implements AutoCloseable {
}
} catch (IOException e) {
logger.error(
- "Fail to roll wal node-{}'s checkpoint writer, change system mode to read-only.",
+ "Fail to roll wal node-{}'s checkpoint writer, change system mode to error.",
identifier,
e);
- config.setReadOnly(true);
+ config.setSystemStatus(SystemStatus.ERROR);
}
} finally {
infoLock.unlock();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
index 2f7bf35abb..77f53bba0c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/modification/DeletionFileNodeTest.java
@@ -245,7 +245,7 @@ public class DeletionFileNodeTest {
};
File fileNodeDir =
- new File(DirectoryManager.getInstance().getSequenceFileFolder(0), processorName);
+ new File(DirectoryManager.getInstance().getNextFolderForSequenceFile(), processorName);
List<File> modFiles = new ArrayList<>();
for (File directory : fileNodeDir.listFiles()) {
assertTrue(directory.isDirectory());
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index c1b1514f32..e1b7afac89 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.udf.service.UDFRegistrationService;
import org.apache.iotdb.db.auth.AuthorizerManager;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.StorageEngine;
@@ -157,7 +158,7 @@ public class EnvironmentUtils {
fail();
}
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(false);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.NORMAL);
// We must disable MQTT service as it will cost a lot of time to be shutdown, which may slow our
// unit tests.
IoTDBDescriptor.getInstance().getConfig().setEnableMQTTService(false);
diff --git a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
index fcd9886ed0..50fafca009 100644
--- a/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
+++ b/spark-iotdb-connector/src/test/scala/org/apache/iotdb/spark/db/EnvironmentUtils.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.commons.auth.authorizer.BasicAuthorizer;
import org.apache.iotdb.commons.auth.authorizer.IAuthorizer;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.SystemStatus;
import org.apache.iotdb.db.conf.directories.DirectoryManager;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.cache.BloomFilterCache;
@@ -111,7 +112,7 @@ public class EnvironmentUtils {
Assert.fail();
}
StorageEngine.getInstance().reset();
- IoTDBDescriptor.getInstance().getConfig().setReadOnly(false);
+ IoTDBDescriptor.getInstance().getConfig().setSystemStatus(SystemStatus.NORMAL);
// clean wal
WALManager.getInstance().stop();