You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ta...@apache.org on 2024/06/19 03:21:06 UTC
(iotdb) branch master updated: Optimize node startup process & Normalize system.properties management (#12752)
This is an automated email from the ASF dual-hosted git repository.
tanxinyu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 5d552729ae8 Optimize node startup process & Normalize system.properties management (#12752)
5d552729ae8 is described below
commit 5d552729ae881f865c8b22c166c23f1ec4074d92
Author: Li Yu Heng <li...@126.com>
AuthorDate: Wed Jun 19 11:21:01 2024 +0800
Optimize node startup process & Normalize system.properties management (#12752)
---
.../iotdb/it/env/cluster/ClusterConstant.java | 3 -
.../it/env/cluster/node/ConfigNodeWrapper.java | 4 +-
.../iotdb/it/env/cluster/node/DataNodeWrapper.java | 4 +-
.../org/apache/iotdb/db/it/IoTDBRecoverIT.java | 2 +-
.../conf/ConfigNodeSystemPropertiesHandler.java | 50 ++++++
.../confignode/conf/SystemPropertiesUtils.java | 86 +++-------
.../response/datanode/DataNodeRegisterResp.java | 6 +-
.../iotdb/confignode/manager/ConfigManager.java | 2 +-
.../manager/node/ClusterNodeStartUtils.java | 20 ++-
.../iotdb/confignode/manager/node/NodeManager.java | 12 +-
.../iotdb/confignode/persistence/NodeInfoTest.java | 3 +-
.../db/conf/DataNodeSystemPropertiesHandler.java | 52 ++++++
.../org/apache/iotdb/db/conf/IoTDBStartCheck.java | 173 ++++----------------
.../iotdb/db/protocol/client/ConfigNodeInfo.java | 66 ++------
.../java/org/apache/iotdb/db/service/DataNode.java | 50 +++---
iotdb-core/node-commons/pom.xml | 4 +
.../iotdb/commons/file/SystemFileFactory.java | 4 -
.../commons/file/SystemPropertiesHandler.java | 174 +++++++++++++++++++++
.../src/main/thrift/confignode.thrift | 1 +
19 files changed, 399 insertions(+), 317 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
index d688f2169d5..12af0d025f7 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/ClusterConstant.java
@@ -120,9 +120,6 @@ public class ClusterConstant {
public static final String COMMON_PROPERTIES_FILE = "iotdb-common.properties";
public static final String IOTDB_SYSTEM_PROPERTIES_FILE = "iotdb-system.properties";
- public static final String SYSTEM_PROPERTIES_FILE = "system.properties";
- public static final String CONFIG_NODE_SYSTEM_PROPERTIES_FILE = "confignode-system.properties";
-
// Properties' keys
// Common
public static final String CONFIG_NODE_CONSENSUS_PROTOCOL_CLASS =
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java
index a457149f598..a93c54c52b8 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/ConfigNodeWrapper.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.it.env.cluster.node;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.confignode.conf.ConfigNodeConstant;
import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.config.MppBaseConfig;
import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
@@ -37,7 +38,6 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_CONSEN
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_INIT_HEAP_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_DIRECT_MEMORY_SIZE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_MAX_HEAP_SIZE;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.CONFIG_NODE_SYSTEM_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REGION_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DATA_REPLICATION_FACTOR;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.DEFAULT_CONFIG_NODE_COMMON_PROPERTIES;
@@ -105,7 +105,7 @@ public class ConfigNodeWrapper extends AbstractNodeWrapper {
@Override
public String getSystemPropertiesPath() {
- return workDirFilePath("data/confignode/system", CONFIG_NODE_SYSTEM_PROPERTIES_FILE);
+ return workDirFilePath("data/confignode/system", ConfigNodeConstant.SYSTEM_FILE_NAME);
}
@Override
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java
index 1135f5cd055..647bd9b9fa0 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/node/DataNodeWrapper.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.it.env.cluster.node;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBStartCheck;
import org.apache.iotdb.it.env.cluster.EnvUtils;
import org.apache.iotdb.it.env.cluster.config.MppBaseConfig;
import org.apache.iotdb.it.env.cluster.config.MppJVMConfig;
@@ -61,7 +62,6 @@ import static org.apache.iotdb.it.env.cluster.ClusterConstant.PIPE_AIR_GAP_RECEI
import static org.apache.iotdb.it.env.cluster.ClusterConstant.REST_SERVICE_PORT;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REGION_CONSENSUS_PROTOCOL_CLASS;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.SCHEMA_REPLICATION_FACTOR;
-import static org.apache.iotdb.it.env.cluster.ClusterConstant.SYSTEM_PROPERTIES_FILE;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.TARGET;
import static org.apache.iotdb.it.env.cluster.ClusterConstant.USER_DIR;
@@ -139,7 +139,7 @@ public class DataNodeWrapper extends AbstractNodeWrapper {
@Override
public String getSystemPropertiesPath() {
- return workDirFilePath("data/datanode/system/schema", SYSTEM_PROPERTIES_FILE);
+ return workDirFilePath("data/datanode/system/schema", IoTDBStartCheck.PROPERTIES_FILE_NAME);
}
@Override
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java
index 7c4d87511d8..41bcc24a375 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBRecoverIT.java
@@ -98,7 +98,7 @@ public class IoTDBRecoverIT {
}
@Test
- public void RecoverTest1() {
+ public void recoverTest1() {
// stop cluster
EnvFactory.getEnv().shutdownAllDataNodes();
logger.info("All DataNodes are shut down");
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java
new file mode 100644
index 00000000000..b92e2e57e05
--- /dev/null
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeSystemPropertiesHandler.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.confignode.conf;
+
+import org.apache.iotdb.commons.file.SystemPropertiesHandler;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+public class ConfigNodeSystemPropertiesHandler extends SystemPropertiesHandler {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(ConfigNodeSystemPropertiesHandler.class);
+
+ private static ConfigNodeSystemPropertiesHandler INSTANCE;
+
+ private ConfigNodeSystemPropertiesHandler(String filePath) {
+ super(filePath);
+ }
+
+ public static synchronized SystemPropertiesHandler getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE =
+ new ConfigNodeSystemPropertiesHandler(
+ ConfigNodeDescriptor.getInstance().getConf().getSystemDir()
+ + File.separator
+ + ConfigNodeConstant.SYSTEM_FILE_NAME);
+ INSTANCE.recover();
+ }
+ return INSTANCE;
+ }
+}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
index ba9d74a641c..a56f4a63f72 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
@@ -24,18 +24,14 @@ import org.apache.iotdb.commons.conf.CommonConfig;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.BadNodeUrlException;
+import org.apache.iotdb.commons.file.SystemPropertiesHandler;
import org.apache.iotdb.commons.utils.NodeUrlUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
@@ -48,11 +44,8 @@ public class SystemPropertiesUtils {
private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesUtils.class);
- private static File systemPropertiesFile =
- new File(
- ConfigNodeDescriptor.getInstance().getConf().getSystemDir()
- + File.separator
- + ConfigNodeConstant.SYSTEM_FILE_NAME);
+ private static SystemPropertiesHandler systemPropertiesHandler =
+ ConfigNodeSystemPropertiesHandler.getInstance();
private static final ConfigNodeConfig conf = ConfigNodeDescriptor.getInstance().getConf();
private static final CommonConfig COMMON_CONFIG = CommonDescriptor.getInstance().getConfig();
@@ -74,11 +67,10 @@ public class SystemPropertiesUtils {
// TODO: This needs removal of statics ...
public static void reinitializeStatics() {
- systemPropertiesFile =
- new File(
- ConfigNodeDescriptor.getInstance().getConf().getSystemDir()
- + File.separator
- + ConfigNodeConstant.SYSTEM_FILE_NAME);
+ systemPropertiesHandler.resetFilePath(
+ ConfigNodeDescriptor.getInstance().getConf().getSystemDir()
+ + File.separator
+ + ConfigNodeConstant.SYSTEM_FILE_NAME);
}
/**
@@ -87,7 +79,7 @@ public class SystemPropertiesUtils {
* @return True if confignode-system.properties file exist.
*/
public static boolean isRestarted() {
- return systemPropertiesFile.exists();
+ return !systemPropertiesHandler.isFirstStart();
}
/**
@@ -97,7 +89,7 @@ public class SystemPropertiesUtils {
* @throws IOException When read the confignode-system.properties file failed
*/
public static void checkSystemProperties() throws IOException {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
boolean needReWrite = false;
final String format =
"[SystemProperties] The parameter \"{}\" can't be modified after first startup."
@@ -234,11 +226,6 @@ public class SystemPropertiesUtils {
COMMON_CONFIG.setTimePartitionInterval(timePartitionInterval);
}
}
-
- if (needReWrite) {
- // Re-write special parameters if necessary
- storeSystemParameters();
- }
}
/**
@@ -251,7 +238,7 @@ public class SystemPropertiesUtils {
*/
public static List<TConfigNodeLocation> loadConfigNodeList()
throws IOException, BadNodeUrlException {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
String addresses = systemProperties.getProperty("config_node_list", null);
if (addresses != null && !addresses.isEmpty()) {
@@ -265,10 +252,10 @@ public class SystemPropertiesUtils {
* The system parameters can't be changed after the ConfigNode first started. Therefore, store
* them in confignode-system.properties during the first startup.
*
- * @throws IOException getSystemProperties()
+ * @throws IOException systemPropertiesHandler.read()
*/
public static void storeSystemParameters() throws IOException {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
systemProperties.setProperty("iotdb_version", IoTDBConstant.VERSION);
systemProperties.setProperty("commit_id", IoTDBConstant.BUILD_INFO);
@@ -311,7 +298,7 @@ public class SystemPropertiesUtils {
systemProperties.setProperty(
"tag_attribute_total_size", String.valueOf(COMMON_CONFIG.getTagAttributeTotalSize()));
- storeSystemProperties(systemProperties);
+ systemPropertiesHandler.overwrite(systemProperties);
}
/**
@@ -321,7 +308,7 @@ public class SystemPropertiesUtils {
* @throws IOException When store confignode-system.properties file failed
*/
public static void storeConfigNodeList(List<TConfigNodeLocation> configNodes) throws IOException {
- if (!systemPropertiesFile.exists()) {
+ if (!systemPropertiesHandler.fileExist()) {
// Avoid creating confignode-system.properties files during
// synchronizing the ApplyConfigNode logs from the ConsensusLayer.
// 1. For the Non-Seed-ConfigNode, We don't need to create confignode-system.properties file
@@ -330,12 +317,8 @@ public class SystemPropertiesUtils {
// in which case the latest config_node_list will be updated.
return;
}
-
- Properties systemProperties = getSystemProperties();
- systemProperties.setProperty(
+ systemPropertiesHandler.put(
"config_node_list", NodeUrlUtils.convertTConfigNodeUrls(configNodes));
-
- storeSystemProperties(systemProperties);
}
/**
@@ -346,7 +329,7 @@ public class SystemPropertiesUtils {
* @throws IOException When load confignode-system.properties file failed
*/
public static String loadClusterNameWhenRestarted() throws IOException {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
String clusterName = systemProperties.getProperty(CLUSTER_NAME, null);
if (clusterName == null) {
LOGGER.warn(
@@ -366,7 +349,7 @@ public class SystemPropertiesUtils {
* @throws IOException When load confignode-system.properties file failed
*/
public static int loadConfigNodeIdWhenRestarted() throws IOException {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
try {
return Integer.parseInt(systemProperties.getProperty("config_node_id", null));
} catch (NumberFormatException e) {
@@ -387,7 +370,7 @@ public class SystemPropertiesUtils {
*/
public static boolean isSeedConfigNode() {
try {
- Properties systemProperties = getSystemProperties();
+ Properties systemProperties = systemPropertiesHandler.read();
boolean isSeedConfigNode =
Boolean.parseBoolean(systemProperties.getProperty("is_seed_config_node", null));
if (isSeedConfigNode) {
@@ -399,37 +382,4 @@ public class SystemPropertiesUtils {
return false;
}
}
-
- private static synchronized Properties getSystemProperties() throws IOException {
- // Create confignode-system.properties file if necessary
- if (!systemPropertiesFile.exists()) {
- if (systemPropertiesFile.createNewFile()) {
- LOGGER.info(
- "System properties file {} for ConfigNode is created.",
- systemPropertiesFile.getAbsolutePath());
- } else {
- LOGGER.error(
- "Can't create the system properties file {} for ConfigNode. "
- + "IoTDB-ConfigNode is shutdown.",
- systemPropertiesFile.getAbsolutePath());
- throw new IOException("Can't create system properties file");
- }
- }
-
- Properties systemProperties = new Properties();
- try (FileInputStream inputStream = new FileInputStream(systemPropertiesFile)) {
- systemProperties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
- }
- return systemProperties;
- }
-
- private static synchronized void storeSystemProperties(Properties systemProperties)
- throws IOException {
- try (FileOutputStream fileOutputStream = new FileOutputStream(systemPropertiesFile)) {
- systemProperties.store(
- new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8),
- " THIS FILE IS AUTOMATICALLY GENERATED. PLEASE DO NOT MODIFY THIS FILE !!!");
- fileOutputStream.getFD().sync();
- }
- }
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java
index ef7d4f212b2..2cbcb15a901 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/datanode/DataNodeRegisterResp.java
@@ -32,13 +32,13 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
public class DataNodeRegisterResp implements DataSet {
private TSStatus status;
private List<TConfigNodeLocation> configNodeList;
private Integer dataNodeId;
-
private TRuntimeConfiguration runtimeConfiguration;
public DataNodeRegisterResp() {
@@ -85,8 +85,8 @@ public class DataNodeRegisterResp implements DataSet {
resp.setConfigNodeList(configNodeList);
if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
- resp.setDataNodeId(dataNodeId);
- resp.setRuntimeConfiguration(runtimeConfiguration);
+ Optional.ofNullable(dataNodeId).ifPresent(resp::setDataNodeId);
+ Optional.ofNullable(runtimeConfiguration).ifPresent(resp::setRuntimeConfiguration);
}
return resp;
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
index 421235beeb3..20dddc56b09 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java
@@ -407,7 +407,7 @@ public class ConfigManager implements IManager {
req.getClusterName(),
req.getDataNodeConfiguration().getLocation(),
this);
- if (status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
+ if (!req.isPreCheck() && status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
return nodeManager.registerDataNode(req);
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java
index 13cd42393fa..6b809928fa4 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/ClusterNodeStartUtils.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TSStatus;
import org.apache.iotdb.commons.cluster.NodeType;
import org.apache.iotdb.commons.conf.CommonConfig;
+import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.confignode.conf.ConfigNodeDescriptor;
import org.apache.iotdb.confignode.manager.ConfigManager;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -112,10 +113,23 @@ public class ClusterNodeStartUtils {
nodeType.getNodeType(),
CommonConfig.SYSTEM_CONFIG_NAME));
return status;
- } else {
- /* Accept registration if all TEndPoints aren't conflict */
- return ACCEPT_NODE_REGISTRATION;
}
+
+ // Check cluster id
+ final String clusterId =
+ configManager
+ .getClusterManager()
+ .getClusterIdWithRetry(
+ CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2);
+ if (clusterId == null) {
+ status
+ .setCode(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode())
+ .setMessage("clusterId has not generated, please try again later");
+ return status;
+ }
+
+ /* Accept registration if all TEndPoints aren't conflict */
+ return ACCEPT_NODE_REGISTRATION;
}
public static TSStatus confirmNodeRestart(
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
index 0877acd336c..c8b946cf226 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/node/NodeManager.java
@@ -265,17 +265,6 @@ public class NodeManager {
public DataSet registerDataNode(TDataNodeRegisterReq req) {
DataNodeRegisterResp resp = new DataNodeRegisterResp();
resp.setConfigNodeList(getRegisteredConfigNodes());
- final String clusterId =
- configManager
- .getClusterManager()
- .getClusterIdWithRetry(
- CommonDescriptor.getInstance().getConfig().getConnectionTimeoutInMS() / 2);
- if (clusterId == null) {
- resp.setStatus(
- new TSStatus(TSStatusCode.GET_CLUSTER_ID_ERROR.getStatusCode())
- .setMessage("clusterId has not generated, please try again later"));
- return resp;
- }
// Create a new DataNodeHeartbeatCache and force update NodeStatus
int dataNodeId = nodeInfo.generateNextNodeId();
@@ -311,6 +300,7 @@ public class NodeManager {
resp.setStatus(ClusterNodeStartUtils.ACCEPT_NODE_REGISTRATION);
resp.setDataNodeId(
registerDataNodePlan.getDataNodeConfiguration().getLocation().getDataNodeId());
+ String clusterId = configManager.getClusterManager().getClusterId();
resp.setRuntimeConfiguration(getRuntimeConfiguration().setClusterId(clusterId));
return resp;
}
diff --git a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
index 959eeba114d..5ce94932036 100644
--- a/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
+++ b/iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/persistence/NodeInfoTest.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.common.rpc.thrift.TDataNodeConfiguration;
import org.apache.iotdb.common.rpc.thrift.TDataNodeLocation;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.common.rpc.thrift.TNodeResource;
+import org.apache.iotdb.commons.exception.StartupException;
import org.apache.iotdb.confignode.consensus.request.write.confignode.ApplyConfigNodePlan;
import org.apache.iotdb.confignode.consensus.request.write.datanode.RegisterDataNodePlan;
import org.apache.iotdb.confignode.persistence.node.NodeInfo;
@@ -46,7 +47,7 @@ public class NodeInfoTest {
private static final File snapshotDir = new File(BASE_OUTPUT_PATH, "snapshot");
@BeforeClass
- public static void setup() {
+ public static void setup() throws StartupException {
nodeInfo = new NodeInfo();
if (!snapshotDir.exists()) {
snapshotDir.mkdirs();
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java
new file mode 100644
index 00000000000..2c9d6e5e535
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/DataNodeSystemPropertiesHandler.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.conf;
+
+import org.apache.iotdb.commons.file.SystemPropertiesHandler;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+
+import static org.apache.iotdb.db.conf.IoTDBStartCheck.PROPERTIES_FILE_NAME;
+
+public class DataNodeSystemPropertiesHandler extends SystemPropertiesHandler {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(DataNodeSystemPropertiesHandler.class);
+
+ private static DataNodeSystemPropertiesHandler INSTANCE;
+
+ private DataNodeSystemPropertiesHandler(String filePath) {
+ super(filePath);
+ }
+
+ public static synchronized SystemPropertiesHandler getInstance() {
+ if (INSTANCE == null) {
+ INSTANCE =
+ new DataNodeSystemPropertiesHandler(
+ IoTDBDescriptor.getInstance().getConfig().getSystemDir()
+ + File.separator
+ + PROPERTIES_FILE_NAME);
+ INSTANCE.recover();
+ }
+ return INSTANCE;
+ }
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
index c5d22f36e3f..64cdbe20e76 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBStartCheck.java
@@ -24,23 +24,17 @@ import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.ConfigurationException;
import org.apache.iotdb.commons.file.SystemFileFactory;
+import org.apache.iotdb.commons.file.SystemPropertiesHandler;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALMode;
import org.apache.iotdb.db.storageengine.rescon.disk.DirectoryChecker;
import org.apache.commons.io.FileUtils;
-import org.apache.tsfile.common.conf.TSFileConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
@@ -61,13 +55,10 @@ public class IoTDBStartCheck {
private boolean isFirstStart = false;
- private final File propertiesFile;
- private final File oldPropertiesFile;
- private final File tmpPropertiesFile;
-
- private final Properties properties = new Properties();
+ private Properties properties = new Properties();
private final Map<String, Supplier<String>> systemProperties = new HashMap<>();
+ private final SystemPropertiesHandler systemPropertiesHandler;
// region params need checking, determined when first start
private static final String SYSTEM_PROPERTIES_STRING = "System properties:";
@@ -148,15 +139,7 @@ public class IoTDBStartCheck {
}
}
- oldPropertiesFile =
- SystemFileFactory.INSTANCE.getFile(
- IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME);
- propertiesFile =
- SystemFileFactory.INSTANCE.getFile(
- config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME);
- tmpPropertiesFile =
- SystemFileFactory.INSTANCE.getFile(
- IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME + ".tmp");
+ systemPropertiesHandler = DataNodeSystemPropertiesHandler.getInstance();
systemProperties.put(IOTDB_VERSION_STRING, () -> IoTDBConstant.VERSION);
systemProperties.put(COMMIT_ID_STRING, () -> IoTDBConstant.BUILD_INFO);
@@ -165,43 +148,6 @@ public class IoTDBStartCheck {
}
}
- /** check configuration in system.properties when starting IoTDB */
- public boolean checkIsFirstStart() throws IOException {
- // system init first time, no need to check, write system.properties and return
- if (!propertiesFile.exists() && !tmpPropertiesFile.exists()) {
- // create system.properties
- if (propertiesFile.createNewFile()) {
- logger.info(" {} has been created.", propertiesFile.getAbsolutePath());
- } else {
- logger.error("can not create {}", propertiesFile.getAbsolutePath());
- System.exit(-1);
- }
-
- // write properties to system.properties
- try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
- systemProperties.forEach((k, v) -> properties.setProperty(k, v.get()));
- properties.store(
- new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING);
- }
- isFirstStart = true;
- return true;
- }
-
- if (!propertiesFile.exists() && tmpPropertiesFile.exists()) {
- // rename tmp file to system.properties, no need to check
- FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
- logger.info("rename {} to {}", tmpPropertiesFile, propertiesFile);
- isFirstStart = false;
- return false;
- } else if (propertiesFile.exists() && tmpPropertiesFile.exists()) {
- // both files exist, remove tmp file
- FileUtils.forceDelete(tmpPropertiesFile);
- logger.info("remove {}", tmpPropertiesFile);
- }
- isFirstStart = false;
- return false;
- }
-
/**
* check and create directory before start IoTDB.
*
@@ -240,14 +186,20 @@ public class IoTDBStartCheck {
*
* @throws IOException If copy fail or delete fail
*/
- public void checkOldSystemConfig() throws IOException {
+ public static void checkOldSystemConfig() throws IOException {
+ File oldPropertiesFile =
+ SystemFileFactory.INSTANCE.getFile(
+ IoTDBStartCheck.SCHEMA_DIR + File.separator + PROPERTIES_FILE_NAME);
if (oldPropertiesFile.exists()) {
- FileUtils.copyFile(oldPropertiesFile, propertiesFile);
+ File correctPropertiesFile =
+ SystemFileFactory.INSTANCE.getFile(
+ config.getSystemDir() + File.separator + PROPERTIES_FILE_NAME);
+ FileUtils.copyFile(oldPropertiesFile, correctPropertiesFile);
FileUtils.delete(oldPropertiesFile);
logger.info(
"system.properties file has been moved successfully: {} -> {}",
oldPropertiesFile.getAbsolutePath(),
- propertiesFile.getAbsolutePath());
+ correctPropertiesFile.getAbsolutePath());
}
}
@@ -261,19 +213,11 @@ public class IoTDBStartCheck {
*/
public void checkSystemConfig() throws ConfigurationException, IOException {
// read properties from system.properties
- try (FileInputStream inputStream = new FileInputStream(propertiesFile);
- InputStreamReader inputStreamReader =
- new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) {
- properties.load(inputStreamReader);
- }
+ properties = systemPropertiesHandler.read();
- if (isFirstStart) {
+ if (systemPropertiesHandler.isFirstStart()) {
// overwrite system.properties when first start
- try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
- systemProperties.forEach((k, v) -> properties.setProperty(k, v.get()));
- properties.store(
- new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING);
- }
+ generateOrOverwriteSystemPropertiesFile();
if (config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS)
&& config.getWalMode().equals(WALMode.DISABLE)) {
throw new ConfigurationException(
@@ -298,32 +242,15 @@ public class IoTDBStartCheck {
/** repair broken properties */
private void upgradePropertiesFileFromBrokenFile() throws IOException {
- // create an empty tmpPropertiesFile
- if (tmpPropertiesFile.createNewFile()) {
- logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
- } else {
- logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
- System.exit(-1);
- }
-
- try (FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString())) {
- systemProperties.forEach(
- (k, v) -> {
- if (!properties.containsKey(k)) {
- properties.setProperty(k, v.get());
- }
- });
- properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION);
- properties.setProperty(COMMIT_ID_STRING, IoTDBConstant.BUILD_INFO);
- properties.store(
- new OutputStreamWriter(tmpFOS, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING);
- // upgrade finished, delete old system.properties file
- if (propertiesFile.exists()) {
- Files.delete(propertiesFile.toPath());
- }
- }
- // rename system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
+ systemProperties.forEach(
+ (k, v) -> {
+ if (!properties.containsKey(k)) {
+ properties.setProperty(k, v.get());
+ }
+ });
+ properties.setProperty(IOTDB_VERSION_STRING, IoTDBConstant.VERSION);
+ properties.setProperty(COMMIT_ID_STRING, IoTDBConstant.BUILD_INFO);
+ systemPropertiesHandler.overwrite(properties);
}
/** Check all immutable properties */
@@ -362,44 +289,11 @@ public class IoTDBStartCheck {
parameter + "can't be modified after first startup");
}
- // reload properties from system.properties
- private void reloadProperties() throws IOException {
- try (FileInputStream inputStream = new FileInputStream(propertiesFile);
- InputStreamReader inputStreamReader =
- new InputStreamReader(inputStream, TSFileConfig.STRING_CHARSET)) {
- properties.load(inputStreamReader);
- }
- }
-
/** call this method to serialize ClusterName and DataNodeId */
public void serializeClusterNameAndDataNodeId(String clusterName, int dataNodeId)
throws IOException {
- // create an empty tmpPropertiesFile
- if (tmpPropertiesFile.createNewFile()) {
- logger.info("Create system.properties.tmp {}.", tmpPropertiesFile);
- } else {
- logger.error("Create system.properties.tmp {} failed.", tmpPropertiesFile);
- System.exit(-1);
- }
-
- reloadProperties();
- FileOutputStream tmpFOS = new FileOutputStream(tmpPropertiesFile.toString());
- try {
- properties.setProperty(IoTDBConstant.CLUSTER_NAME, clusterName);
- properties.setProperty(DATA_NODE_ID, String.valueOf(dataNodeId));
- properties.store(
- new OutputStreamWriter(tmpFOS, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING);
- // serialize finished, delete old system.properties file
- if (propertiesFile.exists()) {
- Files.delete(propertiesFile.toPath());
- }
- } finally {
- tmpFOS.flush();
- tmpFOS.getFD().sync();
- tmpFOS.close();
- }
- // rename system.properties.tmp to system.properties
- FileUtils.moveFile(tmpPropertiesFile, propertiesFile);
+ systemPropertiesHandler.put(
+ IoTDBConstant.CLUSTER_NAME, clusterName, DATA_NODE_ID, String.valueOf(dataNodeId));
}
public boolean checkConsensusProtocolExists(TConsensusGroupType type) {
@@ -417,21 +311,22 @@ public class IoTDBStartCheck {
long startTime = System.currentTimeMillis();
boolean needsSerialize = false;
for (String param : variableParamValueTable.keySet()) {
- if (!(properties.getProperty(param).equals(getVal(param)))) {
+ if (!properties.getProperty(param).equals(getVal(param))) {
needsSerialize = true;
}
}
if (needsSerialize) {
- try (FileOutputStream outputStream = new FileOutputStream(propertiesFile)) {
- systemProperties.forEach((k, v) -> properties.setProperty(k, v.get()));
- properties.store(
- new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), SYSTEM_PROPERTIES_STRING);
- }
+ generateOrOverwriteSystemPropertiesFile();
}
long endTime = System.currentTimeMillis();
logger.info(
"Serialize mutable system properties successfully, which takes {} ms.",
(endTime - startTime));
}
+
+ public void generateOrOverwriteSystemPropertiesFile() throws IOException {
+ systemProperties.forEach((k, v) -> properties.setProperty(k, v.get()));
+ systemPropertiesHandler.overwrite(properties);
+ }
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java
index d9fe03d80bc..256b8a0be17 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeInfo.java
@@ -22,21 +22,14 @@ package org.apache.iotdb.db.protocol.client;
import org.apache.iotdb.common.rpc.thrift.TEndPoint;
import org.apache.iotdb.commons.consensus.ConfigRegionId;
import org.apache.iotdb.commons.exception.BadNodeUrlException;
-import org.apache.iotdb.commons.file.SystemFileFactory;
-import org.apache.iotdb.commons.utils.FileUtils;
+import org.apache.iotdb.commons.file.SystemPropertiesHandler;
import org.apache.iotdb.commons.utils.NodeUrlUtils;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.conf.DataNodeSystemPropertiesHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -45,13 +38,10 @@ import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
public class ConfigNodeInfo {
-
private static final Logger logger = LoggerFactory.getLogger(ConfigNodeInfo.class);
private static final String CONFIG_NODE_LIST = "config_node_list";
- private static final String PROPERTIES_FILE_NAME = "system.properties";
-
private final ReentrantReadWriteLock configNodeInfoReadWriteLock;
/** latest config nodes. */
@@ -59,26 +49,11 @@ public class ConfigNodeInfo {
public static final ConfigRegionId CONFIG_REGION_ID = new ConfigRegionId(0);
- private final File propertiesFile;
- private final File propertiesFileTmp;
+ SystemPropertiesHandler systemPropertiesHandler = DataNodeSystemPropertiesHandler.getInstance();
private ConfigNodeInfo() {
this.configNodeInfoReadWriteLock = new ReentrantReadWriteLock();
this.onlineConfigNodes = new HashSet<>();
- propertiesFile =
- SystemFileFactory.INSTANCE.getFile(
- IoTDBDescriptor.getInstance().getConfig().getSystemDir()
- + File.separator
- + PROPERTIES_FILE_NAME);
- propertiesFileTmp =
- SystemFileFactory.INSTANCE.getFile(propertiesFile.getAbsolutePath() + ".tmp");
- if (propertiesFileTmp.exists()) {
- try {
- updatePropertiesFile();
- } catch (IOException e) {
- logger.error("Update properties file fail", e);
- }
- }
}
// TODO: This needs removal of statics ...
@@ -105,7 +80,7 @@ public class ConfigNodeInfo {
try {
onlineConfigNodes.clear();
onlineConfigNodes.addAll(latestConfigNodes);
- storeConfigNode();
+ storeConfigNodeList();
long endTime = System.currentTimeMillis();
logger.info(
"Update ConfigNode Successfully: {}, which takes {} ms.",
@@ -125,27 +100,13 @@ public class ConfigNodeInfo {
*
* @throws IOException if properties deserialization or configNode list serialization failed.
*/
- private void storeConfigNode() throws IOException {
- Properties properties = new Properties();
- try (FileInputStream inputStream = new FileInputStream(propertiesFile)) {
- properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+ public void storeConfigNodeList() throws IOException {
+ if (!systemPropertiesHandler.fileExist()) {
+ logger.info("System properties file not exist, not necessary to store ConfigNode list");
+ return;
}
- properties.setProperty(
+ systemPropertiesHandler.put(
CONFIG_NODE_LIST, NodeUrlUtils.convertTEndPointUrls(new ArrayList<>(onlineConfigNodes)));
- try (FileOutputStream fileOutputStream = new FileOutputStream(propertiesFileTmp)) {
- properties.store(new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8), "");
- }
- updatePropertiesFile();
- }
-
- private void updatePropertiesFile() throws IOException {
- if (!propertiesFile.delete()) {
- String msg =
- String.format(
- "Update %s file fail: %s", PROPERTIES_FILE_NAME, propertiesFile.getAbsoluteFile());
- throw new IOException(msg);
- }
- FileUtils.moveFileSafe(propertiesFileTmp, propertiesFile);
}
public void loadConfigNodeList() {
@@ -153,12 +114,7 @@ public class ConfigNodeInfo {
// properties contain CONFIG_NODE_LIST only when start as Data node
try {
configNodeInfoReadWriteLock.writeLock().lock();
- Properties properties = new Properties();
- try (FileInputStream inputStream = new FileInputStream(propertiesFile)) {
- properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ Properties properties = systemPropertiesHandler.read();
if (properties.containsKey(CONFIG_NODE_LIST)) {
onlineConfigNodes.clear();
@@ -170,6 +126,8 @@ public class ConfigNodeInfo {
"Load ConfigNode successfully: {}, which takes {} ms.",
onlineConfigNodes,
(endTime - startTime));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
} catch (BadNodeUrlException e) {
logger.error("Cannot parse config node list in system.properties");
} finally {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
index 255349cab89..c057c3392e5 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/DataNode.java
@@ -33,7 +33,6 @@ import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.consensus.ConsensusGroupId;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.exception.StartupException;
-import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.commons.pipe.config.PipeConfig;
import org.apache.iotdb.commons.pipe.plugin.meta.PipePluginMeta;
import org.apache.iotdb.commons.service.JMXService;
@@ -60,6 +59,7 @@ import org.apache.iotdb.confignode.rpc.thrift.TRuntimeConfiguration;
import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp;
import org.apache.iotdb.consensus.ConsensusFactory;
import org.apache.iotdb.db.conf.DataNodeStartupCheck;
+import org.apache.iotdb.db.conf.DataNodeSystemPropertiesHandler;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.IoTDBStartCheck;
@@ -123,6 +123,7 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.apache.iotdb.commons.conf.IoTDBConstant.DEFAULT_CLUSTER_NAME;
+import static org.apache.iotdb.db.conf.IoTDBStartCheck.PROPERTIES_FILE_NAME;
public class DataNode implements DataNodeMBean {
@@ -136,10 +137,6 @@ public class DataNode implements DataNodeMBean {
IoTDBConstant.JMX_TYPE,
ServiceType.DATA_NODE.getJmxName());
- private static File SYSTEM_PROPERTIES =
- SystemFileFactory.INSTANCE.getFile(
- config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME);
-
/**
* When joining a cluster or getting configuration this node will retry at most "DEFAULT_RETRY"
* times before returning a failure to the client.
@@ -170,10 +167,12 @@ public class DataNode implements DataNodeMBean {
// TODO: This needs removal of statics ...
public static void reinitializeStatics() {
- SYSTEM_PROPERTIES =
- SystemFileFactory.INSTANCE.getFile(
- config.getSystemDir() + File.separator + IoTDBStartCheck.PROPERTIES_FILE_NAME);
registerManager = new RegisterManager();
+ DataNodeSystemPropertiesHandler.getInstance()
+ .resetFilePath(
+ IoTDBDescriptor.getInstance().getConfig().getSystemDir()
+ + File.separator
+ + PROPERTIES_FILE_NAME);
}
private static RegisterManager registerManager = new RegisterManager();
@@ -195,7 +194,6 @@ public class DataNode implements DataNodeMBean {
isFirstStart = prepareDataNode();
if (isFirstStart) {
- // Set target ConfigNodeList from iotdb-system.properties file
ConfigNodeInfo.getInstance()
.updateConfigNodeList(Collections.singletonList(config.getSeedConfigNode()));
} else {
@@ -207,8 +205,11 @@ public class DataNode implements DataNodeMBean {
pullAndCheckSystemConfigurations();
if (isFirstStart) {
+ sendRegisterRequestToConfigNode(true);
+ IoTDBStartCheck.getInstance().generateOrOverwriteSystemPropertiesFile();
+ ConfigNodeInfo.getInstance().storeConfigNodeList();
// Register this DataNode to the cluster when first start
- sendRegisterRequestToConfigNode();
+ sendRegisterRequestToConfigNode(false);
} else {
// Send restart request of this DataNode
sendRestartRequestToConfigNode();
@@ -233,11 +234,6 @@ public class DataNode implements DataNodeMBean {
} catch (StartupException | IOException e) {
logger.error("Fail to start server", e);
- if (isFirstStart) {
- // Delete the system.properties file when first start failed.
- // Therefore, the next time this DataNode is start will still be seen as the first time.
- SYSTEM_PROPERTIES.deleteOnExit();
- }
stop();
System.exit(-1);
}
@@ -247,9 +243,8 @@ public class DataNode implements DataNodeMBean {
private boolean prepareDataNode() throws StartupException, IOException {
long startTime = System.currentTimeMillis();
- // Notice: Consider this DataNode as first start if the system.properties file doesn't exist
- IoTDBStartCheck.getInstance().checkOldSystemConfig();
- boolean isFirstStart = IoTDBStartCheck.getInstance().checkIsFirstStart();
+ // Init system properties handler
+ IoTDBStartCheck.checkOldSystemConfig();
// Set this node
thisNode.setIp(config.getInternalAddress());
@@ -260,7 +255,7 @@ public class DataNode implements DataNodeMBean {
checks.startUpCheck();
long endTime = System.currentTimeMillis();
logger.info("The DataNode is prepared successfully, which takes {} ms", (endTime - startTime));
- return isFirstStart;
+ return DataNodeSystemPropertiesHandler.getInstance().isFirstStart();
}
/**
@@ -286,9 +281,7 @@ public class DataNode implements DataNodeMBean {
configurationResp = configNodeClient.getSystemConfiguration();
break;
} catch (TException | ClientManagerException e) {
- logger.warn(
- "Cannot pull system configurations from ConfigNode-leader, because: {}",
- e.getMessage());
+ logger.warn("Cannot pull system configurations from ConfigNode-leader", e);
retry--;
}
@@ -397,15 +390,18 @@ public class DataNode implements DataNodeMBean {
/**
* Register this DataNode into cluster.
*
+ * @param isPreCheck do pre-check before formal registration
* @throws StartupException if register failed.
- * @throws IOException if serialize cluster name and dataNode Id failed.
+ * @throws IOException if serialize cluster name and datanode id failed.
*/
- private void sendRegisterRequestToConfigNode() throws StartupException, IOException {
+ private void sendRegisterRequestToConfigNode(boolean isPreCheck)
+ throws StartupException, IOException {
logger.info("Sending register request to ConfigNode-leader...");
long startTime = System.currentTimeMillis();
/* Send register request */
int retry = DEFAULT_RETRY;
TDataNodeRegisterReq req = new TDataNodeRegisterReq();
+ req.setPreCheck(isPreCheck);
req.setDataNodeConfiguration(generateDataNodeConfiguration());
req.setClusterName(config.getClusterName());
req.setVersionInfo(new TNodeVersionInfo(IoTDBConstant.VERSION, IoTDBConstant.BUILD_INFO));
@@ -438,7 +434,10 @@ public class DataNode implements DataNodeMBean {
}
if (dataNodeRegisterResp.getStatus().getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode()) {
-
+ if (isPreCheck) {
+ logger.info("Successfully pass the precheck, will do the formal registration soon.");
+ return;
+ }
/* Store runtime configurations when register success */
int dataNodeID = dataNodeRegisterResp.getDataNodeId();
config.setDataNodeId(dataNodeID);
@@ -448,6 +447,7 @@ public class DataNode implements DataNodeMBean {
storeRuntimeConfigurations(
dataNodeRegisterResp.getConfigNodeList(), dataNodeRegisterResp.getRuntimeConfiguration());
long endTime = System.currentTimeMillis();
+
logger.info(
"Successfully register to the cluster: {} , which takes {} ms.",
config.getClusterName(),
diff --git a/iotdb-core/node-commons/pom.xml b/iotdb-core/node-commons/pom.xml
index e75e207d030..c44cd19ecd2 100644
--- a/iotdb-core/node-commons/pom.xml
+++ b/iotdb-core/node-commons/pom.xml
@@ -178,6 +178,10 @@
<artifactId>awaitility</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ratis</groupId>
+ <artifactId>ratis-common</artifactId>
+ </dependency>
</dependencies>
<build>
<resources>
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java
index fe7973bff96..f9d2a3bb567 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemFileFactory.java
@@ -38,7 +38,6 @@ public enum SystemFileFactory {
public File getFile(String pathname) {
if (fsType.equals(FSType.HDFS)) {
throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name());
- // return new HDFSFile(pathname);
} else {
return new File(pathname);
}
@@ -47,7 +46,6 @@ public enum SystemFileFactory {
public File getFile(String parent, String child) {
if (fsType.equals(FSType.HDFS)) {
throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name());
- // return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
@@ -56,7 +54,6 @@ public enum SystemFileFactory {
public File getFile(File parent, String child) {
if (fsType.equals(FSType.HDFS)) {
throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name());
- // return new HDFSFile(parent, child);
} else {
return new File(parent, child);
}
@@ -65,7 +62,6 @@ public enum SystemFileFactory {
public File getFile(URI uri) {
if (fsType.equals(FSType.HDFS)) {
throw new UnsupportedOperationException(UNSUPPORTED_FILE_SYSTEM + fsType.name());
- // return new HDFSFile(uri);
} else {
return new File(uri);
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java
new file mode 100644
index 00000000000..61bcbb49b0c
--- /dev/null
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/file/SystemPropertiesHandler.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.commons.file;
+
+import org.apache.ratis.util.AutoCloseableLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Properties;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+public abstract class SystemPropertiesHandler {
+ private static final Logger LOGGER = LoggerFactory.getLogger(SystemPropertiesHandler.class);
+
+ private File formalFile;
+ private File tmpFile;
+ ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ public SystemPropertiesHandler(String filePath) {
+ this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath);
+ this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp");
+ }
+
+ /**
+ * Put key and value into properties
+ *
+ * @param keyOrValue [key0, value0, key1, value1, ... ]
+ */
+ public void put(Object... keyOrValue) throws IOException {
+ if (keyOrValue.length % 2 != 0) {
+ throw new IllegalArgumentException(
+ "Length of parameters should be evenly divided by 2, but the actual length is "
+ + keyOrValue.length
+ + " : "
+ + Arrays.toString(keyOrValue));
+ }
+ try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock());
+ FileOutputStream outputStream = new FileOutputStream(tmpFile)) {
+ Properties properties = new Properties();
+ // read old properties
+ if (formalFile.exists()) {
+ try (FileInputStream inputStream = new FileInputStream(formalFile)) {
+ properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+ }
+ }
+ // store new properties
+ for (int i = 0; i < keyOrValue.length; i += 2) {
+ properties.put(keyOrValue[i], keyOrValue[i + 1]);
+ }
+ properties.store(new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), "");
+ outputStream.getFD().sync();
+ replaceFormalFile();
+ }
+ }
+
+ public void overwrite(Properties properties) throws IOException {
+ try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) {
+ if (!formalFile.exists()) {
+ writeImpl(properties, formalFile);
+ return;
+ }
+ writeImpl(properties, tmpFile);
+ replaceFormalFile();
+ }
+ }
+
+ public Properties read() throws IOException {
+ try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock())) {
+ Properties properties = new Properties();
+ if (formalFile.exists()) {
+ try (FileInputStream inputStream = new FileInputStream(formalFile)) {
+ properties.load(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+ }
+ }
+ return properties;
+ }
+ }
+
+ public boolean fileExist() {
+ try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.readLock())) {
+ return formalFile.exists() || tmpFile.exists();
+ }
+ }
+
+ public boolean isFirstStart() {
+ return !fileExist();
+ }
+
+ protected void recover() {
+ try (AutoCloseableLock ignore = AutoCloseableLock.acquire(lock.writeLock())) {
+ if (formalFile.exists() && !tmpFile.exists()) {
+ // No need to recover
+ return;
+ }
+ if (!formalFile.exists() && !tmpFile.exists()) {
+ // First start
+ return;
+ }
+ if (formalFile.exists() && tmpFile.exists()) {
+ if (!tmpFile.delete()) {
+ LOGGER.warn(
+ "Delete system.properties tmp file fail, you may manually delete it: {}",
+ tmpFile.getAbsoluteFile());
+ }
+ return;
+ }
+ if (!formalFile.exists() && tmpFile.exists()) {
+ replaceFormalFile();
+ return;
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ throw new UnsupportedOperationException("Should never touch here");
+ }
+
+ private void writeImpl(Properties properties, File file) throws IOException {
+ try (FileOutputStream fileOutputStream = new FileOutputStream(file)) {
+ properties.store(new OutputStreamWriter(fileOutputStream, StandardCharsets.UTF_8), "");
+ }
+ }
+
+ private void replaceFormalFile() throws IOException {
+ if (!tmpFile.exists()) {
+ throw new FileNotFoundException(
+ "Tmp system properties file must exist when call replaceFormalFile");
+ }
+ if (formalFile.exists() && !formalFile.delete()) {
+ String msg =
+ String.format(
+ "Delete formal system properties file fail: %s", formalFile.getAbsoluteFile());
+ throw new IOException(msg);
+ }
+ if (!tmpFile.renameTo(formalFile)) {
+ String msg =
+ String.format(
+ "Failed to replace formal system properties file, you may manually rename it: %s -> %s",
+ tmpFile.getAbsolutePath(), formalFile.getAbsolutePath());
+ throw new IOException(msg);
+ }
+ }
+
+ public void resetFilePath(String filePath) {
+ this.formalFile = SystemFileFactory.INSTANCE.getFile(filePath);
+ this.tmpFile = SystemFileFactory.INSTANCE.getFile(filePath + ".tmp");
+ }
+}
diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
index aa4562f81d8..1ebfc0979a7 100644
--- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
+++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift
@@ -111,6 +111,7 @@ struct TDataNodeRegisterReq {
1: required string clusterName
2: required common.TDataNodeConfiguration dataNodeConfiguration
3: optional TNodeVersionInfo versionInfo
+ 4: optional bool preCheck
}
struct TDataNodeRegisterResp {