You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/11/22 13:39:03 UTC
[iotdb] branch master updated: Add dn_sync_dir in iotdb-datanode.properties (#8094)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new fe48dff712 Add dn_sync_dir in iotdb-datanode.properties (#8094)
fe48dff712 is described below
commit fe48dff7125802c3129d783fb760c6c8deca21c2
Author: Chen YZ <43...@users.noreply.github.com>
AuthorDate: Tue Nov 22 21:38:55 2022 +0800
Add dn_sync_dir in iotdb-datanode.properties (#8094)
---
.../java/org/apache/iotdb/commons/conf/CommonConfig.java | 12 ++++++------
.../org/apache/iotdb/commons/conf/CommonDescriptor.java | 2 +-
.../org/apache/iotdb/commons/sync/utils/SyncPathUtil.java | 6 +++---
.../src/assembly/resources/conf/iotdb-datanode.properties | 15 ++++++++++++++-
.../org/apache/iotdb/db/sync/sender/pipe/TsFilePipe.java | 14 ++++++++++++--
.../java/org/apache/iotdb/db/utils/EnvironmentUtils.java | 4 ++--
.../resources/datanode1conf/iotdb-datanode.properties | 1 +
.../resources/datanode2conf/iotdb-datanode.properties | 1 +
.../resources/datanode3conf/iotdb-datanode.properties | 3 ++-
9 files changed, 42 insertions(+), 16 deletions(-)
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index 7421412955..3b3b3359f1 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -73,7 +73,7 @@ public class CommonConfig {
+ "procedure";
/** Sync directory, including the log and hardlink tsfiles */
- private String syncFolder =
+ private String syncDir =
IoTDBConstant.DEFAULT_BASE_DIR + File.separator + IoTDBConstant.SYNC_FOLDER_NAME;
/** WAL directories */
@@ -125,7 +125,7 @@ public class CommonConfig {
userFolder = addHomeDir(userFolder, homeDir);
roleFolder = addHomeDir(roleFolder, homeDir);
procedureWalFolder = addHomeDir(procedureWalFolder, homeDir);
- syncFolder = addHomeDir(syncFolder, homeDir);
+ syncDir = addHomeDir(syncDir, homeDir);
for (int i = 0; i < walDirs.length; i++) {
walDirs[i] = addHomeDir(walDirs[i], homeDir);
}
@@ -214,12 +214,12 @@ public class CommonConfig {
this.procedureWalFolder = procedureWalFolder;
}
- public String getSyncFolder() {
- return syncFolder;
+ public String getSyncDir() {
+ return syncDir;
}
- public void setSyncFolder(String syncFolder) {
- this.syncFolder = syncFolder;
+ public void setSyncDir(String syncDir) {
+ this.syncDir = syncDir;
}
public String[] getWalDirs() {
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index 869cd348a0..6ef3e0cc01 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -85,7 +85,7 @@ public class CommonDescriptor {
properties
.getProperty("default_ttl_in_ms", String.valueOf(config.getDefaultTTLInMs()))
.trim()));
- config.setSyncFolder(properties.getProperty("sync_dir", config.getSyncFolder()).trim());
+ config.setSyncDir(properties.getProperty("dn_sync_dir", config.getSyncDir()).trim());
config.setWalDirs(
properties
diff --git a/node-commons/src/main/java/org/apache/iotdb/commons/sync/utils/SyncPathUtil.java b/node-commons/src/main/java/org/apache/iotdb/commons/sync/utils/SyncPathUtil.java
index 497c8d74a3..086f5bda32 100644
--- a/node-commons/src/main/java/org/apache/iotdb/commons/sync/utils/SyncPathUtil.java
+++ b/node-commons/src/main/java/org/apache/iotdb/commons/sync/utils/SyncPathUtil.java
@@ -47,7 +47,7 @@ public class SyncPathUtil {
/** sender */
public static String getSenderDir() {
- return CommonDescriptor.getInstance().getConfig().getSyncFolder()
+ return CommonDescriptor.getInstance().getConfig().getSyncDir()
+ File.separator
+ SyncConstant.SENDER_DIR_NAME;
}
@@ -96,7 +96,7 @@ public class SyncPathUtil {
/** receiver */
public static String getReceiverDir() {
- return CommonDescriptor.getInstance().getConfig().getSyncFolder()
+ return CommonDescriptor.getInstance().getConfig().getSyncDir()
+ File.separator
+ SyncConstant.RECEIVER_DIR_NAME;
}
@@ -135,7 +135,7 @@ public class SyncPathUtil {
/** common */
public static String getSysDir() {
- return CommonDescriptor.getInstance().getConfig().getSyncFolder()
+ return CommonDescriptor.getInstance().getConfig().getSyncDir()
+ File.separator
+ SyncConstant.SYNC_SYS_DIR;
}
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 0fab8965a6..019be9bad9 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -145,4 +145,17 @@ dn_target_config_node_list=127.0.0.1:22277
# dn_tracing_dir=datanode\\tracing
# For Linux platform
# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
-# dn_tracing_dir=datanode/tracing
\ No newline at end of file
+# dn_tracing_dir=datanode/tracing
+
+# sync dir
+# If this property is unset, system will save the data in the default relative path directory under the IoTDB folder(i.e., %IOTDB_HOME%/data/datanode).
+# If it is absolute, system will save the data in the exact location it points to.
+# If it is relative, system will save the data in the relative path directory it indicates under the IoTDB folder.
+# Note: If sync_dir is assigned an empty string(i.e.,zero-size), it will be handled as a relative path.
+# For windows platform
+# If its prefix is a drive specifier followed by "\\", or if its prefix is "\\\\", then the path is absolute. Otherwise, it is relative.
+# dn_sync_dir=data\\datanode\\sync
+# For Linux platform
+# If its prefix is "/", then the path is absolute. Otherwise, it is relative.
+# dn_sync_dir=data/datanode/sync
+
diff --git a/server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipe.java b/server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipe.java
index 3249fb66a8..19426575a6 100644
--- a/server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipe.java
+++ b/server/src/main/java/org/apache/iotdb/db/sync/sender/pipe/TsFilePipe.java
@@ -111,7 +111,7 @@ public class TsFilePipe implements Pipe {
private void recover() {
File dir =
new File(
- SyncPathUtil.getSenderRealTimePipeLogDir(
+ SyncPathUtil.getSenderHistoryPipeLogDir(
pipeInfo.getPipeName(), pipeInfo.getCreateTime()));
if (dir.exists()) {
File[] fileList = dir.listFiles();
@@ -121,11 +121,21 @@ public class TsFilePipe implements Pipe {
new BufferedPipeDataQueue(
SyncPathUtil.getSenderDataRegionHistoryPipeLogDir(
pipeInfo.getPipeName(), pipeInfo.getCreateTime(), dataRegionId));
+ historyQueueMap.put(dataRegionId, historyQueue);
+ }
+ }
+ dir =
+ new File(
+ SyncPathUtil.getSenderRealTimePipeLogDir(
+ pipeInfo.getPipeName(), pipeInfo.getCreateTime()));
+ if (dir.exists()) {
+ File[] fileList = dir.listFiles();
+ for (File file : fileList) {
+ String dataRegionId = file.getName();
BufferedPipeDataQueue realTimeQueue =
new BufferedPipeDataQueue(
SyncPathUtil.getSenderDataRegionRealTimePipeLogDir(
pipeInfo.getPipeName(), pipeInfo.getCreateTime(), dataRegionId));
- historyQueueMap.put(dataRegionId, historyQueue);
realTimeQueueMap.put(dataRegionId, realTimeQueue);
this.maxSerialNumber.set(
Math.max(this.maxSerialNumber.get(), realTimeQueue.getLastMaxSerialNumber()));
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index f3a7e776ab..067b63637d 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -264,7 +264,7 @@ public class EnvironmentUtils {
cleanDir(walDir);
}
// delete sync dir
- cleanDir(commonConfig.getSyncFolder());
+ cleanDir(commonConfig.getSyncDir());
// delete data files
for (String dataDir : config.getDataDirs()) {
cleanDir(dataDir);
@@ -356,7 +356,7 @@ public class EnvironmentUtils {
String sgDir = FilePathUtils.regularizePath(config.getSystemDir()) + "databases";
createDir(sgDir);
// create sync
- createDir(commonConfig.getSyncFolder());
+ createDir(commonConfig.getSyncDir());
// create query
createDir(config.getQueryDir());
createDir(TestConstant.OUTPUT_DATA_DIR);
diff --git a/server/src/test/resources/datanode1conf/iotdb-datanode.properties b/server/src/test/resources/datanode1conf/iotdb-datanode.properties
index e1d46f1e5a..87f10978bf 100644
--- a/server/src/test/resources/datanode1conf/iotdb-datanode.properties
+++ b/server/src/test/resources/datanode1conf/iotdb-datanode.properties
@@ -33,3 +33,4 @@ dn_data_dirs=target/datanode1/data
dn_wal_dirs=target/datanode1/wal
dn_tracing_dir=target/datanode1/data/tracing
dn_consensus_dir=target/datanode1/consensus
+dn_sync_dir=target/datanode1/sync
diff --git a/server/src/test/resources/datanode2conf/iotdb-datanode.properties b/server/src/test/resources/datanode2conf/iotdb-datanode.properties
index 151dfd93da..591713bd0c 100644
--- a/server/src/test/resources/datanode2conf/iotdb-datanode.properties
+++ b/server/src/test/resources/datanode2conf/iotdb-datanode.properties
@@ -33,3 +33,4 @@ dn_data_dirs=target/datanode2/data
dn_wal_dirs=target/datanode2/wal
dn_tracing_dir=target/datanode2/data/tracing
dn_consensus_dir=target/datanode2/consensus
+dn_sync_dir=target/datanode2/sync
diff --git a/server/src/test/resources/datanode3conf/iotdb-datanode.properties b/server/src/test/resources/datanode3conf/iotdb-datanode.properties
index 08baa5e8da..11cf8c4bd5 100644
--- a/server/src/test/resources/datanode3conf/iotdb-datanode.properties
+++ b/server/src/test/resources/datanode3conf/iotdb-datanode.properties
@@ -32,4 +32,5 @@ dn_system_dir=target/datanode3/system
dn_data_dirs=target/datanode3/data
dn_wal_dirs=target/datanode3/wal
dn_tracing_dir=target/datanode3/data/tracing
-dn_consensus_dir=target/datanode3/consensus
\ No newline at end of file
+dn_consensus_dir=target/datanode3/consensus
+dn_sync_dir=target/datanode3/sync
\ No newline at end of file