You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/10/10 06:18:11 UTC
[iotdb] branch Add-fsync-in-ConfigNode-persistence-modules created (now 18e3396f1dc)
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a change to branch Add-fsync-in-ConfigNode-persistence-modules
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 18e3396f1dc Finish
This branch includes the following new commits:
new 18e3396f1dc Finish
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Finish
Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yongzao pushed a commit to branch Add-fsync-in-ConfigNode-persistence-modules
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 18e3396f1dcef2a49cfa7142ab7f91970da69b3c
Author: YongzaoDan <53...@qq.com>
AuthorDate: Tue Oct 10 14:17:57 2023 +0800
Finish
---
.../confignode/conf/SystemPropertiesUtils.java | 1 +
.../confignode/persistence/node/NodeInfo.java | 5 ++--
.../persistence/partition/PartitionInfo.java | 28 ++++++++++++----------
.../confignode/procedure/store/ProcedureWAL.java | 3 +++
4 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
index 45d7ba0d672..0f7b9e4f85f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
@@ -404,6 +404,7 @@ public class SystemPropertiesUtils {
systemProperties.store(
fileOutputStream,
" THIS FILE IS AUTOMATICALLY GENERATED. PLEASE DO NOT MODIFY THIS FILE !!!");
+ fileOutputStream.getFD().sync();
}
}
}
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
index aa5e0ed29c3..22590048012 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
@@ -458,9 +458,8 @@ public class NodeInfo implements SnapshotProcessor {
serializeVersionInfo(fileOutputStream);
- fileOutputStream.flush();
-
- fileOutputStream.close();
+ tioStreamTransport.flush();
+ fileOutputStream.getFD().sync();
return tmpFile.renameTo(snapshotFile);
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
index d61396a4c7a..e38e11d22f7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
@@ -74,6 +74,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.util.ArrayList;
@@ -849,34 +850,37 @@ public class PartitionInfo implements SnapshotProcessor {
// snapshot operation.
File tmpFile = new File(snapshotFile.getAbsolutePath() + "-" + UUID.randomUUID());
- try (BufferedOutputStream fileOutputStream =
- new BufferedOutputStream(
- Files.newOutputStream(tmpFile.toPath()), PARTITION_TABLE_BUFFER_SIZE);
- TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileOutputStream)) {
+ try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile);
+ BufferedOutputStream bufferedOutputStream =
+ new BufferedOutputStream(fileOutputStream, PARTITION_TABLE_BUFFER_SIZE);
+ TIOStreamTransport tioStreamTransport = new TIOStreamTransport(bufferedOutputStream)) {
TProtocol protocol = new TBinaryProtocol(tioStreamTransport);
// serialize nextRegionGroupId
- ReadWriteIOUtils.write(nextRegionGroupId.get(), fileOutputStream);
+ ReadWriteIOUtils.write(nextRegionGroupId.get(), bufferedOutputStream);
// serialize StorageGroupPartitionTable
- ReadWriteIOUtils.write(databasePartitionTables.size(), fileOutputStream);
+ ReadWriteIOUtils.write(databasePartitionTables.size(), bufferedOutputStream);
for (Map.Entry<String, DatabasePartitionTable> storageGroupPartitionTableEntry :
databasePartitionTables.entrySet()) {
- ReadWriteIOUtils.write(storageGroupPartitionTableEntry.getKey(), fileOutputStream);
- storageGroupPartitionTableEntry.getValue().serialize(fileOutputStream, protocol);
+ ReadWriteIOUtils.write(storageGroupPartitionTableEntry.getKey(), bufferedOutputStream);
+ storageGroupPartitionTableEntry.getValue().serialize(bufferedOutputStream, protocol);
}
// serialize regionCleanList
- ReadWriteIOUtils.write(regionMaintainTaskList.size(), fileOutputStream);
+ ReadWriteIOUtils.write(regionMaintainTaskList.size(), bufferedOutputStream);
for (RegionMaintainTask task : regionMaintainTaskList) {
- task.serialize(fileOutputStream, protocol);
+ task.serialize(bufferedOutputStream, protocol);
}
// write to file
- fileOutputStream.flush();
- fileOutputStream.close();
+ tioStreamTransport.flush();
+ bufferedOutputStream.flush();
+ fileOutputStream.getFD().sync();
+
// rename file
return tmpFile.renameTo(snapshotFile);
+
} finally {
// with or without success, delete temporary files anyway
for (int retry = 0; retry < 5; retry++) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
index 3f1b4eba687..d4638dd0b23 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
@@ -67,6 +67,9 @@ public class ProcedureWAL {
DataOutputStream dataOutputStream = new DataOutputStream(publicBAOS)) {
procedure.serialize(dataOutputStream);
channel.write(ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size()));
+
+ channel.force(true);
+ fos.getFD().sync();
}
Files.deleteIfExists(walFilePath);
Files.move(walTmpPath, walFilePath);