You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yo...@apache.org on 2023/10/10 06:18:11 UTC

[iotdb] branch Add-fsync-in-ConfigNode-persistence-modules created (now 18e3396f1dc)

This is an automated email from the ASF dual-hosted git repository.

yongzao pushed a change to branch Add-fsync-in-ConfigNode-persistence-modules
in repository https://gitbox.apache.org/repos/asf/iotdb.git


      at 18e3396f1dc Finish

This branch includes the following new commits:

     new 18e3396f1dc Finish

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



[iotdb] 01/01: Finish

Posted by yo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

yongzao pushed a commit to branch Add-fsync-in-ConfigNode-persistence-modules
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit 18e3396f1dcef2a49cfa7142ab7f91970da69b3c
Author: YongzaoDan <53...@qq.com>
AuthorDate: Tue Oct 10 14:17:57 2023 +0800

    Finish
---
 .../confignode/conf/SystemPropertiesUtils.java     |  1 +
 .../confignode/persistence/node/NodeInfo.java      |  5 ++--
 .../persistence/partition/PartitionInfo.java       | 28 ++++++++++++----------
 .../confignode/procedure/store/ProcedureWAL.java   |  3 +++
 4 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
index 45d7ba0d672..0f7b9e4f85f 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/SystemPropertiesUtils.java
@@ -404,6 +404,7 @@ public class SystemPropertiesUtils {
       systemProperties.store(
           fileOutputStream,
           " THIS FILE IS AUTOMATICALLY GENERATED. PLEASE DO NOT MODIFY THIS FILE !!!");
+      fileOutputStream.getFD().sync();
     }
   }
 }
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
index aa5e0ed29c3..22590048012 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/node/NodeInfo.java
@@ -458,9 +458,8 @@ public class NodeInfo implements SnapshotProcessor {
 
       serializeVersionInfo(fileOutputStream);
 
-      fileOutputStream.flush();
-
-      fileOutputStream.close();
+      tioStreamTransport.flush();
+      fileOutputStream.getFD().sync();
 
       return tmpFile.renameTo(snapshotFile);
 
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
index d61396a4c7a..e38e11d22f7 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/partition/PartitionInfo.java
@@ -74,6 +74,7 @@ import org.slf4j.LoggerFactory;
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.ArrayList;
@@ -849,34 +850,37 @@ public class PartitionInfo implements SnapshotProcessor {
     // snapshot operation.
     File tmpFile = new File(snapshotFile.getAbsolutePath() + "-" + UUID.randomUUID());
 
-    try (BufferedOutputStream fileOutputStream =
-            new BufferedOutputStream(
-                Files.newOutputStream(tmpFile.toPath()), PARTITION_TABLE_BUFFER_SIZE);
-        TIOStreamTransport tioStreamTransport = new TIOStreamTransport(fileOutputStream)) {
+    try (FileOutputStream fileOutputStream = new FileOutputStream(tmpFile);
+        BufferedOutputStream bufferedOutputStream =
+            new BufferedOutputStream(fileOutputStream, PARTITION_TABLE_BUFFER_SIZE);
+        TIOStreamTransport tioStreamTransport = new TIOStreamTransport(bufferedOutputStream)) {
       TProtocol protocol = new TBinaryProtocol(tioStreamTransport);
 
       // serialize nextRegionGroupId
-      ReadWriteIOUtils.write(nextRegionGroupId.get(), fileOutputStream);
+      ReadWriteIOUtils.write(nextRegionGroupId.get(), bufferedOutputStream);
 
       // serialize StorageGroupPartitionTable
-      ReadWriteIOUtils.write(databasePartitionTables.size(), fileOutputStream);
+      ReadWriteIOUtils.write(databasePartitionTables.size(), bufferedOutputStream);
       for (Map.Entry<String, DatabasePartitionTable> storageGroupPartitionTableEntry :
           databasePartitionTables.entrySet()) {
-        ReadWriteIOUtils.write(storageGroupPartitionTableEntry.getKey(), fileOutputStream);
-        storageGroupPartitionTableEntry.getValue().serialize(fileOutputStream, protocol);
+        ReadWriteIOUtils.write(storageGroupPartitionTableEntry.getKey(), bufferedOutputStream);
+        storageGroupPartitionTableEntry.getValue().serialize(bufferedOutputStream, protocol);
       }
 
       // serialize regionCleanList
-      ReadWriteIOUtils.write(regionMaintainTaskList.size(), fileOutputStream);
+      ReadWriteIOUtils.write(regionMaintainTaskList.size(), bufferedOutputStream);
       for (RegionMaintainTask task : regionMaintainTaskList) {
-        task.serialize(fileOutputStream, protocol);
+        task.serialize(bufferedOutputStream, protocol);
       }
 
       // write to file
-      fileOutputStream.flush();
-      fileOutputStream.close();
+      tioStreamTransport.flush();
+      bufferedOutputStream.flush();
+      fileOutputStream.getFD().sync();
+
       // rename file
       return tmpFile.renameTo(snapshotFile);
+
     } finally {
       // with or without success, delete temporary files anyway
       for (int retry = 0; retry < 5; retry++) {
diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
index 3f1b4eba687..d4638dd0b23 100644
--- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
+++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/store/ProcedureWAL.java
@@ -67,6 +67,9 @@ public class ProcedureWAL {
         DataOutputStream dataOutputStream = new DataOutputStream(publicBAOS)) {
       procedure.serialize(dataOutputStream);
       channel.write(ByteBuffer.wrap(publicBAOS.getBuf(), 0, publicBAOS.size()));
+
+      channel.force(true);
+      fos.getFD().sync();
     }
     Files.deleteIfExists(walFilePath);
     Files.move(walTmpPath, walFilePath);