[incubator-iotdb] 01/01: avoid unnecessary close in timed merge

[qi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch optimize_merge_sync_close
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git

commit cc8d172a351fc11554a54df1deb72d212f1a149b
Author: qiaojialin <64...@qq.com>
AuthorDate: Fri Apr 17 11:31:17 2020 +0800

    avoid unnecessary close in timed merge
---
 .../engine/storagegroup/StorageGroupProcessor.java |  3 +-
 .../db/engine/storagegroup/TsFileProcessor.java    | 35 ++++++++++++++--------
 2 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 273148d..1ac1e85 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -1349,11 +1349,12 @@ public class StorageGroupProcessor {
       }
       logger.info("{} will close all files for starting a merge (fullmerge = {})", storageGroupName,
           fullMerge);
-      syncCloseAllWorkingTsFileProcessors();
+
       if (unSequenceFileList.isEmpty() || sequenceFileTreeSet.isEmpty()) {
         logger.info("{} no files to be merged", storageGroupName);
         return;
       }
+      syncCloseAllWorkingTsFileProcessors();
 
       long budget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget();
       long timeLowerBound = System.currentTimeMillis() - dataTTL;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 7d7dafe..b4cfcbc 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -275,7 +275,11 @@ public class TsFileProcessor {
     long fileSize = tsFileResource.getFileSize();
     long fileSizeThreshold = IoTDBDescriptor.getInstance().getConfig()
         .getTsFileSizeThreshold();
-    return fileSize > fileSizeThreshold;
+    if (fileSize >= fileSizeThreshold) {
+      logger.info("{} fileSize {} >= fileSizeThreshold {}", tsFileResource.getPath(),
+          fileSize, fileSizeThreshold);
+    }
+    return fileSize >= fileSizeThreshold;
   }
 
   void syncClose() {
@@ -317,7 +321,17 @@ public class TsFileProcessor {
       logger.debug(FLUSH_QUERY_WRITE_LOCKED, storageGroupName, tsFileResource.getFile().getName());
     }
     try {
-      logger.info("Async close the file: {}", tsFileResource.getFile().getAbsolutePath());
+
+      if (workMemTable != null) {
+        logger.info(
+            "{}: flush a working memtable in async close tsfile {}, memtable size: {}, tsfile size: {}",
+            storageGroupName, tsFileResource.getFile().getAbsolutePath(), workMemTable.memSize(),
+            tsFileResource.getFileSize());
+      } else {
+        logger.info("{}: flush a NotifyFlushMemTable in async close tsfile {}, tsfile size: {}",
+            storageGroupName, tsFileResource.getFile().getAbsolutePath(), tsFileResource.getFileSize());
+      }
+
       if (shouldClose) {
         return;
       }
@@ -332,11 +346,7 @@ public class TsFileProcessor {
       //we have to add the memtable into flushingList first and then set the shouldClose tag.
       // see https://issues.apache.org/jira/browse/IOTDB-510
       IMemTable tmpMemTable = workMemTable == null ? new NotifyFlushMemTable() : workMemTable;
-      if (logger.isDebugEnabled()) {
-        logger
-            .debug("{}: {} async flush a memtable (signal = {}) when async close",
-                storageGroupName, tsFileResource.getFile().getName(), tmpMemTable.isSignalMemTable());
-      }
+
       try {
         addAMemtableIntoFlushingList(tmpMemTable);
         shouldClose = true;
@@ -575,7 +585,7 @@ public class TsFileProcessor {
         endFile();
       } catch (Exception e) {
         logger.error("{} meet error when flush FileMetadata to {}, change system mode to read-only",
-            storageGroupName, tsFileResource.getFile().getAbsolutePath());
+            storageGroupName, tsFileResource.getFile().getAbsolutePath(), e);
         IoTDBDescriptor.getInstance().getConfig().setReadOnly(true);
         try {
           writer.reset();
@@ -607,17 +617,16 @@ public class TsFileProcessor {
     // mark the TsFileResource closed, no need writer anymore
     closeTsFileCallback.call(this);
 
-    writer = null;
-
     if (logger.isInfoEnabled()) {
       long closeEndTime = System.currentTimeMillis();
-      logger.info("Storage group {} close the file {}, start time is {}, end time is {}, "
+      logger.info("Storage group {} close the file {}, TsFile size is {}, "
               + "time consumption of flushing metadata is {}ms",
           storageGroupName, tsFileResource.getFile().getAbsoluteFile(),
-          DatetimeUtils.convertMillsecondToZonedDateTime(closeStartTime),
-          DatetimeUtils.convertMillsecondToZonedDateTime(closeEndTime),
+          writer.getFile().length(),
           closeEndTime - closeStartTime);
     }
+
+    writer = null;
   }