You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/09/26 08:28:22 UTC
[iotdb] 01/01: reset currentChunkSize when flush out chunk metadata
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch fix-cmt-slow
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 843ade31bd51083a644d41e14187c58775d66b70
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Mon Sep 26 16:28:04 2022 +0800
reset currentChunkSize when flush out chunk metadata
---
.../org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java | 9 +++++----
.../org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +++++++++
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 9ee1f7f566..76ead0f1a7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -285,10 +285,11 @@ public class ChunkMetadata implements IChunkMetadata {
}
public long calculateRamSize() {
- return CHUNK_METADATA_FIXED_RAM_SIZE
- + RamUsageEstimator.sizeOf(tsFilePrefixPath)
- + RamUsageEstimator.sizeOf(measurementUid)
- + statistics.calculateRamSize();
+ long memSize = CHUNK_METADATA_FIXED_RAM_SIZE;
+ memSize += RamUsageEstimator.sizeOf(tsFilePrefixPath);
+ memSize += RamUsageEstimator.sizeOf(measurementUid);
+ memSize += statistics.calculateRamSize();
+ return memSize;
}
public static long calculateRamSize(String measurementId, TSDataType dataType) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 6cf00f8b84..2a3b2afbb6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -117,6 +117,7 @@ public class TsFileIOWriter implements AutoCloseable {
protected boolean enableMemoryControl = false;
private Path lastSerializePath = null;
protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>();
+ private volatile int chunkMetadataCount = 0;
public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta";
/** empty construct function. */
@@ -287,6 +288,7 @@ public class TsFileIOWriter implements AutoCloseable {
if (enableMemoryControl) {
this.currentChunkMetadataSize += currentChunkMetadata.calculateRamSize();
}
+ chunkMetadataCount++;
chunkMetadataList.add(currentChunkMetadata);
currentChunkMetadata = null;
}
@@ -609,7 +611,14 @@ public class TsFileIOWriter implements AutoCloseable {
// This function should be called after all data of an aligned device has been written
if (enableMemoryControl && currentChunkMetadataSize > maxMetadataSize) {
try {
+ logger.debug(
+ "Flushing chunk metadata, total size is {}, count is {}, avg size is {}",
+ currentChunkMetadataSize,
+ chunkMetadataCount,
+ currentChunkMetadataSize / chunkMetadataCount);
sortAndFlushChunkMetadata();
+ chunkMetadataCount = 0;
+ currentChunkMetadataSize = 0;
} catch (IOException e) {
logger.error("Meets exception when flushing metadata to temp file for {}", file, e);
throw e;