You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/09/26 08:28:21 UTC
[iotdb] branch fix-cmt-slow created (now 843ade31bd)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a change to branch fix-cmt-slow
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 843ade31bd reset currentChunkSize when flush out chunk metadata
This branch includes the following new commits:
new 843ade31bd reset currentChunkSize when flush out chunk metadata
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: reset currentChunkSize when flush out chunk metadata
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch fix-cmt-slow
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 843ade31bd51083a644d41e14187c58775d66b70
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Mon Sep 26 16:28:04 2022 +0800
reset currentChunkSize when flush out chunk metadata
---
.../org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java | 9 +++++----
.../org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +++++++++
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
index 9ee1f7f566..76ead0f1a7 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java
@@ -285,10 +285,11 @@ public class ChunkMetadata implements IChunkMetadata {
}
public long calculateRamSize() {
- return CHUNK_METADATA_FIXED_RAM_SIZE
- + RamUsageEstimator.sizeOf(tsFilePrefixPath)
- + RamUsageEstimator.sizeOf(measurementUid)
- + statistics.calculateRamSize();
+ long memSize = CHUNK_METADATA_FIXED_RAM_SIZE;
+ memSize += RamUsageEstimator.sizeOf(tsFilePrefixPath);
+ memSize += RamUsageEstimator.sizeOf(measurementUid);
+ memSize += statistics.calculateRamSize();
+ return memSize;
}
public static long calculateRamSize(String measurementId, TSDataType dataType) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index 6cf00f8b84..2a3b2afbb6 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -117,6 +117,7 @@ public class TsFileIOWriter implements AutoCloseable {
protected boolean enableMemoryControl = false;
private Path lastSerializePath = null;
protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>();
+ private volatile int chunkMetadataCount = 0;
public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta";
/** empty construct function. */
@@ -287,6 +288,7 @@ public class TsFileIOWriter implements AutoCloseable {
if (enableMemoryControl) {
this.currentChunkMetadataSize += currentChunkMetadata.calculateRamSize();
}
+ chunkMetadataCount++;
chunkMetadataList.add(currentChunkMetadata);
currentChunkMetadata = null;
}
@@ -609,7 +611,14 @@ public class TsFileIOWriter implements AutoCloseable {
// This function should be called after all data of an aligned device has been written
if (enableMemoryControl && currentChunkMetadataSize > maxMetadataSize) {
try {
+ logger.debug(
+ "Flushing chunk metadata, total size is {}, count is {}, avg size is {}",
+ currentChunkMetadataSize,
+ chunkMetadataCount,
+ currentChunkMetadataSize / chunkMetadataCount);
sortAndFlushChunkMetadata();
+ chunkMetadataCount = 0;
+ currentChunkMetadataSize = 0;
} catch (IOException e) {
logger.error("Meets exception when flushing metadata to temp file for {}", file, e);
throw e;