You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/10/13 14:33:57 UTC
[iotdb] branch master updated: [IOTDB-4584] Correct the ChunkMetadata proportion in writing TsFile (#7555)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ad81f58a74 [IOTDB-4584] Correct the ChunkMetadata proportion in writing TsFile (#7555)
ad81f58a74 is described below
commit ad81f58a74b40ceb165ce5a7136487c0e9d603a8
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Thu Oct 13 22:33:49 2022 +0800
[IOTDB-4584] Correct the ChunkMetadata proportion in writing TsFile (#7555)
---
.../resources/conf/iotdb-datanode.properties | 8 ++-----
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 25 ++++++++--------------
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 12 +++--------
.../RewriteCrossSpaceCompactionSelector.java | 7 ++++--
.../impl/ReadChunkCompactionPerformer.java | 4 +---
.../writer/CrossSpaceCompactionWriter.java | 4 +---
.../writer/InnerSpaceCompactionWriter.java | 4 +---
.../iotdb/db/engine/flush/MemTableFlushTask.java | 1 -
.../db/engine/storagegroup/TsFileProcessor.java | 9 +-------
.../file/AbstractTsFileRecoverPerformer.java | 10 +--------
10 files changed, 24 insertions(+), 60 deletions(-)
diff --git a/server/src/assembly/resources/conf/iotdb-datanode.properties b/server/src/assembly/resources/conf/iotdb-datanode.properties
index 38c50a32c6..32d732d310 100644
--- a/server/src/assembly/resources/conf/iotdb-datanode.properties
+++ b/server/src/assembly/resources/conf/iotdb-datanode.properties
@@ -466,9 +466,9 @@ timestamp_precision=ms
# Datatype: int
# primitive_array_size=32
-# the percentage of write memory for chunk metadata remains in a single file writer when flushing memtable
+# size proportion for chunk metadata maintains in memory when writing tsfile
# Datatype: double
-# chunk_metadata_size_proportion_in_write=0.1
+# chunk_metadata_size_proportion=0.1
# Ratio of write memory for invoking flush disk, 0.4 by default
# If you have extremely high write load (like batch=1000), it can be set lower than the default value like 0.2
@@ -586,10 +586,6 @@ timestamp_precision=ms
# BALANCE: alternate two compaction types
# compaction_priority=BALANCE
-# size proportion for chunk metadata maintains in memory when compacting
-# Datatype: double
-# chunk_metadata_size_proportion_in_compaction=0.05
-
# The target tsfile size in compaction
# Datatype: long, Unit: byte
# target_compaction_file_size=1073741824
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index b9260ca254..15c9531e79 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -152,8 +152,6 @@ public class IoTDBConfig {
/** The proportion of write memory for memtable */
private double writeProportion = 0.8;
- private double chunkMetadataSizeProportionInWrite = 0.1;
-
/** The proportion of write memory for compaction */
private double compactionProportion = 0.2;
@@ -441,7 +439,7 @@ public class IoTDBConfig {
*/
private CompactionPriority compactionPriority = CompactionPriority.BALANCE;
- private double chunkMetadataSizeProportionInCompaction = 0.05;
+ private double chunkMetadataSizeProportion = 0.1;
/** The target tsfile size in compaction, 1 GB by default */
private long targetCompactionFileSize = 1073741824L;
@@ -3241,21 +3239,12 @@ public class IoTDBConfig {
this.throttleThreshold = throttleThreshold;
}
- public double getChunkMetadataSizeProportionInWrite() {
- return chunkMetadataSizeProportionInWrite;
- }
-
- public void setChunkMetadataSizeProportionInWrite(double chunkMetadataSizeProportionInWrite) {
- this.chunkMetadataSizeProportionInWrite = chunkMetadataSizeProportionInWrite;
+ public double getChunkMetadataSizeProportion() {
+ return chunkMetadataSizeProportion;
}
- public double getChunkMetadataSizeProportionInCompaction() {
- return chunkMetadataSizeProportionInCompaction;
- }
-
- public void setChunkMetadataSizeProportionInCompaction(
- double chunkMetadataSizeProportionInCompaction) {
- this.chunkMetadataSizeProportionInCompaction = chunkMetadataSizeProportionInCompaction;
+ public void setChunkMetadataSizeProportion(double chunkMetadataSizeProportion) {
+ this.chunkMetadataSizeProportion = chunkMetadataSizeProportion;
}
public long getCacheWindowTimeInMs() {
@@ -3428,4 +3417,8 @@ public class IoTDBConfig {
this.schemaRatisConsensusLeaderElectionTimeoutMaxMs =
schemaRatisConsensusLeaderElectionTimeoutMaxMs;
}
+
+ public double getUsableCompactionMemoryProportion() {
+ return 1.0d - chunkMetadataSizeProportion;
+ }
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 4509cb44a3..1e3e13a509 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -630,11 +630,11 @@ public class IoTDBDescriptor {
properties.getProperty(
"concurrent_compaction_thread",
Integer.toString(conf.getConcurrentCompactionThread()))));
- conf.setChunkMetadataSizeProportionInCompaction(
+ conf.setChunkMetadataSizeProportion(
Double.parseDouble(
properties.getProperty(
- "chunk_metadata_size_proportion_in_compaction",
- Double.toString(conf.getChunkMetadataSizeProportionInCompaction()))));
+ "chunk_metadata_size_proportion",
+ Double.toString(conf.getChunkMetadataSizeProportion()))));
conf.setTargetCompactionFileSize(
Long.parseLong(
properties.getProperty(
@@ -1473,12 +1473,6 @@ public class IoTDBDescriptor {
// update tsfile-format config
loadTsFileProps(properties);
- conf.setChunkMetadataSizeProportionInWrite(
- Double.parseDouble(
- properties.getProperty(
- "chunk_metadata_size_proportion_in_write",
- Double.toString(conf.getChunkMetadataSizeProportionInWrite()))));
-
// update max_deduplicated_path_num
conf.setMaxQueryDeduplicatedPathNum(
Integer.parseInt(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
index b314757ef8..50beb34a19 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
@@ -81,8 +81,11 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector
this.timePartition = timePartition;
this.tsFileManager = tsFileManager;
this.memoryBudget =
- SystemInfo.getInstance().getMemorySizeForCompaction()
- / IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread();
+ (long)
+ ((double)
+ (SystemInfo.getInstance().getMemorySizeForCompaction()
+ / IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread())
+ * config.getUsableCompactionMemoryProportion());
this.maxCrossCompactionFileNum =
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
this.maxCrossCompactionFileSize =
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java
index ac0fa1ddc4..582cc82e15 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/performer/impl/ReadChunkCompactionPerformer.java
@@ -69,9 +69,7 @@ public class ReadChunkCompactionPerformer implements ISeqCompactionPerformer {
(long)
(SystemInfo.getInstance().getMemorySizeForCompaction()
/ IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread()
- * IoTDBDescriptor.getInstance()
- .getConfig()
- .getChunkMetadataSizeProportionInCompaction());
+ * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
try (MultiTsFileDeviceIterator deviceIterator = new MultiTsFileDeviceIterator(seqFiles);
TsFileIOWriter writer =
new TsFileIOWriter(targetResource.getTsFile(), true, sizeForFileWriter)) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/CrossSpaceCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/CrossSpaceCompactionWriter.java
index 07cc2019aa..b85cbffb73 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/CrossSpaceCompactionWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/CrossSpaceCompactionWriter.java
@@ -79,9 +79,7 @@ public class CrossSpaceCompactionWriter extends AbstractCompactionWriter {
(long)
(SystemInfo.getInstance().getMemorySizeForCompaction()
/ IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread()
- * IoTDBDescriptor.getInstance()
- .getConfig()
- .getChunkMetadataSizeProportionInCompaction()
+ * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion()
/ targetResources.size());
for (int i = 0; i < targetResources.size(); i++) {
this.fileWriterList.add(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/InnerSpaceCompactionWriter.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/InnerSpaceCompactionWriter.java
index 978067e800..d48e609316 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/InnerSpaceCompactionWriter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/writer/InnerSpaceCompactionWriter.java
@@ -48,9 +48,7 @@ public class InnerSpaceCompactionWriter extends AbstractCompactionWriter {
(long)
(SystemInfo.getInstance().getMemorySizeForCompaction()
/ IoTDBDescriptor.getInstance().getConfig().getConcurrentCompactionThread()
- * IoTDBDescriptor.getInstance()
- .getConfig()
- .getChunkMetadataSizeProportionInCompaction());
+ * IoTDBDescriptor.getInstance().getConfig().getChunkMetadataSizeProportion());
this.fileWriter = new TsFileIOWriter(targetFileResource.getTsFile(), true, sizeForFileWriter);
isEmptyFile = true;
resource = targetFileResource;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index 6c6ab965b6..a86380749d 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -282,7 +282,6 @@ public class MemTableFlushTask {
this.writer.endChunkGroup();
} else {
((IChunkWriter) ioMessage).writeToFileWriter(this.writer);
- writer.checkMetadataSizeAndMayFlush();
}
} catch (IOException e) {
LOGGER.error(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
index 7036be7e0d..841055e460 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessor.java
@@ -179,14 +179,7 @@ public class TsFileProcessor {
this.storageGroupName = storageGroupName;
this.tsFileResource = new TsFileResource(tsfile, this);
this.storageGroupInfo = storageGroupInfo;
- this.writer =
- new RestorableTsFileIOWriter(
- tsfile,
- (long)
- (IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold()
- * IoTDBDescriptor.getInstance()
- .getConfig()
- .getChunkMetadataSizeProportionInWrite()));
+ this.writer = new RestorableTsFileIOWriter(tsfile);
this.updateLatestFlushTimeCallback = updateLatestFlushTimeCallback;
this.sequence = sequence;
this.walNode = WALManager.getInstance().applyForWALNode(storageGroupName);
diff --git a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
index e506d66c3a..2378b81332 100644
--- a/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
+++ b/server/src/main/java/org/apache/iotdb/db/wal/recover/file/AbstractTsFileRecoverPerformer.java
@@ -18,7 +18,6 @@
*/
package org.apache.iotdb.db.wal.recover.file;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.utils.FileLoaderUtils;
@@ -77,14 +76,7 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable {
// try to remove corrupted part of the TsFile
try {
- writer =
- new RestorableTsFileIOWriter(
- tsFile,
- (long)
- (IoTDBDescriptor.getInstance().getConfig().getMemtableSizeThreshold()
- * IoTDBDescriptor.getInstance()
- .getConfig()
- .getChunkMetadataSizeProportionInWrite()));
+ writer = new RestorableTsFileIOWriter(tsFile);
} catch (NotCompatibleTsFileException e) {
boolean result = tsFile.delete();
logger.warn(