You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/09/24 08:23:08 UTC
[iotdb] branch rel/0.13 updated: [To rel/0.13][IOTDB-4492] Control total size of cross space compaction task (#7420)
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new a48dca2bc2 [To rel/0.13][IOTDB-4492] Control total size of cross space compaction task (#7420)
a48dca2bc2 is described below
commit a48dca2bc27301c004ec9ff38babb31d0ff293e7
Author: Mrquan <50...@users.noreply.github.com>
AuthorDate: Sat Sep 24 16:23:02 2022 +0800
[To rel/0.13][IOTDB-4492] Control total size of cross space compaction task (#7420)
---
server/src/assembly/resources/conf/iotdb-engine.properties | 5 +++++
.../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++
.../main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++++
.../rewrite/selector/RewriteCompactionFileSelector.java | 12 +++++++++++-
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties
index 9e5735ea3b..e17a900a25 100644
--- a/server/src/assembly/resources/conf/iotdb-engine.properties
+++ b/server/src/assembly/resources/conf/iotdb-engine.properties
@@ -477,6 +477,11 @@ timestamp_precision=ms
# Datatype: int
# max_cross_compaction_candidate_file_num=1000
+# The max total size when selecting cross space compaction candidate files
+# At least one unseq file with it's overlapped seq files will be selected even exceeded this number
+# Datatype: long, Unit: byte
+# max_cross_compaction_candidate_file_size=5368709120
+
# If one merge file selection runs for more than this time, it will be ended and its current
# selection will be used as final selection.
# When < 0, it means time is unbounded.
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index dcf401dfdd..f4cfc65920 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -431,6 +431,9 @@ public class IoTDBConfig {
/** The max candidate file num in cross space compaction */
private int maxCrossCompactionCandidateFileNum = 1000;
+ /** The max total size of candidate files in cross space compaction */
+ private long maxCrossCompactionCandidateFileSize = 1024 * 1024 * 1024 * 5L;
+
/** The interval of compaction task schedulation in each virtual storage group. The unit is ms. */
private long compactionScheduleIntervalInMs = 60_000L;
@@ -2571,6 +2574,14 @@ public class IoTDBConfig {
this.maxCrossCompactionCandidateFileNum = maxCrossCompactionCandidateFileNum;
}
+ public long getMaxCrossCompactionCandidateFileSize() {
+ return maxCrossCompactionCandidateFileSize;
+ }
+
+ public void setMaxCrossCompactionCandidateFileSize(long maxCrossCompactionCandidateFileSize) {
+ this.maxCrossCompactionCandidateFileSize = maxCrossCompactionCandidateFileSize;
+ }
+
public long getCompactionSubmissionIntervalInMs() {
return compactionSubmissionIntervalInMs;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index e39cbb820f..815113f6b4 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -567,6 +567,12 @@ public class IoTDBDescriptor {
"max_cross_compaction_candidate_file_num",
Integer.toString(conf.getMaxCrossCompactionCandidateFileNum()))));
+ conf.setMaxCrossCompactionCandidateFileSize(
+ Long.parseLong(
+ properties.getProperty(
+ "max_cross_compaction_candidate_file_size",
+ Long.toString(conf.getMaxCrossCompactionCandidateFileSize()))));
+
conf.setCompactionWriteThroughputMbPerSec(
Integer.parseInt(
properties.getProperty(
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
index 396edb7f5e..be9cad8395 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
@@ -68,7 +68,8 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
private Collection<Integer> tmpSelectedSeqFiles;
private long tempMaxSeqFileCost;
-
+ private long totalSize;
+ private final long maxCrossCompactionFileSize;
private boolean[] seqSelected;
private int seqSelectedNum;
@@ -77,6 +78,8 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
this.memoryBudget = memoryBudget;
this.maxCrossCompactionFileNum =
IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileNum();
+ this.maxCrossCompactionFileSize =
+ IoTDBDescriptor.getInstance().getConfig().getMaxCrossCompactionCandidateFileSize();
}
/**
@@ -104,6 +107,7 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
@Override
public List[] select() throws MergeException {
long startTime = System.currentTimeMillis();
+ totalSize = 0;
try {
logger.debug(
"Selecting merge candidates from {} seqFile, {} unseqFiles",
@@ -185,6 +189,11 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
}
}
+ for (int seqIndex : tmpSelectedSeqFiles) {
+ totalSize += resource.getSeqFiles().get(seqIndex).getTsFileSize();
+ }
+ totalSize += unseqFile.getTsFileSize();
+
tempMaxSeqFileCost = maxSeqFileCost;
long newCost =
useTightBound
@@ -210,6 +219,7 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
if (selectedUnseqFiles.size() == 0
|| (seqSelectedNum + selectedUnseqFiles.size() + 1 + tmpSelectedSeqFiles.size()
<= maxCrossCompactionFileNum
+ && totalSize <= maxCrossCompactionFileSize
&& totalCost + newCost < memoryBudget)) {
selectedUnseqFiles.add(unseqFile);
maxSeqFileCost = tempMaxSeqFileCost;