You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2023/02/23 07:44:52 UTC
[iotdb] branch rel/1.0 updated: [To rel/1.0][IOTDB-5481][IOTDB-5509]change default cross performer and inner unseq performer to FAST and Add level control for cross compaction selecting (#9117)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/1.0
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/1.0 by this push:
new 1698548e07 [To rel/1.0][IOTDB-5481][IOTDB-5509]change default cross performer and inner unseq performer to FAST and Add level control for cross compaction selecting (#9117)
1698548e07 is described below
commit 1698548e07a9eb9268fb71745d05c9434783c0fa
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Thu Feb 23 15:44:46 2023 +0800
[To rel/1.0][IOTDB-5481][IOTDB-5509]change default cross performer and inner unseq performer to FAST and Add level control for cross compaction selecting (#9117)
---
.../resources/conf/iotdb-common.properties | 4 +--
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 18 ++++++++++--
.../impl/RewriteCrossSpaceCompactionSelector.java | 15 ++++++++--
.../engine/compaction/AbstractCompactionTest.java | 6 ++++
.../engine/compaction/CompactionSchedulerTest.java | 1 +
.../CompactionSchedulerWithFastPerformerTest.java | 1 +
.../CrossSpaceCompactionWithFastPerformerTest.java | 2 ++
...eCompactionWithFastPerformerValidationTest.java | 1 +
...sSpaceCompactionWithReadPointPerformerTest.java | 2 ++
...actionWithReadPointPerformerValidationTest.java | 1 +
.../engine/compaction/cross/MergeUpgradeTest.java | 2 ++
.../cross/RewriteCompactionFileSelectorTest.java | 33 +++++++++++++++-------
.../compaction/utils/CompactionConfigRestorer.java | 4 +++
13 files changed, 74 insertions(+), 16 deletions(-)
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 92b135eb87..1254cca9a4 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -549,7 +549,7 @@ cluster_name=defaultCluster
# the compaction performer of cross space compaction task
# Options: read_point, fast (under test)
-# cross_performer=read_point
+# cross_performer=fast
# the selector of inner sequence space compaction task
# Options: size_tiered
@@ -565,7 +565,7 @@ cluster_name=defaultCluster
# the performer of inner unsequence space compaction task
# Options: read_point, fast (under test)
-# inner_unseq_performer=read_point
+# inner_unseq_performer=fast
# The priority of compaction execution
# INNER_CROSS: prioritize inner space compaction, reduce the number of files first
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 2ba70d2f05..06cf83ce33 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -422,7 +422,7 @@ public class IoTDBConfig {
InnerUnsequenceCompactionSelector.SIZE_TIERED;
private InnerUnseqCompactionPerformer innerUnseqCompactionPerformer =
- InnerUnseqCompactionPerformer.READ_POINT;
+ InnerUnseqCompactionPerformer.FAST;
/**
* The strategy of cross space compaction task. There are just one cross space compaction strategy
@@ -430,7 +430,7 @@ public class IoTDBConfig {
*/
private CrossCompactionSelector crossCompactionSelector = CrossCompactionSelector.REWRITE;
- private CrossCompactionPerformer crossCompactionPerformer = CrossCompactionPerformer.READ_POINT;
+ private CrossCompactionPerformer crossCompactionPerformer = CrossCompactionPerformer.FAST;
/**
* The priority of compaction task execution. There are three priority strategy INNER_CROSS:
@@ -478,6 +478,12 @@ public class IoTDBConfig {
/** The max total size of candidate files in cross space compaction */
private long maxCrossCompactionCandidateFileSize = 1024 * 1024 * 1024 * 5L;
+ /**
+ * Only the unseq files whose level of inner space compaction reaches this value can be selected
+ * to participate in the cross space compaction.
+ */
+ private int minCrossCompactionUnseqFileLevel = 1;
+
/** The interval of compaction task schedulation in each virtual database. The unit is ms. */
private long compactionScheduleIntervalInMs = 60_000L;
@@ -2866,6 +2872,14 @@ public class IoTDBConfig {
this.maxCrossCompactionCandidateFileSize = maxCrossCompactionCandidateFileSize;
}
+ public int getMinCrossCompactionUnseqFileLevel() {
+ return minCrossCompactionUnseqFileLevel;
+ }
+
+ public void setMinCrossCompactionUnseqFileLevel(int minCrossCompactionUnseqFileLevel) {
+ this.minCrossCompactionUnseqFileLevel = minCrossCompactionUnseqFileLevel;
+ }
+
public long getCompactionSubmissionIntervalInMs() {
return compactionSubmissionIntervalInMs;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index 6a671ebc79..9298f3bb5c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.engine.compaction.selector.estimator.AbstractCompacti
import org.apache.iotdb.db.engine.compaction.selector.impl.CrossSpaceCompactionCandidate.CrossCompactionTaskResourceSplit;
import org.apache.iotdb.db.engine.compaction.selector.impl.CrossSpaceCompactionCandidate.TsFileResourceCandidate;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
+import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.MergeException;
import org.apache.iotdb.db.rescon.SystemInfo;
@@ -185,10 +186,20 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector
CrossCompactionTaskResource taskResource,
TsFileResource unseqFile,
List<TsFileResource> seqFiles,
- long memoryCost) {
+ long memoryCost)
+ throws IOException {
+ TsFileNameGenerator.TsFileName unseqFileName =
+ TsFileNameGenerator.getTsFileName(unseqFile.getTsFile().getName());
+ // we add a hard limit for cross compaction that selected unseqFile should be compacted in inner
+ // space at least once. This is used to make to improve the priority of inner compaction and
+ // avoid too much cross compaction with small files.
+ if (unseqFileName.getInnerCompactionCnt() < config.getMinCrossCompactionUnseqFileLevel()) {
+ return false;
+ }
// currently, we must allow at least one unseqFile be selected to handle the situation that
// an unseqFile has huge time range but few data points.
- if (taskResource.getUnseqFiles().size() == 0) {
+ // IMPORTANT: this logic is opposite to previous level control
+ if (taskResource.getUnseqFiles().isEmpty()) {
return true;
}
long totalFileSize = unseqFile.getTsFileSize();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
index dbf41d5390..19ea62e569 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/AbstractCompactionTest.java
@@ -106,6 +106,9 @@ public class AbstractCompactionTest {
private final long oldLowerTargetChunkPointNum =
IoTDBDescriptor.getInstance().getConfig().getChunkPointNumLowerBoundInCompaction();
+ private int oldMinCrossCompactionUnseqLevel =
+ IoTDBDescriptor.getInstance().getConfig().getMinCrossCompactionUnseqFileLevel();
+
protected static File STORAGE_GROUP_DIR =
new File(
TestConstant.BASE_OUTPUT_PATH
@@ -374,6 +377,9 @@ public class AbstractCompactionTest {
CompactionTaskManager.getInstance().stop();
seqResources.clear();
unseqResources.clear();
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMinCrossCompactionUnseqFileLevel(oldMinCrossCompactionUnseqLevel);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(oldTargetChunkSize);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkPointNum(oldTargetChunkPointNum);
IoTDBDescriptor.getInstance()
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
index 8638fd1607..d8ebdf1121 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
@@ -99,6 +99,7 @@ public class CompactionSchedulerTest {
IoTDBDescriptor.getInstance()
.getConfig()
.setInnerUnseqCompactionPerformer(InnerUnseqCompactionPerformer.READ_POINT);
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
CompactionTaskManager.getInstance().start();
while (CompactionTaskManager.getInstance().getExecutingTaskCount() > 0) {
try {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
index fdffe757a1..15fa53426e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerWithFastPerformerTest.java
@@ -99,6 +99,7 @@ public class CompactionSchedulerWithFastPerformerTest {
IoTDBDescriptor.getInstance()
.getConfig()
.setInnerUnseqCompactionPerformer(InnerUnseqCompactionPerformer.FAST);
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
CompactionTaskManager.getInstance().start();
while (CompactionTaskManager.getInstance().getExecutingTaskCount() > 0) {
try {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java
index 2aed739cf0..817bbb9d36 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerTest.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.cache.BloomFilterCache;
import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
@@ -96,6 +97,7 @@ public class CrossSpaceCompactionWithFastPerformerTest {
@Before
public void setUp() throws MetadataException {
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
CompactionTaskManager.getInstance().start();
Thread.currentThread().setName("pool-1-IoTDB-Compaction-1");
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
index 9aced0d1c5..b43120691c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithFastPerformerValidationTest.java
@@ -69,6 +69,7 @@ public class CrossSpaceCompactionWithFastPerformerValidationTest extends Abstrac
public void setUp()
throws IOException, WriteProcessException, MetadataException, InterruptedException {
super.setUp();
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-1");
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java
index 7de635c197..7b1535f715 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerTest.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.exception.MetadataException;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.cache.BloomFilterCache;
import org.apache.iotdb.db.engine.cache.ChunkCache;
import org.apache.iotdb.db.engine.cache.TimeSeriesMetadataCache;
@@ -96,6 +97,7 @@ public class CrossSpaceCompactionWithReadPointPerformerTest {
@Before
public void setUp() throws MetadataException {
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
CompactionTaskManager.getInstance().start();
Thread.currentThread().setName("pool-1-IoTDB-Compaction-1");
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
index a772891801..b04e838a8b 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionWithReadPointPerformerValidationTest.java
@@ -67,6 +67,7 @@ public class CrossSpaceCompactionWithReadPointPerformerValidationTest
public void setUp()
throws IOException, WriteProcessException, MetadataException, InterruptedException {
super.setUp();
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
IoTDBDescriptor.getInstance().getConfig().setTargetChunkSize(1024);
TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(30);
Thread.currentThread().setName("pool-1-IoTDB-Compaction-1");
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java
index 73e00bb31e..2c829716e2 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/MergeUpgradeTest.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.compaction.selector.impl.CrossCompactionTaskResource;
import org.apache.iotdb.db.engine.compaction.selector.impl.RewriteCrossSpaceCompactionSelector;
@@ -66,6 +67,7 @@ public class MergeUpgradeTest {
@Before
public void setUp() throws IOException, WriteProcessException {
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
prepareSeries();
prepareFiles();
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
index acdac11994..4c8f19b0c8 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.compaction.cross;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.constant.TestConstant;
import org.apache.iotdb.db.engine.compaction.selector.impl.CrossCompactionTaskResource;
@@ -29,6 +30,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.engine.storagegroup.timeindex.ITimeIndex;
import org.apache.iotdb.db.exception.MergeException;
+import org.apache.iotdb.db.exception.StorageEngineException;
import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.read.common.Path;
@@ -38,7 +40,9 @@ import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.apache.commons.io.FileUtils;
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -57,6 +61,20 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
private static final Logger logger =
LoggerFactory.getLogger(RewriteCompactionFileSelectorTest.class);
+ private int oldMinCrossCompactionUnseqLevel =
+ IoTDBDescriptor.getInstance().getConfig().getMinCrossCompactionUnseqFileLevel();
+
+ @Before
+ public void setUp() throws IOException, MetadataException, WriteProcessException {
+ super.setUp();
+ IoTDBDescriptor.getInstance().getConfig().setMinCrossCompactionUnseqFileLevel(0);
+ }
+
+ @After
+ public void tearDown() throws StorageEngineException, IOException {
+ super.tearDown();
+ }
+
@Test
public void testFullSelection() throws MergeException, IOException {
RewriteCrossSpaceCompactionSelector selector =
@@ -120,8 +138,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
File file =
new File(
TestConstant.BASE_OUTPUT_PATH.concat(
- 10
- + "unseq"
+ System.currentTimeMillis()
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
@@ -172,8 +189,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
File file =
new File(
TestConstant.BASE_OUTPUT_PATH.concat(
- 10
- + "unseq"
+ System.currentTimeMillis()
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
@@ -224,8 +240,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
File file =
new File(
TestConstant.BASE_OUTPUT_PATH.concat(
- 10
- + "unseq"
+ System.currentTimeMillis()
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ 10
+ IoTDBConstant.FILE_NAME_SEPARATOR
@@ -492,8 +507,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
File file =
new File(
TestConstant.OUTPUT_DATA_DIR.concat(
- 10
- + "seq"
+ System.currentTimeMillis()
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
@@ -512,8 +526,7 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
File file =
new File(
TestConstant.OUTPUT_DATA_DIR.concat(
- 10
- + "unseq"
+ System.currentTimeMillis()
+ IoTDBConstant.FILE_NAME_SEPARATOR
+ i
+ IoTDBConstant.FILE_NAME_SEPARATOR
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
index 5896693a21..acfc5175d4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/utils/CompactionConfigRestorer.java
@@ -54,6 +54,9 @@ public class CompactionConfigRestorer {
private InnerUnseqCompactionPerformer oldInnerUnseqPerformer =
IoTDBDescriptor.getInstance().getConfig().getInnerUnseqCompactionPerformer();
+ private int oldMinCrossCompactionUnseqLevel =
+ IoTDBDescriptor.getInstance().getConfig().getMinCrossCompactionUnseqFileLevel();
+
public CompactionConfigRestorer() {}
public void restoreCompactionConfig() {
@@ -78,5 +81,6 @@ public class CompactionConfigRestorer {
config.setCrossCompactionPerformer(oldCrossPerformer);
config.setInnerSeqCompactionPerformer(oldInnerSeqPerformer);
config.setInnerUnseqCompactionPerformer(oldInnerUnseqPerformer);
+ config.setMinCrossCompactionUnseqFileLevel(oldMinCrossCompactionUnseqLevel);
}
}