You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2022/05/10 01:24:13 UTC
[iotdb] branch rel/0.12 updated: use param MaxSelectUnseqFileNumInEachUnseqCompaction (#5849)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 69156ec987 use param MaxSelectUnseqFileNumInEachUnseqCompaction (#5849)
69156ec987 is described below
commit 69156ec987180f8538f06af460089cce7902aa21
Author: Liu Xuxin <37...@users.noreply.github.com>
AuthorDate: Tue May 10 09:24:08 2022 +0800
use param MaxSelectUnseqFileNumInEachUnseqCompaction (#5849)
---
client-cpp/pom.xml | 4 +-
compile-tools/pom.xml | 6 +-
distribution/pom.xml | 2 +-
example/client-cpp-example/pom.xml | 2 +-
example/udf/pom.xml | 2 +-
grafana/pom.xml | 2 +-
jdbc/pom.xml | 2 +-
pom.xml | 8 +-
.../merge/selector/MaxFileMergeFileSelector.java | 6 +-
.../engine/merge/MaxFileMergeFileSelectorTest.java | 232 +++++++++++++++++++++
10 files changed, 251 insertions(+), 15 deletions(-)
diff --git a/client-cpp/pom.xml b/client-cpp/pom.xml
index 8876f57b06..124dc27555 100644
--- a/client-cpp/pom.xml
+++ b/client-cpp/pom.xml
@@ -98,8 +98,8 @@
<cmake.root.dir>${project.parent.basedir}/compile-tools/thrift/target/cmake-${cmake-version}-win64-x64/</cmake.root.dir>
<thrift.exec.absolute.path>${project.parent.basedir}/compile-tools/thrift/target/build/compiler/cpp/bin/${cmake.build.type}/thrift.exe</thrift.exec.absolute.path>
<iotdb.server.script>start-server.bat</iotdb.server.script>
- <boost.include.dir />
- <boost.library.dir />
+ <boost.include.dir/>
+ <boost.library.dir/>
</properties>
</profile>
<profile>
diff --git a/compile-tools/pom.xml b/compile-tools/pom.xml
index 3f31be2692..55a7e25147 100644
--- a/compile-tools/pom.xml
+++ b/compile-tools/pom.xml
@@ -35,7 +35,7 @@
<cmake-version>3.17.3</cmake-version>
<openssl.include.dir>-Dtrue1=true1</openssl.include.dir>
<bison.executable.dir>-Dtrue1=true1</bison.executable.dir>
- <cmake.build.type />
+ <cmake.build.type/>
</properties>
<modules>
<module>thrift</module>
@@ -114,8 +114,8 @@
<thrift.make.executable>make</thrift.make.executable>
<thrift.compiler.executable>thrift.exe</thrift.compiler.executable>
<gradlew.executable>gradlew.bat</gradlew.executable>
- <boost.include.dir />
- <boost.library.dir />
+ <boost.include.dir/>
+ <boost.library.dir/>
</properties>
</profile>
</profiles>
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 0a9242afd3..67bb6467c7 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -29,7 +29,7 @@
</parent>
<artifactId>iotdb-distribution</artifactId>
<name>IoTDB Distribution</name>
- <modules />
+ <modules/>
<build>
<plugins>
<plugin>
diff --git a/example/client-cpp-example/pom.xml b/example/client-cpp-example/pom.xml
index 28adfeb028..7511f52b7e 100644
--- a/example/client-cpp-example/pom.xml
+++ b/example/client-cpp-example/pom.xml
@@ -69,7 +69,7 @@
<properties>
<cmake.generator>Visual Studio 16 2019</cmake.generator>
<cmake.root.dir>${project.parent.basedir}/../compile-tools/thrift/target/cmake-${cmake-version}-win64-x64/</cmake.root.dir>
- <boost.include.dir />
+ <boost.include.dir/>
</properties>
</profile>
<profile>
diff --git a/example/udf/pom.xml b/example/udf/pom.xml
index 09d4a814e9..6b627ae43c 100644
--- a/example/udf/pom.xml
+++ b/example/udf/pom.xml
@@ -77,7 +77,7 @@
<importOrder>
<order>org.apache.iotdb,,javax,java,\#</order>
</importOrder>
- <removeUnusedImports />
+ <removeUnusedImports/>
</java>
</configuration>
<executions>
diff --git a/grafana/pom.xml b/grafana/pom.xml
index 0897bfaaf4..a4c09b1fa8 100644
--- a/grafana/pom.xml
+++ b/grafana/pom.xml
@@ -170,7 +170,7 @@
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
<resource>META-INF/spring.schemas</resource>
</transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>${start-class}</mainClass>
</transformer>
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index 4b579d2782..a9bf04079e 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -198,7 +198,7 @@
</goals>
</pluginExecutionFilter>
<action>
- <ignore />
+ <ignore/>
</action>
</pluginExecution>
</pluginExecutions>
diff --git a/pom.xml b/pom.xml
index 2ecee0d20b..5318a29530 100644
--- a/pom.xml
+++ b/pom.xml
@@ -151,7 +151,7 @@
<sonar.junit.reportPaths>target/surefire-reports,target/failsafe-reports</sonar.junit.reportPaths>
<!-- By default, the argLine is empty-->
<gson.version>2.8.6</gson.version>
- <argLine />
+ <argLine/>
<!-- whether enable compiling the cpp client-->
<client-cpp>false</client-cpp>
<!-- disable enforcer by default-->
@@ -688,7 +688,7 @@
<importOrder>
<order>org.apache.iotdb,,javax,java,\#</order>
</importOrder>
- <removeUnusedImports />
+ <removeUnusedImports/>
</java>
<lineEndings>UNIX</lineEndings>
</configuration>
@@ -760,7 +760,7 @@
<phase>validate</phase>
<configuration>
<rules>
- <dependencyConvergence />
+ <dependencyConvergence/>
</rules>
</configuration>
<goals>
@@ -806,7 +806,7 @@
</requireJavaVersion>
<!-- Disabled for now as it breaks the ability to build single modules -->
<!--reactorModuleConvergence/-->
- <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies" />
+ <banVulnerable implementation="org.sonatype.ossindex.maven.enforcer.BanVulnerableDependencies"/>
</rules>
</configuration>
</execution>
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
index 9a4f672ad4..f3da117f2e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
@@ -145,10 +145,14 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
long startTime = System.currentTimeMillis();
long timeConsumption = 0;
long timeLimit = IoTDBDescriptor.getInstance().getConfig().getMergeFileSelectionTimeBudget();
+ int maxSelectedUnseqFileNum =
+ IoTDBDescriptor.getInstance().getConfig().getMaxSelectUnseqFileNumInEachUnseqCompaction();
if (timeLimit < 0) {
timeLimit = Long.MAX_VALUE;
}
- while (unseqIndex < resource.getUnseqFiles().size() && timeConsumption < timeLimit) {
+ while (unseqIndex < resource.getUnseqFiles().size()
+ && timeConsumption < timeLimit
+ && unseqIndex < maxSelectedUnseqFileNum) {
// select next unseq files
TsFileResource unseqFile = resource.getUnseqFiles().get(unseqIndex);
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
index 6e8a4c1ae3..b90a10ff35 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
@@ -861,4 +861,236 @@ public class MaxFileMergeFileSelectorTest extends MergeTest {
Assert.assertEquals(2, result[0].size());
Assert.assertEquals(2, result[1].size());
}
+
+ @Test
+ public void testSelectLimitedNumFiles()
+ throws IOException, WriteProcessException, MergeException {
+ int originNum =
+ IoTDBDescriptor.getInstance().getConfig().getMaxSelectUnseqFileNumInEachUnseqCompaction();
+ IoTDBDescriptor.getInstance().getConfig().setMaxSelectUnseqFileNumInEachUnseqCompaction(1);
+ try {
+ List<TsFileResource> seqList = new ArrayList<>();
+ List<TsFileResource> unseqList = new ArrayList<>();
+ // first file [0, 10]
+ // first device [0, 5]
+ // second device [0, 10]
+ File firstFile =
+ new File(
+ TestConstant.OUTPUT_DATA_DIR.concat(
+ 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource firstTsFileResource = new TsFileResource(firstFile);
+ firstTsFileResource.setClosed(true);
+ firstTsFileResource.setMinPlanIndex(1);
+ firstTsFileResource.setMaxPlanIndex(1);
+ firstTsFileResource.setVersion(1);
+ seqList.add(firstTsFileResource);
+ if (!firstFile.getParentFile().exists()) {
+ Assert.assertTrue(firstFile.getParentFile().mkdirs());
+ }
+
+ TsFileWriter fileWriter = new TsFileWriter(firstFile);
+ for (String deviceId : deviceIds) {
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ fileWriter.registerTimeseries(
+ new Path(deviceId, measurementSchema.getMeasurementId()), measurementSchema);
+ }
+ }
+ for (long i = 0; i < 10; ++i) {
+ for (int j = 0; j < deviceNum; j++) {
+ if (j == 3 && i > 5) {
+ continue;
+ }
+ TSRecord record = new TSRecord(i, deviceIds[j]);
+ for (int k = 0; k < measurementNum; ++k) {
+ record.addTuple(
+ DataPoint.getDataPoint(
+ measurementSchemas[k].getType(),
+ measurementSchemas[k].getMeasurementId(),
+ String.valueOf(i)));
+ }
+ fileWriter.write(record);
+ firstTsFileResource.updateStartTime(deviceIds[j], i);
+ firstTsFileResource.updateEndTime(deviceIds[j], i);
+ }
+ }
+
+ fileWriter.flushAllChunkGroups();
+ fileWriter.close();
+
+ // second file time range: [11, 20]
+ // first measurement: [11, 20]
+ // second measurement: [11, 20]
+ File secondFile =
+ new File(
+ TestConstant.OUTPUT_DATA_DIR.concat(
+ 2
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 2
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource secondTsFileResource = new TsFileResource(secondFile);
+ secondTsFileResource.setClosed(true);
+ secondTsFileResource.setMinPlanIndex(2);
+ secondTsFileResource.setMaxPlanIndex(2);
+ secondTsFileResource.setVersion(2);
+ seqList.add(secondTsFileResource);
+
+ if (!secondFile.getParentFile().exists()) {
+ Assert.assertTrue(secondFile.getParentFile().mkdirs());
+ }
+ fileWriter = new TsFileWriter(secondFile);
+ for (String deviceId : deviceIds) {
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ fileWriter.registerTimeseries(
+ new Path(deviceId, measurementSchema.getMeasurementId()), measurementSchema);
+ }
+ }
+ for (long i = 11; i < 21; ++i) {
+ for (int j = 0; j < deviceNum; j++) {
+ TSRecord record = new TSRecord(i, deviceIds[j]);
+ for (int k = 0; k < measurementNum; k++) {
+ record.addTuple(
+ DataPoint.getDataPoint(
+ measurementSchemas[k].getType(),
+ measurementSchemas[k].getMeasurementId(),
+ String.valueOf(i)));
+ }
+ fileWriter.write(record);
+ secondTsFileResource.updateStartTime(deviceIds[j], i);
+ secondTsFileResource.updateEndTime(deviceIds[j], i);
+ }
+ }
+ fileWriter.flushAllChunkGroups();
+ fileWriter.close();
+
+ // unseq file: [0, 1]
+ File thirdFile =
+ new File(
+ TestConstant.OUTPUT_DATA_DIR.concat(
+ 3
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 3
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource thirdTsFileResource = new TsFileResource(thirdFile);
+ thirdTsFileResource.setClosed(true);
+ thirdTsFileResource.setMinPlanIndex(3);
+ thirdTsFileResource.setMaxPlanIndex(3);
+ thirdTsFileResource.setVersion(3);
+ unseqList.add(thirdTsFileResource);
+
+ if (!secondFile.getParentFile().exists()) {
+ Assert.assertTrue(thirdFile.getParentFile().mkdirs());
+ }
+ fileWriter = new TsFileWriter(thirdFile);
+ for (String deviceId : deviceIds) {
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ fileWriter.registerTimeseries(
+ new Path(deviceId, measurementSchema.getMeasurementId()), measurementSchema);
+ }
+ }
+ for (long i = 0; i < 2; ++i) {
+ for (int j = 0; j < deviceNum; j++) {
+ TSRecord record = new TSRecord(i, deviceIds[j]);
+ for (int k = 0; k < measurementNum; k++) {
+ record.addTuple(
+ DataPoint.getDataPoint(
+ measurementSchemas[k].getType(),
+ measurementSchemas[k].getMeasurementId(),
+ String.valueOf(i)));
+ }
+ fileWriter.write(record);
+ thirdTsFileResource.updateStartTime(deviceIds[j], i);
+ thirdTsFileResource.updateEndTime(deviceIds[j], i);
+ }
+ }
+ fileWriter.flushAllChunkGroups();
+ fileWriter.close();
+
+ // unseq file: [6, 8]
+ File fourthFile =
+ new File(
+ TestConstant.OUTPUT_DATA_DIR.concat(
+ 4
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 4
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource fourthTsFileResource = new TsFileResource(fourthFile);
+ fourthTsFileResource.setClosed(true);
+ fourthTsFileResource.setMinPlanIndex(4);
+ fourthTsFileResource.setMaxPlanIndex(4);
+ fourthTsFileResource.setVersion(4);
+ unseqList.add(fourthTsFileResource);
+
+ if (!fourthFile.getParentFile().exists()) {
+ Assert.assertTrue(fourthFile.getParentFile().mkdirs());
+ }
+ fileWriter = new TsFileWriter(fourthFile);
+ for (String deviceId : deviceIds) {
+ for (MeasurementSchema measurementSchema : measurementSchemas) {
+ fileWriter.registerTimeseries(
+ new Path(deviceId, measurementSchema.getMeasurementId()), measurementSchema);
+ }
+ }
+ for (long i = 6; i < 15; ++i) {
+ for (int j = 0; j < deviceNum; j++) {
+ if (j == 3) {
+ continue;
+ }
+ TSRecord record = new TSRecord(i, deviceIds[j]);
+ for (int k = 0; k < measurementNum; k++) {
+ record.addTuple(
+ DataPoint.getDataPoint(
+ measurementSchemas[k].getType(),
+ measurementSchemas[k].getMeasurementId(),
+ String.valueOf(i)));
+ }
+ fileWriter.write(record);
+ fourthTsFileResource.updateStartTime(deviceIds[j], i);
+ fourthTsFileResource.updateEndTime(deviceIds[j], i);
+ }
+ }
+ TSRecord record = new TSRecord(1, deviceIds[3]);
+ for (int k = 0; k < measurementNum; k++) {
+ record.addTuple(
+ DataPoint.getDataPoint(
+ measurementSchemas[k].getType(),
+ measurementSchemas[k].getMeasurementId(),
+ String.valueOf(1)));
+ }
+ fileWriter.write(record);
+ fourthTsFileResource.updateStartTime(deviceIds[3], 1);
+ fourthTsFileResource.updateEndTime(deviceIds[3], 1);
+ fileWriter.flushAllChunkGroups();
+ fileWriter.close();
+
+ MergeResource resource = new MergeResource(seqList, unseqList);
+ IMergeFileSelector mergeFileSelector =
+ new MaxFileMergeFileSelector(resource, 500 * 1024 * 1024);
+ List[] result = mergeFileSelector.select();
+ Assert.assertEquals(1, result[0].size());
+ Assert.assertEquals(1, result[1].size());
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMaxSelectUnseqFileNumInEachUnseqCompaction(originNum);
+ }
+ }
}