You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/11/03 08:40:00 UTC
[iotdb] branch IOTDB-4835 updated: add test
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-4835
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/IOTDB-4835 by this push:
new ea012abb44 add test
ea012abb44 is described below
commit ea012abb44bafc76930989a8ffae421394f7b0d7
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Thu Nov 3 16:39:51 2022 +0800
add test
---
.../engine/compaction/CompactionTaskManager.java | 1 +
.../sizetiered/SizeTieredCompactionSelector.java | 2 +-
.../engine/compaction/CompactionSchedulerTest.java | 103 ++++++++++++++++++++-
3 files changed, 104 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java
index a63f15e974..d7aeaa873b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManager.java
@@ -384,6 +384,7 @@ public class CompactionTaskManager implements IService {
candidateCompactionTaskQueue.clear();
}
currentTaskNum = new AtomicInteger(0);
+ init = true;
logger.info("Compaction task manager started.");
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
index 13a1157ae3..bb4eae414b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
@@ -106,10 +106,10 @@ public class SizeTieredCompactionSelector
if (currentName.getInnerCompactionCnt() != level) {
if (selectedFileList.size() > 1) {
taskPriorityQueue.add(new Pair<>(new ArrayList<>(selectedFileList), selectedFileSize));
+ shouldContinueToSearch = false;
}
selectedFileList = new ArrayList<>();
selectedFileSize = 0L;
- shouldContinueToSearch = false;
} else if (currentFile.getStatus() != TsFileResourceStatus.CLOSED) {
selectedFileList.clear();
selectedFileSize = 0L;
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
index 8fe816e4ec..93d0f27e9e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionSchedulerTest.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.compaction;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.MetadataException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -1818,7 +1819,6 @@ public class CompactionSchedulerTest {
Collections.emptyMap());
}
for (int i = 0; i < 100; i++) {
-
List<List<Long>> chunkPagePointsNum = new ArrayList<>();
List<Long> pagePointsNum = new ArrayList<>();
pagePointsNum.add(100L);
@@ -1896,6 +1896,107 @@ public class CompactionSchedulerTest {
}
}
+ @Test
+ public void testLargeFileInLowerLevel() throws Exception {
+ logger.warn("Running test16");
+ int prevMaxCompactionCandidateFileNum =
+ IoTDBDescriptor.getInstance().getConfig().getMaxInnerCompactionCandidateFileNum();
+ IoTDBDescriptor.getInstance().getConfig().setMaxInnerCompactionCandidateFileNum(2);
+ long originTargetSize = IoTDBDescriptor.getInstance().getConfig().getTargetCompactionFileSize();
+ IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(1024 * 1024);
+ String sgName = COMPACTION_TEST_SG + "test17";
+ try {
+ CompactionTaskManager.getInstance().restart();
+ TsFileManager tsFileManager = new TsFileManager(sgName, "0", "target");
+ Set<String> fullPath = new HashSet<>();
+ for (String device : fullPaths) {
+ fullPath.add(sgName + device);
+ }
+ for (int i = 0; i < 10; i++) {
+ List<List<Long>> chunkPagePointsNum = new ArrayList<>();
+ List<Long> pagePointsNum = new ArrayList<>();
+ pagePointsNum.add(100L);
+ chunkPagePointsNum.add(pagePointsNum);
+ TsFileResource tsFileResource =
+ new TsFileResource(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ .concat(File.separator)
+ .concat("sequence")
+ .concat(File.separator)
+ .concat(sgName)
+ .concat(File.separator)
+ .concat("0")
+ .concat(File.separator)
+ .concat("0")
+ .concat(File.separator)
+ .concat(
+ (i + 1)
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + (i + 1)
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 1
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile")));
+ CompactionFileGeneratorUtils.writeTsFile(
+ fullPath, chunkPagePointsNum, 100 * i + 100, tsFileResource);
+ tsFileManager.add(tsFileResource, true);
+ }
+
+ List<List<Long>> chunkPagePointsNum = new ArrayList<>();
+ List<Long> pagePointsNum = new ArrayList<>();
+ pagePointsNum.add(100000L);
+ chunkPagePointsNum.add(pagePointsNum);
+ TsFileResource tsFileResource =
+ new TsFileResource(
+ new File(
+ TestConstant.BASE_OUTPUT_PATH
+ .concat(File.separator)
+ .concat("sequence")
+ .concat(File.separator)
+ .concat(sgName)
+ .concat(File.separator)
+ .concat("0")
+ .concat(File.separator)
+ .concat("0")
+ .concat(File.separator)
+ .concat(
+ 11
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 11
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile")));
+ CompactionFileGeneratorUtils.writeTsFile(
+ fullPath, chunkPagePointsNum, 100 * 10 + 100, tsFileResource);
+ tsFileManager.add(tsFileResource, true);
+
+ CompactionScheduler.scheduleCompaction(tsFileManager, 0);
+ Thread.sleep(100);
+ long sleepTime = 0;
+ while (tsFileManager.getTsFileList(true).size() > 3) {
+ CompactionScheduler.scheduleCompaction(tsFileManager, 0);
+ Thread.sleep(100);
+ sleepTime += 100;
+ if (sleepTime >= 20_000) {
+ fail();
+ }
+ }
+
+ stopCompactionTaskManager();
+ tsFileManager.setAllowCompaction(false);
+ assertEquals(3, tsFileManager.getTsFileList(true).size());
+ } finally {
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setMaxInnerCompactionCandidateFileNum(prevMaxCompactionCandidateFileNum);
+ IoTDBDescriptor.getInstance().getConfig().setTargetCompactionFileSize(originTargetSize);
+ }
+ }
+
public void stopCompactionTaskManager() {
CompactionTaskManager.getInstance().clearCandidateQueue();
while (CompactionTaskManager.getInstance().getRunningCompactionTaskList().size() > 0) {