You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2021/05/28 06:11:46 UTC
[iotdb] branch rel/0.12 updated: [To rel/0.12][IOTDB-1398] Do not
select unseq files when there are uncompacted old unseq files (#3284)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.12 by this push:
new 4f7a15c [To rel/0.12][IOTDB-1398] Do not select unseq files when there are uncompacted old unseq files (#3284)
4f7a15c is described below
commit 4f7a15c2f6e0b0e2adbd42af2d2a4ab2443410dc
Author: zhanglingzhe0820 <44...@qq.com>
AuthorDate: Fri May 28 14:11:18 2021 +0800
[To rel/0.12][IOTDB-1398] Do not select unseq files when there are uncompacted old unseq files (#3284)
---
.../db/engine/compaction/TsFileManagement.java | 4 +-
.../merge/selector/MaxFileMergeFileSelector.java | 9 ++-
.../engine/merge/MaxFileMergeFileSelectorTest.java | 73 ++++++++++++++++++++++
.../apache/iotdb/db/engine/merge/MergeTest.java | 9 +--
4 files changed, 86 insertions(+), 9 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
index 4c71ca7..595ddf8 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/TsFileManagement.java
@@ -237,9 +237,7 @@ public abstract class TsFileManagement {
"{} too much unseq files to be merged, reduce it to {}",
storageGroupName,
maxOpenFileNumInEachUnseqCompaction);
- unSeqMergeList =
- unSeqMergeList.subList(
- unSeqMergeList.size() - maxOpenFileNumInEachUnseqCompaction, unSeqMergeList.size());
+ unSeqMergeList = unSeqMergeList.subList(0, maxOpenFileNumInEachUnseqCompaction);
}
long budget = IoTDBDescriptor.getInstance().getConfig().getMergeMemoryBudget();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
index f8ecdc0..3deb763 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
@@ -168,7 +168,10 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
useTightBound
? calculateTightMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit)
: calculateLooseMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit);
- updateSelectedFiles(newCost, unseqFile);
+ if (!updateSelectedFiles(newCost, unseqFile)) {
+ // older unseq files must be merged before newer ones
+ break;
+ }
tmpSelectedSeqFiles.clear();
unseqIndex++;
@@ -181,7 +184,7 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
}
}
- private void updateSelectedFiles(long newCost, TsFileResource unseqFile) {
+ private boolean updateSelectedFiles(long newCost, TsFileResource unseqFile) {
if (totalCost + newCost < memoryBudget) {
selectedUnseqFiles.add(unseqFile);
maxSeqFileCost = tempMaxSeqFileCost;
@@ -198,7 +201,9 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
tmpSelectedSeqFiles,
newCost,
totalCost);
+ return true;
}
+ return false;
}
private boolean checkClosed(TsFileResource unseqFile) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
index 14c72d4..789d58d 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
@@ -236,4 +236,77 @@ public class MaxFileMergeFileSelectorTest extends MergeTest {
assertEquals(2, result[0].size());
resource.clear();
}
+
+ @Test
+ public void testSelectContinuousUnseqFile()
+ throws IOException, WriteProcessException, MergeException {
+ List<TsFileResource> seqList = new ArrayList<>();
+ List<TsFileResource> unseqList = new ArrayList<>();
+ try {
+ // seq files [0,0] [1,1] [2,2] ... [99,99]
+ int seqFileNum = 99;
+ for (int i = 0; i < seqFileNum; i++) {
+ File file =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH.concat(
+ 10
+ + "seq"
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + i
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource fileResource = new TsFileResource(file);
+ fileResource.setClosed(true);
+ prepareFile(fileResource, i, 1, 0);
+ seqList.add(fileResource);
+ }
+ int unseqFileNum = 3;
+ // 3 unseq files [0,0] [0,99] [99,99]
+ for (int i = 0; i < unseqFileNum; i++) {
+ File file =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH.concat(
+ 10
+ + "unseq"
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + i
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource fileResource = new TsFileResource(file);
+ fileResource.setClosed(true);
+ unseqList.add(fileResource);
+ }
+ prepareFile(unseqList.get(0), 0, 1, 10);
+ prepareFile(unseqList.get(1), 0, 100, 20);
+ prepareFile(unseqList.get(2), 99, 1, 30);
+
+ MergeResource resource = new MergeResource(seqList, unseqList);
+ // the budget is enough to select unseq0 and unseq2, but not unseq1
+ // the first selection should only contain seq0 and unseq0
+ IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000);
+ List[] result = mergeFileSelector.select();
+ assertEquals(1, result[0].size());
+ assertEquals(1, result[1].size());
+ assertEquals(seqList.get(0), result[0].get(0));
+ assertEquals(unseqList.get(0), result[1].get(0));
+ resource.clear();
+
+ resource =
+ new MergeResource(
+ seqList.subList(1, seqList.size()), unseqList.subList(1, unseqList.size()));
+ // the second selection should be empty
+ mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000);
+ result = mergeFileSelector.select();
+ assertEquals(0, result.length);
+ resource.clear();
+ } finally {
+ removeFiles(seqList, unseqList);
+ }
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index d6e6735..fa705ca 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -86,7 +86,7 @@ abstract class MergeTest {
@After
public void tearDown() throws IOException, StorageEngineException {
- removeFiles();
+ removeFiles(seqResources, unseqResources);
seqResources.clear();
unseqResources.clear();
IoTDBDescriptor.getInstance()
@@ -186,12 +186,13 @@ abstract class MergeTest {
prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000);
}
- private void removeFiles() throws IOException {
- for (TsFileResource tsFileResource : seqResources) {
+ void removeFiles(List<TsFileResource> seqResList, List<TsFileResource> unseqResList)
+ throws IOException {
+ for (TsFileResource tsFileResource : seqResList) {
tsFileResource.remove();
tsFileResource.getModFile().remove();
}
- for (TsFileResource tsFileResource : unseqResources) {
+ for (TsFileResource tsFileResource : unseqResList) {
tsFileResource.remove();
tsFileResource.getModFile().remove();
}