You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2021/05/27 03:34:55 UTC
[iotdb] branch jira_1398 created (now 9b73a13)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch jira_1398
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 9b73a13 do not select unseq files when there are unmerged old unseq files
This branch includes the following new commits:
new 9b73a13 do not select unseq files when there are unmerged old unseq files
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: do not select unseq files when there are unmerged
old unseq files
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch jira_1398
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9b73a13ac26449fdd2de28d5c9fd65cbde8365fa
Author: jt <jt...@163.com>
AuthorDate: Thu May 27 11:33:44 2021 +0800
do not select unseq files when there are unmerged old unseq files
---
.../merge/selector/MaxFileMergeFileSelector.java | 9 ++-
.../engine/merge/MaxFileMergeFileSelectorTest.java | 73 ++++++++++++++++++++++
.../apache/iotdb/db/engine/merge/MergeTest.java | 9 +--
3 files changed, 85 insertions(+), 6 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
index f8ecdc0..3deb763 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/selector/MaxFileMergeFileSelector.java
@@ -168,7 +168,10 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
useTightBound
? calculateTightMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit)
: calculateLooseMemoryCost(unseqFile, tmpSelectedSeqFiles, startTime, timeLimit);
- updateSelectedFiles(newCost, unseqFile);
+ if (!updateSelectedFiles(newCost, unseqFile)) {
+ // older unseq files must be merged before newer ones
+ break;
+ }
tmpSelectedSeqFiles.clear();
unseqIndex++;
@@ -181,7 +184,7 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
}
}
- private void updateSelectedFiles(long newCost, TsFileResource unseqFile) {
+ private boolean updateSelectedFiles(long newCost, TsFileResource unseqFile) {
if (totalCost + newCost < memoryBudget) {
selectedUnseqFiles.add(unseqFile);
maxSeqFileCost = tempMaxSeqFileCost;
@@ -198,7 +201,9 @@ public class MaxFileMergeFileSelector implements IMergeFileSelector {
tmpSelectedSeqFiles,
newCost,
totalCost);
+ return true;
}
+ return false;
}
private boolean checkClosed(TsFileResource unseqFile) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
index 14c72d4..789d58d 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MaxFileMergeFileSelectorTest.java
@@ -236,4 +236,77 @@ public class MaxFileMergeFileSelectorTest extends MergeTest {
assertEquals(2, result[0].size());
resource.clear();
}
+
+ @Test
+ public void testSelectContinuousUnseqFile()
+ throws IOException, WriteProcessException, MergeException {
+ List<TsFileResource> seqList = new ArrayList<>();
+ List<TsFileResource> unseqList = new ArrayList<>();
+ try {
+ // seq files [0,0] [1,1] [2,2] ... [99,99]
+ int seqFileNum = 99;
+ for (int i = 0; i < seqFileNum; i++) {
+ File file =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH.concat(
+ 10
+ + "seq"
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + i
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource fileResource = new TsFileResource(file);
+ fileResource.setClosed(true);
+ prepareFile(fileResource, i, 1, 0);
+ seqList.add(fileResource);
+ }
+ int unseqFileNum = 3;
+ // 3 unseq files [0,0] [0,99] [99,99]
+ for (int i = 0; i < unseqFileNum; i++) {
+ File file =
+ new File(
+ TestConstant.BASE_OUTPUT_PATH.concat(
+ 10
+ + "unseq"
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + i
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 10
+ + IoTDBConstant.FILE_NAME_SEPARATOR
+ + 0
+ + ".tsfile"));
+ TsFileResource fileResource = new TsFileResource(file);
+ fileResource.setClosed(true);
+ unseqList.add(fileResource);
+ }
+ prepareFile(unseqList.get(0), 0, 1, 10);
+ prepareFile(unseqList.get(1), 0, 100, 20);
+ prepareFile(unseqList.get(2), 99, 1, 30);
+
+ MergeResource resource = new MergeResource(seqList, unseqList);
+ // the budget is enough to select unseq0 and unseq2, but not unseq1
+ // the first selection should only contain seq0 and unseq0
+ IMergeFileSelector mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000);
+ List[] result = mergeFileSelector.select();
+ assertEquals(1, result[0].size());
+ assertEquals(1, result[1].size());
+ assertEquals(seqList.get(0), result[0].get(0));
+ assertEquals(unseqList.get(0), result[1].get(0));
+ resource.clear();
+
+ resource =
+ new MergeResource(
+ seqList.subList(1, seqList.size()), unseqList.subList(1, unseqList.size()));
+ // the second selection should be empty
+ mergeFileSelector = new MaxFileMergeFileSelector(resource, 29000);
+ result = mergeFileSelector.select();
+ assertEquals(0, result.length);
+ resource.clear();
+ } finally {
+ removeFiles(seqList, unseqList);
+ }
+ }
}
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
index d6e6735..fa705ca 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/merge/MergeTest.java
@@ -86,7 +86,7 @@ abstract class MergeTest {
@After
public void tearDown() throws IOException, StorageEngineException {
- removeFiles();
+ removeFiles(seqResources, unseqResources);
seqResources.clear();
unseqResources.clear();
IoTDBDescriptor.getInstance()
@@ -186,12 +186,13 @@ abstract class MergeTest {
prepareFile(tsFileResource, 0, ptNum * unseqFileNum, 20000);
}
- private void removeFiles() throws IOException {
- for (TsFileResource tsFileResource : seqResources) {
+ void removeFiles(List<TsFileResource> seqResList, List<TsFileResource> unseqResList)
+ throws IOException {
+ for (TsFileResource tsFileResource : seqResList) {
tsFileResource.remove();
tsFileResource.getModFile().remove();
}
- for (TsFileResource tsFileResource : unseqResources) {
+ for (TsFileResource tsFileResource : unseqResList) {
tsFileResource.remove();
tsFileResource.getModFile().remove();
}