You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2022/12/26 06:20:45 UTC
[iotdb] branch rel/0.13 updated: filter out the deleted files when selecting files (#8609)
This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch rel/0.13
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.13 by this push:
new 6bc5b93e14 filter out the deleted files when selecting files (#8609)
6bc5b93e14 is described below
commit 6bc5b93e14a455a8ecf90fa2498ef47a7c4ed061
Author: 周沛辰 <45...@users.noreply.github.com>
AuthorDate: Mon Dec 26 14:20:40 2022 +0800
filter out the deleted files when selecting files (#8609)
---
.../manage/CrossSpaceCompactionResource.java | 5 +--
.../selector/RewriteCompactionFileSelector.java | 6 ++--
.../sizetiered/SizeTieredCompactionSelector.java | 5 ++-
.../db/engine/storagegroup/TsFileResource.java | 4 +++
.../cross/RewriteCompactionFileSelectorTest.java | 36 ++++++++++++++++++++++
5 files changed, 48 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java
index 7ced2fbcfc..239547e75a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/manage/CrossSpaceCompactionResource.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction.cross.rewrite.manage;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.metadata.path.PartialPath;
import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.utils.Pair;
@@ -80,9 +81,9 @@ public class CrossSpaceCompactionResource {
*/
private void filterUnseqResource(List<TsFileResource> unseqResources) {
for (TsFileResource resource : unseqResources) {
- if (resource.isCompacting() || resource.isCompactionCandidate() || !resource.isClosed()) {
+ if (resource.getStatus() != TsFileResourceStatus.CLOSED || !resource.getTsFile().exists()) {
return;
- } else if (!resource.isDeleted() && resource.stillLives(ttlLowerBound)) {
+ } else if (resource.stillLives(ttlLowerBound)) {
this.unseqFiles.add(resource);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
index ed77a69858..caf5943a4a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/selector/RewriteCompactionFileSelector.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.compaction.cross.rewrite.manage.CrossSpaceCompactionResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.exception.MergeException;
import org.slf4j.Logger;
@@ -256,9 +257,8 @@ public class RewriteCompactionFileSelector implements ICrossSpaceMergeFileSelect
*/
private boolean checkIsSeqFilesValid() {
for (Integer seqIdx : tmpSelectedSeqFiles) {
- if (resource.getSeqFiles().get(seqIdx).isCompactionCandidate()
- || resource.getSeqFiles().get(seqIdx).isCompacting()
- || !resource.getSeqFiles().get(seqIdx).isClosed()) {
+ if (resource.getSeqFiles().get(seqIdx).getStatus() != TsFileResourceStatus.CLOSED
+ || !resource.getSeqFiles().get(seqIdx).getTsFile().exists()) {
return false;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
index b8d9a4c746..ee4a45d9db 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/sizetiered/SizeTieredCompactionSelector.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.engine.compaction.task.AbstractCompactionTask;
import org.apache.iotdb.db.engine.storagegroup.TsFileManager;
import org.apache.iotdb.db.engine.storagegroup.TsFileNameGenerator;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.tsfile.utils.Pair;
import org.slf4j.Logger;
@@ -124,9 +125,7 @@ public class SizeTieredCompactionSelector extends AbstractInnerSpaceCompactionSe
TsFileNameGenerator.TsFileName currentName =
TsFileNameGenerator.getTsFileName(currentFile.getTsFile().getName());
if (currentName.getInnerCompactionCnt() != level
- || currentFile.isCompactionCandidate()
- || currentFile.isCompacting()
- || !currentFile.isClosed()) {
+ || currentFile.getStatus() != TsFileResourceStatus.CLOSED) {
selectedFileList.clear();
selectedFileSize = 0L;
continue;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 581d05e105..7f6707f98b 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -623,6 +623,10 @@ public class TsFileResource {
return this.status == TsFileResourceStatus.COMPACTION_CANDIDATE;
}
+ public TsFileResourceStatus getStatus() {
+ return this.status;
+ }
+
public void setStatus(TsFileResourceStatus status) {
switch (status) {
case CLOSED:
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
index a885570390..2d20e3220e 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCompactionFileSelectorTest.java
@@ -901,4 +901,40 @@ public class RewriteCompactionFileSelectorTest extends MergeTest {
.getConfig()
.setMaxCrossCompactionCandidateFileNum(maxCrossFilesNum);
}
+
+ /** Test selecting with deleted files. */
+ @Test
+ public void testSelectionWithFileBeingDeleted() throws MergeException, IOException {
+ // cross select files with the last unseq file being deleted
+ unseqResources.get(unseqResources.size() - 1).setStatus(TsFileResourceStatus.DELETED);
+ CrossSpaceCompactionResource resource =
+ new CrossSpaceCompactionResource(seqResources, unseqResources);
+ assertEquals(5, resource.getSeqFiles().size());
+ assertEquals(5, resource.getUnseqFiles().size());
+ ICrossSpaceMergeFileSelector mergeFileSelector =
+ new RewriteCompactionFileSelector(resource, Long.MAX_VALUE);
+ List[] result = mergeFileSelector.select();
+ List<TsFileResource> seqSelected = result[0];
+ List<TsFileResource> unseqSelected = result[1];
+
+ assertEquals(5, seqSelected.size());
+ assertEquals(5, unseqSelected.size());
+ assertEquals(seqResources, seqSelected);
+ assertEquals(resource.getUnseqFiles(), unseqSelected);
+ resource.clear();
+
+ // cross select files with the last unseq file and the last seq file being deleted
+ seqResources.get(seqResources.size() - 1).setStatus(TsFileResourceStatus.DELETED);
+ resource = new CrossSpaceCompactionResource(seqResources, unseqResources);
+ assertEquals(4, resource.getSeqFiles().size());
+ assertEquals(5, resource.getUnseqFiles().size());
+ result = mergeFileSelector.select();
+
+ assertEquals(4, result[0].size());
+ assertEquals(4, result[1].size());
+
+ resource.getUnseqFiles().remove(resource.getUnseqFiles().size() - 1);
+ assertEquals(resource.getSeqFiles(), result[0]);
+ assertEquals(resource.getUnseqFiles(), result[1]);
+ }
}