You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2024/10/22 07:01:21 UTC
(iotdb) branch master updated: Fix compaction task comparator bug & Modify target file name of inner compaction task (#13848)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new d1c54d40dcd Fix compaction task comparator bug & Modify target file name of inner compaction task (#13848)
d1c54d40dcd is described below
commit d1c54d40dcd2f898e91581435079c9509358c820
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Tue Oct 22 15:01:15 2024 +0800
Fix compaction task comparator bug & Modify target file name of inner compaction task (#13848)
* fix compaction task comparator bug
* fix compile
* remove print
* fix bugs
* add ut
---
.../compaction/execute/task/InnerSpaceCompactionTask.java | 9 ++++++++-
.../comparator/DefaultCompactionTaskComparatorImpl.java | 8 ++++----
.../dataregion/tsfile/generator/TsFileNameGenerator.java | 4 +++-
.../dataregion/compaction/CompactionTaskComparatorTest.java | 12 +++++++++++-
4 files changed, 26 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index fd06a8a3816..3b1db8588c9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -362,13 +362,15 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
TsFileResource resource = filesView.sortedAllSourceFilesInTask.get(i);
File file = resource.getTsFile();
File skippedSourceFile = filesView.skippedSourceFiles.get(i).getTsFile();
+ TsFileNameGenerator.TsFileName skippedSourceFileName =
+ TsFileNameGenerator.getTsFileName(skippedSourceFile.getName());
TsFileNameGenerator.TsFileName tsFileName = TsFileNameGenerator.getTsFileName(file.getName());
String newFileName =
String.format(
"%s-%s-%s-%s" + TsFileConstant.TSFILE_SUFFIX,
tsFileName.getTime(),
tsFileName.getVersion(),
- tsFileName.getInnerCompactionCnt(),
+ skippedSourceFileName.getInnerCompactionCnt(),
tsFileName.getCrossCompactionCnt() + 1);
TsFileResource renamedTargetFile =
new TsFileResource(
@@ -602,6 +604,11 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
return filesView.sourceFilesInCompactionPerformer;
}
+ public double getAvgCompactionCount() {
+ return (double) filesView.sumOfCompactionCount
+ / filesView.sourceFilesInCompactionPerformer.size();
+ }
+
@TestOnly
public void setTargetTsFileResource(TsFileResource targetTsFileResource) {
this.filesView.setTargetFileForRecover(targetTsFileResource);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
index bf892924d6a..f80b43528f2 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/comparator/DefaultCompactionTaskComparatorImpl.java
@@ -91,10 +91,10 @@ public class DefaultCompactionTaskComparatorImpl implements ICompactionTaskCompa
// if the sum of compaction count of the selected files are different
// we prefer to execute task with smaller compaction count
// this can reduce write amplification
- if (((double) o1.getSumOfCompactionCount()) / o1.getSelectedTsFileResourceList().size()
- != ((double) o2.getSumOfCompactionCount()) / o2.getSelectedTsFileResourceList().size()) {
- return o1.getSumOfCompactionCount() / o1.getSelectedTsFileResourceList().size()
- - o2.getSumOfCompactionCount() / o2.getSelectedTsFileResourceList().size();
+ double avgCompactionCount1 = o1.getAvgCompactionCount();
+ double avgCompactionCount2 = o2.getAvgCompactionCount();
+ if (Math.abs(avgCompactionCount1 - avgCompactionCount2) > 1e-2) {
+ return Double.compare(avgCompactionCount1, avgCompactionCount2);
}
// if the time partition of o1 and o2 are different
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
index 0245e448144..ea408883553 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/generator/TsFileNameGenerator.java
@@ -319,10 +319,12 @@ public class TsFileNameGenerator {
public static List<TsFileResource> getNewInnerCompactionTargetFileResources(
List<TsFileResource> tsFileResources, boolean sequence)
throws IOException, DiskSpaceInsufficientException {
+ long maxInnerCompactionCount = Long.MIN_VALUE;
long maxCrossMergeCount = Long.MIN_VALUE;
int maxTierLevel = 0;
for (TsFileResource resource : tsFileResources) {
TsFileName tsFileName = getTsFileName(resource.getTsFile().getName());
+ maxInnerCompactionCount = Math.max(tsFileName.innerCompactionCnt, maxInnerCompactionCount);
maxCrossMergeCount = Math.max(tsFileName.crossCompactionCnt, maxCrossMergeCount);
maxTierLevel = Math.max(resource.getTierLevel(), maxTierLevel);
}
@@ -339,7 +341,7 @@ public class TsFileNameGenerator {
tsFileResources.get(0).getTimePartition(),
tsFileName.time,
tsFileName.version,
- tsFileName.innerCompactionCnt + 1,
+ (int) maxInnerCompactionCount + 1,
(int) maxCrossMergeCount,
maxTierLevel,
IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX)),
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
index 730bc4287e6..90edb22ebbb 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/CompactionTaskComparatorTest.java
@@ -42,6 +42,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -118,7 +119,7 @@ public class CompactionTaskComparatorTest {
/** Test comparation of task with same file num and file size, different compaction count */
@Test
- public void testFileCompactCountCompare() throws InterruptedException {
+ public void testFileCompactCountCompare1() throws InterruptedException {
AbstractCompactionTask[] compactionTasks = new AbstractCompactionTask[100];
for (int i = 0; i < 100; ++i) {
List<TsFileResource> resources = new ArrayList<>();
@@ -138,6 +139,15 @@ public class CompactionTaskComparatorTest {
}
}
+ @Test
+ public void testFileCompactionCountCompare2() {
+ InnerSpaceCompactionTask task1 = Mockito.mock(InnerSpaceCompactionTask.class);
+ Mockito.when(task1.getAvgCompactionCount()).thenReturn(0.1);
+ InnerSpaceCompactionTask task2 = Mockito.mock(InnerSpaceCompactionTask.class);
+ Mockito.when(task2.getAvgCompactionCount()).thenReturn(0.2);
+ Assert.assertTrue(new DefaultCompactionTaskComparatorImpl().compare(task1, task2) < 0);
+ }
+
@Test
public void testPriorityQueueSizeLimit() {
MinMaxPriorityQueue<AbstractCompactionTask> limitQueue =