You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ma...@apache.org on 2022/08/08 09:00:55 UTC
[iotdb] branch IOTDB-3455-O updated: add cross compaction memory budget to system info
This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-3455-O
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/IOTDB-3455-O by this push:
new 044ef7c027 add cross compaction memory budget to system info
044ef7c027 is described below
commit 044ef7c027723f8f99848a96bee7bf389d9bcdce
Author: Liu Xuxin <li...@outlook.com>
AuthorDate: Mon Aug 8 17:00:44 2022 +0800
add cross compaction memory budget to system info
---
.../db/engine/compaction/CompactionScheduler.java | 8 +++--
.../compaction/cross/CrossSpaceCompactionTask.java | 5 +++
.../RewriteCrossSpaceCompactionSelector.java | 6 ++--
.../compaction/task/ICompactionSelector.java | 5 +--
.../compaction/CompactionTaskManagerTest.java | 1 +
.../compaction/cross/CrossSpaceCompactionTest.java | 3 ++
.../cross/CrossSpaceCompactionValidationTest.java | 39 ++++++++++++++++++++++
.../cross/RewriteCrossSpaceCompactionTest.java | 4 +++
8 files changed, 63 insertions(+), 8 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
index cc4ad3d66e..2f6340c40c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionScheduler.java
@@ -149,19 +149,21 @@ public class CompactionScheduler {
crossSpaceCompactionSelector.selectCrossSpaceTask(
tsFileManager.getSequenceListByTimePartition(timePartition),
tsFileManager.getUnsequenceListByTimePartition(timePartition));
- for (Pair<List<TsFileResource>, List<TsFileResource>> selectedFilesPair : taskList) {
+ List<Long> memoryCost = crossSpaceCompactionSelector.getCompactionMemoryCost();
+ for (int i = 0, size = taskList.size(); i < size; ++i) {
CompactionTaskManager.getInstance()
.addTaskToWaitingQueue(
new CrossSpaceCompactionTask(
timePartition,
tsFileManager,
- selectedFilesPair.left,
- selectedFilesPair.right,
+ taskList.get(i).left,
+ taskList.get(i).right,
IoTDBDescriptor.getInstance()
.getConfig()
.getCrossCompactionPerformer()
.createInstance(),
CompactionTaskManager.currentTaskNum,
+ memoryCost.get(i),
tsFileManager.getNextCompactionTaskId()));
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
index cd0e1d7704..01a5e7a247 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java
@@ -31,6 +31,7 @@ import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceList;
import org.apache.iotdb.db.engine.storagegroup.TsFileResourceStatus;
import org.apache.iotdb.db.query.control.FileReaderManager;
+import org.apache.iotdb.db.rescon.SystemInfo;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
@@ -66,6 +67,7 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
List<TsFileResource> selectedUnsequenceFiles,
ICrossCompactionPerformer performer,
AtomicInteger currentTaskNum,
+ long memoryCost,
long serialId) {
super(
tsFileManager.getStorageGroupName(),
@@ -80,11 +82,13 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
this.unseqTsFileResourceList = tsFileManager.getUnsequenceListByTimePartition(timePartition);
this.performer = performer;
this.hashCode = this.toString().hashCode();
+ this.memoryCost = memoryCost;
}
@Override
protected void doCompaction() {
try {
+ SystemInfo.getInstance().addCompactionMemoryCost(memoryCost);
if (!tsFileManager.isAllowCompaction()) {
return;
}
@@ -193,6 +197,7 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask {
false,
true);
} finally {
+ SystemInfo.getInstance().resetCompactionMemoryCost(memoryCost);
releaseAllLock();
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
index f6322cc536..ca918b6bba 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/rewrite/RewriteCrossSpaceCompactionSelector.java
@@ -105,7 +105,7 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector
* @return two lists of TsFileResource, the former is selected seqFiles and the latter is selected
* unseqFiles or an empty array if there are no proper candidates by the budget.
*/
- public List[] select() throws MergeException {
+ private List[] select() throws MergeException {
long startTime = System.currentTimeMillis();
try {
LOGGER.debug(
@@ -352,7 +352,7 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector
}
@Override
- public long getSelectedFileMemoryCost() {
- return totalCost;
+ public List<Long> getCompactionMemoryCost() {
+ return Collections.singletonList(totalCost);
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java
index 95a5f88d6c..08983cc7d3 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/task/ICompactionSelector.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.db.engine.compaction.cross.utils.ReadPointCrossCompactio
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.tsfile.utils.Pair;
+import java.util.Collections;
import java.util.List;
/**
@@ -55,8 +56,8 @@ public interface ICompactionSelector {
}
}
- default long getSelectedFileMemoryCost() {
- return 0L;
+ default List<Long> getCompactionMemoryCost() {
+ return Collections.emptyList();
}
static AbstractCompactionEstimator getCompactionEstimator(
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java
index eda36fca69..8ddd23d3f4 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/CompactionTaskManagerTest.java
@@ -297,6 +297,7 @@ public class CompactionTaskManagerTest extends InnerCompactionTest {
unseqResources,
new ReadPointCompactionPerformer(),
new AtomicInteger(0),
+ 0,
0);
for (TsFileResource resource : seqResources) {
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java
index 554c1d8181..47a86d215c 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTest.java
@@ -442,6 +442,7 @@ public class CrossSpaceCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
0);
compactionTask.start();
List<TsFileResource> targetTsfileResourceList = new ArrayList<>();
@@ -748,6 +749,7 @@ public class CrossSpaceCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
0);
compactionTask.start();
List<TsFileResource> targetTsfileResourceList = new ArrayList<>();
@@ -1053,6 +1055,7 @@ public class CrossSpaceCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
0);
compactionTask.start();
List<TsFileResource> targetTsfileResourceList = new ArrayList<>();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java
index d275dd9dd4..ccd4c3601a 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionValidationTest.java
@@ -104,6 +104,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -151,6 +152,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -198,6 +200,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -253,6 +256,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -305,6 +309,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -355,6 +360,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -408,6 +414,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -460,6 +467,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -512,6 +520,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -565,6 +574,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -618,6 +628,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -671,6 +682,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -725,6 +737,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -780,6 +793,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -835,6 +849,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -891,6 +906,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -948,6 +964,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1005,6 +1022,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1062,6 +1080,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1119,6 +1138,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1175,6 +1195,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1227,6 +1248,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1280,6 +1302,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1333,6 +1356,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1386,6 +1410,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1440,6 +1465,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1495,6 +1521,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1550,6 +1577,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1606,6 +1634,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1663,6 +1692,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1720,6 +1750,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1777,6 +1808,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1834,6 +1866,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1890,6 +1923,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1944,6 +1978,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -1994,6 +2029,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -2049,6 +2085,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -2105,6 +2142,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
@@ -2161,6 +2199,7 @@ public class CrossSpaceCompactionValidationTest extends AbstractCompactionTest {
.getCrossCompactionPerformer()
.createInstance(),
new AtomicInteger(0),
+ 0,
tsFileManager.getNextCompactionTaskId())
.doCompaction();
diff --git a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
index c5f3cb132d..40b6b8d17f 100644
--- a/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/engine/compaction/cross/RewriteCrossSpaceCompactionTest.java
@@ -227,6 +227,7 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
unseqResources,
new ReadPointCompactionPerformer(),
new AtomicInteger(0),
+ 0,
0);
task.start();
@@ -464,6 +465,7 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
unseqResources,
new ReadPointCompactionPerformer(),
new AtomicInteger(0),
+ 0,
0);
task.start();
@@ -611,6 +613,7 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
unseqResources,
new ReadPointCompactionPerformer(),
new AtomicInteger(0),
+ 0,
0);
task.setSourceFilesToCompactionCandidate();
task.checkValidAndSetMerging();
@@ -731,6 +734,7 @@ public class RewriteCrossSpaceCompactionTest extends AbstractCompactionTest {
unseqResources,
new ReadPointCompactionPerformer(),
new AtomicInteger(0),
+ 0,
0);
task.setSourceFilesToCompactionCandidate();
task.checkValidAndSetMerging();