You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by sh...@apache.org on 2025/08/15 01:59:25 UTC
(iotdb) branch usePatternTreeMapInCompaction-1.3 created (now d89e6cf2950)
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a change to branch usePatternTreeMapInCompaction-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at d89e6cf2950 Using PatternTreeMap to cache mod entries in ReadChunkCompactionPerformer
This branch includes the following new commits:
new d89e6cf2950 Using PatternTreeMap to cache mod entries in ReadChunkCompactionPerformer
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: Using PatternTreeMap to cache mod entries in ReadChunkCompactionPerformer
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuwenwei pushed a commit to branch usePatternTreeMapInCompaction-1.3
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit d89e6cf2950c822d0ff5361e371e030f36608d2c
Author: shuwenwei <s1...@gmail.com>
AuthorDate: Thu Aug 14 18:35:17 2025 +0800
Using PatternTreeMap to cache mod entries in ReadChunkCompactionPerformer
---
.../performer/impl/FastCompactionPerformer.java | 5 +--
.../compaction/execute/utils/CompactionUtils.java | 28 +++++++++++++++
.../execute/utils/MultiTsFileDeviceIterator.java | 41 +++++++++-------------
.../fast/FastAlignedSeriesCompactionExecutor.java | 5 +--
.../FastNonAlignedSeriesCompactionExecutor.java | 5 +--
.../executor/fast/SeriesCompactionExecutor.java | 14 +++-----
6 files changed, 52 insertions(+), 46 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
index b8347c5b91e..fb99acd99cd 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/performer/impl/FastCompactionPerformer.java
@@ -362,10 +362,7 @@ public class FastCompactionPerformer
}
// read mods
PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> modifications =
- PatternTreeMapFactory.getModsPatternTreeMap();
- for (Modification modification : resource.getModFile().getModificationsIter()) {
- modifications.append(modification.getPath(), modification);
- }
+ CompactionUtils.buildModEntryPatternTreeMap(resource);
modificationCache.put(resource.getTsFile().getName(), modifications);
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
index d0f79cb992b..1ace2bde3a0 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/CompactionUtils.java
@@ -21,6 +21,9 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils;
import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
+import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.commons.service.metric.MetricService;
import org.apache.iotdb.commons.service.metric.enums.Tag;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
@@ -31,6 +34,7 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.DeviceTimeIndex;
+import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.iotdb.metrics.utils.MetricLevel;
import org.apache.iotdb.metrics.utils.SystemMetric;
@@ -45,6 +49,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -357,6 +362,29 @@ public class CompactionUtils {
}
}
+ public static PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer>
+ buildModEntryPatternTreeMap(TsFileResource resource) {
+ PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> patternTreeMap =
+ PatternTreeMapFactory.getModsPatternTreeMap();
+ Iterable<Modification> iterator = resource.getModFile().getModificationsIter();
+ for (Modification modification : iterator) {
+ patternTreeMap.append(modification.getPath(), modification);
+ }
+ return patternTreeMap;
+ }
+
+ public static List<Modification> getMatchedModifications(
+ PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> patternTreeMap,
+ IDeviceID deviceID,
+ String measurement)
+ throws IllegalPathException {
+ if (patternTreeMap == null) {
+ return Collections.emptyList();
+ }
+ PartialPath path = CompactionPathUtils.getPath(deviceID, measurement);
+ return ModificationFile.sortAndMerge(patternTreeMap.getOverlapped(path));
+ }
+
public static boolean isDiskHasSpace() {
return isDiskHasSpace(0d);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
index cbc6ac1b686..90a507ced53 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/MultiTsFileDeviceIterator.java
@@ -22,16 +22,17 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
+import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.storageengine.dataregion.compaction.io.CompactionTsFileReader;
import org.apache.iotdb.db.storageengine.dataregion.compaction.schedule.constant.CompactionType;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
-import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.AlignedChunkMetadata;
@@ -69,7 +70,9 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
private List<TsFileResource> tsFileResourcesSortedByAsc;
private Map<TsFileResource, TsFileSequenceReader> readerMap = new HashMap<>();
private final Map<TsFileResource, TsFileDeviceIterator> deviceIteratorMap = new HashMap<>();
- private final Map<TsFileResource, List<Modification>> modificationCache = new HashMap<>();
+ private final Map<
+ TsFileResource, PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer>>
+ modificationCache = new HashMap<>();
private Pair<IDeviceID, Boolean> currentDevice = null;
private long ttlForCurrentDevice;
private long timeLowerBoundForCurrentDevice;
@@ -433,10 +436,9 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
timeLowerBoundForCurrentDevice);
}
- List<Modification> modifications =
+ PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> modifications =
modificationCache.computeIfAbsent(
- tsFileResource,
- r -> new LinkedList<>(ModificationFile.getNormalMods(r).getModifications()));
+ tsFileResource, CompactionUtils::buildModEntryPatternTreeMap);
// construct the input params List<List<Modification>> for QueryUtils.modifyAlignedChunkMetaData
AlignedChunkMetadata alignedChunkMetadata = alignedChunkMetadataList.get(0);
@@ -447,15 +449,9 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
modificationForCurDevice.add(Collections.emptyList());
continue;
}
- List<Modification> modificationList = new ArrayList<>();
- PartialPath path =
- CompactionPathUtils.getPath(
- currentDevice.getLeft(), valueChunkMetadata.getMeasurementUid());
- for (Modification modification : modifications) {
- if (modification.getPath().matchFullPath(path)) {
- modificationList.add(modification);
- }
- }
+ List<Modification> modificationList =
+ CompactionUtils.getMatchedModifications(
+ modifications, device, valueChunkMetadata.getMeasurementUid());
if (ttlDeletion != null) {
modificationList.add(ttlDeletion);
}
@@ -663,17 +659,14 @@ public class MultiTsFileDeviceIterator implements AutoCloseable {
chunkMetadataListMap.get(currentCompactingSeries);
chunkMetadataListMap.remove(currentCompactingSeries);
- List<Modification> modificationsInThisResource =
- modificationCache.computeIfAbsent(
- resource,
- r -> new LinkedList<>(ModificationFile.getNormalMods(r).getModifications()));
- LinkedList<Modification> modificationForCurrentSeries = new LinkedList<>();
+ PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer>
+ modificationsInThisResource =
+ modificationCache.computeIfAbsent(
+ resource, CompactionUtils::buildModEntryPatternTreeMap);
// collect the modifications for current series
- for (Modification modification : modificationsInThisResource) {
- if (modification.getPath().matchFullPath(path)) {
- modificationForCurrentSeries.add(modification);
- }
- }
+ List<Modification> modificationForCurrentSeries =
+ CompactionUtils.getMatchedModifications(
+ modificationsInThisResource, device, currentCompactingSeries);
// add ttl deletion for current series
if (ttlDeletion != null) {
modificationForCurrentSeries.add(ttlDeletion);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
index fa5f53f4209..edf127b20de 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastAlignedSeriesCompactionExecutor.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.batch.utils.AlignedSeriesBatchCompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.AlignedPageElement;
@@ -242,9 +241,7 @@ public class FastAlignedSeriesCompactionExecutor extends SeriesCompactionExecuto
valueModifications.add(null);
} else {
valueModifications.add(
- getModificationsFromCache(
- resource,
- CompactionPathUtils.getPath(deviceId, x.getMeasurementUid())));
+ getModificationsFromCache(resource, deviceId, x.getMeasurementUid()));
}
} catch (IllegalPathException e) {
throw new RuntimeException(e);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
index ab7d509c0ec..860ca3927d6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/FastNonAlignedSeriesCompactionExecutor.java
@@ -23,7 +23,6 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
-import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionPathUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.FileElement;
@@ -147,9 +146,7 @@ public class FastNonAlignedSeriesCompactionExecutor extends SeriesCompactionExec
ModificationUtils.modifyChunkMetaData(
iChunkMetadataList,
getModificationsFromCache(
- resource,
- CompactionPathUtils.getPath(
- deviceId, iChunkMetadataList.get(0).getMeasurementUid())));
+ resource, deviceId, iChunkMetadataList.get(0).getMeasurementUid()));
if (iChunkMetadataList.isEmpty()) {
// all chunks has been deleted in this file, just remove it
removeFile(fileElement);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
index a44453645c0..9bf2e43d4cb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/utils/executor/fast/SeriesCompactionExecutor.java
@@ -20,10 +20,10 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast;
import org.apache.iotdb.commons.exception.IllegalPathException;
-import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.path.PatternTreeMap;
import org.apache.iotdb.db.exception.WriteProcessException;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
+import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.ModifiedStatus;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.ChunkMetadataElement;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.executor.fast.element.FileElement;
@@ -31,7 +31,6 @@ import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.exe
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.reader.PointPriorityReader;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.writer.AbstractCompactionWriter;
import org.apache.iotdb.db.storageengine.dataregion.modification.Modification;
-import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
@@ -46,7 +45,6 @@ import org.apache.tsfile.read.common.TimeRange;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -474,17 +472,13 @@ public abstract class SeriesCompactionExecutor {
/**
* Get the modifications of a timeseries in the ModificationFile of a TsFile. Create ttl
* modification from ttl cache.
- *
- * @param path name of the time series
*/
protected List<Modification> getModificationsFromCache(
- TsFileResource tsFileResource, PartialPath path) {
+ TsFileResource tsFileResource, IDeviceID deviceId, String measurement)
+ throws IllegalPathException {
PatternTreeMap<Modification, PatternTreeMapFactory.ModsSerializer> allModifications =
modificationCacheMap.get(tsFileResource.getTsFile().getName());
- if (allModifications == null) {
- return Collections.emptyList();
- }
- return ModificationFile.sortAndMerge(allModifications.getOverlapped(path));
+ return CompactionUtils.getMatchedModifications(allModifications, deviceId, measurement);
}
@SuppressWarnings("squid:S3776")