You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/07/19 02:39:41 UTC
[iotdb] branch cp_fix_delete_speed created (now 2d469bd5749)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch cp_fix_delete_speed
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 2d469bd5749 Optimize the speed of delete by pattern (#10566)
This branch includes the following new commits:
new 2d469bd5749 Optimize the speed of delete by pattern (#10566)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: Optimize the speed of delete by pattern (#10566)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch cp_fix_delete_speed
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 2d469bd5749aff7a48c6d0a8448596e12d29561c
Author: Haonan <hh...@outlook.com>
AuthorDate: Tue Jul 18 16:22:23 2023 +0800
Optimize the speed of delete by pattern (#10566)
---
.../db/storageengine/dataregion/DataRegion.java | 44 ++++++++++++++++++----
.../dataregion/tsfile/TsFileResource.java | 5 ++-
.../tsfile/timeindex/DeviceTimeIndex.java | 18 ++++++++-
.../dataregion/tsfile/timeindex/FileTimeIndex.java | 3 +-
.../dataregion/tsfile/timeindex/ITimeIndex.java | 3 +-
5 files changed, 60 insertions(+), 13 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 3f018a8c113..9db5eaba0ca 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -1851,14 +1851,26 @@ public class DataRegion implements IDataRegionForQuery {
List<TsFileResource> sealedTsFileResource = new ArrayList<>();
List<TsFileResource> unsealedTsFileResource = new ArrayList<>();
separateTsFile(sealedTsFileResource, unsealedTsFileResource);
-
+ // deviceMatchInfo is used for filter the matched deviceId in TsFileResource
+ // deviceMatchInfo contains the DeviceId means this device matched the pattern
+ Set<String> deviceMatchInfo = new HashSet<>();
deleteDataInFiles(
- unsealedTsFileResource, deletion, devicePaths, updatedModFiles, timePartitionFilter);
+ unsealedTsFileResource,
+ deletion,
+ devicePaths,
+ updatedModFiles,
+ timePartitionFilter,
+ deviceMatchInfo);
writeUnlock();
hasReleasedLock = true;
deleteDataInFiles(
- sealedTsFileResource, deletion, devicePaths, updatedModFiles, timePartitionFilter);
+ sealedTsFileResource,
+ deletion,
+ devicePaths,
+ updatedModFiles,
+ timePartitionFilter,
+ deviceMatchInfo);
} catch (Exception e) {
// roll back
@@ -1916,16 +1928,32 @@ public class DataRegion implements IDataRegionForQuery {
Set<PartialPath> devicePaths,
long deleteStart,
long deleteEnd,
- TimePartitionFilter timePartitionFilter) {
+ TimePartitionFilter timePartitionFilter,
+ Set<String> deviceMatchInfo) {
if (timePartitionFilter != null
&& !timePartitionFilter.satisfy(databaseName, tsFileResource.getTimePartition())) {
return true;
}
+ long fileStartTime = tsFileResource.getTimeIndex().getMinStartTime();
+ long fileEndTime = tsFileResource.getTimeIndex().getMaxEndTime();
for (PartialPath device : devicePaths) {
long deviceStartTime, deviceEndTime;
if (device.hasWildcard()) {
- Pair<Long, Long> startAndEndTime = tsFileResource.getPossibleStartTimeAndEndTime(device);
+ if (!tsFileResource.isClosed() && fileEndTime == Long.MIN_VALUE) {
+ // unsealed seq file
+ if (deleteEnd < fileStartTime) {
+ // time range of file has not overlapped with the deletion
+ return true;
+ }
+ } else {
+ if (deleteEnd < fileStartTime || deleteStart > fileEndTime) {
+ // time range of file has not overlapped with the deletion
+ return true;
+ }
+ }
+ Pair<Long, Long> startAndEndTime =
+ tsFileResource.getPossibleStartTimeAndEndTime(device, deviceMatchInfo);
if (startAndEndTime == null) {
continue;
}
@@ -1962,7 +1990,8 @@ public class DataRegion implements IDataRegionForQuery {
Deletion deletion,
Set<PartialPath> devicePaths,
List<ModificationFile> updatedModFiles,
- TimePartitionFilter timePartitionFilter)
+ TimePartitionFilter timePartitionFilter,
+ Set<String> deviceMatchInfo)
throws IOException {
for (TsFileResource tsFileResource : tsFileResourceList) {
if (canSkipDelete(
@@ -1970,7 +1999,8 @@ public class DataRegion implements IDataRegionForQuery {
devicePaths,
deletion.getStartTime(),
deletion.getEndTime(),
- timePartitionFilter)) {
+ timePartitionFilter,
+ deviceMatchInfo)) {
continue;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index adbf8a59b39..32c8fa9ab26 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -503,8 +503,9 @@ public class TsFileResource {
* Get the min start time and max end time of devices matched by given devicePattern. If there's
* no device matched by given pattern, return null.
*/
- public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
- return timeIndex.getPossibleStartTimeAndEndTime(devicePattern);
+ public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+ PartialPath devicePattern, Set<String> deviceMatchInfo) {
+ return timeIndex.getPossibleStartTimeAndEndTime(devicePattern, deviceMatchInfo);
}
public boolean isClosed() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
index dc6f280f358..f1324582d9b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/DeviceTimeIndex.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.utils.SerializeUtils;
import org.apache.iotdb.db.exception.PartitionViolationException;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
@@ -377,13 +378,14 @@ public class DeviceTimeIndex implements ITimeIndex {
}
@Override
- public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
+ public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+ PartialPath devicePattern, Set<String> deviceMatchInfo) {
boolean hasMatchedDevice = false;
long startTime = Long.MAX_VALUE;
long endTime = Long.MIN_VALUE;
for (Entry<String, Integer> entry : deviceToIndex.entrySet()) {
try {
- if (devicePattern.matchFullPath(new PartialPath(entry.getKey()))) {
+ if (deviceMatchInfo.contains(entry.getKey())) {
hasMatchedDevice = true;
if (startTimes[entry.getValue()] < startTime) {
startTime = startTimes[entry.getValue()];
@@ -391,6 +393,18 @@ public class DeviceTimeIndex implements ITimeIndex {
if (endTimes[entry.getValue()] > endTime) {
endTime = endTimes[entry.getValue()];
}
+ } else {
+ if (devicePattern.matchFullPath(
+ DataNodeDevicePathCache.getInstance().getPartialPath(entry.getKey()))) {
+ deviceMatchInfo.add(entry.getKey());
+ hasMatchedDevice = true;
+ if (startTimes[entry.getValue()] < startTime) {
+ startTime = startTimes[entry.getValue()];
+ }
+ if (endTimes[entry.getValue()] > endTime) {
+ endTime = endTimes[entry.getValue()];
+ }
+ }
}
} catch (IllegalPathException e) {
// won't reach here
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
index cfa14e6521b..954459135fa 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java
@@ -234,7 +234,8 @@ public class FileTimeIndex implements ITimeIndex {
}
@Override
- public Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern) {
+ public Pair<Long, Long> getPossibleStartTimeAndEndTime(
+ PartialPath devicePattern, Set<String> deviceMatchInfo) {
return new Pair<>(startTime, endTime);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
index e2703ad455b..f867ac32a6f 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ITimeIndex.java
@@ -198,7 +198,8 @@ public interface ITimeIndex {
*/
long[] getStartAndEndTime(String deviceId);
- Pair<Long, Long> getPossibleStartTimeAndEndTime(PartialPath devicePattern);
+ Pair<Long, Long> getPossibleStartTimeAndEndTime(
+ PartialPath devicePattern, Set<String> deviceMatchInfo);
/**
* Get TimeIndex Type