You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yc...@apache.org on 2024/07/09 15:36:43 UTC
(iotdb) branch ttlInActive created (now 0ac380e4a3d)
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a change to branch ttlInActive
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 0ac380e4a3d feat: support new filter for ttl
This branch includes the following new commits:
new 0ac380e4a3d feat: support new filter for ttl
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(iotdb) 01/01: feat: support new filter for ttl
Posted by yc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ycycse pushed a commit to branch ttlInActive
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 0ac380e4a3dd31fc01a5a7546c7ea07002e117b3
Author: YangCaiyin <yc...@gmail.com>
AuthorDate: Tue Jul 9 23:36:28 2024 +0800
feat: support new filter for ttl
---
.../source/ActiveDeviceRegionScanOperator.java | 1 -
.../source/RegionScanForActiveDeviceUtil.java | 12 ++++-
.../iotdb/db/utils/TimeFilterForDeviceTTL.java | 54 ++++++++++++++++++++++
3 files changed, 65 insertions(+), 2 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java
index 6437fe673fa..e74d1c675df 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java
@@ -91,7 +91,6 @@ public class ActiveDeviceRegionScanOperator extends AbstractRegionScanDataSource
int templateId = deviceContext.getTemplateId();
// TODO: use IDeviceID interface to get ttl
long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
-
// TODO: make it more readable, like "30 days" or "10 hours"
String ttlStr = ttl == Long.MAX_VALUE ? IoTDBConstant.TTL_INFINITE : String.valueOf(ttl);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
index 11faa55f394..cb66e0aeee6 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/RegionScanForActiveDeviceUtil.java
@@ -20,7 +20,9 @@
package org.apache.iotdb.db.queryengine.execution.operator.source;
import org.apache.iotdb.commons.exception.IllegalPathException;
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.queryengine.common.DeviceContext;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractChunkOffset;
import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.AbstractDeviceChunkMetaData;
import org.apache.iotdb.db.storageengine.dataregion.read.filescan.model.DeviceStartEndTime;
@@ -77,7 +79,7 @@ public class RegionScanForActiveDeviceUtil extends AbstractRegionScanForActiveDa
while (iterator.hasNext()) {
DeviceStartEndTime deviceStartEndTime = iterator.next();
IDeviceID deviceID = deviceStartEndTime.getDevicePath();
- long startTime = deviceStartEndTime.getStartTime();
+ long startTime = getStartTimeConsideringTTL(deviceID, deviceStartEndTime.getStartTime());
long endTime = deviceStartEndTime.getEndTime();
// If this device has already been removed by another TsFileHandle, we should skip it.
// If the time range is filtered, the devicePath is not active in this time range.
@@ -100,6 +102,14 @@ public class RegionScanForActiveDeviceUtil extends AbstractRegionScanForActiveDa
return true;
}
+ private long getStartTimeConsideringTTL(IDeviceID deviceID, long originalStartTime){
+ long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
+ if(ttl != Long.MAX_VALUE){
+ return Math.max(originalStartTime, CommonDateTimeUtils.currentTime() - ttl);
+ }
+ return originalStartTime;
+ }
+
@Override
public void processDeviceChunkMetadata(AbstractDeviceChunkMetaData deviceChunkMetaData)
throws IllegalPathException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
new file mode 100644
index 00000000000..8e3505a871b
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimeFilterForDeviceTTL.java
@@ -0,0 +1,54 @@
+package org.apache.iotdb.db.utils;
+
+import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
+import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeTTLCache;
+import org.apache.tsfile.file.metadata.IDeviceID;
+import org.apache.tsfile.read.filter.basic.Filter;
+
+import java.util.Map;
+
+public class TimeFilterForDeviceTTL {
+
+ private Filter timeFilter;
+
+ private Map<IDeviceID, Long> ttlCached;
+
+ public TimeFilterForDeviceTTL(Filter timeFilter) {
+ this.timeFilter = timeFilter;
+ }
+
+ public boolean satisfyStartEndTIme(long startTime, long endTime, IDeviceID deviceID){
+ long ttl = getTTL(deviceID);
+ if(ttl != Long.MAX_VALUE){
+ long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
+ if(validStartTime > endTime){
+ return false;
+ }
+ return timeFilter.satisfyStartEndTime(validStartTime, endTime);
+ }
+ return timeFilter.satisfyStartEndTime(startTime, endTime);
+ }
+
+ public boolean satisfy(long time, IDeviceID deviceID){
+ long ttl = getTTL(deviceID);
+ if(ttl != Long.MAX_VALUE) {
+ long validStartTime = CommonDateTimeUtils.currentTime() - ttl;
+ if(validStartTime > time){
+ return false;
+ }
+ return timeFilter.satisfy(validStartTime, null);
+ }
+ return timeFilter.satisfy(time, null);
+ }
+
+ private long getTTL(IDeviceID deviceID){
+ if(ttlCached.containsKey(deviceID)){
+ return ttlCached.get(deviceID);
+ }
+ long ttl = DataNodeTTLCache.getInstance().getTTL(deviceID);
+ ttlCached.put(deviceID, ttl);
+ return ttl;
+ }
+
+
+}