You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2022/10/13 00:36:27 UTC
[iotdb] branch master updated: [IOTDB-4615] TTL supports timestamp precision (#7577)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new fa0ed4dd82 [IOTDB-4615] TTL supports timestamp precision (#7577)
fa0ed4dd82 is described below
commit fa0ed4dd82d0f27ec0133336c37c3f264975c67d
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Oct 13 08:36:20 2022 +0800
[IOTDB-4615] TTL supports timestamp precision (#7577)
---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 20 ++---------------
.../org/apache/iotdb/db/engine/StorageEngine.java | 21 ++----------------
.../apache/iotdb/db/engine/StorageEngineV2.java | 8 +++----
.../db/engine/querycontext/QueryDataSource.java | 5 +++--
.../iotdb/db/engine/storagegroup/DataRegion.java | 25 +++++++++++++++-------
.../db/engine/storagegroup/TsFileResource.java | 3 ++-
.../dataregion/StorageGroupManager.java | 2 +-
.../db/mpp/common/header/ColumnHeaderConstant.java | 2 +-
.../apache/iotdb/db/qp/utils/DatetimeUtils.java | 16 ++++++++++++++
9 files changed, 47 insertions(+), 55 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 38eb3e3c47..6be34e457d 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -1005,7 +1005,7 @@ public class IoTDBDescriptor {
loadAuthorCache(properties);
conf.setTimePartitionIntervalForStorage(
- convertMilliWithPrecision(conf.getTimePartitionIntervalForStorage()));
+ DatetimeUtils.convertMilliTimeWithPrecision(conf.getTimePartitionIntervalForStorage()));
}
private void loadAuthorCache(Properties properties) {
@@ -1918,7 +1918,7 @@ public class IoTDBDescriptor {
conf.setSeriesPartitionExecutorClass(globalConfig.getSeriesPartitionExecutorClass());
conf.setSeriesPartitionSlotNum(globalConfig.getSeriesPartitionSlotNum());
conf.setTimePartitionIntervalForRouting(
- convertMilliWithPrecision(globalConfig.timePartitionInterval));
+ DatetimeUtils.convertMilliTimeWithPrecision(globalConfig.timePartitionInterval));
conf.setReadConsistencyLevel(globalConfig.getReadConsistencyLevel());
}
@@ -1991,22 +1991,6 @@ public class IoTDBDescriptor {
logger.info("Cluster allocateMemoryForLastCache = {}", conf.getAllocateMemoryForLastCache());
}
- public long convertMilliWithPrecision(long milliTime) {
- long result = milliTime;
- String timePrecision = conf.getTimestampPrecision();
- switch (timePrecision) {
- case "ns":
- result = milliTime * 1000_000L;
- break;
- case "us":
- result = milliTime * 1000L;
- break;
- default:
- break;
- }
- return result;
- }
-
private static class IoTDBDescriptorHolder {
private static final IoTDBDescriptor INSTANCE = new IoTDBDescriptor();
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 08c3dc3b2c..31d781d334 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@ -147,24 +147,7 @@ public class StorageEngine implements IService {
private static void initTimePartition() {
timePartitionInterval =
- convertMilliWithPrecision(
- IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForStorage());
- }
-
- public static long convertMilliWithPrecision(long milliTime) {
- long result = milliTime;
- String timePrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
- switch (timePrecision) {
- case "ns":
- result = milliTime * 1000_000L;
- break;
- case "us":
- result = milliTime * 1000L;
- break;
- default:
- break;
- }
- return result;
+ IoTDBDescriptor.getInstance().getConfig().getTimePartitionIntervalForStorage();
}
public static long getTimePartitionInterval() {
@@ -590,7 +573,7 @@ public class StorageEngine implements IService {
virtualStorageGroupId,
fileFlushPolicy,
storageGroupMNode.getFullPath());
- processor.setDataTTL(storageGroupMNode.getDataTTL());
+ processor.setDataTTLWithTimePrecisionCheck(storageGroupMNode.getDataTTL());
processor.setCustomFlushListeners(customFlushListeners);
processor.setCustomCloseFileListeners(customCloseFileListeners);
return processor;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
index 0033d1b963..e978f5091e 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngineV2.java
@@ -104,8 +104,6 @@ public class StorageEngineV2 implements IService {
/** whether enable data partition if disabled, all data belongs to partition 0 */
@ServerConfigConsistent private static boolean enablePartition = config.isEnablePartition();
- private final boolean enableMemControl = config.isEnableMemControl();
-
/**
* a folder (system/storage_groups/ by default) that persist system info. Each Storage Processor
* will have a subfolder under the systemDir.
@@ -480,7 +478,7 @@ public class StorageEngineV2 implements IService {
String.valueOf(dataRegionId.getId()),
fileFlushPolicy,
logicalStorageGroupName);
- dataRegion.setDataTTL(ttl);
+ dataRegion.setDataTTLWithTimePrecisionCheck(ttl);
dataRegion.setCustomFlushListeners(customFlushListeners);
dataRegion.setCustomCloseFileListeners(customCloseFileListeners);
return dataRegion;
@@ -566,7 +564,7 @@ public class StorageEngineV2 implements IService {
for (DataRegionId dataRegionId : dataRegionIdList) {
DataRegion dataRegion = dataRegionMap.get(dataRegionId);
if (dataRegion != null) {
- dataRegion.setDataTTL(dataTTL);
+ dataRegion.setDataTTLWithTimePrecisionCheck(dataTTL);
}
}
}
@@ -700,7 +698,7 @@ public class StorageEngineV2 implements IService {
for (DataRegionId dataRegionId : dataRegionIdList) {
DataRegion dataRegion = dataRegionMap.get(dataRegionId);
if (dataRegion != null) {
- dataRegion.setDataTTL(req.TTL);
+ dataRegion.setDataTTLWithTimePrecisionCheck(req.TTL);
}
}
return RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
index 175fe567be..64ffa01198 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/QueryDataSource.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.engine.querycontext;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
+import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.tsfile.read.filter.TimeFilter;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.read.filter.operator.AndFilter;
@@ -77,9 +78,9 @@ public class QueryDataSource {
public Filter updateFilterUsingTTL(Filter filter) {
if (dataTTL != Long.MAX_VALUE) {
if (filter != null) {
- filter = new AndFilter(filter, TimeFilter.gtEq(System.currentTimeMillis() - dataTTL));
+ filter = new AndFilter(filter, TimeFilter.gtEq(DatetimeUtils.currentTime() - dataTTL));
} else {
- filter = TimeFilter.gtEq(System.currentTimeMillis() - dataTTL);
+ filter = TimeFilter.gtEq(DatetimeUtils.currentTime() - dataTTL);
}
}
return filter;
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
index 188bea0a9e..4851fd097e 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/DataRegion.java
@@ -80,6 +80,7 @@ import org.apache.iotdb.db.qp.physical.crud.DeletePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
import org.apache.iotdb.db.qp.physical.crud.InsertRowsOfOneDevicePlan;
import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
+import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.query.context.QueryContext;
import org.apache.iotdb.db.query.control.FileReaderManager;
import org.apache.iotdb.db.query.control.QueryFileManager;
@@ -853,7 +854,7 @@ public class DataRegion {
throws WriteProcessException, TriggerExecutionException {
// reject insertions that are out of ttl
if (!isAlive(insertRowPlan.getTime())) {
- throw new OutOfTTLException(insertRowPlan.getTime(), (System.currentTimeMillis() - dataTTL));
+ throw new OutOfTTLException(insertRowPlan.getTime(), (DatetimeUtils.currentTime() - dataTTL));
}
writeLock("InsertRow");
try {
@@ -896,7 +897,7 @@ public class DataRegion {
throws WriteProcessException, TriggerExecutionException {
// reject insertions that are out of ttl
if (!isAlive(insertRowNode.getTime())) {
- throw new OutOfTTLException(insertRowNode.getTime(), (System.currentTimeMillis() - dataTTL));
+ throw new OutOfTTLException(insertRowNode.getTime(), (DatetimeUtils.currentTime() - dataTTL));
}
if (enableMemControl) {
StorageEngineV2.blockInsertionIfReject(null);
@@ -1090,7 +1091,7 @@ public class DataRegion {
if (loc == insertTabletNode.getRowCount()) {
throw new OutOfTTLException(
insertTabletNode.getTimes()[insertTabletNode.getTimes().length - 1],
- (System.currentTimeMillis() - dataTTL));
+ (DatetimeUtils.currentTime() - dataTTL));
}
// TODO(Trigger)// fire trigger before insertion
@@ -1156,7 +1157,7 @@ public class DataRegion {
* @return whether the given time falls in ttl
*/
private boolean isAlive(long time) {
- return dataTTL == Long.MAX_VALUE || (System.currentTimeMillis() - time) <= dataTTL;
+ return dataTTL == Long.MAX_VALUE || (DatetimeUtils.currentTime() - time) <= dataTTL;
}
/**
@@ -1747,7 +1748,7 @@ public class DataRegion {
logger.debug("{}: TTL not set, ignore the check", storageGroupName + "-" + dataRegionId);
return;
}
- long ttlLowerBound = System.currentTimeMillis() - dataTTL;
+ long ttlLowerBound = DatetimeUtils.currentTime() - dataTTL;
logger.debug(
"{}: TTL removing files before {}",
storageGroupName + "-" + dataRegionId,
@@ -2039,7 +2040,7 @@ public class DataRegion {
List<TsFileResource> tsfileResourcesForQuery = new ArrayList<>();
long timeLowerBound =
- dataTTL != Long.MAX_VALUE ? System.currentTimeMillis() - dataTTL : Long.MIN_VALUE;
+ dataTTL != Long.MAX_VALUE ? DatetimeUtils.currentTime() - dataTTL : Long.MIN_VALUE;
context.setQueryTimeLowerBound(timeLowerBound);
// for upgrade files and old files must be closed
@@ -3295,6 +3296,13 @@ public class DataRegion {
return workUnsequenceTsFileProcessors.values();
}
+ public void setDataTTLWithTimePrecisionCheck(long dataTTL) {
+ if (dataTTL != Long.MAX_VALUE) {
+ dataTTL = DatetimeUtils.convertMilliTimeWithPrecision(dataTTL);
+ }
+ this.dataTTL = dataTTL;
+ }
+
public void setDataTTL(long dataTTL) {
this.dataTTL = dataTTL;
}
@@ -3543,8 +3551,9 @@ public class DataRegion {
TSStatusCode.OUT_OF_TTL_ERROR.getStatusCode(),
String.format(
"Insertion time [%s] is less than ttl time bound [%s]",
- new Date(insertRowNode.getTime()),
- new Date(System.currentTimeMillis() - dataTTL))));
+ DatetimeUtils.convertMillsecondToZonedDateTime(insertRowNode.getTime()),
+ DatetimeUtils.convertMillsecondToZonedDateTime(
+ DatetimeUtils.currentTime() - dataTTL))));
continue;
}
// init map
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 5fc6d4a06a..75a296c7db 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.engine.storagegroup.timeindex.V012FileTimeIndex;
import org.apache.iotdb.db.engine.upgrade.UpgradeTask;
import org.apache.iotdb.db.exception.PartitionViolationException;
import org.apache.iotdb.db.metadata.utils.ResourceByPathUtils;
+import org.apache.iotdb.db.qp.utils.DatetimeUtils;
import org.apache.iotdb.db.query.filter.TsFileFilter;
import org.apache.iotdb.db.service.UpgradeSevice;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
@@ -730,7 +731,7 @@ public class TsFileResource {
/** @return whether the given time falls in ttl */
private boolean isAlive(long time, long dataTTL) {
- return dataTTL == Long.MAX_VALUE || (System.currentTimeMillis() - time) <= dataTTL;
+ return dataTTL == Long.MAX_VALUE || (DatetimeUtils.currentTime() - time) <= dataTTL;
}
public void setProcessor(TsFileProcessor processor) {
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/dataregion/StorageGroupManager.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/dataregion/StorageGroupManager.java
index b1cbd047a3..91717b2e61 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/dataregion/StorageGroupManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/dataregion/StorageGroupManager.java
@@ -390,7 +390,7 @@ public class StorageGroupManager {
public void setTTL(long dataTTL) {
for (DataRegion dataRegion : this.dataRegion) {
if (dataRegion != null) {
- dataRegion.setDataTTL(dataTTL);
+ dataRegion.setDataTTLWithTimePrecisionCheck(dataTTL);
}
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
index d91eb57ca7..a9dcc297c0 100644
--- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/ColumnHeaderConstant.java
@@ -44,7 +44,7 @@ public class ColumnHeaderConstant {
public static final String COLUMN_ATTRIBUTES = "attributes";
public static final String COLUMN_IS_ALIGNED = "isAligned";
public static final String COLUMN_COUNT = "count";
- public static final String COLUMN_TTL = "ttl";
+ public static final String COLUMN_TTL = "ttl(ms)";
public static final String COLUMN_SCHEMA_REPLICATION_FACTOR = "SchemaReplicationFactor";
public static final String COLUMN_DATA_REPLICATION_FACTOR = "DataReplicationFactor";
public static final String COLUMN_TIME_PARTITION_INTERVAL = "TimePartitionInterval";
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java b/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
index e7ed8e6478..ddae6eb9c1 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/utils/DatetimeUtils.java
@@ -686,6 +686,22 @@ public class DatetimeUtils {
return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millisecond), ZoneId.systemDefault());
}
+ public static long convertMilliTimeWithPrecision(long milliTime) {
+ long result = milliTime;
+ String timePrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ switch (timePrecision) {
+ case "ns":
+ result = milliTime * 1000_000L;
+ break;
+ case "us":
+ result = milliTime * 1000L;
+ break;
+ default:
+ break;
+ }
+ return result;
+ }
+
public enum DurationUnit {
y,
mo,