You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2023/10/07 03:37:16 UTC
[iotdb] branch timestamp_check12 created (now 7d5469791b4)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch timestamp_check12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
at 7d5469791b4 [IOTDB-6157] Check timestamp precision when insert (#11151)
This branch includes the following new commits:
new 7d5469791b4 [IOTDB-6157] Check timestamp precision when insert (#11151)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: [IOTDB-6157] Check timestamp precision when insert (#11151)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch timestamp_check12
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 7d5469791b4a99c8e952825a2767acc3ad2976d4
Author: Haonan <hh...@outlook.com>
AuthorDate: Mon Sep 18 17:26:51 2023 +0800
[IOTDB-6157] Check timestamp precision when insert (#11151)
---
.../iotdb/it/env/cluster/MppCommonConfig.java | 7 ++
.../it/env/cluster/MppSharedCommonConfig.java | 7 ++
.../iotdb/it/env/remote/RemoteCommonConfig.java | 5 ++
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java | 38 +++++++++-
.../apache/iotdb/db/it/IoTDBDatetimeFormatIT.java | 1 +
.../apache/iotdb/db/it/IoTDBInsertMultiRowIT.java | 11 +++
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 36 +++++++++
.../iotdb/db/protocol/mqtt/MPPPublishHandler.java | 2 +
.../v1/handler/StatementConstructionHandler.java | 3 +
.../v2/handler/StatementConstructionHandler.java | 6 +-
.../plan/analyze/LoadTsfileAnalyzer.java | 5 +-
.../db/queryengine/plan/parser/ASTVisitor.java | 2 +
.../plan/parser/StatementGenerator.java | 25 ++++++-
.../iotdb/db/utils/TimestampPrecisionUtils.java | 36 ++++++++-
.../db/utils/TimestampPrecisionUtilsTest.java | 87 ++++++++++++++++++++++
.../resources/conf/iotdb-common.properties | 6 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 13 ++++
.../iotdb/commons/conf/CommonDescriptor.java | 10 +++
pom.xml | 4 +-
20 files changed, 293 insertions(+), 13 deletions(-)
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
index 5d1714f6347..34eae508be8 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppCommonConfig.java
@@ -217,6 +217,13 @@ public class MppCommonConfig extends MppBaseConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) {
+ setProperty(
+ "timestamp_precision_check_enabled", String.valueOf(timestampPrecisionCheckEnabled));
+ return this;
+ }
+
@Override
public CommonConfig setEnableMemControl(boolean enableMemControl) {
setProperty("enable_mem_control", String.valueOf(enableMemControl));
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
index 0b1daea98d3..e21dfca3b00 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/cluster/MppSharedCommonConfig.java
@@ -228,6 +228,13 @@ public class MppSharedCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) {
+ cnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled);
+ dnConfig.setTimestampPrecisionCheckEnabled(timestampPrecisionCheckEnabled);
+ return this;
+ }
+
@Override
public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(
int ratisSnapshotTriggerThreshold) {
diff --git a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
index 177b4b1af3d..7c6615753eb 100644
--- a/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/it/env/remote/RemoteCommonConfig.java
@@ -163,6 +163,11 @@ public class RemoteCommonConfig implements CommonConfig {
return this;
}
+ @Override
+ public CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) {
+ return this;
+ }
+
@Override
public CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(
int ratisSnapshotTriggerThreshold) {
diff --git a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
index df302e79891..943ab1d7c24 100644
--- a/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
+++ b/integration-test/src/main/java/org/apache/iotdb/itbase/env/CommonConfig.java
@@ -78,6 +78,8 @@ public interface CommonConfig {
CommonConfig setTimestampPrecision(String timestampPrecision);
+ CommonConfig setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled);
+
CommonConfig setConfigNodeRatisSnapshotTriggerThreshold(int ratisSnapshotTriggerThreshold);
CommonConfig setMaxDegreeOfIndexNode(int maxDegreeOfIndexNode);
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
index 780aa75e8c5..19170f7964c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IOTDBLoadTsFileIT.java
@@ -133,8 +133,7 @@ public class IOTDBLoadTsFileIT {
statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_0));
statement.execute(String.format("delete database %s", SchemaConfig.STORAGE_GROUP_1));
- } catch (IoTDBSQLException e) {
- LOGGER.info(String.format("delete storage group message : %s", e.getMessage()));
+ } catch (IoTDBSQLException ignored) {
}
}
@@ -570,7 +569,7 @@ public class IOTDBLoadTsFileIT {
@Test
public void testLoadWithEmptyTsFile() throws Exception {
- try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {}
+ try (TsFileGenerator ignored = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {}
try (Connection connection = EnvFactory.getEnv().getConnection();
Statement statement = connection.createStatement()) {
@@ -583,6 +582,39 @@ public class IOTDBLoadTsFileIT {
}
}
+ @Test
+ public void testLoadTsFileWithWrongTimestampPrecision() throws Exception {
+ try (TsFileGenerator generator = new TsFileGenerator(new File(tmpDir, "1-0-0-0.tsfile"))) {
+ generator.registerTimeseries(
+ SchemaConfig.DEVICE_0,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_00,
+ SchemaConfig.MEASUREMENT_01,
+ SchemaConfig.MEASUREMENT_02,
+ SchemaConfig.MEASUREMENT_03));
+ generator.registerAlignedTimeseries(
+ SchemaConfig.DEVICE_1,
+ Arrays.asList(
+ SchemaConfig.MEASUREMENT_10,
+ SchemaConfig.MEASUREMENT_11,
+ SchemaConfig.MEASUREMENT_12,
+ SchemaConfig.MEASUREMENT_13));
+ // generate ns timestamp
+ generator.generateData(
+ SchemaConfig.DEVICE_0, 100000, PARTITION_INTERVAL / 10_000, false, 1694689856546000000L);
+ generator.generateData(
+ SchemaConfig.DEVICE_1, 100000, PARTITION_INTERVAL / 10_000, true, 1694689856546000000L);
+ }
+
+ try (Connection connection = EnvFactory.getEnv().getConnection();
+ Statement statement = connection.createStatement()) {
+
+ statement.execute(String.format("load \"%s\"", tmpDir.getAbsolutePath()));
+ } catch (IoTDBSQLException e) {
+ Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
+ }
+ }
+
private static class SchemaConfig {
private static final String STORAGE_GROUP_0 = "root.sg.test_0";
private static final String STORAGE_GROUP_1 = "root.sg.test_1";
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
index 5812d6b432e..bb3a8de7d12 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBDatetimeFormatIT.java
@@ -44,6 +44,7 @@ public class IoTDBDatetimeFormatIT {
@Before
public void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getCommonConfig().setTimestampPrecisionCheckEnabled(false);
EnvFactory.getEnv().initClusterEnvironment();
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
index ef3eec20ad1..faff5a18172 100644
--- a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBInsertMultiRowIT.java
@@ -159,4 +159,15 @@ public class IoTDBInsertMultiRowIT {
fail();
}
}
+
+ @Test
+ public void testInsertMultiRowWithWrongTimestampPrecision() {
+ try (Statement st1 = connection.createStatement()) {
+ st1.execute(
+ "insert into root.t1.d99.wt01(timestamp, s1, s2) values(1618283005586000, 1, 1), (1618283005586001, 1, 2)");
+ fail();
+ } catch (SQLException e) {
+ assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
+ }
+ }
}
diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
index acf80a2dfc4..d637fed1f5c 100644
--- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
+++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java
@@ -537,6 +537,42 @@ public class IoTDBSessionSimpleIT {
}
}
+ @Test
+ @Category({LocalStandaloneIT.class, ClusterIT.class})
+ public void insertTabletWithWrongTimestampPrecisionTest() {
+ try (ISession session = EnvFactory.getEnv().getSessionConnection()) {
+ List<MeasurementSchema> schemaList = new ArrayList<>();
+ schemaList.add(new MeasurementSchema("s0", TSDataType.DOUBLE, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s3", TSDataType.INT32, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s4", TSDataType.BOOLEAN, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s5", TSDataType.TEXT, TSEncoding.RLE));
+ schemaList.add(new MeasurementSchema("s6", TSDataType.TEXT, TSEncoding.RLE));
+
+ Tablet tablet = new Tablet("root.sg1.d1", schemaList);
+ for (long time = 1694689856546000000L; time < 1694689856546000010L; time++) {
+ int rowIndex = tablet.rowSize++;
+ tablet.addTimestamp(rowIndex, time);
+
+ tablet.addValue(schemaList.get(0).getMeasurementId(), rowIndex, (double) time);
+ tablet.addValue(schemaList.get(1).getMeasurementId(), rowIndex, (float) time);
+ tablet.addValue(schemaList.get(2).getMeasurementId(), rowIndex, time);
+ tablet.addValue(schemaList.get(3).getMeasurementId(), rowIndex, (int) time);
+ tablet.addValue(schemaList.get(4).getMeasurementId(), rowIndex, time % 2 == 0);
+ tablet.addValue(schemaList.get(5).getMeasurementId(), rowIndex, new Binary("Text" + time));
+ tablet.addValue(schemaList.get(6).getMeasurementId(), rowIndex, "Text" + time);
+ }
+
+ if (tablet.rowSize != 0) {
+ session.insertTablet(tablet);
+ tablet.reset();
+ }
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("Current system timestamp precision is ms"));
+ }
+ }
+
@Test
@Category({LocalStandaloneIT.class, ClusterIT.class})
public void createTimeSeriesWithDoubleTicksTest() {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
index a23df1d55d8..4063c7c44d1 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/mqtt/MPPPublishHandler.java
@@ -34,6 +34,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.schema.ISchemaFetcher;
import org.apache.iotdb.db.queryengine.plan.execution.ExecutionResult;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.utils.CommonUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.service.rpc.thrift.TSProtocolVersion;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -134,6 +135,7 @@ public class MPPPublishHandler extends AbstractInterceptHandler {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(
DataNodeDevicePathCache.getInstance().getPartialPath(event.getDevice()));
+ TimestampPrecisionUtils.checkTimestampPrecision(event.getTimestamp());
statement.setTime(event.getTimestamp());
statement.setMeasurements(event.getMeasurements().toArray(new String[0]));
if (event.getDataTypes() == null) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
index 9e91aedaa32..eb2d32df72e 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v1/handler/StatementConstructionHandler.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.db.exception.WriteProcessRejectException;
import org.apache.iotdb.db.protocol.rest.v1.model.InsertTabletRequest;
import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDevicePathCache;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.BitMap;
@@ -37,6 +38,8 @@ public class StatementConstructionHandler {
public static InsertTabletStatement constructInsertTabletStatement(
InsertTabletRequest insertTabletRequest)
throws MetadataException, WriteProcessRejectException {
+ TimestampPrecisionUtils.checkTimestampPrecision(
+ insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size() - 1));
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
insertStatement.setDevicePath(
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
index 2f7fb2bc2b9..ed5f1b05ca4 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/rest/v2/handler/StatementConstructionHandler.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.cache.schema.DataNodeDeviceP
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowsStatement;
import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertTabletStatement;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.utils.Binary;
@@ -42,7 +43,9 @@ public class StatementConstructionHandler {
public static InsertTabletStatement constructInsertTabletStatement(
InsertTabletRequest insertTabletRequest)
- throws MetadataException, WriteProcessRejectException {
+ throws IllegalPathException, WriteProcessRejectException {
+ TimestampPrecisionUtils.checkTimestampPrecision(
+ insertTabletRequest.getTimestamps().get(insertTabletRequest.getTimestamps().size() - 1));
// construct insert statement
InsertTabletStatement insertStatement = new InsertTabletStatement();
insertStatement.setDevicePath(
@@ -210,6 +213,7 @@ public class StatementConstructionHandler {
PathUtils.checkIsLegalSingleMeasurementsAndUpdate(
insertRecordsRequest.getMeasurementsList().get(i))
.toArray(new String[0]));
+ TimestampPrecisionUtils.checkTimestampPrecision(insertRecordsRequest.getTimestamps().get(i));
statement.setTime(insertRecordsRequest.getTimestamps().get(i));
statement.setDataTypes(dataTypesList.get(i).toArray(new TSDataType[0]));
List<Object> values =
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
index 13bd528e56b..5f507dbe1d9 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/LoadTsfileAnalyzer.java
@@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.DatabaseSchemaSta
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.db.utils.FileLoaderUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
@@ -125,7 +126,8 @@ public class LoadTsfileAnalyzer {
} catch (Exception e) {
LOGGER.warn(String.format("Parse file %s to resource error.", tsFile.getPath()), e);
throw new SemanticException(
- String.format("Parse file %s to resource error", tsFile.getPath()));
+ String.format(
+ "Parse file %s to resource error, because %s", tsFile.getPath(), e.getMessage()));
}
}
@@ -171,6 +173,7 @@ public class LoadTsfileAnalyzer {
} else {
tsFileResource.deserialize();
}
+ TimestampPrecisionUtils.checkTimestampPrecision(tsFileResource.getFileEndTime());
tsFileResource.setStatus(TsFileResourceStatus.NORMAL);
loadTsFileStatement.addTsFileResource(tsFileResource);
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
index f3e394c5808..2a22cbb2342 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/ASTVisitor.java
@@ -196,6 +196,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowSpaceQuotaSt
import org.apache.iotdb.db.queryengine.plan.statement.sys.quota.ShowThrottleQuotaStatement;
import org.apache.iotdb.db.schemaengine.template.TemplateAlterOperationType;
import org.apache.iotdb.db.utils.DateTimeUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.trigger.api.enums.TriggerEvent;
import org.apache.iotdb.trigger.api.enums.TriggerType;
@@ -1844,6 +1845,7 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> {
} else {
timestamp =
parseTimeValue(insertMultiValues.get(i).timeValue(), DateTimeUtils.currentTime());
+ TimestampPrecisionUtils.checkTimestampPrecision(timestamp);
}
} else {
if (!isTimeDefault) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
index 74cb4fb8aa2..4e90fdf6e24 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/parser/StatementGenerator.java
@@ -66,6 +66,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.metadata.template.UnsetSch
import org.apache.iotdb.db.schemaengine.schemaregion.utils.MetaFormatUtils;
import org.apache.iotdb.db.schemaengine.template.TemplateQueryType;
import org.apache.iotdb.db.utils.QueryDataSetUtils;
+import org.apache.iotdb.db.utils.TimestampPrecisionUtils;
import org.apache.iotdb.db.utils.constant.SqlConstant;
import org.apache.iotdb.mpp.rpc.thrift.TDeleteModelMetricsReq;
import org.apache.iotdb.mpp.rpc.thrift.TFetchTimeseriesReq;
@@ -283,6 +284,7 @@ public class StatementGenerator {
InsertRowStatement insertStatement = new InsertRowStatement();
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath()));
+ TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp());
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0]));
insertStatement.setAligned(insertRecordReq.isAligned);
@@ -298,6 +300,7 @@ public class StatementGenerator {
InsertRowStatement insertStatement = new InsertRowStatement();
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertRecordReq.getPrefixPath()));
+ TimestampPrecisionUtils.checkTimestampPrecision(insertRecordReq.getTimestamp());
insertStatement.setTime(insertRecordReq.getTimestamp());
insertStatement.setMeasurements(insertRecordReq.getMeasurements().toArray(new String[0]));
insertStatement.setDataTypes(new TSDataType[insertStatement.getMeasurements().length]);
@@ -316,8 +319,12 @@ public class StatementGenerator {
insertStatement.setDevicePath(
DEVICE_PATH_CACHE.getPartialPath(insertTabletReq.getPrefixPath()));
insertStatement.setMeasurements(insertTabletReq.getMeasurements().toArray(new String[0]));
- insertStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size));
+ long[] timestamps =
+ QueryDataSetUtils.readTimesFromBuffer(insertTabletReq.timestamps, insertTabletReq.size);
+ if (timestamps.length != 0) {
+ TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length - 1]);
+ }
+ insertStatement.setTimes(timestamps);
insertStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
insertTabletReq.values,
@@ -349,8 +356,12 @@ public class StatementGenerator {
InsertTabletStatement insertTabletStatement = new InsertTabletStatement();
insertTabletStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPaths.get(i)));
insertTabletStatement.setMeasurements(req.measurementsList.get(i).toArray(new String[0]));
- insertTabletStatement.setTimes(
- QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i)));
+ long[] timestamps =
+ QueryDataSetUtils.readTimesFromBuffer(req.timestampsList.get(i), req.sizeList.get(i));
+ if (timestamps.length != 0) {
+ TimestampPrecisionUtils.checkTimestampPrecision(timestamps[timestamps.length - 1]);
+ }
+ insertTabletStatement.setTimes(timestamps);
insertTabletStatement.setColumns(
QueryDataSetUtils.readTabletValuesFromBuffer(
req.valuesList.get(i),
@@ -389,6 +400,7 @@ public class StatementGenerator {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.getPrefixPaths().get(i)));
statement.setMeasurements(req.getMeasurementsList().get(i).toArray(new String[0]));
+ TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i));
statement.setTime(req.getTimestamps().get(i));
statement.fillValues(req.valuesList.get(i));
statement.setAligned(req.isAligned);
@@ -415,6 +427,7 @@ public class StatementGenerator {
addMeasurementAndValue(
statement, req.getMeasurementsList().get(i), req.getValuesList().get(i));
statement.setDataTypes(new TSDataType[statement.getMeasurements().length]);
+ TimestampPrecisionUtils.checkTimestampPrecision(req.getTimestamps().get(i));
statement.setTime(req.getTimestamps().get(i));
statement.setNeedInferType(true);
statement.setAligned(req.isAligned);
@@ -436,6 +449,8 @@ public class StatementGenerator {
InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement();
insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
+ // req.timestamps sorted on session side
+ TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size() - 1));
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(insertStatement.getDevicePath());
@@ -461,6 +476,8 @@ public class StatementGenerator {
InsertRowsOfOneDeviceStatement insertStatement = new InsertRowsOfOneDeviceStatement();
insertStatement.setDevicePath(DEVICE_PATH_CACHE.getPartialPath(req.prefixPath));
List<InsertRowStatement> insertRowStatementList = new ArrayList<>();
+ // req.timestamps sorted on session side
+ TimestampPrecisionUtils.checkTimestampPrecision(req.timestamps.get(req.timestamps.size() - 1));
for (int i = 0; i < req.timestamps.size(); i++) {
InsertRowStatement statement = new InsertRowStatement();
statement.setDevicePath(insertStatement.getDevicePath());
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
index e613dacf908..5b7bafc3b34 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/TimestampPrecisionUtils.java
@@ -19,12 +19,15 @@
package org.apache.iotdb.db.utils;
import org.apache.iotdb.commons.conf.CommonDescriptor;
+import org.apache.iotdb.db.exception.sql.SemanticException;
import java.util.concurrent.TimeUnit;
public class TimestampPrecisionUtils {
- private static final String timestampPrecision =
+ static String TIMESTAMP_PRECISION =
CommonDescriptor.getInstance().getConfig().getTimestampPrecision();
+ private static boolean isTimestampPrecisionCheckEnabled =
+ CommonDescriptor.getInstance().getConfig().isTimestampPrecisionCheckEnabled();
@FunctionalInterface
private interface ConvertFunction<T1, T2, R> {
@@ -54,4 +57,35 @@ public class TimestampPrecisionUtils {
public static long convertToCurrPrecision(long sourceTime, TimeUnit sourceUnit) {
return convertFunction.apply(sourceTime, sourceUnit);
}
+
+ /** check whether the input timestamp match the current system timestamp precision. */
+ public static void checkTimestampPrecision(long time) {
+ if (!isTimestampPrecisionCheckEnabled) {
+ return;
+ }
+ switch (TIMESTAMP_PRECISION) {
+ case "ms":
+ if (time > 10_000_000_000_000L) {
+ throw new SemanticException(
+ String.format(
+ "Current system timestamp precision is %s, "
+ + "please check whether the timestamp %s is correct.",
+ TIMESTAMP_PRECISION, time));
+ }
+ break;
+ case "us":
+ if (time > 10_000_000_000_000_000L) {
+ throw new SemanticException(
+ String.format(
+ "Current system timestamp precision is %s, "
+ + "please check whether the timestamp %s is correct.",
+ TIMESTAMP_PRECISION, time));
+ }
+ break;
+ // Long.MaxValue is 19 digits, therefore no problem when the precision is ns.
+ case "ns":
+ default:
+ break;
+ }
+ }
}
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java
new file mode 100644
index 00000000000..87585cc663f
--- /dev/null
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/utils/TimestampPrecisionUtilsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.utils;
+
+import org.junit.Test;
+
+import static org.junit.Assert.fail;
+
+public class TimestampPrecisionUtilsTest {
+
+ @Test
+ public void testCheckMsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCheckIllegalMsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ms";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ fail();
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void testCheckUsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+
+ @Test
+ public void testCheckIllegalUsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "us";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L);
+ fail();
+ } catch (Exception ignored) {
+ }
+ }
+
+ @Test
+ public void testCheckNsTimestampPrecision() {
+ TimestampPrecisionUtils.TIMESTAMP_PRECISION = "ns";
+ try {
+ TimestampPrecisionUtils.checkTimestampPrecision(-1L);
+ TimestampPrecisionUtils.checkTimestampPrecision(0L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000L);
+ TimestampPrecisionUtils.checkTimestampPrecision(1694689856546000000L);
+ TimestampPrecisionUtils.checkTimestampPrecision(Long.MAX_VALUE);
+ } catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+}
diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
index e27e572b213..149c97e5674 100644
--- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -449,7 +449,11 @@ cluster_name=defaultCluster
# Use this value to set timestamp precision as "ms", "us" or "ns".
# Once the precision has been set, it can not be changed.
# Datatype: String
-# timestamp_precision=ms
+timestamp_precision=ms
+
+# When the timestamp precision check is enabled, the timestamps those are over 13 digits for ms precision, or over 16 digits for us precision are not allowed to be inserted.
+# Datatype: Boolean
+# timestamp_precision_check_enabled=true
# Default TTL for databases that are not set TTL by statements, If not set (default), the TTL will be unlimited.
# Negative value means the TTL is unlimited.
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
index adc5cee2158..2556b6155d4 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonConfig.java
@@ -137,6 +137,11 @@ public class CommonConfig {
/** This variable set timestamp precision as millisecond, microsecond or nanosecond. */
private String timestampPrecision = "ms";
+ private boolean timestampPrecisionCheckEnabled = true;
+
+ /** The number of threads in the thread pool that execute model inference tasks. */
+ private int modelInferenceExecutionThreadCount = 5;
+
/**
* The name of the directory that stores the tsfiles temporarily hold or generated by the pipe
* module. The directory is located in the data directory of IoTDB.
@@ -472,6 +477,14 @@ public class CommonConfig {
return timestampPrecision;
}
+ public void setTimestampPrecisionCheckEnabled(boolean timestampPrecisionCheckEnabled) {
+ this.timestampPrecisionCheckEnabled = timestampPrecisionCheckEnabled;
+ }
+
+ public boolean isTimestampPrecisionCheckEnabled() {
+ return timestampPrecisionCheckEnabled;
+ }
+
public String getPipeHardlinkBaseDirName() {
return pipeHardlinkBaseDirName;
}
diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
index 76925891899..5431647a7e4 100644
--- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
+++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/conf/CommonDescriptor.java
@@ -200,6 +200,16 @@ public class CommonDescriptor {
config.setTimestampPrecision(
properties.getProperty("timestamp_precision", config.getTimestampPrecision()).trim());
+ config.setTimestampPrecisionCheckEnabled(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "timestamp_precision_check_enabled",
+ String.valueOf(config.isTimestampPrecisionCheckEnabled()))));
+
+ config.setDatanodeTokenTimeoutMS(
+ Integer.parseInt(
+ properties.getProperty("datanode_token_timeout", String.valueOf(3 * 60 * 1000))));
+
String endPointUrl =
properties.getProperty(
"target_ml_node_endpoint",
diff --git a/pom.xml b/pom.xml
index ded8698fcf8..9530aa5fcd9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1194,6 +1194,7 @@
</activation>
<properties>
<maven.compiler.release>8</maven.compiler.release>
+ <argLine>--illegal-access=permit --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.java [...]
<!-- change to 1.15.0 will modify many codes (all are javadocs), we change it to 1.15.0
until: iotdb decides to do not support jdk8-->
<google.java.format.version>1.7</google.java.format.version>
@@ -1223,8 +1224,7 @@
<jdk>[16,20)</jdk>
</activation>
<properties>
- <maven.compiler.release>8</maven.compiler.release>
- <argLine>--illegal-access=permit --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.java [...]
+ <argLine>--add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add [...]
</properties>
</profile>
<!--