You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2019/07/23 11:06:42 UTC
[incubator-iotdb] branch master updated: Add micro and nano
timestamp precision (#285)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ab7d2db Add micro and nano timestamp precision (#285)
ab7d2db is described below
commit ab7d2db4394b2c4449c6f57e7c3d3af43e9725fd
Author: 1160300922 <36...@users.noreply.github.com>
AuthorDate: Tue Jul 23 20:06:37 2019 +0900
Add micro and nano timestamp precision (#285)
* add micro and nano precision
---
.../apache/iotdb/cli/client/AbstractClient.java | 63 ++++-
.../java/org/apache/iotdb/cli/client/Client.java | 2 +
.../org/apache/iotdb/cli/client/WinClient.java | 2 +
.../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 4 +-
server/iotdb/conf/iotdb-engine.properties | 9 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 ++-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 94 +++++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 5 +
.../apache/iotdb/db/qp/constant/DatetimeUtils.java | 297 +++++++++++++++++++--
.../iotdb/db/qp/strategy/LogicalGenerator.java | 2 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 1 +
.../db/sql/DatetimeQueryDataSetUtilsTest.java | 8 +-
service-rpc/src/main/thrift/rpc.thrift | 1 +
14 files changed, 494 insertions(+), 30 deletions(-)
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java b/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
index 2ded013..9dcb7ab 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
+++ b/client/src/main/java/org/apache/iotdb/cli/client/AbstractClient.java
@@ -82,7 +82,7 @@ public abstract class AbstractClient {
protected static final String SHOW_METADATA_COMMAND = "show timeseries";
protected static final int MAX_HELP_CONSOLE_WIDTH = 88;
protected static final String TIMESTAMP_STR = "Time";
- protected static final int ISO_DATETIME_LEN = 26;
+ protected static final int ISO_DATETIME_LEN = 35;
protected static final String IMPORT_CMD = "import";
private static final String NEED_NOT_TO_PRINT_TIMESTAMP = "AGGREGATION";
private static final String DEFAULT_TIME_FORMAT = "default";
@@ -92,6 +92,8 @@ public abstract class AbstractClient {
protected static int maxTimeLength = ISO_DATETIME_LEN;
protected static int maxValueLength = 15;
protected static boolean isQuit = false;
+ protected static String TIMESTAMP_PRECISION = "ms";
+
/**
* control the width of columns for 'show timeseries path' and 'show storage group'.
* <p>
@@ -187,7 +189,8 @@ public abstract class AbstractClient {
boolean isShow = res instanceof IoTDBMetadataResultSet;
if (!isShow && resultSetMetaData.getColumnTypeName(0) != null) {
- printTimestamp = !res.getMetaData().getColumnTypeName(0).equalsIgnoreCase(NEED_NOT_TO_PRINT_TIMESTAMP);
+ printTimestamp = !res.getMetaData().getColumnTypeName(0)
+ .equalsIgnoreCase(NEED_NOT_TO_PRINT_TIMESTAMP);
}
if (res instanceof IoTDBQueryResultSet) {
printTimestamp = printTimestamp && !((IoTDBQueryResultSet) res).isIgnoreTimeStamp();
@@ -219,6 +222,10 @@ public abstract class AbstractClient {
printCount(isShow, res, cnt);
}
+ protected static String getTimestampPrecision() {
+ return TIMESTAMP_PRECISION;
+ }
+
protected static void printCount(boolean isShow, ResultSet res, int cnt) throws SQLException {
if (isShow) {
int type = res.getType();
@@ -341,6 +348,50 @@ public abstract class AbstractClient {
return options;
}
+ public static String parseLongToDateWithPrecision(DateTimeFormatter formatter,
+ long timestamp, ZoneId zoneid, String timestampPrecision) {
+ if (timestampPrecision.equals("ms")) {
+ long integerofDate = timestamp / 1000;
+ String digits = Long.toString(timestamp % 1000);
+ ZonedDateTime dateTime = ZonedDateTime
+ .ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
+ String datetime = dateTime.format(formatter);
+ int length = digits.length();
+ if (length != 3) {
+ for (int i = 0; i < 3 - length; i++) {
+ digits = "0" + digits;
+ }
+ }
+ return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
+ } else if (timestampPrecision.equals("us")) {
+ long integerofDate = timestamp / 1000_000;
+ String digits = Long.toString(timestamp % 1000_000);
+ ZonedDateTime dateTime = ZonedDateTime
+ .ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
+ String datetime = dateTime.format(formatter);
+ int length = digits.length();
+ if (length != 6) {
+ for (int i = 0; i < 6 - length; i++) {
+ digits = "0" + digits;
+ }
+ }
+ return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
+ } else {
+ long integerofDate = timestamp / 1000_000_000L;
+ String digits = Long.toString(timestamp % 1000_000_000L);
+ ZonedDateTime dateTime = ZonedDateTime
+ .ofInstant(Instant.ofEpochSecond(integerofDate), zoneid);
+ String datetime = dateTime.format(formatter);
+ int length = digits.length();
+ if (length != 9) {
+ for (int i = 0; i < 9 - length; i++) {
+ digits = "0" + digits;
+ }
+ }
+ return datetime.substring(0, 19) + "." + digits + datetime.substring(19);
+ }
+ }
+
private static String formatDatetime(long timestamp, ZoneId zoneId) {
ZonedDateTime dateTime;
switch (timeFormat) {
@@ -349,8 +400,8 @@ public abstract class AbstractClient {
return Long.toString(timestamp);
case DEFAULT_TIME_FORMAT:
case "iso8601":
- dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), zoneId);
- return dateTime.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+ return parseLongToDateWithPrecision(
+ DateTimeFormatter.ISO_OFFSET_DATE_TIME, timestamp, zoneId, getTimestampPrecision());
default:
dateTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timestamp), zoneId);
return dateTime.format(DateTimeFormatter.ofPattern(timeFormat));
@@ -712,11 +763,11 @@ public abstract class AbstractClient {
enum OperationResult {
STOP_OPER, CONTINUE_OPER, NO_OPER
}
-
+
protected static void printf(String format, Object ... args) {
SCREEN_PRINTER.printf(format, args);
}
-
+
protected static void print(String msg) {
SCREEN_PRINTER.println(msg);
}
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/Client.java b/client/src/main/java/org/apache/iotdb/cli/client/Client.java
index 8115e8e..f287959 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/Client.java
+++ b/client/src/main/java/org/apache/iotdb/cli/client/Client.java
@@ -130,6 +130,8 @@ public class Client extends AbstractClient {
String s;
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
+ TIMESTAMP_PRECISION = properties.getTimestampPrecision();
+
displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
diff --git a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java b/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java
index 57885dc..b66f658 100644
--- a/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java
+++ b/client/src/main/java/org/apache/iotdb/cli/client/WinClient.java
@@ -135,6 +135,8 @@ public class WinClient extends AbstractClient {
.getConnection(Config.IOTDB_URL_PREFIX + host + ":" + port + "/", username, password)) {
properties = connection.getServerProperties();
AGGREGRATE_TIME_LIST.addAll(properties.getSupportedTimeAggregationOperations());
+ TIMESTAMP_PRECISION = properties.getTimestampPrecision();
+
displayLogo(properties.getVersion());
println(IOTDB_CLI_PREFIX + "> login successfully");
while (true) {
diff --git a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
index bc511b0..e374064 100644
--- a/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
+++ b/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBConnectionTest.java
@@ -82,13 +82,15 @@ public class IoTDBConnectionTest {
add("min_time");
}
};
+ final String timestampPrecision = "ms";
when(client.getProperties())
- .thenReturn(new ServerProperties(version, supportedAggregationTime));
+ .thenReturn(new ServerProperties(version, supportedAggregationTime, timestampPrecision));
connection.client = client;
assertEquals(connection.getServerProperties().getVersion(), version);
for (int i = 0; i < supportedAggregationTime.size(); i++) {
assertEquals(connection.getServerProperties().getSupportedTimeAggregationOperations().get(i),
supportedAggregationTime.get(i));
}
+ assertEquals(connection.getServerProperties().getTimestampPrecision(), timestampPrecision);
}
}
diff --git a/server/iotdb/conf/iotdb-engine.properties b/server/iotdb/conf/iotdb-engine.properties
index 952b404..01dea2c 100644
--- a/server/iotdb/conf/iotdb-engine.properties
+++ b/server/iotdb/conf/iotdb-engine.properties
@@ -53,6 +53,15 @@ flush_wal_threshold=10000
# Set this parameter to 0 may slow down the ingestion on slow disk.
force_wal_period_in_ms=10
+
+####################
+### Timestamp Precision Configuration
+####################
+# Use this value to set timestamp precision as "ms", "us" or "ns".
+# Once the precision is been set, it can not be changed.
+timestamp_precision=ms
+
+
####################
### Directory Configuration
####################
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 6c72ed5..86c3622 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -72,6 +72,11 @@ public class IoTDBConfig {
private int flushWalThreshold = 10000;
/**
+ * this variable set timestamp precision as millisecond, microsecond or nanosecond
+ */
+ private String timestampPrecision = "ms";
+
+ /**
* The cycle when write ahead log is periodically forced to be written to disk(in milliseconds) If
* set this parameter to 0 it means call outputStream.force(true) after every each insert
*/
@@ -94,6 +99,11 @@ public class IoTDBConfig {
private String systemDir = "data/system";
/**
+ * Schema directory, including storage set of values.
+ */
+ private String schemaDir = "data/system/schema";
+
+ /**
* Data directory of data. It can be settled as dataDirs = {"data1", "data2", "data3"};
*/
private String[] dataDirs = {"data/data"};
@@ -240,6 +250,7 @@ public class IoTDBConfig {
List<String> dirs = new ArrayList<>();
dirs.add(baseDir);
dirs.add(systemDir);
+ dirs.add(schemaDir);
dirs.add(walFolder);
dirs.add(indexFileDir);
dirs.addAll(Arrays.asList(dataDirs));
@@ -258,10 +269,11 @@ public class IoTDBConfig {
}
baseDir = dirs.get(0);
systemDir = dirs.get(1);
- walFolder = dirs.get(2);
- indexFileDir = dirs.get(3);
+ schemaDir = dirs.get(2);
+ walFolder = dirs.get(3);
+ indexFileDir = dirs.get(4);
for (int i = 0; i < dataDirs.length; i++) {
- dataDirs[i] = dirs.get(i + 4);
+ dataDirs[i] = dirs.get(i + 5);
}
}
@@ -303,6 +315,14 @@ public class IoTDBConfig {
this.rpcPort = rpcPort;
}
+ public void setTimestampPrecision(String timestampPrecision) {
+ this.timestampPrecision = timestampPrecision;
+ }
+
+ public String getTimestampPrecision() {
+ return timestampPrecision;
+ }
+
public boolean isEnableWal() {
return enableWal;
}
@@ -335,6 +355,14 @@ public class IoTDBConfig {
this.systemDir = systemDir;
}
+ public String getSchemaDir() {
+ return schemaDir;
+ }
+
+ void setSchemaDir(String schemaDir) {
+ this.schemaDir = schemaDir;
+ }
+
public String getWalFolder() {
return walFolder;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
new file mode 100644
index 0000000..59270e2
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfigCheck.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.conf;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.Properties;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class IoTDBConfigCheck {
+
+ // this file is located in data/system/schema/system_properties.
+ // If user delete folder "data", system_properties can reset.
+ public static final String PROPERTIES_FILE_NAME = "system.properties";
+ public static final String SCHEMA_DIR =
+ IoTDBDescriptor.getInstance().getConfig().getSchemaDir();
+ private static final IoTDBConfigCheck INSTANCE = new IoTDBConfigCheck();
+ private static final Logger logger = LoggerFactory.getLogger(IoTDBDescriptor.class);
+ private Properties properties = new Properties();
+ // this is a initial parameter.
+ private static String TIMESTAMP_PRECISION = "ms";
+
+ public static final IoTDBConfigCheck getInstance() {
+ return IoTDBConfigCheck.INSTANCE;
+ }
+
+ public void checkConfig() {
+ TIMESTAMP_PRECISION = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+ createDir(SCHEMA_DIR);
+ checkFile(SCHEMA_DIR);
+ logger.info("System configuration is ok.");
+ }
+
+ public void createDir(String filepath) {
+ File dir = new File(filepath);
+ if (!dir.exists()) {
+ dir.mkdirs();
+ logger.info(" {} dir has been created.", SCHEMA_DIR);
+ }
+ }
+
+ public void checkFile(String filepath) {
+ // create file : read timestamp precision from engine.properties, create system_properties.txt
+ // use output stream to write timestamp precision to file.
+ File file = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
+ try {
+ if (!file.exists()) {
+ file.createNewFile();
+ logger.info(" {} has been created.", file.getAbsolutePath());
+ try (FileOutputStream outputStream = new FileOutputStream(file.toString())) {
+ properties.setProperty("timestamp_precision", TIMESTAMP_PRECISION);
+ properties.store(outputStream, "System properties:");
+ }
+ }
+ } catch (IOException e) {
+ logger.error("Can not create {}.", file.getAbsolutePath(), e);
+ }
+ // get existed properties from system_properties.txt
+ File inputFile = new File(filepath + File.separator + PROPERTIES_FILE_NAME);
+ try (FileInputStream inputStream = new FileInputStream(inputFile.toString())) {
+ properties.load(new InputStreamReader(inputStream, TSFileConfig.STRING_ENCODING));
+ if (!properties.getProperty("timestamp_precision").equals(TIMESTAMP_PRECISION)) {
+ logger.error("Wrong timestamp precision, please set as: " + properties
+ .getProperty("timestamp_precision") + " !");
+ System.exit(-1);
+ }
+ } catch (IOException e) {
+ logger.error("Load system.properties from {} failed.", file.getAbsolutePath(), e);
+ }
+ }
+}
+
+
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index ef0ce40..c916114 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -123,6 +123,9 @@ public class IoTDBDescriptor {
conf.setRpcPort(Integer.parseInt(properties.getProperty("rpc_port",
Integer.toString(conf.getRpcPort()))));
+ conf.setTimestampPrecision(properties.getProperty("timestamp_precision",
+ conf.getTimestampPrecision()));
+
conf.setEnableParameterAdapter(
Boolean.parseBoolean(properties.getProperty("enable_parameter_adapter",
Boolean.toString(conf.isEnableParameterAdapter()))));
@@ -136,6 +139,8 @@ public class IoTDBDescriptor {
conf.setSystemDir(FilePathUtils.regularizePath(conf.getBaseDir()) + "system");
+ conf.setSchemaDir(FilePathUtils.regularizePath(conf.getSystemDir()) + "schema");
+
conf.setDataDirs(properties.getProperty("data_dirs", conf.getDataDirs()[0])
.split(","));
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/constant/DatetimeUtils.java b/server/src/main/java/org/apache/iotdb/db/qp/constant/DatetimeUtils.java
index 2df79e8..ca70be4 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/constant/DatetimeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/constant/DatetimeUtils.java
@@ -25,6 +25,7 @@ import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
import java.time.format.SignStyle;
import java.time.temporal.ChronoField;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
@@ -82,6 +83,30 @@ public class DatetimeUtils {
}
/**
+ * such as '10:15:30' or '10:15:30.123456'.
+ */
+ public static final DateTimeFormatter ISO_LOCAL_TIME_WITH_US;
+
+ static {
+ ISO_LOCAL_TIME_WITH_US = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart().appendLiteral('.')
+ .appendValue(ChronoField.MICRO_OF_SECOND, 6).optionalEnd().toFormatter();
+ }
+
+ /**
+ * such as '10:15:30' or '10:15:30.123456789'.
+ */
+ public static final DateTimeFormatter ISO_LOCAL_TIME_WITH_NS;
+
+ static {
+ ISO_LOCAL_TIME_WITH_NS = new DateTimeFormatterBuilder().appendValue(ChronoField.HOUR_OF_DAY, 2)
+ .appendLiteral(':').appendValue(ChronoField.MINUTE_OF_HOUR, 2).appendLiteral(':')
+ .appendValue(ChronoField.SECOND_OF_MINUTE, 2).optionalStart().appendLiteral('.')
+ .appendValue(ChronoField.NANO_OF_SECOND, 9).optionalEnd().toFormatter();
+ }
+
+ /**
* such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_MS;
@@ -94,6 +119,30 @@ public class DatetimeUtils {
}
/**
+ * such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_US = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WIDTH_1_2).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
+ * such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_NS = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WIDTH_1_2).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
* such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH;
@@ -106,6 +155,30 @@ public class DatetimeUtils {
}
/**
+ * such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_US = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
+ * such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_NS = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
* such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT;
@@ -118,6 +191,30 @@ public class DatetimeUtils {
}
/**
+ * such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_US = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
+ * such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_NS = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT).appendLiteral('T').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
* such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE;
@@ -129,6 +226,28 @@ public class DatetimeUtils {
}
/**
+ * such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SPACE_US = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId().toFormatter();
+ }
+
+ /**
+ * such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SPACE_NS = new DateTimeFormatterBuilder().parseCaseInsensitive()
+ .append(DateTimeFormatter.ISO_LOCAL_DATE).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId().toFormatter();
+ }
+
+ /**
* such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE;
@@ -142,6 +261,32 @@ public class DatetimeUtils {
}
/**
+ * such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US = new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
+ * such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS = new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_SLASH).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
* such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123+01:00'.
*/
public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE;
@@ -153,59 +298,181 @@ public class DatetimeUtils {
.toFormatter();
}
+ /**
+ * such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US = new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_US)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
+ /**
+ * such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456789+01:00'.
+ */
+ public static final DateTimeFormatter ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS;
+
+ static {
+ ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS = new DateTimeFormatterBuilder()
+ .parseCaseInsensitive()
+ .append(ISO_LOCAL_DATE_WITH_DOT).appendLiteral(' ').append(ISO_LOCAL_TIME_WITH_NS)
+ .appendOffsetId()
+ .toFormatter();
+ }
+
public static final DateTimeFormatter formatter = new DateTimeFormatterBuilder()
/**
* The ISO date-time formatter that formats or parses a date-time with an offset, such as
* '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123+01:00'.
*/
.appendOptional(ISO_OFFSET_DATE_TIME_WITH_MS)
+
+ /**
+ * such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_US)
+
+ /**
+ * such as '2011-12-03T10:15:30+01:00' or '2011-12-03T10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_NS)
+
/**
* such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123+01:00'.
*/
.appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH)
+
+ /**
+ * such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_US)
+
+ /**
+ * such as '2011/12/03T10:15:30+01:00' or '2011/12/03T10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_NS)
+
/**
* such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123+01:00'.
*/
.appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT)
+
+ /**
+ * such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_US)
+
+ /**
+ * such as '2011.12.03T10:15:30+01:00' or '2011.12.03T10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_NS)
+
/**
* such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123+01:00'.
*/
.appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE)
+
+ /**
+ * such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE_US)
+
+ /**
+ * such as '2011-12-03 10:15:30+01:00' or '2011-12-03 10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SPACE_NS)
+
/**
* such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123+01:00'.
*/
.appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE)
+
+ /**
+ * such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_US)
+
+ /**
+ * such as '2011/12/03 10:15:30+01:00' or '2011/12/03 10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_SLASH_WITH_SPACE_NS)
+
/**
* such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123+01:00'.
*/
- .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE).toFormatter();
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE)
+
+ /**
+ * such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_US)
+
+ /**
+ * such as '2011.12.03 10:15:30+01:00' or '2011.12.03 10:15:30.123456789+01:00'.
+ */
+ .appendOptional(ISO_OFFSET_DATE_TIME_WITH_DOT_WITH_SPACE_NS).toFormatter();
+
+ public static long convertDatetimeStrToLong(String str, ZoneId zoneId)
+ throws LogicalOperatorException {
+ return convertDatetimeStrToLong(str, toZoneOffset(zoneId), 0);
+ }
- public static long convertDatetimeStrToMillisecond(String str, ZoneId zoneId)
+ public static long getInstantWithPrecision(String str, String timestampPrecision)
throws LogicalOperatorException {
- return convertDatetimeStrToMillisecond(str, toZoneOffset(zoneId), 0);
+ try {
+ ZonedDateTime zonedDateTime = ZonedDateTime.parse(str, formatter);
+ Instant instant = zonedDateTime.toInstant();
+ if (timestampPrecision.equals("us")) {
+ if (instant.getEpochSecond() < 0 && instant.getNano() > 0) {
+ // adjustment can reduce the loss of the division
+ long millis = Math.multiplyExact(instant.getEpochSecond() + 1, 1000_000);
+ long adjustment = instant.getNano() / 1000 - 1;
+ return Math.addExact(millis, adjustment);
+ } else {
+ long millis = Math.multiplyExact(instant.getEpochSecond(), 1000_000);
+ return Math
+ .addExact(millis, instant.getNano() / 1000);
+ }
+ } else if (timestampPrecision.equals("ns")) {
+ long millis = Math.multiplyExact(instant.getEpochSecond(), 1000_000_000L);
+ return Math
+ .addExact(millis, instant.getNano());
+ }
+ return instant.toEpochMilli();
+ } catch (DateTimeParseException e) {
+ throw new LogicalOperatorException(e);
+ }
}
/**
- * convert date time string to millisecond.
+ * convert date time string to millisecond, microsecond or nanosecond.
*/
- public static long convertDatetimeStrToMillisecond(String str, ZoneOffset offset, int depth)
+ public static long convertDatetimeStrToLong(String str, ZoneOffset offset, int depth)
throws LogicalOperatorException {
- if (depth >= 2){
+
+ String timestampPrecision = IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision();
+
+ if (depth >= 2) {
throw new DateTimeException(
- String.format("Failed to convert %s to millisecond, zone offset is %s, "
- + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00", str, offset));
+ String.format("Failed to convert %s to millisecond, zone offset is %s, "
+ + "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00", str, offset));
}
- if (str.contains("Z")){
- return convertDatetimeStrToMillisecond(str.substring(0, str.indexOf('Z')) + "+00:00", offset, depth);
- } else if (str.length() - str.lastIndexOf('+') != 6 && str.length() - str.lastIndexOf('-') != 6) {
- return convertDatetimeStrToMillisecond(str + offset, offset, depth + 1);
- } else if (str.contains("[") || str.contains("]")) {
+ if (str.contains("Z")) {
+ return convertDatetimeStrToLong(str.substring(0, str.indexOf('Z')) + "+00:00", offset,
+ depth);
+ } else if (str.length() - str.lastIndexOf('+') != 6
+ && str.length() - str.lastIndexOf('-') != 6) {
+ return convertDatetimeStrToLong(str + offset, offset, depth + 1);
+ } else if (str.contains("[") || str.contains("]")) {
throw new DateTimeException(
String.format("%s with [time-region] at end is not supported now, "
+ "please input like 2011-12-03T10:15:30 or 2011-12-03T10:15:30+01:00", str));
}
- ZonedDateTime zonedDateTime = ZonedDateTime.parse(str, formatter);
- return zonedDateTime.toInstant().toEpochMilli();
+ return getInstantWithPrecision(str, timestampPrecision);
}
public static ZoneOffset toZoneOffset(ZoneId zoneId) {
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index 91dc42e..8334039 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -865,7 +865,7 @@ public class LogicalGenerator {
return System.currentTimeMillis();
}
try {
- return DatetimeUtils.convertDatetimeStrToMillisecond(timestampStr, zoneId);
+ return DatetimeUtils.convertDatetimeStrToLong(timestampStr, zoneId);
} catch (Exception e) {
throw new LogicalOperatorException(String
.format("Input time format %s error. "
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index 6d871e3..2a9d19a 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.service;
import org.apache.iotdb.db.concurrent.IoTDBDefaultThreadExceptionHandler;
+import org.apache.iotdb.db.conf.IoTDBConfigCheck;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
@@ -45,6 +46,7 @@ public class IoTDB implements IoTDBMBean {
}
public static void main(String[] args) {
+ IoTDBConfigCheck.getInstance().checkConfig();
IoTDB daemon = IoTDB.getInstance();
daemon.active();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 4aa0a9e..06e857b 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@ -892,6 +892,7 @@ public class TSServiceImpl implements TSIService.Iface, ServerContext {
properties.setSupportedTimeAggregationOperations(new ArrayList<>());
properties.getSupportedTimeAggregationOperations().add(IoTDBConstant.MAX_TIME);
properties.getSupportedTimeAggregationOperations().add(IoTDBConstant.MIN_TIME);
+ properties.setTimestampPrecision(IoTDBDescriptor.getInstance().getConfig().getTimestampPrecision());
return properties;
}
diff --git a/server/src/test/java/org/apache/iotdb/db/sql/DatetimeQueryDataSetUtilsTest.java b/server/src/test/java/org/apache/iotdb/db/sql/DatetimeQueryDataSetUtilsTest.java
index 693b813..d51979d 100644
--- a/server/src/test/java/org/apache/iotdb/db/sql/DatetimeQueryDataSetUtilsTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/sql/DatetimeQueryDataSetUtilsTest.java
@@ -76,11 +76,11 @@ public class DatetimeQueryDataSetUtilsTest {
"2019.01.02 15:13:27" + zoneOffset, "2019-01-02T15:13:27" + zoneOffset,
"2019/01/02T15:13:27" + zoneOffset, "2019.01.02T15:13:27" + zoneOffset,};
for (String str : timeFormatWithoutMs) {
- Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToMillisecond(str, zoneOffset, 0));
+ Assert.assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0));
}
for (String str : timeFormatWithoutMs) {
- assertEquals(res, DatetimeUtils.convertDatetimeStrToMillisecond(str, zoneId));
+ assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneId));
}
}
@@ -94,11 +94,11 @@ public class DatetimeQueryDataSetUtilsTest {
"2019-01-02T15:13:27.689" + zoneOffset, "2019/01/02T15:13:27.689" + zoneOffset,
"2019.01.02T15:13:27.689" + zoneOffset,};
for (String str : timeFormatWithoutMs) {
- assertEquals(res, DatetimeUtils.convertDatetimeStrToMillisecond(str, zoneOffset, 0));
+ assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneOffset, 0));
}
for (String str : timeFormatWithoutMs) {
- assertEquals(res, DatetimeUtils.convertDatetimeStrToMillisecond(str, zoneId));
+ assertEquals(res, DatetimeUtils.convertDatetimeStrToLong(str, zoneId));
}
}
diff --git a/service-rpc/src/main/thrift/rpc.thrift b/service-rpc/src/main/thrift/rpc.thrift
index e64c659..e0f7ddc 100644
--- a/service-rpc/src/main/thrift/rpc.thrift
+++ b/service-rpc/src/main/thrift/rpc.thrift
@@ -260,6 +260,7 @@ struct TSInsertionReq {
struct ServerProperties {
1: required string version;
2: required list<string> supportedTimeAggregationOperations;
+ 3: required string timestampPrecision;
}
service TSIService {