You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/07/11 14:57:05 UTC
[doris] 23/30: [improvement](column) Support for the default value of current_timestamp in microsecond (#21487)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1c7e7ccaabbd54710d69c9aa6bf8eb5847f90e74
Author: zy-kkk <zh...@gmail.com>
AuthorDate: Tue Jul 11 14:04:13 2023 +0800
[improvement](column) Support for the default value of current_timestamp in microsecond (#21487)
---
fe/fe-core/src/main/cup/sql_parser.cup | 4 +
.../java/org/apache/doris/analysis/ColumnDef.java | 70 ++++++++++++++++-
.../apache/doris/analysis/DefaultValueExprDef.java | 49 +++++++++++-
.../apache/doris/datasource/InternalCatalog.java | 8 +-
.../data/correctness_p0/test_current_timestamp.out | 9 +++
.../correctness_p0/test_current_timestamp.groovy | 89 +++++++++++++++++++++-
6 files changed, 222 insertions(+), 7 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index b89f207575..87570db713 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -3519,6 +3519,10 @@ opt_default_value ::=
{:
RESULT = ColumnDef.DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE;
:}
+ | KW_DEFAULT KW_CURRENT_TIMESTAMP LPAREN INTEGER_LITERAL:precision RPAREN
+ {:
+ RESULT = ColumnDef.DefaultValue.currentTimeStampDefaultValueWithPrecision(precision);
+ :}
;
opt_is_key ::=
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index b8c69b438b..2ae7a04335 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -36,6 +36,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
import java.util.List;
import java.util.stream.Collectors;
@@ -88,6 +89,12 @@ public class ColumnDef {
this.defaultValueExprDef = new DefaultValueExprDef(exprName);
}
+ public DefaultValue(boolean isSet, String value, String exprName, Long precision) {
+ this.isSet = isSet;
+ this.value = value;
+ this.defaultValueExprDef = new DefaultValueExprDef(exprName, precision);
+ }
+
// default "CURRENT_TIMESTAMP", only for DATETIME type
public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
public static String NOW = "now";
@@ -104,14 +111,60 @@ public class ColumnDef {
// default "value", "[]" means empty array
public static DefaultValue ARRAY_EMPTY_DEFAULT_VALUE = new DefaultValue(true, "[]");
+ public static DefaultValue currentTimeStampDefaultValueWithPrecision(Long precision) {
+ if (precision > ScalarType.MAX_DATETIMEV2_SCALE || precision < 0) {
+ throw new IllegalArgumentException("column's default value current_timestamp"
+ + " precision must be between 0 and 6");
+ }
+ if (precision == 0) {
+ return new DefaultValue(true, CURRENT_TIMESTAMP, NOW);
+ }
+ String value = CURRENT_TIMESTAMP + "(" + precision + ")";
+ String exprName = NOW;
+ return new DefaultValue(true, value, exprName, precision);
+ }
+
public boolean isCurrentTimeStamp() {
return "CURRENT_TIMESTAMP".equals(value) && NOW.equals(defaultValueExprDef.getExprName());
}
+ public boolean isCurrentTimeStampWithPrecision() {
+ return defaultValueExprDef != null && value.startsWith(CURRENT_TIMESTAMP + "(")
+ && NOW.equals(defaultValueExprDef.getExprName());
+ }
+
+ public long getCurrentTimeStampPrecision() {
+ if (isCurrentTimeStampWithPrecision()) {
+ return Long.parseLong(value.substring(CURRENT_TIMESTAMP.length() + 1, value.length() - 1));
+ }
+ return 0;
+ }
+
public String getValue() {
- return isCurrentTimeStamp()
- ? LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' ')
- : value;
+ if (isCurrentTimeStamp()) {
+ return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' ');
+ } else if (isCurrentTimeStampWithPrecision()) {
+ long precision = getCurrentTimeStampPrecision();
+ String format = "yyyy-MM-dd HH:mm:ss";
+ if (precision == 0) {
+ return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', ' ');
+ } else if (precision == 1) {
+ format = "yyyy-MM-dd HH:mm:ss.S";
+ } else if (precision == 2) {
+ format = "yyyy-MM-dd HH:mm:ss.SS";
+ } else if (precision == 3) {
+ format = "yyyy-MM-dd HH:mm:ss.SSS";
+ } else if (precision == 4) {
+ format = "yyyy-MM-dd HH:mm:ss.SSSS";
+ } else if (precision == 5) {
+ format = "yyyy-MM-dd HH:mm:ss.SSSSS";
+ } else if (precision == 6) {
+ format = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+ }
+ return LocalDateTime.now(TimeUtils.getTimeZone().toZoneId())
+ .format(DateTimeFormatter.ofPattern(format));
+ }
+ return value;
}
}
@@ -459,6 +512,17 @@ public class ColumnDef {
new DateLiteral(defaultValue, scalarType);
} else {
if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) {
+ if (defaultValueExprDef.getPrecision() != null) {
+ Long defaultValuePrecision = defaultValueExprDef.getPrecision();
+ String typeStr = scalarType.toString();
+ int typePrecision =
+ Integer.parseInt(typeStr.substring(typeStr.indexOf("(") + 1, typeStr.indexOf(")")));
+ if (defaultValuePrecision > typePrecision) {
+ typeStr = typeStr.replace("V2", "");
+ throw new AnalysisException("default value precision: " + defaultValue
+ + " can not be greater than type precision: " + typeStr);
+ }
+ }
break;
} else {
throw new AnalysisException("date literal [" + defaultValue + "] is invalid");
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
index adccf078c6..dcaa944e0d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
@@ -22,29 +22,52 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.io.Text;
+import org.apache.doris.common.io.Writable;
+import org.apache.doris.persist.gson.GsonPostProcessable;
+import org.apache.doris.persist.gson.GsonUtils;
+import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
/**
* This def used for column which defaultValue is an expression.
*/
-public class DefaultValueExprDef {
+public class DefaultValueExprDef implements Writable, GsonPostProcessable {
private static final Logger LOG = LogManager.getLogger(DefaultValueExprDef.class);
@SerializedName("exprName")
private String exprName;
+ @SerializedName("precision")
+ private Long precision;
+
public DefaultValueExprDef(String exprName) {
this.exprName = exprName;
}
+ public DefaultValueExprDef(String exprName, Long precision) {
+ this.exprName = exprName;
+ this.precision = precision;
+ }
+
/**
* generate a FunctionCallExpr
* @return FunctionCallExpr of exprName
*/
public FunctionCallExpr getExpr(Type type) {
- FunctionCallExpr expr = new FunctionCallExpr(exprName, new FunctionParams(null));
+ List<Expr> exprs = null;
+ if (precision != null) {
+ exprs = Lists.newArrayList();
+ exprs.add(new IntLiteral(precision));
+ }
+ FunctionCallExpr expr = new FunctionCallExpr(exprName, new FunctionParams(exprs));
try {
expr.analyzeImplForDefaultValue(type);
} catch (AnalysisException e) {
@@ -56,4 +79,26 @@ public class DefaultValueExprDef {
public String getExprName() {
return exprName;
}
+
+ public Long getPrecision() {
+ return precision;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ String json = GsonUtils.GSON.toJson(this);
+ Text.writeString(out, json);
+ }
+
+ public static DefaultValueExprDef read(DataInput in) throws IOException {
+ String json = Text.readString(in);
+ return GsonUtils.GSON.fromJson(json, DefaultValueExprDef.class);
+ }
+
+ @Override
+ public void gsonPostProcess() throws IOException {
+ if (precision == null) {
+ precision = 0L;
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index 4d0a054943..4f70c919f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1228,8 +1228,14 @@ public class InternalCatalog implements CatalogIf<Database> {
boolean setDefault = StringUtils.isNotBlank(column.getDefaultValue());
DefaultValue defaultValue;
if (column.getDefaultValueExprDef() != null) {
- defaultValue = new DefaultValue(setDefault, column.getDefaultValue(),
+ if (column.getDefaultValueExprDef().getPrecision() != null) {
+ defaultValue = new DefaultValue(setDefault, column.getDefaultValue(),
+ column.getDefaultValueExprDef().getExprName(),
+ column.getDefaultValueExprDef().getPrecision());
+ } else {
+ defaultValue = new DefaultValue(setDefault, column.getDefaultValue(),
column.getDefaultValueExprDef().getExprName());
+ }
} else {
defaultValue = new DefaultValue(setDefault, column.getDefaultValue());
}
diff --git a/regression-test/data/correctness_p0/test_current_timestamp.out b/regression-test/data/correctness_p0/test_current_timestamp.out
index 067ef6f5c6..e66b4f88b9 100644
--- a/regression-test/data/correctness_p0/test_current_timestamp.out
+++ b/regression-test/data/correctness_p0/test_current_timestamp.out
@@ -42,3 +42,12 @@
-- !stream_load_json6 --
2
+-- !insert_sql --
+1
+
+-- !insert_sql --
+1
+
+-- !select_sql --
+2
+
diff --git a/regression-test/suites/correctness_p0/test_current_timestamp.groovy b/regression-test/suites/correctness_p0/test_current_timestamp.groovy
index dcadc4c0a0..e4673259e6 100644
--- a/regression-test/suites/correctness_p0/test_current_timestamp.groovy
+++ b/regression-test/suites/correctness_p0/test_current_timestamp.groovy
@@ -53,7 +53,7 @@ suite("test_current_timestamp") {
DISTRIBUTED BY HASH(id)
PROPERTIES("replication_num" = "1");
"""
-
+
// test insert into.
sql " insert into ${tableName} (id,name,dt_0,dt_2,dt_4,dt_6) values (1,'aa',current_timestamp(),current_timestamp(),current_timestamp(),current_timestamp()); "
sql " insert into ${tableName} (id,name,dt_0,dt_2,dt_4,dt_6) values (2,'bb',current_timestamp(),current_timestamp(),current_timestamp(),current_timestamp()); "
@@ -128,4 +128,91 @@ suite("test_current_timestamp") {
qt_stream_load_json5 """ select id, name from ${tableName2} where dt_1 is not NULL order by id; """
qt_stream_load_json6 """ select count(*) from ${tableName2} where dt_2 is not NULL; """
+
+ def tableName3 = "test_current_timestamp_ms"
+
+ sql """ DROP TABLE IF EXISTS ${tableName3} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName3}
+ (
+ id TINYINT,
+ name CHAR(10) NOT NULL DEFAULT "zs",
+ dt_2 DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3)
+ )
+ COMMENT "test current_timestamp table3"
+ DISTRIBUTED BY HASH(id)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // test insert into.
+ qt_insert_sql " insert into ${tableName3} (id,name,dt_2) values (1,'aa',current_timestamp(3)); "
+ qt_insert_sql " insert into ${tableName3} (id,name) values (2,'bb'); "
+
+ qt_select_sql """ select count(*) from ${tableName3} where to_date(dt_2) = to_date(current_timestamp(3)); """
+
+ // test csv stream load.
+ streamLoad {
+ table "${tableName3}"
+
+ set 'column_separator', ','
+ set 'columns', 'id, name, dt_2 = current_timestamp(3)'
+
+ file 'test_current_timestamp_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ // test json stream load
+ streamLoad {
+ table "${tableName3}"
+
+ set 'columns', 'id, name, dt_2 = current_timestamp(3)'
+ set 'format', 'json'
+ set 'read_json_by_line', 'true'
+ set 'strip_outer_array', 'false'
+
+ file 'test_current_timestamp_streamload.json'
+
+ time 10000 // limit inflight 10s
+ }
+
+ sql "sync"
+
+ // test create
+ def tableName4 = "test_current_timestamp_ms2"
+ test {
+ sql """ DROP TABLE IF EXISTS ${tableName4} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName4}
+ (
+ id TINYINT,
+ name CHAR(10) NOT NULL DEFAULT "zs",
+ dt_2 DATETIME(1) DEFAULT CURRENT_TIMESTAMP(3)
+ )
+ COMMENT "test current_timestamp table4"
+ DISTRIBUTED BY HASH(id)
+ PROPERTIES("replication_num" = "1");
+ """
+ exception "errCode = 2, detailMessage = default value precision: CURRENT_TIMESTAMP(3) can not be greater than type precision: DATETIME(1)"
+ }
+
+ // test create
+ def tableName5 = "test_current_timestamp_ms3"
+ test {
+ sql """ DROP TABLE IF EXISTS ${tableName5} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName5}
+ (
+ id TINYINT,
+ name CHAR(10) NOT NULL DEFAULT "zs",
+ dt_2 DATETIME(6) DEFAULT CURRENT_TIMESTAMP(7)
+ )
+ COMMENT "test current_timestamp table5"
+ DISTRIBUTED BY HASH(id)
+ PROPERTIES("replication_num" = "1");
+ """
+ exception "errCode = 2, detailMessage = Internal Error, maybe syntax error or this is a bug: column's default value current_timestamp precision must be between 0 and 6"
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org