You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/06/10 15:48:34 UTC
[incubator-doris] 01/03: [feature] datetime column type support auto-initialized with default … (#9972)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch dev-1.0.1
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
commit b8323e9f70bd233f8c161e45aa028e7dba74a314
Author: BePPPower <43...@users.noreply.github.com>
AuthorDate: Thu Jun 9 00:28:03 2022 +0800
[feature] datetime column type support auto-initialized with default … (#9972)
---
fe/fe-core/src/main/cup/sql_parser.cup | 8 ++-
.../java/org/apache/doris/analysis/ColumnDef.java | 37 ++++++++++++--
.../org/apache/doris/analysis/DataDescription.java | 4 +-
.../apache/doris/analysis/DefaultValueExprDef.java | 58 ++++++++++++++++++++++
.../apache/doris/analysis/FunctionCallExpr.java | 17 +++++++
.../java/org/apache/doris/analysis/InsertStmt.java | 12 +++--
.../main/java/org/apache/doris/catalog/Column.java | 19 +++++--
.../src/main/java/org/apache/doris/load/Load.java | 12 ++++-
.../org/apache/doris/planner/LoadScanNode.java | 6 ++-
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../data/correctness/test_current_timestamp.out | 7 +++
.../test_current_timestamp_streamload.csv | 4 ++
.../correctness/test_current_timestamp.groovy | 55 ++++++++++++++++++++
13 files changed, 223 insertions(+), 17 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index ce45f1cf56..ce3600ecf4 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -237,7 +237,7 @@ parser code {:
// Total keywords of doris
terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL, KW_ALTER, KW_AND, KW_ANTI, KW_APPEND, KW_AS, KW_ASC, KW_AUTHORS, KW_ARRAY,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG, KW_BITMAP, KW_BITMAP_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY, KW_BUILTIN,
- KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK, KW_CLUSTER, KW_CLUSTERS, KW_CLEAN,
+ KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK, KW_CLUSTER, KW_CLUSTERS, KW_CLEAN, KW_CURRENT_TIMESTAMP,
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT, KW_COMMITTED, KW_COMPACT,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT, KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DAY, KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE,
@@ -2414,6 +2414,10 @@ opt_default_value ::=
{:
RESULT = ColumnDef.DefaultValue.NULL_DEFAULT_VALUE;
:}
+ | KW_DEFAULT KW_CURRENT_TIMESTAMP
+ {:
+ RESULT = ColumnDef.DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE;
+ :}
;
opt_is_key ::=
@@ -5730,6 +5734,8 @@ keyword ::=
{: RESULT = id; :}
| KW_VARCHAR:id
{: RESULT = id; :}
+ | KW_CURRENT_TIMESTAMP:id
+ {: RESULT = id; :}
;
// Identifier that contain keyword
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index cf61e09218..db794c730b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -58,12 +58,31 @@ public class ColumnDef {
public static class DefaultValue {
public boolean isSet;
public String value;
+ // used for column which defaultValue is an expression.
+ public DefaultValueExprDef defaultValueExprDef;
public DefaultValue(boolean isSet, String value) {
this.isSet = isSet;
this.value = value;
+ this.defaultValueExprDef = null;
}
+ /**
+ * used for column which defaultValue is an expression.
+ * @param isSet is Set DefaultValue
+ * @param value default value
+ * @param exprName default value expression
+ */
+ public DefaultValue(boolean isSet, String value, String exprName) {
+ this.isSet = isSet;
+ this.value = value;
+ this.defaultValueExprDef = new DefaultValueExprDef(exprName);
+ }
+
+ // default "CURRENT_TIMESTAMP", only for DATETIME type
+ public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
+ public static String NOW = "now";
+ public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new DefaultValue(true, CURRENT_TIMESTAMP, NOW);
// no default value
public static DefaultValue NOT_SET = new DefaultValue(false, null);
// default null
@@ -245,11 +264,13 @@ public class ColumnDef {
}
if (defaultValue.isSet && defaultValue.value != null) {
- validateDefaultValue(type, defaultValue.value);
+ validateDefaultValue(type, defaultValue.value, defaultValue.defaultValueExprDef);
}
}
- public static void validateDefaultValue(Type type, String defaultValue) throws AnalysisException {
+ @SuppressWarnings("checkstyle:Indentation")
+ public static void validateDefaultValue(Type type, String defaultValue, DefaultValueExprDef defaultValueExprDef)
+ throws AnalysisException {
Preconditions.checkNotNull(defaultValue);
Preconditions.checkArgument(type.isScalarType());
ScalarType scalarType = (ScalarType) type;
@@ -281,7 +302,15 @@ public class ColumnDef {
break;
case DATE:
case DATETIME:
- DateLiteral dateLiteral = new DateLiteral(defaultValue, type);
+ if (defaultValueExprDef == null) {
+ new DateLiteral(defaultValue, type);
+ } else {
+ if (defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) {
+ break;
+ } else {
+ throw new AnalysisException("date literal [" + defaultValue + "] is invalid");
+ }
+ }
break;
case CHAR:
case VARCHAR:
@@ -333,7 +362,7 @@ public class ColumnDef {
public Column toColumn() {
return new Column(name, typeDef.getType(), isKey, aggregateType, isAllowNull, defaultValue.value, comment,
- visible);
+ visible, defaultValue.defaultValueExprDef);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
index fbd8802cca..727c73c0c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
@@ -329,7 +329,7 @@ public class DataDescription {
}
if (args.get(0) != null) {
- ColumnDef.validateDefaultValue(column.getOriginType(), args.get(0));
+ ColumnDef.validateDefaultValue(column.getOriginType(), args.get(0), column.getDefaultValueExprDef());
}
}
@@ -364,7 +364,7 @@ public class DataDescription {
}
if (replaceValue != null) {
- ColumnDef.validateDefaultValue(column.getOriginType(), replaceValue);
+ ColumnDef.validateDefaultValue(column.getOriginType(), replaceValue, column.getDefaultValueExprDef());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
new file mode 100644
index 0000000000..af81840dae
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ColumnDef.java
+// and modified by Doris
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.common.AnalysisException;
+
+import com.google.gson.annotations.SerializedName;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * This def used for column which defaultValue is an expression.
+ */
+public class DefaultValueExprDef {
+ private static final Logger LOG = LogManager.getLogger(DefaultValueExprDef.class);
+ @SerializedName("exprName")
+ private String exprName;
+
+ public DefaultValueExprDef(String exprName) {
+ this.exprName = exprName;
+ }
+
+ /**
+ * generate a FunctionCallExpr
+ * @return FunctionCallExpr of exprName
+ */
+ public FunctionCallExpr getExpr() {
+ FunctionCallExpr expr = new FunctionCallExpr(exprName, new FunctionParams(null));
+ try {
+ expr.analyzeImplForDefaultValue();
+ } catch (AnalysisException e) {
+ LOG.warn("analyzeImplForDefaultValue fail: {}", e);
+ }
+ return expr;
+ }
+
+ public String getExprName() {
+ return exprName;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index fc733bd636..4c0a85a6fe 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -706,6 +706,23 @@ public class FunctionCallExpr extends Expr {
fnName, fnParams.isStar() ? "*" : Joiner.on(", ").join(argTypesSql));
}
+ /**
+ * This analyzeImp used for DefaultValueExprDef
+ * to generate a builtinFunction.
+ * @throws AnalysisException
+ */
+ public void analyzeImplForDefaultValue() throws AnalysisException {
+ fn = getBuiltinFunction(null, fnName.getFunction(), new Type[0],
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ type = fn.getReturnType();
+ for (int i = 0; i < children.size(); ++i) {
+ if (getChild(i).getType().isNull()) {
+ uncheckedCastChild(Type.BOOLEAN, i);
+ }
+ }
+ return;
+ }
+
@Override
public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
if (isMergeAggFn) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
index af1b2921cc..ba063440cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -690,10 +690,14 @@ public class InsertStmt extends DdlStmt {
*/
Preconditions.checkState(col.isAllowNull());
resultExprs.add(NullLiteral.create(col.getType()));
- }
- else {
- StringLiteral defaultValueExpr = new StringLiteral(col.getDefaultValue());
- resultExprs.add(defaultValueExpr.checkTypeCompatibility(col.getType()));
+ } else {
+ if (col.getDefaultValueExprDef() != null) {
+ resultExprs.add(col.getDefaultValueExpr());
+ } else {
+ StringLiteral defaultValueExpr;
+ defaultValueExpr = new StringLiteral(col.getDefaultValue());
+ resultExprs.add(defaultValueExpr.checkTypeCompatibility(col.getType()));
+ }
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index d5b45a83e8..a8b51ec719 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -18,6 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.alter.SchemaChangeHandler;
+import org.apache.doris.analysis.DefaultValueExprDef;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
@@ -87,6 +88,8 @@ public class Column implements Writable {
private Expr defineExpr; // use to define column in materialize view
@SerializedName(value = "visible")
private boolean visible;
+ @SerializedName(value = "defaultValueExprDef")
+ private DefaultValueExprDef defaultValueExprDef; // used for default value
public Column() {
this.name = "";
@@ -95,6 +98,7 @@ public class Column implements Writable {
this.isKey = false;
this.stats = new ColumnStats();
this.visible = true;
+ this.defineExpr = null;
this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH);
}
@@ -117,10 +121,10 @@ public class Column implements Writable {
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
String defaultValue, String comment) {
- this(name, type, isKey, aggregateType, isAllowNull, defaultValue, comment, true);
+ this(name, type, isKey, aggregateType, isAllowNull, defaultValue, comment, true, null);
}
public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull,
- String defaultValue, String comment, boolean visible) {
+ String defaultValue, String comment, boolean visible, DefaultValueExprDef defaultValueExprDef) {
this.name = name;
if (this.name == null) {
this.name = "";
@@ -136,6 +140,7 @@ public class Column implements Writable {
this.isKey = isKey;
this.isAllowNull = isAllowNull;
this.defaultValue = defaultValue;
+ this.defaultValueExprDef = defaultValueExprDef;
this.comment = comment;
this.stats = new ColumnStats();
this.visible = visible;
@@ -151,6 +156,7 @@ public class Column implements Writable {
this.isKey = column.isKey();
this.isAllowNull = column.isAllowNull();
this.defaultValue = column.getDefaultValue();
+ this.defaultValueExprDef = column.defaultValueExprDef;
this.comment = column.getComment();
this.stats = column.getStats();
this.visible = column.visible;
@@ -281,11 +287,15 @@ public class Column implements Writable {
if (getDataType() == PrimitiveType.VARCHAR) {
return defaultValueLiteral;
}
+ if (defaultValueExprDef != null) {
+ return defaultValueExprDef.getExpr();
+ }
Expr result = defaultValueLiteral.castTo(getType());
result.checkValueValid();
return result;
}
+
public void setStats(ColumnStats stats) {
this.stats = stats;
}
@@ -474,6 +484,10 @@ public class Column implements Writable {
defineExpr = expr;
}
+ public DefaultValueExprDef getDefaultValueExprDef() {
+ return defaultValueExprDef;
+ }
+
public SlotRef getRefColumn() {
List<Expr> slots = new ArrayList<>();
if (defineExpr == null) {
@@ -612,7 +626,6 @@ public class Column implements Writable {
}
public static Column read(DataInput in) throws IOException {
-
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, Column.class);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index ea381977e2..f2fe394e72 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -1242,7 +1242,11 @@ public class Load {
exprs.add(funcExpr.getChild(1));
} else {
if (column.getDefaultValue() != null) {
- exprs.add(new StringLiteral(column.getDefaultValue()));
+ if (column.getDefaultValueExprDef() != null) {
+ exprs.add(column.getDefaultValueExpr());
+ } else {
+ exprs.add(new StringLiteral(column.getDefaultValue()));
+ }
} else {
if (column.isAllowNull()) {
exprs.add(NullLiteral.create(Type.VARCHAR));
@@ -1261,7 +1265,11 @@ public class Load {
innerIfExprs.add(funcExpr.getChild(1));
} else {
if (column.getDefaultValue() != null) {
- innerIfExprs.add(new StringLiteral(column.getDefaultValue()));
+ if (column.getDefaultValueExprDef() != null) {
+ innerIfExprs.add(column.getDefaultValueExpr());
+ } else {
+ innerIfExprs.add(new StringLiteral(column.getDefaultValue()));
+ }
} else {
if (column.isAllowNull()) {
innerIfExprs.add(NullLiteral.create(Type.VARCHAR));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
index 7805f8b2d5..62ef18126a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
@@ -144,7 +144,11 @@ public abstract class LoadScanNode extends ScanNode {
} else {
Column column = destSlotDesc.getColumn();
if (column.getDefaultValue() != null) {
- expr = new StringLiteral(destSlotDesc.getColumn().getDefaultValue());
+ if (column.getDefaultValueExprDef() != null) {
+ expr = column.getDefaultValueExpr();
+ } else {
+ expr = new StringLiteral(destSlotDesc.getColumn().getDefaultValue());
+ }
} else {
if (column.isAllowNull()) {
expr = NullLiteral.create(column.getType());
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 1fb4a9d47a..faaf4fdfe5 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -423,6 +423,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE));
keywordMap.put("sql_block_rule", new Integer(SqlParserSymbols.KW_SQL_BLOCK_RULE));
keywordMap.put("tablets", new Integer(SqlParserSymbols.KW_TABLETS));
+ keywordMap.put("current_timestamp", new Integer(SqlParserSymbols.KW_CURRENT_TIMESTAMP));
}
// map from token id to token description
diff --git a/regression-test/data/correctness/test_current_timestamp.out b/regression-test/data/correctness/test_current_timestamp.out
new file mode 100644
index 0000000000..dca4f4dd5a
--- /dev/null
+++ b/regression-test/data/correctness/test_current_timestamp.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !insert_into --
+4
+
+-- !stream_load --
+4
+
diff --git a/regression-test/data/correctness/test_current_timestamp_streamload.csv b/regression-test/data/correctness/test_current_timestamp_streamload.csv
new file mode 100644
index 0000000000..55f24a1c84
--- /dev/null
+++ b/regression-test/data/correctness/test_current_timestamp_streamload.csv
@@ -0,0 +1,4 @@
+5,ee
+6,ff
+7,gg
+8,hh
\ No newline at end of file
diff --git a/regression-test/suites/correctness/test_current_timestamp.groovy b/regression-test/suites/correctness/test_current_timestamp.groovy
new file mode 100644
index 0000000000..3f84c86fd5
--- /dev/null
+++ b/regression-test/suites/correctness/test_current_timestamp.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+
+suite("test_current_timestamp", "correctness") {
+ def tableName = "test_current_timestamp"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ id TINYINT,
+ name CHAR(10) NOT NULL DEFAULT "zs",
+ dt_0 DATETIME,
+ dt_1 DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ COMMENT "test current_timestamp table"
+ DISTRIBUTED BY HASH(id)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // test insert into.
+ sql " insert into ${tableName} (id,name,dt_0) values (1,'aa',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values (2,'bb',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values (3,'cc',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values (4,'dd',current_timestamp()); "
+
+ qt_insert_into """ select count(*) from ${tableName} where to_date(dt_0) = to_date(dt_1); """
+
+ // test stream load.
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'id, name, dt_0 = current_timestamp()'
+
+ file 'test_current_timestamp_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+ qt_stream_load """ select count(*) from ${tableName} where id > 4 and to_date(dt_0) = to_date(dt_1); """
+ }
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org