You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2022/07/27 04:39:03 UTC
[doris] branch master updated: [feature](nereids) support cast and extract date for TPC-H (#10999)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0cdd70e9c9 [feature](nereids) support cast and extract date for TPC-H (#10999)
0cdd70e9c9 is described below
commit 0cdd70e9c945f2016c747da9b2415da7a6cfd1db
Author: yinzhijian <37...@qq.com>
AuthorDate: Wed Jul 27 12:38:56 2022 +0800
[feature](nereids) support cast and extract date for TPC-H (#10999)
support cast and extract date for TPC-H, for example:
select cast(a as datetime) as d from test;
select extract(year from datetime_column) as y from test
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 5 ++
.../java/org/apache/doris/analysis/CastExpr.java | 20 +++++++
.../apache/doris/analysis/FunctionCallExpr.java | 8 ++-
.../glue/translator/ExpressionTranslator.java | 9 +++
.../doris/nereids/parser/LogicalPlanBuilder.java | 16 +++++
.../doris/nereids/rules/analysis/BindFunction.java | 8 ++-
.../doris/nereids/trees/expressions/Cast.java | 69 ++++++++++++++++++++++
.../nereids/trees/expressions/functions/Year.java | 53 +++++++++++++++++
.../expressions/visitor/ExpressionVisitor.java | 5 ++
.../org/apache/doris/nereids/types/DataType.java | 30 ++++++++++
.../trees/expressions/ExpressionParserTest.java | 21 +++++++
12 files changed, 243 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index ff20819e56..21ff03c484 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -145,6 +145,7 @@ DAY: 'DAY';
DATA: 'DATA';
DATABASE: 'DATABASE';
DATABASES: 'DATABASES';
+DATE: 'DATE';
DATEADD: 'DATEADD';
DATE_ADD: 'DATE_ADD';
DATEDIFF: 'DATEDIFF';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 18ad0089af..64fa6c8752 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -196,6 +196,7 @@ valueExpression
primaryExpression
: CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase
| CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase
+ | name=CAST LEFT_PAREN expression AS identifier RIGHT_PAREN #cast
| constant #constantDefault
| ASTERISK #star
| qualifiedName DOT ASTERISK #star
@@ -205,6 +206,8 @@ primaryExpression
| identifier #columnReference
| base=primaryExpression DOT fieldName=identifier #dereference
| LEFT_PAREN expression RIGHT_PAREN #parenthesizedExpression
+ | EXTRACT LEFT_PAREN field=identifier FROM (DATE | TIMESTAMP)?
+ source=valueExpression RIGHT_PAREN #extract
;
qualifiedName
@@ -324,6 +327,7 @@ ansiNonReserved
| DATA
| DATABASE
| DATABASES
+ | DATE
| DATEADD
| DATE_ADD
| DATEDIFF
@@ -577,6 +581,7 @@ nonReserved
| DATA
| DATABASE
| DATABASES
+ | DATE
| DATEADD
| DATE_ADD
| DATEDIFF
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 9455abb74c..523c239cc6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -529,4 +529,24 @@ public class CastExpr extends Expr {
|| (children.get(0).getType().isStringType() && !getType().isStringType())
|| (!children.get(0).getType().isDateType() && getType().isDateType());
}
+
+ @Override
+ public void finalizeImplForNereids() throws AnalysisException {
+ FunctionName fnName = new FunctionName(getFnName(type));
+ Function searchDesc = new Function(fnName, Arrays.asList(collectChildReturnTypes()), Type.INVALID, false);
+ if (type.isScalarType()) {
+ if (isImplicit) {
+ fn = Env.getCurrentEnv().getFunction(
+ searchDesc, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ } else {
+ fn = Env.getCurrentEnv().getFunction(
+ searchDesc, Function.CompareMode.IS_IDENTICAL);
+ }
+ } else if (type.isArrayType()) {
+ fn = ScalarFunction.createBuiltin(getFnName(Type.ARRAY),
+ type, Function.NullableMode.ALWAYS_NULLABLE,
+ Lists.newArrayList(Type.VARCHAR), false,
+ "doris::CastFunctions::cast_to_array_val", null, null, true);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 8407726208..ae70ca3297 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1318,10 +1318,16 @@ public class FunctionCallExpr extends Expr {
fn = getBuiltinFunction(fnName.getFunction(), new Type[]{childType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
- } else if (fnName.getFunction().equalsIgnoreCase("substring")) {
+ } else if (fnName.getFunction().equalsIgnoreCase("substring")
+ || fnName.getFunction().equalsIgnoreCase("cast")) {
Type[] childTypes = getChildren().stream().map(t -> t.type).toArray(Type[]::new);
fn = getBuiltinFunction(fnName.getFunction(), childTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
+ } else if (fnName.getFunction().equalsIgnoreCase("year")) {
+ Type childType = getChild(0).type;
+ fn = getBuiltinFunction(fnName.getFunction(), new Type[]{childType},
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ type = fn.getReturnType();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index 9146380e7f..66f97d6973 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -23,6 +23,7 @@ import org.apache.doris.analysis.BinaryPredicate.Operator;
import org.apache.doris.analysis.BoolLiteral;
import org.apache.doris.analysis.CaseExpr;
import org.apache.doris.analysis.CaseWhenClause;
+import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
@@ -38,6 +39,7 @@ import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
import org.apache.doris.nereids.trees.expressions.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.DoubleLiteral;
@@ -244,6 +246,13 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
return new CaseExpr(null, caseWhenClauses, elseExpr);
}
+ @Override
+ public Expr visitCast(Cast cast, PlanTranslatorContext context) {
+ // left child of cast is expression, right child of cast is target type
+ return new CastExpr(cast.getDataType().toCatalogDataType(),
+ cast.left().accept(this, context));
+ }
+
// TODO: Supports for `distinct`
@Override
public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index b5d7c2faf6..03d77bb6a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -73,6 +73,7 @@ import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
import org.apache.doris.nereids.trees.expressions.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.Divide;
@@ -117,6 +118,7 @@ import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.RuleNode;
import org.antlr.v4.runtime.tree.TerminalNode;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -424,6 +426,20 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
return new CaseWhen(whenClauses, getExpression(context.elseExpression));
}
+ @Override
+ public Expression visitCast(DorisParser.CastContext ctx) {
+ return ParserUtils.withOrigin(ctx, () ->
+ new Cast(getExpression(ctx.expression()), ctx.identifier().getText()));
+ }
+
+ @Override
+ public UnboundFunction visitExtract(DorisParser.ExtractContext ctx) {
+ return ParserUtils.withOrigin(ctx, () -> {
+ String functionName = ctx.field.getText();
+ return new UnboundFunction(functionName, false, Arrays.asList(getExpression(ctx.source)));
+ });
+ }
+
@Override
public UnboundFunction visitFunctionCall(DorisParser.FunctionCallContext ctx) {
return ParserUtils.withOrigin(ctx, () -> {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
index 3a5b2bbeeb..4156dade49 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
@@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
import org.apache.doris.nereids.trees.expressions.functions.Substring;
import org.apache.doris.nereids.trees.expressions.functions.Sum;
+import org.apache.doris.nereids.trees.expressions.functions.Year;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -90,7 +91,6 @@ public class BindFunction implements AnalysisRuleFactory {
}
return new Sum(unboundFunction.getArguments().get(0));
} else if (name.equalsIgnoreCase("substr") || name.equalsIgnoreCase("substring")) {
-
List<Expression> arguments = unboundFunction.getArguments();
if (arguments.size() == 2) {
return new Substring(unboundFunction.getArguments().get(0),
@@ -100,6 +100,12 @@ public class BindFunction implements AnalysisRuleFactory {
unboundFunction.getArguments().get(2));
}
return unboundFunction;
+ } else if (name.equalsIgnoreCase("year")) {
+ List<Expression> arguments = unboundFunction.getArguments();
+ if (arguments.size() != 1) {
+ return unboundFunction;
+ }
+ return new Year(unboundFunction.getArguments().get(0));
}
return unboundFunction;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
new file mode 100644
index 0000000000..63d0fd83c5
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
@@ -0,0 +1,69 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.exceptions.UnboundException;
+import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
+import org.apache.doris.nereids.types.DataType;
+
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+
+/**
+ * cast function.
+ */
+public class Cast extends Expression implements BinaryExpression {
+
+ public Cast(Expression child, String type) {
+ super(child, new StringLiteral(type));
+ }
+
+ @Override
+ public DataType getDataType() {
+ StringLiteral type = (StringLiteral) right();
+ return DataType.convertFromString(type.getValue());
+ }
+
+ @Override
+ public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
+ return visitor.visitCast(this, context);
+ }
+
+ @Override
+ public boolean nullable() {
+ return left().nullable();
+ }
+
+ @Override
+ public Expression withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 2);
+ Preconditions.checkArgument(children.get(1) instanceof StringLiteral);
+ return new Cast(children.get(0), ((StringLiteral) children.get(1)).getValue());
+ }
+
+ @Override
+ public String toSql() throws UnboundException {
+ return "CAST(" + left().toSql() + " AS " + ((StringLiteral) right()).getValue() + ")";
+ }
+
+ @Override
+ public String toString() {
+ return toSql();
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Year.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Year.java
new file mode 100644
index 0000000000..13f52fb8af
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Year.java
@@ -0,0 +1,53 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.UnaryExpression;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.IntegerType;
+
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+
+/**
+ * year function.
+ */
+public class Year extends BoundFunction implements UnaryExpression {
+
+ public Year(Expression child) {
+ super("year", child);
+ }
+
+ @Override
+ public DataType getDataType() {
+ return IntegerType.INSTANCE;
+ }
+
+ @Override
+ public boolean nullable() {
+ return child().nullable();
+ }
+
+ @Override
+ public Expression withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 1);
+ return new Year(children.get(0));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
index 248c32ab29..bb1e3d5f05 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitor.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.Arithmetic;
import org.apache.doris.nereids.trees.expressions.Between;
import org.apache.doris.nereids.trees.expressions.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.CaseWhen;
+import org.apache.doris.nereids.trees.expressions.Cast;
import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
import org.apache.doris.nereids.trees.expressions.CompoundPredicate;
import org.apache.doris.nereids.trees.expressions.DateLiteral;
@@ -179,6 +180,10 @@ public abstract class ExpressionVisitor<R, C> {
return visitStringRegexPredicate(regexp, context);
}
+ public R visitCast(Cast cast, C context) {
+ return visit(cast, context);
+ }
+
public R visitBoundFunction(BoundFunction boundFunction, C context) {
return visit(boundFunction, context);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
index 8a29991a2a..a08fa9759c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java
@@ -76,6 +76,36 @@ public abstract class DataType {
}
}
+ /**
+ * Convert to data type in Nereids.
+ * throw exception when cannot convert to Nereids type
+ *
+ * @param type data type in string representation
+ * @return data type in Nereids
+ */
+ public static DataType convertFromString(String type) {
+ // TODO: use a better way to resolve types
+ switch (type.toLowerCase()) {
+ case "bool":
+ case "boolean":
+ return BooleanType.INSTANCE;
+ case "int":
+ return IntegerType.INSTANCE;
+ case "bigint":
+ return BigIntType.INSTANCE;
+ case "double":
+ return DoubleType.INSTANCE;
+ case "string":
+ return StringType.INSTANCE;
+ case "null":
+ return NullType.INSTANCE;
+ case "datetime":
+ return DateTimeType.INSTANCE;
+ default:
+ throw new AnalysisException("Nereids do not support type: " + type);
+ }
+ }
+
public abstract Type toCatalogDataType();
@Override
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
index d02a76c0ee..7001f3b2ed 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
@@ -206,4 +206,25 @@ public class ExpressionParserTest {
interval = "tt > now() - interval 1+1 day";
assertExpr(interval);
}
+
+ @Test
+ public void testExtract() throws Exception {
+ String extract = "SELECT EXTRACT(YEAR FROM TIMESTAMP '2022-02-21 00:00:00') AS year FROM TEST;";
+ assertSql(extract);
+
+ String extract2 = "SELECT EXTRACT(YEAR FROM DATE '2022-02-21 00:00:00') AS year FROM TEST;";
+ assertSql(extract2);
+
+ String extract3 = "SELECT EXTRACT(YEAR FROM '2022-02-21 00:00:00') AS year FROM TEST;";
+ assertSql(extract3);
+ }
+
+ @Test
+ public void testCast() throws Exception {
+ String cast = "SELECT CAST(A AS STRING) FROM TEST;";
+ assertSql(cast);
+
+ String cast2 = "SELECT CAST(A AS INT) AS I FROM TEST;";
+ assertSql(cast2);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org