You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by li...@apache.org on 2022/07/18 04:39:02 UTC
[doris] branch master updated: [feature](nereids) support substring (#10847)
This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0b177669d9 [feature](nereids) support substring (#10847)
0b177669d9 is described below
commit 0b177669d9c5e67437c8d22d39e442dc90198785
Author: yinzhijian <37...@qq.com>
AuthorDate: Mon Jul 18 12:38:56 2022 +0800
[feature](nereids) support substring (#10847)
support substring, for example:
select substr(a, 2), substring(b ,3 ,4) from test1;
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 +-
.../apache/doris/analysis/FunctionCallExpr.java | 5 ++
.../glue/translator/ExpressionTranslator.java | 21 ++++++++
.../glue/translator/PhysicalPlanTranslator.java | 18 +++++--
.../doris/nereids/rules/analysis/BindFunction.java | 23 +++++---
.../trees/expressions/functions/Substring.java | 61 ++++++++++++++++++++++
.../trees/expressions/ExpressionParserTest.java | 3 ++
7 files changed, 123 insertions(+), 11 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 4eba9d040f..cf8944423c 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -193,7 +193,8 @@ primaryExpression
: constant #constantDefault
| ASTERISK #star
| qualifiedName DOT ASTERISK #star
- | identifier '(' DISTINCT? arguments+=expression* ')' #functionCall
+ | identifier LEFT_PAREN DISTINCT? arguments+=expression
+ (COMMA arguments+=expression)* RIGHT_PAREN #functionCall
| LEFT_PAREN query RIGHT_PAREN #subqueryExpression
| identifier #columnReference
| base=primaryExpression DOT fieldName=identifier #dereference
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 2e1edacea7..83258db7b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1299,12 +1299,17 @@ public class FunctionCallExpr extends Expr {
@Override
public void finalizeImplForNereids() throws AnalysisException {
// TODO: support other functions
+ // TODO: Supports type conversion to match the type of the function's parameters
if (fnName.getFunction().equalsIgnoreCase("sum")) {
// Prevent the cast type in vector exec engine
Type childType = getChild(0).type.getMaxResolutionType();
fn = getBuiltinFunction(fnName.getFunction(), new Type[]{childType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
type = fn.getReturnType();
+ } else if (fnName.getFunction().equalsIgnoreCase("substring")) {
+ Type[] childTypes = getChildren().stream().map(t -> t.type).toArray(Type[]::new);
+ fn = getBuiltinFunction(fnName.getFunction(), childTypes, Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ type = fn.getReturnType();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
index 6991364c0c..47635edc88 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/ExpressionTranslator.java
@@ -25,6 +25,7 @@ import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.IntLiteral;
+import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.nereids.exceptions.AnalysisException;
@@ -45,6 +46,7 @@ import org.apache.doris.nereids.trees.expressions.LessThanEqual;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.expressions.StringRegexPredicate;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionVisitor;
@@ -192,6 +194,25 @@ public class ExpressionTranslator extends DefaultExpressionVisitor<Expr, PlanTra
compoundPredicate.child(1).accept(this, context));
}
+ @Override
+ public Expr visitStringRegexPredicate(StringRegexPredicate stringRegexPredicate, PlanTranslatorContext context) {
+ ExpressionType nodeType = stringRegexPredicate.getType();
+ org.apache.doris.analysis.LikePredicate.Operator staleOp;
+ switch (nodeType) {
+ case LIKE:
+ staleOp = LikePredicate.Operator.LIKE;
+ break;
+ case REGEXP:
+ staleOp = LikePredicate.Operator.REGEXP;
+ break;
+ default:
+ throw new AnalysisException(String.format("Unknown node type: %s", nodeType.name()));
+ }
+ return new org.apache.doris.analysis.LikePredicate(staleOp,
+ stringRegexPredicate.left().accept(this, context),
+ stringRegexPredicate.right().accept(this, context));
+ }
+
// TODO: Supports for `distinct`
@Override
public Expr visitBoundFunction(BoundFunction function, PlanTranslatorContext context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 0467ef4f22..aa23e63c1c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -41,6 +41,7 @@ import org.apache.doris.nereids.trees.expressions.visitor.SlotExtractor;
import org.apache.doris.nereids.trees.plans.AggPhase;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.physical.PhysicalAggregate;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin;
@@ -111,10 +112,19 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (rootFragment.isPartitioned() && rootFragment.getPlanRoot().getNumInstances() > 1) {
rootFragment = exchangeToMergeFragment(rootFragment, context);
}
- List<Expr> outputExprs = Lists.newArrayList();
- physicalPlan.getOutput().stream().map(Slot::getExprId)
- .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId)));
- rootFragment.setOutputExprs(outputExprs);
+ // TODO: trick here, we need push project down
+ if (physicalPlan.getType() == PlanType.PHYSICAL_PROJECT) {
+ PhysicalProject<Plan> physicalProject = (PhysicalProject<Plan>) physicalPlan;
+ List<Expr> outputExprs = physicalProject.getProjects().stream()
+ .map(e -> ExpressionTranslator.translate((Expression) e, context))
+ .collect(Collectors.toList());
+ rootFragment.setOutputExprs(outputExprs);
+ } else {
+ List<Expr> outputExprs = Lists.newArrayList();
+ physicalPlan.getOutput().stream().map(Slot::getExprId)
+ .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId)));
+ rootFragment.setOutputExprs(outputExprs);
+ }
rootFragment.getPlanRoot().convertToVectoriezd();
for (PlanFragment fragment : context.getPlanFragmentList()) {
fragment.finalize(null);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
index 3fb2181da4..ddc56f86d9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindFunction.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.functions.Substring;
import org.apache.doris.nereids.trees.expressions.functions.Sum;
import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -72,15 +73,25 @@ public class BindFunction implements AnalysisRuleFactory {
public Expression visitUnboundFunction(UnboundFunction unboundFunction, Void context) {
String name = unboundFunction.getName();
// TODO: lookup function in the function registry
- if (!name.equalsIgnoreCase("sum")) {
- return unboundFunction;
- }
+ if (name.equalsIgnoreCase("sum")) {
+ List<Expression> arguments = unboundFunction.getArguments();
+ if (arguments.size() != 1) {
+ return unboundFunction;
+ }
+ return new Sum(unboundFunction.getArguments().get(0));
+ } else if (name.equalsIgnoreCase("substr") || name.equalsIgnoreCase("substring")) {
- List<Expression> arguments = unboundFunction.getArguments();
- if (arguments.size() != 1) {
+ List<Expression> arguments = unboundFunction.getArguments();
+ if (arguments.size() == 2) {
+ return new Substring(unboundFunction.getArguments().get(0),
+ unboundFunction.getArguments().get(1));
+ } else if (arguments.size() == 3) {
+ return new Substring(unboundFunction.getArguments().get(0), unboundFunction.getArguments().get(1),
+ unboundFunction.getArguments().get(2));
+ }
return unboundFunction;
}
- return new Sum(unboundFunction.getArguments().get(0));
+ return unboundFunction;
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Substring.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Substring.java
new file mode 100644
index 0000000000..b8c12a9e90
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Substring.java
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions.functions;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Literal;
+import org.apache.doris.nereids.trees.expressions.TernaryExpression;
+import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.types.StringType;
+
+import com.google.common.base.Preconditions;
+
+import java.util.List;
+
+/**
+ * substring function.
+ */
+public class Substring extends BoundFunction implements TernaryExpression {
+
+ public Substring(Expression str, Expression pos, Expression len) {
+ super("substring", str, pos, len);
+ }
+
+ public Substring(Expression str, Expression pos) {
+ super("substring", str, pos, new Literal(Integer.MAX_VALUE));
+ }
+
+ @Override
+ public DataType getDataType() {
+ return StringType.INSTANCE;
+ }
+
+ @Override
+ public boolean nullable() {
+ return first().nullable();
+ }
+
+ @Override
+ public Expression withChildren(List<Expression> children) {
+ Preconditions.checkArgument(children.size() == 2 || children.size() == 3);
+ if (children.size() == 2) {
+ return new Substring(children.get(0), children.get(1));
+ }
+ return new Substring(children.get(0), children.get(1), children.get(2));
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
index efbaef8ad0..1014662cbb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/ExpressionParserTest.java
@@ -106,6 +106,9 @@ public class ExpressionParserTest {
String sumAndAvg = "select sum(a),avg(b) from test1";
assertSql(sumAndAvg);
+
+ String substring = "select substr(a, 1, 2), substring(b ,3 ,4) from test1";
+ assertSql(substring);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org