You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/11 13:41:24 UTC
[doris] branch master updated: [feature](Nereids) parse pipe_concat symbol as concat when sql mode set to PIPES_AS_CONCAT (#15775)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 88a2088c1d [feature](Nereids) parse pipe_concat symbol as concat when sql mode set to PIPES_AS_CONCAT (#15775)
88a2088c1d is described below
commit 88a2088c1df4b075b4031343335a876f5d351400
Author: 谢健 <ji...@gmail.com>
AuthorDate: Wed Jan 11 21:41:14 2023 +0800
[feature](Nereids) parse pipe_concat symbol as concat when sql mode set to PIPES_AS_CONCAT (#15775)
---
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +-
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 13 +++---
.../doris/nereids/parser/LogicalPlanBuilder.java | 52 +++++++++++++++++-----
.../rewrite/rules/FoldConstantRuleOnFE.java | 11 -----
.../expressions/literal/IntegerLikeLiteral.java | 4 ++
.../java/org/apache/doris/qe/SessionVariable.java | 2 +-
.../java/org/apache/doris/qe/SqlModeHelper.java | 14 +++---
.../main/java/org/apache/doris/qe/VariableMgr.java | 49 +++++++++++++++++---
.../suites/nereids_syntax_p0/expression.groovy | 6 +++
.../suites/nereids_syntax_p0/system_var.groovy | 2 +-
10 files changed, 114 insertions(+), 41 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
index 69683b982b..ae81bf6739 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4
@@ -416,7 +416,7 @@ TILDE: '~';
AMPERSAND: '&';
LOGICALAND: '&&';
PIPE: '|';
-CONCAT_PIPE: '||';
+DOUBLEPIPES: '||';
HAT: '^';
COLON: ':';
ARROW: '->';
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 40c88b27af..594b3f7a0b 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -245,13 +245,14 @@ expression
;
booleanExpression
- : NOT booleanExpression #logicalNot
- | EXISTS LEFT_PAREN query RIGHT_PAREN #exist
- | (ISNULL | IS_NULL_PRED) LEFT_PAREN valueExpression RIGHT_PAREN #isnull
- | IS_NOT_NULL_PRED LEFT_PAREN valueExpression RIGHT_PAREN #is_not_null_pred
- | valueExpression predicate? #predicated
+ : NOT booleanExpression #logicalNot
+ | EXISTS LEFT_PAREN query RIGHT_PAREN #exist
+ | (ISNULL | IS_NULL_PRED) LEFT_PAREN valueExpression RIGHT_PAREN #isnull
+ | IS_NOT_NULL_PRED LEFT_PAREN valueExpression RIGHT_PAREN #is_not_null_pred
+ | valueExpression predicate? #predicated
| left=booleanExpression operator=(AND | LOGICALAND) right=booleanExpression #logicalBinary
- | left=booleanExpression operator=(OR | CONCAT_PIPE) right=booleanExpression #logicalBinary
+ | left=booleanExpression operator=OR right=booleanExpression #logicalBinary
+ | left=booleanExpression operator=DOUBLEPIPES right=booleanExpression #doublePipes
;
predicate
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 1485627517..a61f62d1df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
import org.apache.doris.analysis.SetType;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.AggClauseContext;
@@ -143,7 +144,6 @@ import org.apache.doris.nereids.trees.expressions.ScalarSubquery;
import org.apache.doris.nereids.trees.expressions.Subtract;
import org.apache.doris.nereids.trees.expressions.TVFProperties;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
-import org.apache.doris.nereids.trees.expressions.VariableDesc;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.Function;
import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
@@ -171,12 +171,14 @@ import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.IntegerLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Interval;
import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.expressions.literal.SmallIntLiteral;
+import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.trees.plans.JoinHint;
@@ -213,7 +215,13 @@ import org.apache.doris.nereids.types.TinyIntType;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.policy.PolicyTypeEnum;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.SessionVariable;
+import org.apache.doris.qe.SqlModeHelper;
+import org.apache.doris.qe.VariableMgr;
+import org.apache.doris.qe.VariableVarConverters;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
@@ -527,17 +535,29 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
@Override
public Expression visitSystemVariable(SystemVariableContext ctx) {
+ String name = ctx.identifier().getText();
+ SessionVariable sessionVariable = ConnectContext.get().getSessionVariable();
+ Literal literal = null;
if (ctx.kind == null) {
- return new VariableDesc(SetType.SESSION, ctx.identifier().getText());
+ literal = VariableMgr.getLiteral(sessionVariable, name, SetType.DEFAULT);
+ } else if (ctx.kind.getType() == DorisParser.SESSION) {
+ literal = VariableMgr.getLiteral(sessionVariable, name, SetType.SESSION);
+ } else if (ctx.kind.getType() == DorisParser.GLOBAL) {
+ literal = VariableMgr.getLiteral(sessionVariable, name, SetType.GLOBAL);
}
- switch (ctx.kind.getType()) {
- case DorisParser.GLOBAL:
- return new VariableDesc(SetType.GLOBAL, ctx.identifier().getText());
- case DorisParser.SESSION:
- return new VariableDesc(SetType.SESSION, ctx.identifier().getText());
- default:
- throw new ParseException("Unsupported system variable: " + ctx.getText(), ctx);
+ if (literal == null) {
+ throw new ParseException("Unsupported system variable: " + ctx.getText(), ctx);
+ }
+ if (!Strings.isNullOrEmpty(name) && VariableVarConverters.hasConverter(name)) {
+ try {
+ Preconditions.checkArgument(literal instanceof IntegerLikeLiteral);
+ IntegerLikeLiteral integerLikeLiteral = (IntegerLikeLiteral) literal;
+ literal = new StringLiteral(VariableVarConverters.decode(name, integerLikeLiteral.getLongValue()));
+ } catch (DdlException e) {
+ throw new ParseException(e.getMessage(), ctx);
+ }
}
+ return literal;
}
@Override
@@ -607,7 +627,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
case DorisParser.AND:
return new And(left, right);
case DorisParser.OR:
- case DorisParser.CONCAT_PIPE:
return new Or(left, right);
default:
throw new ParseException("Unsupported logical binary type: " + ctx.operator.getText(), ctx);
@@ -808,6 +827,19 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
+ ", supported time unit: YEAR/MONTH/DAY/HOUR/MINUTE/SECOND", ctx);
}
+ @Override
+ public Expression visitDoublePipes(DorisParser.DoublePipesContext ctx) {
+ return ParserUtils.withOrigin(ctx, () -> {
+ Expression left = getExpression(ctx.left);
+ Expression right = getExpression(ctx.right);
+ if (ConnectContext.get().getSessionVariable().getSqlMode() == SqlModeHelper.MODE_PIPES_AS_CONCAT) {
+ return new UnboundFunction("concat", Lists.newArrayList(left, right));
+ } else {
+ return new Or(left, right);
+ }
+ });
+ }
+
/**
* Create a value based [[CaseWhen]] expression. This has the following SQL form:
* {{{
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRuleOnFE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRuleOnFE.java
index c8e005e9e6..2ac4c7fe45 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRuleOnFE.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rewrite/rules/FoldConstantRuleOnFE.java
@@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.expressions.NullSafeEqual;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.TimestampArithmetic;
-import org.apache.doris.nereids.trees.expressions.VariableDesc;
import org.apache.doris.nereids.trees.expressions.WhenClause;
import org.apache.doris.nereids.trees.expressions.functions.BoundFunction;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
@@ -55,13 +54,10 @@ import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.BooleanType;
import org.apache.doris.nereids.util.ExpressionUtils;
-import org.apache.doris.qe.VariableMgr;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
@@ -348,13 +344,6 @@ public class FoldConstantRuleOnFE extends AbstractExpressionRewriteRule {
return Literal.of(isNull.child().nullable());
}
- @Override
- public Expression visitVariableDesc(VariableDesc variableDesc, ExpressionRewriteContext context) {
- Preconditions.checkArgument(variableDesc.isSystemVariable());
- return new StringLiteral(VariableMgr.getValue(context.connectContext.getSessionVariable(), variableDesc));
-
- }
-
@Override
public Expression visitTimestampArithmetic(TimestampArithmetic arithmetic, ExpressionRewriteContext context) {
return ExpressionEvaluator.INSTANCE.eval(arithmetic);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLikeLiteral.java
index c4e60e4c7a..9f8a08a00f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLikeLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/IntegerLikeLiteral.java
@@ -34,5 +34,9 @@ public abstract class IntegerLikeLiteral extends Literal {
return getNumber().intValue();
}
+ public long getLongValue() {
+ return getNumber().longValue();
+ }
+
public abstract Number getNumber();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 1ec875a851..2c9390185d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -294,7 +294,7 @@ public class SessionVariable implements Serializable, Writable {
// Set sqlMode to empty string
@VariableMgr.VarAttr(name = SQL_MODE, needForward = true)
- public long sqlMode = 0L;
+ public long sqlMode = SqlModeHelper.MODE_DEFAULT;
@VariableMgr.VarAttr(name = RESOURCE_VARIABLE)
public String resourceGroup = "normal";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SqlModeHelper.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SqlModeHelper.java
index f014ebd631..7709ac6143 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SqlModeHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SqlModeHelper.java
@@ -40,7 +40,8 @@ public class SqlModeHelper {
// of how they works and to be compatible with MySQL, so for now they are not
// really meaningful.
/* Bits for different SQL MODE modes, you can add custom SQL MODE here */
- public static final long MODE_REAL_AS_FLOAT = 1L;
+ /* When a new session is created, its sql mode is set to MODE_DEFAULT */
+ public static final long MODE_DEFAULT = 1L;
public static final long MODE_PIPES_AS_CONCAT = 2L;
public static final long MODE_ANSI_QUOTES = 4L;
public static final long MODE_IGNORE_SPACE = 8L;
@@ -70,9 +71,8 @@ public class SqlModeHelper {
public static final long MODE_TRADITIONAL = 1L << 27;
public static final long MODE_LAST = 1L << 33;
+ public static final long MODE_REAL_AS_FLOAT = 1L << 34;
- /* When a new session is create, its sql mode is set to MODE_DEFAULT */
- public static final long MODE_DEFAULT = 0L;
public static final long MODE_ALLOWED_MASK =
(MODE_REAL_AS_FLOAT | MODE_PIPES_AS_CONCAT | MODE_ANSI_QUOTES | MODE_IGNORE_SPACE | MODE_NOT_USED
@@ -80,7 +80,8 @@ public class SqlModeHelper {
| MODE_NO_AUTO_VALUE_ON_ZERO | MODE_NO_BACKSLASH_ESCAPES | MODE_STRICT_TRANS_TABLES
| MODE_STRICT_ALL_TABLES | MODE_NO_ZERO_IN_DATE | MODE_NO_ZERO_DATE | MODE_INVALID_DATES
| MODE_ERROR_FOR_DIVISION_BY_ZERO | MODE_HIGH_NOT_PRECEDENCE | MODE_NO_ENGINE_SUBSTITUTION
- | MODE_PAD_CHAR_TO_FULL_LENGTH | MODE_TRADITIONAL | MODE_ANSI | MODE_TIME_TRUNCATE_FRACTIONAL);
+ | MODE_PAD_CHAR_TO_FULL_LENGTH | MODE_TRADITIONAL | MODE_ANSI | MODE_TIME_TRUNCATE_FRACTIONAL
+ | MODE_DEFAULT);
public static final long MODE_COMBINE_MASK = (MODE_ANSI | MODE_TRADITIONAL);
@@ -89,6 +90,7 @@ public class SqlModeHelper {
private static final Map<String, Long> combineModeSet = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
static {
+ sqlModeSet.put("DEFAULT", MODE_DEFAULT);
sqlModeSet.put("REAL_AS_FLOAT", MODE_REAL_AS_FLOAT);
sqlModeSet.put("PIPES_AS_CONCAT", MODE_PIPES_AS_CONCAT);
sqlModeSet.put("ANSI_QUOTES", MODE_ANSI_QUOTES);
@@ -121,8 +123,8 @@ public class SqlModeHelper {
// convert long type SQL MODE to string type that user can read
public static String decode(Long sqlMode) throws DdlException {
- // 0 parse to empty string
- if (sqlMode == 0) {
+ if (sqlMode == MODE_DEFAULT) {
+ //For compatibility with older versions， return empty string
return "";
}
if ((sqlMode & ~MODE_ALLOWED_MASK) != 0) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
index b66a9b1df6..36468bc6f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/VariableMgr.java
@@ -27,7 +27,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.PatternMatcher;
-import org.apache.doris.nereids.trees.expressions.VariableDesc;
+import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.persist.GlobalVarPersistInfo;
import com.google.common.base.Preconditions;
@@ -54,6 +54,7 @@ import java.util.Map;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import javax.annotation.Nullable;
/**
* Variable manager, merge session variable and global variable.
@@ -442,12 +443,50 @@ public class VariableMgr {
}
// For Nereids optimizer
- public static String getValue(SessionVariable var, VariableDesc desc) throws RuntimeException {
+ public static @Nullable Literal getLiteral(SessionVariable var, String name, SetType setType) {
+ VarContext ctx = ctxByVarName.get(name);
+ if (ctx == null) {
+ return null;
+ }
+
+ if (setType == SetType.GLOBAL) {
+ rlock.lock();
+ try {
+ return getLiteral(ctx.getObj(), ctx.getField());
+ } finally {
+ rlock.unlock();
+ }
+ } else {
+ return getLiteral(var, ctx.getField());
+ }
+ }
+
+ private static Literal getLiteral(Object obj, Field field) {
try {
- return getValue(var, desc.getName(), desc.getSetType());
- } catch (AnalysisException e) {
- throw new RuntimeException(e);
+ switch (field.getType().getSimpleName()) {
+ case "boolean":
+ return Literal.of(field.getBoolean(obj));
+ case "byte":
+ return Literal.of(field.getByte(obj));
+ case "short":
+ return Literal.of(field.getShort(obj));
+ case "int":
+ return Literal.of(field.getInt(obj));
+ case "long":
+ return Literal.of(field.getLong(obj));
+ case "float":
+ return Literal.of(field.getFloat(obj));
+ case "double":
+ return Literal.of(field.getDouble(obj));
+ case "String":
+ return Literal.of((String) field.get(obj));
+ default:
+ return Literal.of("");
+ }
+ } catch (IllegalAccessException e) {
+ LOG.warn("Access failed.", e);
}
+ return Literal.of("");
}
private static String getValue(Object obj, Field field) {
diff --git a/regression-test/suites/nereids_syntax_p0/expression.groovy b/regression-test/suites/nereids_syntax_p0/expression.groovy
index 59c373ba0b..05eab65a3e 100644
--- a/regression-test/suites/nereids_syntax_p0/expression.groovy
+++ b/regression-test/suites/nereids_syntax_p0/expression.groovy
@@ -28,4 +28,10 @@ suite("nereids_syntax_expression_test") {
sql "select true or false, true || false"
result([[true, true]])
}
+
+ sql """ SET sql_mode = "PIPES_AS_CONCAT" """
+ test {
+ sql "select @@session.sql_mode, 1 || 2"
+ result([["PIPES_AS_CONCAT", "12"]])
+ }
}
diff --git a/regression-test/suites/nereids_syntax_p0/system_var.groovy b/regression-test/suites/nereids_syntax_p0/system_var.groovy
index 3ae1580b8a..3a3714e8c8 100644
--- a/regression-test/suites/nereids_syntax_p0/system_var.groovy
+++ b/regression-test/suites/nereids_syntax_p0/system_var.groovy
@@ -44,6 +44,6 @@ suite("nereids_sys_var") {
sql "select @@session.time_zone"
test {
sql "select @@session.enable_nereids_planner"
- result ([["true"]])
+ result ([[true]])
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org