You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/10/20 14:29:29 UTC
[doris] branch master updated: [feature](Nereids) generate ExprId from 0 for each statement (#13382)
This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 483a46d17c [feature](Nereids) generate ExprId from 0 for each statement (#13382)
483a46d17c is described below
commit 483a46d17ce8c0f494de880226c1ab885150a5de
Author: morrySnow <10...@users.noreply.github.com>
AuthorDate: Thu Oct 20 22:29:22 2022 +0800
[feature](Nereids) generate ExprId from 0 for each statement (#13382)
Currently, ExprId in Nereids is generated by a global gnerator and shared by all statement. There are three problems:
1. ExprId could out of bound
2. hard to debug
3. could not use bitset to present ExprId set
This PR solve this problem by new Id generator for each statement. after this PR ExprId always start from 0 for each statement.
TODO:
1. refactor all place that new StatementContext in test code to ensure the logic is same with main code.
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 3 +-
.../org/apache/doris/nereids/StatementContext.java | 30 +++++++++++++++----
.../doris/nereids/glue/LogicalPlanAdapter.java | 19 ++++++++----
.../glue/translator/PhysicalPlanTranslator.java | 1 +
.../doris/nereids/parser/LogicalPlanBuilder.java | 18 +++++++++--
.../apache/doris/nereids/parser/NereidsParser.java | 35 +++++++++++-----------
.../doris/nereids/rules/analysis/BindRelation.java | 4 +--
.../doris/nereids/trees/expressions/Alias.java | 2 --
.../trees/expressions/NamedExpressionUtil.java | 25 +++++++++-------
.../java/org/apache/doris/qe/ConnectContext.java | 11 +++++++
.../java/org/apache/doris/qe/ConnectProcessor.java | 6 ++--
.../java/org/apache/doris/qe/StmtExecutor.java | 15 ++++++++--
.../doris/nereids/parser/NereidsParserTest.java | 4 ++-
.../logical/PushdownFilterThroughJoinTest.java | 34 ++++++++++-----------
.../doris/nereids/stats/StatsCalculatorTest.java | 32 ++++++++------------
.../apache/doris/nereids/util/MemoTestUtils.java | 4 ++-
.../apache/doris/utframe/TestWithFeService.java | 4 ++-
17 files changed, 154 insertions(+), 93 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index fa81545d31..868a2496d7 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -50,8 +50,7 @@ singleStatement
statement
: query #statementDefault
- | (EXPLAIN | DESC | DESCRIBE) level=(VERBOSE | GRAPH)?
- query #explain
+ | (EXPLAIN | DESC | DESCRIBE) level=(VERBOSE | GRAPH)? query #explain
;
// -----------------Query-----------------
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
index fc632cd9a6..32c57590e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.IdGenerator;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.plans.RelationId;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.OriginStatement;
@@ -27,22 +28,37 @@ import org.apache.doris.qe.OriginStatement;
* Statement context for nereids
*/
public class StatementContext {
- private final ConnectContext connectContext;
- private final OriginStatement originStatement;
- private final IdGenerator<RelationId> idGenerator = RelationId.createGenerator();
+ private ConnectContext connectContext;
+
+ private OriginStatement originStatement;
+
+ private final IdGenerator<ExprId> exprIdGenerator = ExprId.createGenerator();
+
+ private final IdGenerator<RelationId> relationIdGenerator = RelationId.createGenerator();
private StatementBase parsedStatement;
+ public StatementContext() {
+ }
+
public StatementContext(ConnectContext connectContext, OriginStatement originStatement) {
this.connectContext = connectContext;
this.originStatement = originStatement;
}
+ public void setConnectContext(ConnectContext connectContext) {
+ this.connectContext = connectContext;
+ }
+
public ConnectContext getConnectContext() {
return connectContext;
}
+ public void setOriginStatement(OriginStatement originStatement) {
+ this.originStatement = originStatement;
+ }
+
public OriginStatement getOriginStatement() {
return originStatement;
}
@@ -51,8 +67,12 @@ public class StatementContext {
return parsedStatement;
}
- public RelationId getNextId() {
- return idGenerator.getNextId();
+ public ExprId getNextExprId() {
+ return exprIdGenerator.getNextId();
+ }
+
+ public RelationId getNextRelationId() {
+ return relationIdGenerator.getNextId();
}
public void setParsedStatement(StatementBase parsedStatement) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
index 76d101148d..6e0a2cdb71 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/LogicalPlanAdapter.java
@@ -22,6 +22,7 @@ import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.Queriable;
import org.apache.doris.analysis.RedirectStatus;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -35,12 +36,14 @@ import java.util.List;
*/
public class LogicalPlanAdapter extends StatementBase implements Queriable {
+ private final StatementContext statementContext;
private final LogicalPlan logicalPlan;
private List<Expr> resultExprs;
private ArrayList<String> colLabels;
- public LogicalPlanAdapter(LogicalPlan logicalPlan) {
+ public LogicalPlanAdapter(LogicalPlan logicalPlan, StatementContext statementContext) {
this.logicalPlan = logicalPlan;
+ this.statementContext = statementContext;
}
@Override
@@ -62,15 +65,15 @@ public class LogicalPlanAdapter extends StatementBase implements Queriable {
return null;
}
+ public ArrayList<String> getColLabels() {
+ return colLabels;
+ }
+
@Override
public List<Expr> getResultExprs() {
return resultExprs;
}
- public ArrayList<String> getColLabels() {
- return colLabels;
- }
-
public void setResultExprs(List<Expr> resultExprs) {
this.resultExprs = resultExprs;
}
@@ -79,12 +82,16 @@ public class LogicalPlanAdapter extends StatementBase implements Queriable {
this.colLabels = colLabels;
}
+ public StatementContext getStatementContext() {
+ return statementContext;
+ }
+
public String toDigest() {
// TODO: generate real digest
return "";
}
public static LogicalPlanAdapter of(Plan plan) {
- return new LogicalPlanAdapter((LogicalPlan) plan);
+ return new LogicalPlanAdapter((LogicalPlan) plan, null);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 818fa16f62..1b973daf47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -163,6 +163,7 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla
if (e instanceof SlotReference && outputExpressionList.stream().anyMatch(o -> o.anyMatch(e::equals))) {
groupSlotList.add((SlotReference) e);
} else {
+ // TODO: review this, should not
groupSlotList.add(new SlotReference(e.toSql(), e.getDataType(), e.nullable(), Collections.emptyList()));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 0a9f7ebefc..1f93a2ba44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.ArithmeticExpr.Operator;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisParser;
import org.apache.doris.nereids.DorisParser.AggClauseContext;
import org.apache.doris.nereids.DorisParser.AliasedQueryContext;
@@ -71,6 +72,7 @@ import org.apache.doris.nereids.DorisParser.TypeConstructorContext;
import org.apache.doris.nereids.DorisParser.UnitIdentifierContext;
import org.apache.doris.nereids.DorisParser.WhereClauseContext;
import org.apache.doris.nereids.DorisParserBaseVisitor;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.analyzer.UnboundAlias;
import org.apache.doris.nereids.analyzer.UnboundFunction;
import org.apache.doris.nereids.analyzer.UnboundOneRowRelation;
@@ -140,6 +142,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias;
import org.apache.doris.nereids.types.DataType;
import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -167,6 +170,8 @@ import java.util.stream.Collectors;
*/
public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
+ private StatementContext statementContext = new StatementContext();
+
protected <T> T typedVisit(ParseTree ctx) {
return (T) ctx.accept(this);
}
@@ -194,8 +199,17 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> {
* Visit multi-statements.
*/
@Override
- public List<LogicalPlan> visitMultiStatements(MultiStatementsContext ctx) {
- return visit(ctx.statement(), LogicalPlan.class);
+ public List<Pair<LogicalPlan, StatementContext>> visitMultiStatements(MultiStatementsContext ctx) {
+ List<Pair<LogicalPlan, StatementContext>> logicalPlans = Lists.newArrayList();
+ for (org.apache.doris.nereids.DorisParser.StatementContext statement : ctx.statement()) {
+ statementContext = new StatementContext();
+ if (ConnectContext.get() != null) {
+ ConnectContext.get().setStatementContext(statementContext);
+ }
+ logicalPlans.add(Pair.of(
+ ParserUtils.withOrigin(ctx, () -> (LogicalPlan) visit(statement)), statementContext));
+ }
+ return logicalPlans;
}
/* ********************************************************************************************
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
index e1077d13c3..22476d361d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/NereidsParser.java
@@ -19,21 +19,23 @@ package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.common.Pair;
import org.apache.doris.nereids.DorisLexer;
import org.apache.doris.nereids.DorisParser;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import com.google.common.collect.Lists;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.ParserRuleContext;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.misc.ParseCancellationException;
-import java.util.ArrayList;
import java.util.List;
import java.util.function.Function;
@@ -45,26 +47,25 @@ public class NereidsParser {
private static final PostProcessor POST_PROCESSOR = new PostProcessor();
/**
- * In MySQL protocol, client could send multi-statement in.
- * a single packet.
- * https://dev.mysql.com/doc/internals/en/com-set-option.html
+ * In MySQL protocol, client could send multi-statement in a single packet.
+ * see <a href="https://dev.mysql.com/doc/internals/en/com-set-option.html">docs</a> for more information.
*/
public List<StatementBase> parseSQL(String originStr) {
- List<LogicalPlan> logicalPlans = parseMultiple(originStr);
- List<StatementBase> statementBases = new ArrayList<>();
- for (LogicalPlan logicalPlan : logicalPlans) {
+ List<Pair<LogicalPlan, StatementContext>> logicalPlans = parseMultiple(originStr);
+ List<StatementBase> statementBases = Lists.newArrayList();
+ for (Pair<LogicalPlan, StatementContext> logicalPlan : logicalPlans) {
// TODO: this is a trick to support explain. Since we do not support any other command in a short time.
// It is acceptable. In the future, we need to refactor this.
- if (logicalPlan instanceof ExplainCommand) {
- ExplainCommand explainCommand = (ExplainCommand) logicalPlan;
+ if (logicalPlan.first instanceof ExplainCommand) {
+ ExplainCommand explainCommand = (ExplainCommand) logicalPlan.first;
LogicalPlan innerPlan = explainCommand.getLogicalPlan();
- LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(innerPlan);
+ LogicalPlanAdapter logicalPlanAdapter = new LogicalPlanAdapter(innerPlan, logicalPlan.second);
logicalPlanAdapter.setIsExplain(new ExplainOptions(
explainCommand.getLevel() == ExplainLevel.VERBOSE,
explainCommand.getLevel() == ExplainLevel.GRAPH));
statementBases.add(logicalPlanAdapter);
} else {
- statementBases.add(new LogicalPlanAdapter(logicalPlan));
+ statementBases.add(new LogicalPlanAdapter(logicalPlan.first, logicalPlan.second));
}
}
return statementBases;
@@ -77,21 +78,21 @@ public class NereidsParser {
* @return logical plan
*/
public LogicalPlan parseSingle(String sql) {
- return (LogicalPlan) parse(sql, DorisParser::singleStatement);
+ return parse(sql, DorisParser::singleStatement);
}
- public List<LogicalPlan> parseMultiple(String sql) {
- return (List<LogicalPlan>) parse(sql, DorisParser::multiStatements);
+ public List<Pair<LogicalPlan, StatementContext>> parseMultiple(String sql) {
+ return parse(sql, DorisParser::multiStatements);
}
public Expression parseExpression(String expression) {
- return (Expression) parse(expression, DorisParser::expression);
+ return parse(expression, DorisParser::expression);
}
- private Object parse(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
+ private <T> T parse(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
ParserRuleContext tree = toAst(sql, parseFunction);
LogicalPlanBuilder logicalPlanBuilder = new LogicalPlanBuilder();
- return logicalPlanBuilder.visit(tree);
+ return (T) logicalPlanBuilder.visit(tree);
}
private ParserRuleContext toAst(String sql, Function<DorisParser, ParserRuleContext> parseFunction) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index 9ca1e1b43e..a6e09beb1b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -78,7 +78,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
Table table = getTable(dbName, nameParts.get(0), cascadesContext.getConnectContext().getEnv());
// TODO: should generate different Scan sub class according to table's type
if (table.getType() == TableType.OLAP) {
- return new LogicalOlapScan(cascadesContext.getStatementContext().getNextId(),
+ return new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(),
(OlapTable) table, ImmutableList.of(dbName));
} else if (table.getType() == TableType.VIEW) {
Plan viewPlan = parseAndAnalyzeView(table.getDdlSql(), cascadesContext);
@@ -96,7 +96,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
}
Table table = getTable(dbName, nameParts.get(1), connectContext.getEnv());
if (table.getType() == TableType.OLAP) {
- return new LogicalOlapScan(cascadesContext.getStatementContext().getNextId(),
+ return new LogicalOlapScan(cascadesContext.getStatementContext().getNextRelationId(),
(OlapTable) table, ImmutableList.of(dbName));
} else if (table.getType() == TableType.VIEW) {
Plan viewPlan = parseAndAnalyzeView(table.getDdlSql(), cascadesContext);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
index d9f8735103..add793b070 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Alias.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -48,7 +47,6 @@ public class Alias extends NamedExpression implements UnaryExpression {
this(NamedExpressionUtil.newExprId(), child, name);
}
- @VisibleForTesting
public Alias(ExprId exprId, Expression child, String name) {
super(child);
this.exprId = exprId;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpressionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpressionUtil.java
index 12ab04ce92..27cd0d3d9d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpressionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpressionUtil.java
@@ -17,23 +17,25 @@
package org.apache.doris.nereids.trees.expressions;
-import org.apache.doris.common.IdGenerator;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.annotations.VisibleForTesting;
-import java.lang.reflect.Field;
-
/**
* The util of named expression.
*/
public class NamedExpressionUtil {
- /**
- * Tool class for generate next ExprId.
- */
- private static final IdGenerator<ExprId> ID_GENERATOR = ExprId.createGenerator();
+
+ // for test only
+ private static StatementContext statementContext = new StatementContext();
public static ExprId newExprId() {
- return ID_GENERATOR.getNextId();
+ // this branch is for test only
+ if (ConnectContext.get() == null || ConnectContext.get().getStatementContext() == null) {
+ return statementContext.getNextExprId();
+ }
+ return ConnectContext.get().getStatementContext().getNextExprId();
}
/**
@@ -41,8 +43,9 @@ public class NamedExpressionUtil {
*/
@VisibleForTesting
public static void clear() throws Exception {
- Field nextId = ID_GENERATOR.getClass().getSuperclass().getDeclaredField("nextId");
- nextId.setAccessible(true);
- nextId.setInt(ID_GENERATOR, 0);
+ if (ConnectContext.get() != null) {
+ ConnectContext.get().setStatementContext(new StatementContext());
+ }
+ statementContext = new StatementContext();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
index 0ad83d5176..b6e7ac9d84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java
@@ -31,6 +31,7 @@ import org.apache.doris.mysql.MysqlCapability;
import org.apache.doris.mysql.MysqlChannel;
import org.apache.doris.mysql.MysqlCommand;
import org.apache.doris.mysql.MysqlSerializer;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
import org.apache.doris.resource.Tag;
import org.apache.doris.thrift.TResourceInfo;
@@ -146,6 +147,8 @@ public class ConnectContext {
private SessionContext sessionContext;
+ private StatementContext statementContext;
+
public SessionContext getSessionContext() {
return sessionContext;
}
@@ -512,6 +515,14 @@ public class ConnectContext {
this.tracer = Telemetry.getOpenTelemetry().getTracer(name);
}
+ public StatementContext getStatementContext() {
+ return statementContext;
+ }
+
+ public void setStatementContext(StatementContext statementContext) {
+ this.statementContext = statementContext;
+ }
+
// kill operation with no protect.
public void kill(boolean killConnection) {
LOG.warn("kill query from {}, kill connection: {}", getMysqlChannel().getRemoteHostPortString(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 6e37ebde04..e184a16789 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -264,10 +264,10 @@ public class ConnectProcessor {
try {
stmts = nereidsParser.parseSQL(originStmt);
} catch (Exception e) {
- nereidsParseException = e;
// TODO: We should catch all exception here until we support all query syntax.
+ nereidsParseException = e;
LOG.info(" Fallback to stale planner."
- + " Nereids cannot process this statement: \"{}\".", originStmt.toString());
+ + " Nereids cannot process this statement: \"{}\".", originStmt);
}
}
// stmts == null when Nereids cannot planner this query or Nereids is disabled.
@@ -283,7 +283,7 @@ public class ConnectProcessor {
parsedStmt = stmts.get(i);
if (parsedStmt instanceof SelectStmt) {
if (!ctx.getSessionVariable().enableFallbackToOriginalPlanner) {
- throw new Exception(String.format("SQL: {}", parsedStmt.toSql()), nereidsParseException);
+ throw new Exception(String.format("SQL: %s", parsedStmt.toSql()), nereidsParseException);
}
}
parsedStmt.setOrigStmt(new OriginStatement(originStmt, i));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index c1f8a0983e..d6c5648b53 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -164,7 +164,7 @@ public class StmtExecutor implements ProfileWriter {
private static final String NULL_VALUE_FOR_LOAD = "\\N";
private final Object writeProfileLock = new Object();
private ConnectContext context;
- private StatementContext statementContext;
+ private final StatementContext statementContext;
private MysqlSerializer serializer;
private OriginStatement originStmt;
private StatementBase parsedStmt;
@@ -191,6 +191,7 @@ public class StmtExecutor implements ProfileWriter {
this.serializer = context.getSerializer();
this.isProxy = isProxy;
this.statementContext = new StatementContext(context, originStmt);
+ this.context.setStatementContext(statementContext);
}
// this constructor is only for test now.
@@ -205,8 +206,16 @@ public class StmtExecutor implements ProfileWriter {
this.originStmt = parsedStmt.getOrigStmt();
this.serializer = context.getSerializer();
this.isProxy = false;
- this.statementContext = new StatementContext(ctx, originStmt);
- this.statementContext.setParsedStatement(parsedStmt);
+ if (parsedStmt instanceof LogicalPlanAdapter) {
+ this.statementContext = ((LogicalPlanAdapter) parsedStmt).getStatementContext();
+ this.statementContext.setConnectContext(ctx);
+ this.statementContext.setOriginStatement(originStmt);
+ this.statementContext.setParsedStatement(parsedStmt);
+ } else {
+ this.statementContext = new StatementContext(ctx, originStmt);
+ this.statementContext.setParsedStatement(parsedStmt);
+ }
+ this.context.setStatementContext(statementContext);
}
public static InternalService.PDataRow getRowStringValue(List<Expr> cols) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
index 2fd3d757ac..6fb74728d9 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java
@@ -19,6 +19,8 @@ package org.apache.doris.nereids.parser;
import org.apache.doris.analysis.ExplainOptions;
import org.apache.doris.analysis.StatementBase;
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.exceptions.ParseException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.trees.expressions.literal.DecimalLiteral;
@@ -42,7 +44,7 @@ public class NereidsParserTest extends ParserTestBase {
public void testParseMultiple() {
NereidsParser nereidsParser = new NereidsParser();
String sql = "SELECT b FROM test;SELECT a FROM test;";
- List<LogicalPlan> logicalPlanList = nereidsParser.parseMultiple(sql);
+ List<Pair<LogicalPlan, StatementContext>> logicalPlanList = nereidsParser.parseMultiple(sql);
Assertions.assertEquals(2, logicalPlanList.size());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java
index c10b0b05b1..ceb9b7a7d3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughJoinTest.java
@@ -34,7 +34,7 @@ import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.nereids.util.PlanConstructor;
import com.google.common.collect.ImmutableList;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
@@ -50,14 +50,26 @@ public class PushdownFilterThroughJoinTest implements PatternMatchSupported {
/**
* ut before.
*/
- @BeforeAll
- public final void beforeAll() {
+ @BeforeEach
+ public final void beforeEach() {
rStudent = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.student,
ImmutableList.of(""));
rScore = new LogicalOlapScan(PlanConstructor.getNextRelationId(), PlanConstructor.score, ImmutableList.of(""));
}
- public void testLeft(JoinType joinType) {
+ @Test
+ public void oneSide() {
+ testLeft(JoinType.CROSS_JOIN);
+ testLeft(JoinType.INNER_JOIN);
+ testLeft(JoinType.LEFT_OUTER_JOIN);
+ testLeft(JoinType.LEFT_SEMI_JOIN);
+ testLeft(JoinType.LEFT_ANTI_JOIN);
+ testRight(JoinType.RIGHT_OUTER_JOIN);
+ testRight(JoinType.RIGHT_SEMI_JOIN);
+ testRight(JoinType.RIGHT_ANTI_JOIN);
+ }
+
+ private void testLeft(JoinType joinType) {
Expression whereCondition1 = new GreaterThan(rStudent.getOutput().get(1), Literal.of(18));
Expression whereCondition2 = new GreaterThan(rStudent.getOutput().get(1), Literal.of(50));
Expression whereCondition = ExpressionUtils.and(whereCondition1, whereCondition2);
@@ -78,7 +90,7 @@ public class PushdownFilterThroughJoinTest implements PatternMatchSupported {
);
}
- public void testRight(JoinType joinType) {
+ private void testRight(JoinType joinType) {
Expression whereCondition1 = new GreaterThan(rStudent.getOutput().get(1), Literal.of(18));
Expression whereCondition2 = new GreaterThan(rStudent.getOutput().get(1), Literal.of(50));
Expression whereCondition = ExpressionUtils.and(whereCondition1, whereCondition2);
@@ -99,18 +111,6 @@ public class PushdownFilterThroughJoinTest implements PatternMatchSupported {
);
}
- @Test
- public void oneSide() {
- testLeft(JoinType.CROSS_JOIN);
- testLeft(JoinType.INNER_JOIN);
- testLeft(JoinType.LEFT_OUTER_JOIN);
- testLeft(JoinType.LEFT_SEMI_JOIN);
- testLeft(JoinType.LEFT_ANTI_JOIN);
- testRight(JoinType.RIGHT_OUTER_JOIN);
- testRight(JoinType.RIGHT_SEMI_JOIN);
- testRight(JoinType.RIGHT_ANTI_JOIN);
- }
-
@Test
public void bothSideToBothSide() {
bothSideToBothSide(JoinType.INNER_JOIN);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 8647886853..b9fa35a335 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -43,8 +43,9 @@ import org.apache.doris.statistics.StatisticsManager;
import org.apache.doris.statistics.StatsDeriveResult;
import org.apache.doris.statistics.TableStats;
-import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import mockit.Expectations;
import mockit.Mocked;
import org.junit.jupiter.api.Assertions;
@@ -59,13 +60,6 @@ import java.util.Optional;
public class StatsCalculatorTest {
- @Mocked
- ConnectContext context;
- @Mocked
- Env env;
- @Mocked
- StatisticsManager statisticsManager;
-
// TODO: temporary disable this test, until we could get column stats
// @Test
// public void testAgg() {
@@ -107,7 +101,7 @@ public class StatsCalculatorTest {
@Test
public void testFilter() {
- List<String> qualifier = new ArrayList<>();
+ List<String> qualifier = Lists.newArrayList();
qualifier.add("test");
qualifier.add("t");
SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier);
@@ -118,8 +112,8 @@ public class StatsCalculatorTest {
columnStat1.setNumNulls(5);
ColumnStat columnStat2 = new ColumnStat();
columnStat2.setNdv(20);
- columnStat1.setNumNulls(10);
- Map<Slot, ColumnStat> slotColumnStatsMap = new HashMap<>();
+ columnStat2.setNumNulls(10);
+ Map<Slot, ColumnStat> slotColumnStatsMap = Maps.newHashMap();
slotColumnStatsMap.put(slot1, columnStat1);
slotColumnStatsMap.put(slot2, columnStat2);
StatsDeriveResult childStats = new StatsDeriveResult(10000, slotColumnStatsMap);
@@ -131,16 +125,11 @@ public class StatsCalculatorTest {
Or or = new Or(eq1, eq2);
Group childGroup = new Group();
- childGroup.setLogicalProperties(new LogicalProperties(new Supplier<List<Slot>>() {
- @Override
- public List<Slot> get() {
- return Collections.emptyList();
- }
- }));
+ childGroup.setLogicalProperties(new LogicalProperties(Collections::emptyList));
GroupPlan groupPlan = new GroupPlan(childGroup);
childGroup.setStatistics(childStats);
- LogicalFilter logicalFilter = new LogicalFilter(and, groupPlan);
+ LogicalFilter<GroupPlan> logicalFilter = new LogicalFilter<>(and, groupPlan);
GroupExpression groupExpression = new GroupExpression(logicalFilter);
groupExpression.addChild(childGroup);
Group ownerGroup = new Group();
@@ -148,7 +137,7 @@ public class StatsCalculatorTest {
StatsCalculator.estimate(groupExpression);
Assertions.assertEquals((long) (10000 * 0.1 * 0.05), ownerGroup.getStatistics().getRowCount(), 0.001);
- LogicalFilter logicalFilterOr = new LogicalFilter(or, groupPlan);
+ LogicalFilter<GroupPlan> logicalFilterOr = new LogicalFilter<>(or, groupPlan);
GroupExpression groupExpressionOr = new GroupExpression(logicalFilterOr);
groupExpressionOr.addChild(childGroup);
Group ownerGroupOr = new Group();
@@ -197,7 +186,10 @@ public class StatsCalculatorTest {
// }
@Test
- public void testOlapScan() {
+ public void testOlapScan(
+ @Mocked ConnectContext context,
+ @Mocked Env env,
+ @Mocked StatisticsManager statisticsManager) {
ColumnStat columnStat1 = new ColumnStat();
columnStat1.setNdv(10);
columnStat1.setNumNulls(5);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java
index faa9bdc10d..4999cde601 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/MemoTestUtils.java
@@ -50,7 +50,9 @@ public class MemoTestUtils {
}
public static StatementContext createStatementContext(ConnectContext connectContext, String sql) {
- return new StatementContext(connectContext, new OriginStatement(sql, 0));
+ StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(sql, 0));
+ connectContext.setStatementContext(statementContext);
+ return statementContext;
}
public static CascadesContext createCascadesContext(String sql) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index fc04e524f2..0b7cbf1272 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -149,7 +149,9 @@ public abstract class TestWithFeService {
}
protected StatementContext createStatementCtx(String sql) {
- return new StatementContext(connectContext, new OriginStatement(sql, 0));
+ StatementContext statementContext = new StatementContext(connectContext, new OriginStatement(sql, 0));
+ connectContext.setStatementContext(statementContext);
+ return statementContext;
}
protected CascadesContext createCascadesContext(String sql) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org