You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2022/07/20 05:54:00 UTC
[doris] branch master updated: [Feature](Nereids) Reorder join to eliminate cross join. (#10890)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9b91f86c38 [Feature](Nereids) Reorder join to eliminate cross join. (#10890)
9b91f86c38 is described below
commit 9b91f86c385512b956c5d0951385fe17c5792c6c
Author: Shuo Wang <wa...@gmail.com>
AuthorDate: Wed Jul 20 13:53:54 2022 +0800
[Feature](Nereids) Reorder join to eliminate cross join. (#10890)
Try to eliminate cross join via finding join conditions in filters and changing the join orders.
For example:
-- input:
SELECT * FROM t1, t2, t3 WHERE t1.id=t3.id AND t2.id=t3.id
-- output:
SELECT * FROM t1 JOIN t3 ON t1.id=t3.id JOIN t2 ON t2.id=t3.id
This feature is controlled by session variable enable_nereids_reorder_to_eliminate_cross_join with true by default.
Simplify usage of Memo and rewrite rule application.
Before this PR, if we want to apply a rewrite rule to a plan, the code is like the below:
Memo memo = new Memo();
memo.initialize(root);
PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
ImmutableList.of(new AggregateDisassemble().build()), jobContext);
plannerContext.pushJob(rewriteTopDownJob);
plannerContext.getJobScheduler().executeJobPool(plannerContext);
Plan after = memo.copyOut();
After this PR, we could use chain style calling:
new Memo(plan)
.newPlannerContext(connectContext)
.setDefaultJobContext()
.topDownRewrite(new AggregateDisassemble())
.getMemo()
.copyOut();
Rename the session variable enable_nereids to enable_nereids_planner to make it more meaningful.
---
.../org/apache/doris/nereids/NereidsPlanner.java | 47 +++--
.../org/apache/doris/nereids/PlannerContext.java | 49 +++++
.../apache/doris/nereids/analyzer/UnboundStar.java | 9 +-
.../doris/nereids/jobs/batch/BatchRulesJob.java | 2 +-
.../batch/JoinReorderRulesJob.java} | 27 +--
.../java/org/apache/doris/nereids/memo/Memo.java | 13 +-
.../nereids/pattern/GroupExpressionMatching.java | 72 ++++---
.../org/apache/doris/nereids/pattern/Patterns.java | 6 +-
.../doris/nereids/pattern/SubTreePattern.java | 45 +++++
.../org/apache/doris/nereids/rules/RuleType.java | 2 +
.../doris/nereids/rules/analysis/BindRelation.java | 19 +-
.../nereids/rules/analysis/BindSlotReference.java | 10 +-
.../nereids/rules/rewrite/logical/ReorderJoin.java | 220 +++++++++++++++++++++
.../trees/expressions/ComparisonPredicate.java | 9 +-
.../trees/expressions/CompoundPredicate.java | 3 +-
.../doris/nereids/trees/expressions/EqualTo.java | 2 +-
.../nereids/trees/expressions/GreaterThan.java | 2 +-
.../trees/expressions/GreaterThanEqual.java | 2 +-
.../doris/nereids/trees/expressions/LessThan.java | 2 +-
.../nereids/trees/expressions/LessThanEqual.java | 2 +-
.../nereids/trees/expressions/NamedExpression.java | 9 +-
.../nereids/trees/expressions/NullSafeEqual.java | 2 +-
.../nereids/trees/expressions/SlotReference.java | 9 +-
.../org/apache/doris/nereids/trees/plans/Plan.java | 4 +-
.../trees/plans/logical/LogicalOlapScan.java | 6 +-
.../trees/plans/logical/LogicalRelation.java | 23 ++-
.../nereids/trees/plans/logical/LogicalSort.java | 2 +-
.../trees/plans/physical/PhysicalAggregate.java | 2 +-
.../trees/plans/physical/PhysicalFilter.java | 2 +-
.../trees/plans/physical/PhysicalHeapSort.java | 8 +
.../trees/plans/physical/PhysicalOlapScan.java | 9 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 4 -
.../java/org/apache/doris/nereids/util/Utils.java | 19 ++
.../java/org/apache/doris/qe/ConnectProcessor.java | 2 +-
.../java/org/apache/doris/qe/SessionVariable.java | 28 ++-
.../doris/nereids/jobs/RewriteTopDownJobTest.java | 17 +-
.../org/apache/doris/nereids/memo/MemoTest.java | 5 +-
.../pattern/GroupExpressionMatchingTest.java | 155 +++++++++++++--
.../apache/doris/nereids/plan/TestPlanOutput.java | 10 +-
.../nereids/rules/analysis/BindRelationTest.java | 68 +++++++
.../LogicalProjectToPhysicalProjectTest.java | 12 +-
.../rewrite/logical/AggregateDisassembleTest.java | 58 +-----
.../rules/rewrite/logical/AnalyzeUtils.java | 62 ------
.../rules/rewrite/logical/ColumnPruningTest.java | 75 +++----
.../rewrite/logical/PushDownPredicateTest.java | 80 +++-----
.../rules/rewrite/logical/TestAnalyzer.java | 60 ++++++
.../doris/nereids/{ => ssb}/AnalyzeSSBTest.java | 68 +------
.../doris/nereids/ssb/SSBJoinReorderTest.java | 167 ++++++++++++++++
.../org/apache/doris/nereids/ssb/SSBTestBase.java} | 24 +--
.../org/apache/doris/nereids/ssb/SSBUtils.java | 6 +-
.../apache/doris/nereids/util/PlanRewriter.java | 77 ++++++++
.../apache/doris/utframe/TestWithFeService.java | 2 +
52 files changed, 1130 insertions(+), 488 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 7880cb8f48..2b32416680 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -24,9 +24,9 @@ import org.apache.doris.common.UserException;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
-import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob;
import org.apache.doris.nereids.jobs.batch.DisassembleRulesJob;
+import org.apache.doris.nereids.jobs.batch.JoinReorderRulesJob;
import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob;
import org.apache.doris.nereids.jobs.batch.PredicatePushDownRulesJob;
import org.apache.doris.nereids.memo.Group;
@@ -99,13 +99,9 @@ public class NereidsPlanner extends Planner {
// TODO: refactor, just demo code here
public PhysicalPlan plan(LogicalPlan plan, PhysicalProperties outputProperties, ConnectContext connectContext)
throws AnalysisException {
- Memo memo = new Memo();
- memo.initialize(plan);
-
- plannerContext = new PlannerContext(memo, connectContext);
- JobContext jobContext = new JobContext(plannerContext, outputProperties, Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
-
+ plannerContext = new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setJobContext(outputProperties);
// Get plan directly. Just for SSB.
return doPlan();
}
@@ -115,19 +111,34 @@ public class NereidsPlanner extends Planner {
* @return PhysicalPlan.
*/
private PhysicalPlan doPlan() {
- AnalyzeRulesJob analyzeRulesJob = new AnalyzeRulesJob(plannerContext);
- analyzeRulesJob.execute();
-
- PredicatePushDownRulesJob predicatePushDownRulesJob = new PredicatePushDownRulesJob(plannerContext);
- predicatePushDownRulesJob.execute();
+ analyze();
+ rewrite();
+ optimize();
+ return getRoot().extractPlan();
+ }
- DisassembleRulesJob disassembleRulesJob = new DisassembleRulesJob(plannerContext);
- disassembleRulesJob.execute();
+ /**
+ * Analyze: bind references according to metadata in the catalog, perform semantic analysis, etc.
+ */
+ private void analyze() {
+ new AnalyzeRulesJob(plannerContext).execute();
+ }
- OptimizeRulesJob optimizeRulesJob = new OptimizeRulesJob(plannerContext);
- optimizeRulesJob.execute();
+ /**
+ * Logical plan rewrite based on a series of heuristic rules.
+ */
+ private void rewrite() {
+ new JoinReorderRulesJob(plannerContext).execute();
+ new PredicatePushDownRulesJob(plannerContext).execute();
+ new DisassembleRulesJob(plannerContext).execute();
+ }
- return getRoot().extractPlan();
+ /**
+ * Cascades style optimize: perform equivalent logical plan exploration and physical implementation enumeration,
+ * try to find best plan under the guidance of statistic information and cost model.
+ */
+ private void optimize() {
+ new OptimizeRulesJob(plannerContext).execute();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java
index 150caa3df9..7d623b85d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/PlannerContext.java
@@ -19,14 +19,23 @@ package org.apache.doris.nereids;
import org.apache.doris.nereids.jobs.Job;
import org.apache.doris.nereids.jobs.JobContext;
+import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob;
+import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
import org.apache.doris.nereids.jobs.scheduler.JobPool;
import org.apache.doris.nereids.jobs.scheduler.JobScheduler;
import org.apache.doris.nereids.jobs.scheduler.JobStack;
import org.apache.doris.nereids.jobs.scheduler.SimpleJobScheduler;
import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.qe.ConnectContext;
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
/**
* Context used in memo.
*/
@@ -90,4 +99,44 @@ public class PlannerContext {
public void setCurrentJobContext(JobContext currentJobContext) {
this.currentJobContext = currentJobContext;
}
+
+ public PlannerContext setDefaultJobContext() {
+ this.currentJobContext = new JobContext(this, new PhysicalProperties(), Double.MAX_VALUE);
+ return this;
+ }
+
+ public PlannerContext setJobContext(PhysicalProperties physicalProperties) {
+ this.currentJobContext = new JobContext(this, physicalProperties, Double.MAX_VALUE);
+ return this;
+ }
+
+ public PlannerContext bottomUpRewrite(RuleFactory... rules) {
+ return execute(new RewriteBottomUpJob(memo.getRoot(), currentJobContext, ImmutableList.copyOf(rules)));
+ }
+
+ public PlannerContext bottomUpRewrite(Rule... rules) {
+ return bottomUpRewrite(ImmutableList.copyOf(rules));
+ }
+
+ public PlannerContext bottomUpRewrite(List<Rule> rules) {
+ return execute(new RewriteBottomUpJob(memo.getRoot(), rules, currentJobContext));
+ }
+
+ public PlannerContext topDownRewrite(RuleFactory... rules) {
+ return execute(new RewriteTopDownJob(memo.getRoot(), currentJobContext, ImmutableList.copyOf(rules)));
+ }
+
+ public PlannerContext topDownRewrite(Rule... rules) {
+ return topDownRewrite(ImmutableList.copyOf(rules));
+ }
+
+ public PlannerContext topDownRewrite(List<Rule> rules) {
+ return execute(new RewriteTopDownJob(memo.getRoot(), rules, currentJobContext));
+ }
+
+ private PlannerContext execute(Job job) {
+ pushJob(job);
+ jobScheduler.executeJobPool(this);
+ return this;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
index 6546d82826..2059cd696f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundStar.java
@@ -24,8 +24,6 @@ import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.util.Utils;
-import org.apache.commons.lang.StringUtils;
-
import java.util.List;
/**
@@ -41,12 +39,7 @@ public class UnboundStar extends NamedExpression implements LeafExpression, Unbo
@Override
public String toSql() {
- String qualified = qualifier.stream().map(Utils::quoteIfNeeded).reduce((t1, t2) -> t1 + "." + t2).orElse("");
- if (StringUtils.isNotEmpty(qualified)) {
- return qualified + ".*";
- } else {
- return "*";
- }
+ return Utils.qualifiedName(qualifier, "*");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java
index ab5f88d1ec..5510647431 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java
@@ -35,7 +35,7 @@ import java.util.Objects;
*
* Each batch of rules will be uniformly executed.
*/
-public class BatchRulesJob {
+public abstract class BatchRulesJob {
protected PlannerContext plannerContext;
protected List<Job> rulesJob = new ArrayList<>();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/JoinReorderRulesJob.java
similarity index 61%
copy from fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
copy to fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/JoinReorderRulesJob.java
index 89cbac83bd..b1a5ce9b0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/JoinReorderRulesJob.java
@@ -15,21 +15,22 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids.util;
+package org.apache.doris.nereids.jobs.batch;
+
+import org.apache.doris.nereids.PlannerContext;
+import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
+
+import com.google.common.collect.ImmutableList;
/**
- * Utils for Nereids.
+ * JoinReorderRulesJob
*/
-public class Utils {
- /**
- * Quoted string if it contains special character or all characters are digit.
- *
- * @param part string to be quoted
- * @return quoted string
- */
- public static String quoteIfNeeded(String part) {
- // We quote strings except the ones which consist of digits only.
- return part.matches("\\w*[\\w&&[^\\d]]+\\w*")
- ? part : part.replace("`", "``");
+public class JoinReorderRulesJob extends BatchRulesJob {
+
+ public JoinReorderRulesJob(PlannerContext plannerContext) {
+ super(plannerContext);
+ rulesJob.addAll(ImmutableList.of(
+ topDownBatch(ImmutableList.of(new ReorderJoin()))
+ ));
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
index 5c8d6d914d..aac059f839 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java
@@ -18,10 +18,12 @@
package org.apache.doris.nereids.memo;
import org.apache.doris.common.IdGenerator;
+import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -44,8 +46,8 @@ public class Memo {
private final Map<GroupExpression, GroupExpression> groupExpressions = Maps.newHashMap();
private Group root;
- public void initialize(Plan node) {
- root = copyIn(node, null, false).getParent();
+ public Memo(Plan plan) {
+ root = copyIn(plan, null, false).getParent();
}
public Group getRoot() {
@@ -96,6 +98,13 @@ public class Memo {
return groupToTreeNode(root);
}
+ /**
+ * Utility function to create a new {@link PlannerContext} with this Memo.
+ */
+ public PlannerContext newPlannerContext(ConnectContext connectContext) {
+ return new PlannerContext(this, connectContext);
+ }
+
private Plan groupToTreeNode(Group group) {
GroupExpression logicalExpression = group.getLogicalExpression();
List<Plan> childrenNode = Lists.newArrayList();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java
index 341f52f305..ec5ca0ff6a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/GroupExpressionMatching.java
@@ -20,6 +20,7 @@ package org.apache.doris.nereids.pattern;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import com.google.common.collect.ImmutableList;
@@ -66,19 +67,21 @@ public class GroupExpressionMatching implements Iterable<Plan> {
return;
}
- // (logicalFilter(), multi()) match (logicalFilter()),
- // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter())
- boolean extraMulti = pattern.arity() == groupExpression.arity() + 1
- && (pattern.hasMultiChild() || pattern.hasMultiGroupChild());
- if (pattern.arity() > groupExpression.arity() && !extraMulti) {
- return;
- }
+ if (!(pattern instanceof SubTreePattern)) {
+ // (logicalFilter(), multi()) match (logicalFilter()),
+ // but (logicalFilter(), logicalFilter(), multi()) not match (logicalFilter())
+ boolean extraMulti = pattern.arity() == groupExpression.arity() + 1
+ && (pattern.hasMultiChild() || pattern.hasMultiGroupChild());
+ if (pattern.arity() > groupExpression.arity() && !extraMulti) {
+ return;
+ }
- // (multi()) match (logicalFilter(), logicalFilter()),
- // but (logicalFilter()) not match (logicalFilter(), logicalFilter())
- if (!pattern.isAny() && pattern.arity() < groupExpression.arity()
- && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) {
- return;
+ // (multi()) match (logicalFilter(), logicalFilter()),
+ // but (logicalFilter()) not match (logicalFilter(), logicalFilter())
+ if (!pattern.isAny() && pattern.arity() < groupExpression.arity()
+ && !pattern.hasMultiChild() && !pattern.hasMultiGroupChild()) {
+ return;
+ }
}
// Pattern.GROUP / Pattern.MULTI / Pattern.MULTI_GROUP can not match GroupExpression
@@ -89,7 +92,7 @@ public class GroupExpressionMatching implements Iterable<Plan> {
// getPlan return the plan with GroupPlan as children
Plan root = groupExpression.getPlan();
// pattern.arity() == 0 equals to root.arity() == 0
- if (pattern.arity() == 0) {
+ if (pattern.arity() == 0 && !(pattern instanceof SubTreePattern)) {
if (pattern.matchPredicates(root)) {
// if no children pattern, we treat all children as GROUP. e.g. Pattern.ANY.
// leaf plan will enter this branch too, e.g. logicalRelation().
@@ -103,29 +106,38 @@ public class GroupExpressionMatching implements Iterable<Plan> {
for (int i = 0; i < groupExpression.arity(); ++i) {
Group childGroup = groupExpression.child(i);
List<Plan> childrenPlan = matchingChildGroup(pattern, childGroup, i);
- childrenPlans.add(childrenPlan);
+
if (childrenPlan.isEmpty()) {
- // current pattern is match but children patterns not match
- return;
+ if (pattern instanceof SubTreePattern) {
+ childrenPlan = ImmutableList.of(new GroupPlan(childGroup));
+ } else {
+ // current pattern is match but children patterns not match
+ return;
+ }
}
+ childrenPlans.add(childrenPlan);
}
-
assembleAllCombinationPlanTree(root, pattern, groupExpression, childrenPlans);
}
}
private List<Plan> matchingChildGroup(Pattern<? extends Plan> parentPattern,
- Group childGroup, int childIndex) {
- boolean isLastPattern = childIndex + 1 >= parentPattern.arity();
- int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex;
- Pattern<? extends Plan> childPattern = parentPattern.child(patternChildIndex);
-
- // translate MULTI and MULTI_GROUP to ANY and GROUP
- if (isLastPattern) {
- if (childPattern.isMulti()) {
- childPattern = Pattern.ANY;
- } else if (childPattern.isMultiGroup()) {
- childPattern = Pattern.GROUP;
+ Group childGroup, int childIndex) {
+ Pattern<? extends Plan> childPattern;
+ if (parentPattern instanceof SubTreePattern) {
+ childPattern = parentPattern;
+ } else {
+ boolean isLastPattern = childIndex + 1 >= parentPattern.arity();
+ int patternChildIndex = isLastPattern ? parentPattern.arity() - 1 : childIndex;
+
+ childPattern = parentPattern.child(patternChildIndex);
+ // translate MULTI and MULTI_GROUP to ANY and GROUP
+ if (isLastPattern) {
+ if (childPattern.isMulti()) {
+ childPattern = Pattern.ANY;
+ } else if (childPattern.isMultiGroup()) {
+ childPattern = Pattern.GROUP;
+ }
}
}
@@ -135,8 +147,8 @@ public class GroupExpressionMatching implements Iterable<Plan> {
}
private void assembleAllCombinationPlanTree(Plan root, Pattern<Plan> rootPattern,
- GroupExpression groupExpression,
- List<List<Plan>> childrenPlans) {
+ GroupExpression groupExpression,
+ List<List<Plan>> childrenPlans) {
int[] childrenPlanIndex = new int[childrenPlans.size()];
int offset = 0;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java
index fe31becf2f..77c3287751 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/Patterns.java
@@ -59,7 +59,11 @@ public interface Patterns {
return new PatternDescriptor<>(Pattern.MULTI_GROUP, defaultPromise());
}
- /* abstract plan patterns */
+ default <T extends Plan> PatternDescriptor<T> subTree(Class<? extends Plan>... subTreeNodeTypes) {
+ return new PatternDescriptor<>(new SubTreePattern(subTreeNodeTypes), defaultPromise());
+ }
+
+ /* abstract plan operator patterns */
/**
* create a leafPlan pattern.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/SubTreePattern.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/SubTreePattern.java
new file mode 100644
index 0000000000..02e1208ed6
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/SubTreePattern.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.pattern;
+
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.Set;
+
+/**
+ * Pattern to match a subtree.
+ * Match a subtree of plan, plan nodes in the matched result are the subset of specified plan types when
+ * declare the pattern.
+ */
+public class SubTreePattern<TYPE extends Plan> extends Pattern<TYPE> {
+ private final Set<Class<? extends Plan>> subTreeNodeTypes;
+
+ public SubTreePattern(Class<? extends Plan>... subTreeNodeTypes) {
+ super(PlanType.UNKNOWN, ImmutableList.of());
+ this.subTreeNodeTypes = ImmutableSet.copyOf(subTreeNodeTypes);
+ }
+
+ @Override
+ public boolean matchRoot(Plan plan) {
+ return plan != null && subTreeNodeTypes.contains(plan.getClass());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index d0e8c9e975..2119a1e265 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -47,6 +47,8 @@ public enum RuleType {
COLUMN_PRUNE_SORT_CHILD(RuleTypeClass.REWRITE),
COLUMN_PRUNE_JOIN_CHILD(RuleTypeClass.REWRITE),
+ REORDER_JOIN(RuleTypeClass.REWRITE),
+
REWRITE_SENTINEL(RuleTypeClass.REWRITE),
// exploration rules
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index d1096e33ec..a9b3bb4009 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -25,7 +25,7 @@ import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.qe.ConnectContext;
-import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
import java.util.List;
@@ -40,14 +40,17 @@ public class BindRelation extends OneAnalysisRuleFactory {
List<String> nameParts = ctx.root.getNameParts();
switch (nameParts.size()) {
case 1: {
- List<String> qualifier = Lists.newArrayList(connectContext.getDatabase(), nameParts.get(0));
- Table table = getTable(qualifier, connectContext.getCatalog());
+ // Use current database name from catalog.
+ String dbName = connectContext.getDatabase();
+ Table table = getTable(dbName, nameParts.get(0), connectContext.getCatalog());
// TODO: should generate different Scan sub class according to table's type
- return new LogicalOlapScan(table, qualifier);
+ return new LogicalOlapScan(table, ImmutableList.of(dbName));
}
case 2: {
- Table table = getTable(nameParts, connectContext.getCatalog());
- return new LogicalOlapScan(table, nameParts);
+ // Use database name from table name parts.
+ String dbName = connectContext.getClusterName() + ":" + nameParts.get(0);
+ Table table = getTable(dbName, nameParts.get(1), connectContext.getCatalog());
+ return new LogicalOlapScan(table, ImmutableList.of(dbName));
}
default:
throw new IllegalStateException("Table name [" + ctx.root.getTableName() + "] is invalid.");
@@ -55,13 +58,11 @@ public class BindRelation extends OneAnalysisRuleFactory {
}).toRule(RuleType.BINDING_RELATION);
}
- private Table getTable(List<String> qualifier, Catalog catalog) {
- String dbName = qualifier.get(0);
+ private Table getTable(String dbName, String tableName, Catalog catalog) {
Database db = catalog.getInternalDataSource().getDb(dbName)
.orElseThrow(() -> new RuntimeException("Database [" + dbName + "] does not exist."));
db.readLock();
try {
- String tableName = qualifier.get(1);
return db.getTable(tableName).orElseThrow(() -> new RuntimeException(
"Table [" + tableName + "] does not exist in database [" + dbName + "]."));
} finally {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
index 07527e5168..ce24b3000d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSlotReference.java
@@ -229,18 +229,22 @@ public class BindSlotReference implements AnalysisRuleFactory {
case 2:
// Unbound slot name is `table`.`column`
List<String> qualifier = boundSlot.getQualifier();
+ String name = boundSlot.getName();
switch (qualifier.size()) {
case 2:
// qualifier is `db`.`table`
return nameParts.get(0).equalsIgnoreCase(qualifier.get(1))
- && nameParts.get(1).equalsIgnoreCase(boundSlot.getName());
+ && nameParts.get(1).equalsIgnoreCase(name);
case 1:
// qualifier is `table`
return nameParts.get(0).equalsIgnoreCase(qualifier.get(0))
- && nameParts.get(1).equalsIgnoreCase(boundSlot.getName());
+ && nameParts.get(1).equalsIgnoreCase(name);
+ case 0:
+ // has no qualifiers
+ return nameParts.get(1).equalsIgnoreCase(name);
default:
throw new AnalysisException("Not supported qualifier: "
- + StringUtils.join(qualifier, "."));
+ + StringUtils.join(qualifier, "."));
}
default:
throw new AnalysisException("Not supported name: "
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
new file mode 100644
index 0000000000..0405c2975b
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/ReorderJoin.java
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.visitor.SlotExtractor;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Try to eliminate cross join via finding join conditions in filters and change the join orders.
+ * <p>
+ * <pre>
+ * For example:
+ *
+ * input:
+ * SELECT * FROM t1, t2, t3 WHERE t1.id=t3.id AND t2.id=t3.id
+ *
+ * output:
+ * SELECT * FROM t1 JOIN t3 ON t1.id=t3.id JOIN t2 ON t2.id=t3.id
+ * </pre>
+ * </p>
+ * TODO: This is tested by SSB queries currently, add more `unit` test for this rule
+ * when we have a plan building and comparing framework.
+ */
+public class ReorderJoin extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalFilter(subTree(LogicalJoin.class, LogicalFilter.class)).thenApply(ctx -> {
+ LogicalFilter<Plan> filter = ctx.root;
+ if (!ctx.plannerContext.getConnectContext().getSessionVariable()
+ .isEnableNereidsReorderToEliminateCrossJoin()) {
+ return filter;
+ }
+ PlanCollector collector = new PlanCollector();
+ filter.accept(collector, null);
+ List<Plan> joinInputs = collector.joinInputs;
+ List<Expression> conjuncts = collector.conjuncts;
+
+ if (joinInputs.size() >= 3 && !conjuncts.isEmpty()) {
+ return reorderJoinsAccordingToConditions(joinInputs, conjuncts);
+ } else {
+ return filter;
+ }
+ }).toRule(RuleType.REORDER_JOIN);
+ }
+
+ /**
+ * Reorder join orders according to join conditions to eliminate cross join.
+ * <p/>
+ * Let's say we have input join tables: [t1, t2, t3] and
+ * conjunctive predicates: [t1.id=t3.id, t2.id=t3.id]
+ * The input join for t1 and t2 is cross join.
+ * <p/>
+ * The algorithm split join inputs into two groups: `left input` t1 and `candidate right input` [t2, t3].
+ * Try to find an inner join from t1 and candidate right inputs [t2, t3], if any combination
+ * of [Join(t1, t2), Join(t1, t3)] could be optimized to inner join according to the join conditions.
+ * <p/>
+ * As a result, Join(t1, t3) is an inner join.
+ * Then the logic is applied to the rest of [Join(t1, t3), t2] recursively.
+ */
+ private Plan reorderJoinsAccordingToConditions(List<Plan> joinInputs, List<Expression> conjuncts) {
+ if (joinInputs.size() == 2) {
+ Set<Slot> joinOutput = getJoinOutput(joinInputs.get(0), joinInputs.get(1));
+ Map<Boolean, List<Expression>> split = splitConjuncts(conjuncts, joinOutput);
+ List<Expression> joinConditions = split.get(true);
+ List<Expression> nonJoinConditions = split.get(false);
+
+ Optional<Expression> cond;
+ if (joinConditions.isEmpty()) {
+ cond = Optional.empty();
+ } else {
+ cond = Optional.of(ExpressionUtils.and(joinConditions));
+ }
+
+ LogicalJoin join = new LogicalJoin(JoinType.INNER_JOIN, cond, joinInputs.get(0), joinInputs.get(1));
+ if (nonJoinConditions.isEmpty()) {
+ return join;
+ } else {
+ return new LogicalFilter(ExpressionUtils.and(nonJoinConditions), join);
+ }
+ } else {
+ Plan left = joinInputs.get(0);
+ List<Plan> candidate = joinInputs.subList(1, joinInputs.size());
+
+ List<Slot> leftOutput = left.getOutput();
+ Optional<Plan> rightOpt = candidate.stream().filter(right -> {
+ List<Slot> rightOutput = right.getOutput();
+
+ Set<Slot> joinOutput = getJoinOutput(left, right);
+ Optional<Expression> joinCond = conjuncts.stream()
+ .filter(expr -> {
+ Set<Slot> exprInputSlots = SlotExtractor.extractSlot(expr);
+ if (exprInputSlots.isEmpty()) {
+ return false;
+ }
+
+ if (new HashSet<>(leftOutput).containsAll(exprInputSlots)) {
+ return false;
+ }
+
+ if (new HashSet<>(rightOutput).containsAll(exprInputSlots)) {
+ return false;
+ }
+
+ return joinOutput.containsAll(exprInputSlots);
+ }).findFirst();
+ return joinCond.isPresent();
+ }).findFirst();
+
+ Plan right = rightOpt.orElseGet(() -> candidate.get(1));
+ Set<Slot> joinOutput = getJoinOutput(left, right);
+ Map<Boolean, List<Expression>> split = splitConjuncts(conjuncts, joinOutput);
+ List<Expression> joinConditions = split.get(true);
+ List<Expression> nonJoinConditions = split.get(false);
+
+ Optional<Expression> cond;
+ if (joinConditions.isEmpty()) {
+ cond = Optional.empty();
+ } else {
+ cond = Optional.of(ExpressionUtils.and(joinConditions));
+ }
+
+ LogicalJoin join = new LogicalJoin(JoinType.INNER_JOIN, cond, left, right);
+
+ List<Plan> newInputs = new ArrayList<>();
+ newInputs.add(join);
+ newInputs.addAll(candidate.stream().filter(plan -> !right.equals(plan)).collect(Collectors.toList()));
+ return reorderJoinsAccordingToConditions(newInputs, nonJoinConditions);
+ }
+ }
+
+ private Set<Slot> getJoinOutput(Plan left, Plan right) {
+ HashSet<Slot> joinOutput = new HashSet<>();
+ joinOutput.addAll(left.getOutput());
+ joinOutput.addAll(right.getOutput());
+ return joinOutput;
+ }
+
+ private Map<Boolean, List<Expression>> splitConjuncts(List<Expression> conjuncts, Set<Slot> slots) {
+ return conjuncts.stream().collect(Collectors.partitioningBy(
+ // TODO: support non equal to conditions.
+ expr -> expr instanceof EqualTo && slots.containsAll(SlotExtractor.extractSlot(expr))));
+ }
+
+ private class PlanCollector extends PlanVisitor<Void, Void> {
+ public final List<Plan> joinInputs = new ArrayList<>();
+ public final List<Expression> conjuncts = new ArrayList<>();
+
+ @Override
+ public Void visit(Plan plan, Void context) {
+ for (Plan child : plan.children()) {
+ child.accept(this, context);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<Plan> filter, Void context) {
+ Plan child = filter.child();
+ if (child instanceof LogicalJoin) {
+ conjuncts.addAll(ExpressionUtils.extractConjunct(filter.getPredicates()));
+ }
+
+ child.accept(this, context);
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalJoin(LogicalJoin<Plan, Plan> join, Void context) {
+ if (join.getJoinType() != JoinType.CROSS_JOIN && join.getJoinType() != JoinType.INNER_JOIN) {
+ return null;
+ }
+
+ join.left().accept(this, context);
+ join.right().accept(this, context);
+
+ join.getCondition().ifPresent(cond -> conjuncts.addAll(ExpressionUtils.extractConjunct(cond)));
+ if (!(join.left() instanceof LogicalJoin)) {
+ joinInputs.add(join.left());
+ }
+ if (!(join.right() instanceof LogicalJoin)) {
+ joinInputs.add(join.right());
+ }
+ return null;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
index 51b35f4c63..5f3b1a210c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/ComparisonPredicate.java
@@ -29,6 +29,9 @@ import java.util.Objects;
* Such as: "=", "<", "<=", ">", ">=", "<=>"
*/
public abstract class ComparisonPredicate extends Expression implements BinaryExpression {
+
+ protected final String symbol;
+
/**
* Constructor of ComparisonPredicate.
*
@@ -36,8 +39,9 @@ public abstract class ComparisonPredicate extends Expression implements BinaryEx
* @param left left child of comparison predicate
* @param right right child of comparison predicate
*/
- public ComparisonPredicate(ExpressionType nodeType, Expression left, Expression right) {
+ public ComparisonPredicate(ExpressionType nodeType, Expression left, Expression right, String symbol) {
super(nodeType, left, right);
+ this.symbol = symbol;
}
@Override
@@ -52,8 +56,7 @@ public abstract class ComparisonPredicate extends Expression implements BinaryEx
@Override
public String toSql() {
- String nodeType = getType().toString();
- return left().toSql() + ' ' + nodeType + ' ' + right().toSql();
+ return "(" + left().toSql() + ' ' + symbol + ' ' + right().toSql() + ")";
}
public <R, C> R accept(ExpressionVisitor<R, C> visitor, C context) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
index ca60ef8e38..3bfe654ff4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/CompoundPredicate.java
@@ -76,8 +76,7 @@ public class CompoundPredicate extends Expression implements BinaryExpression {
@Override
public String toString() {
- String nodeType = getType().toString();
- return nodeType + "(" + left() + ", " + right() + ")";
+ return "(" + left().toString() + " " + getType().toString() + " " + right().toString() + ")";
}
public ExpressionType flip() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
index ae98950e8d..be860cbe0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/EqualTo.java
@@ -30,7 +30,7 @@ import java.util.List;
public class EqualTo extends ComparisonPredicate {
public EqualTo(Expression left, Expression right) {
- super(ExpressionType.EQUAL_TO, left, right);
+ super(ExpressionType.EQUAL_TO, left, right, "=");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
index bad29aae63..2ad8ff1454 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThan.java
@@ -35,7 +35,7 @@ public class GreaterThan extends ComparisonPredicate {
* @param right right child of greater than
*/
public GreaterThan(Expression left, Expression right) {
- super(ExpressionType.GREATER_THAN, left, right);
+ super(ExpressionType.GREATER_THAN, left, right, ">");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
index 1b5ca42628..a4ce4885d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/GreaterThanEqual.java
@@ -35,7 +35,7 @@ public class GreaterThanEqual extends ComparisonPredicate {
* @param right right child of Greater Than And Equal
*/
public GreaterThanEqual(Expression left, Expression right) {
- super(ExpressionType.GREATER_THAN_EQUAL, left, right);
+ super(ExpressionType.GREATER_THAN_EQUAL, left, right, ">=");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
index af49c7dcd1..2223a8fb3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThan.java
@@ -35,7 +35,7 @@ public class LessThan extends ComparisonPredicate {
* @param right right child of Less Than
*/
public LessThan(Expression left, Expression right) {
- super(ExpressionType.LESS_THAN, left, right);
+ super(ExpressionType.LESS_THAN, left, right, "<");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
index 3d994c1332..865ffa26e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/LessThanEqual.java
@@ -35,7 +35,7 @@ public class LessThanEqual extends ComparisonPredicate {
* @param right right child of Less Than And Equal
*/
public LessThanEqual(Expression left, Expression right) {
- super(ExpressionType.LESS_THAN_EQUAL, left, right);
+ super(ExpressionType.LESS_THAN_EQUAL, left, right, "<=");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpression.java
index 4ab8221544..26c711ae0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpression.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NamedExpression.java
@@ -18,8 +18,7 @@
package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.nereids.exceptions.UnboundException;
-
-import org.apache.commons.collections.CollectionUtils;
+import org.apache.doris.nereids.util.Utils;
import java.util.List;
@@ -60,10 +59,6 @@ public abstract class NamedExpression extends Expression {
* @throws UnboundException throw this exception if this expression is unbound
*/
public String getQualifiedName() throws UnboundException {
- String qualifiedName = "";
- if (CollectionUtils.isNotEmpty(getQualifier())) {
- qualifiedName = String.join(".", getQualifier()) + ".";
- }
- return qualifiedName + getName();
+ return Utils.qualifiedName(getQualifier(), getName());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
index 8b43970674..a4648640fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NullSafeEqual.java
@@ -36,7 +36,7 @@ public class NullSafeEqual extends ComparisonPredicate {
* @param right right child of Null Safe Equal
*/
public NullSafeEqual(Expression left, Expression right) {
- super(ExpressionType.NULL_SAFE_EQUAL, left, right);
+ super(ExpressionType.NULL_SAFE_EQUAL, left, right, "<=>");
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
index d57c723c64..aaa782e3a8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SlotReference.java
@@ -20,9 +20,9 @@ package org.apache.doris.nereids.trees.expressions;
import org.apache.doris.catalog.Column;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.Utils;
import com.google.common.base.Preconditions;
-import org.apache.commons.lang.StringUtils;
import java.util.List;
import java.util.Objects;
@@ -96,12 +96,7 @@ public class SlotReference extends Slot {
@Override
public String toString() {
- String uniqueName = name + "#" + exprId;
- if (qualifier.isEmpty()) {
- return uniqueName;
- } else {
- return StringUtils.join(qualifier, ".") + "." + uniqueName;
- }
+ return Utils.qualifiedName(qualifier, name + "#" + exprId);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
index cf2f448dcf..029e0f0c0e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/Plan.java
@@ -35,7 +35,9 @@ public interface Plan extends TreeNode<Plan>, PlanStats {
PlanType getType();
- <R, C> R accept(PlanVisitor<R, C> visitor, C context);
+ default <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ throw new RuntimeException("accept() is not implemented by plan " + this.getClass().getSimpleName());
+ }
List<Expression> getExpressions();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 1912c09c06..0d3738b025 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -24,8 +24,6 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
-import org.apache.commons.lang3.StringUtils;
-
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -43,7 +41,7 @@ public class LogicalOlapScan extends LogicalRelation {
* Constructor for LogicalOlapScan.
*
* @param table Doris table
- * @param qualifier qualified relation name
+ * @param qualifier table name qualifier
*/
public LogicalOlapScan(Table table, List<String> qualifier,
Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties) {
@@ -52,7 +50,7 @@ public class LogicalOlapScan extends LogicalRelation {
@Override
public String toString() {
- return "ScanOlapTable([" + StringUtils.join(qualifier, ".") + "." + table.getName() + "])";
+ return "ScanOlapTable (" + qualifiedName() + ")";
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java
index 499bd5e604..31020755be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRelation.java
@@ -25,9 +25,9 @@ import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
-import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
@@ -66,11 +66,6 @@ public abstract class LogicalRelation extends LogicalLeaf {
return qualifier;
}
- @Override
- public String toString() {
- return "LogicalRelation (" + StringUtils.join(qualifier, ".") + ")";
- }
-
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -92,7 +87,7 @@ public abstract class LogicalRelation extends LogicalLeaf {
public List<Slot> computeOutput() {
return table.getBaseSchema()
.stream()
- .map(col -> SlotReference.fromColumn(col, qualifier))
+ .map(col -> SlotReference.fromColumn(col, qualified()))
.collect(ImmutableList.toImmutableList());
}
@@ -105,4 +100,18 @@ public abstract class LogicalRelation extends LogicalLeaf {
public List<Expression> getExpressions() {
return ImmutableList.of();
}
+
+ /**
+ * Full qualified name parts, i.e., concat qualifier and name into a list.
+ */
+ public List<String> qualified() {
+ return Utils.qualifiedNameParts(qualifier, table.getName());
+ }
+
+ /**
+ * Full qualified table name, concat qualifier and name with `.` as separator.
+ */
+ public String qualifiedName() {
+ return Utils.qualifiedName(qualifier, table.getName());
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java
index a3cc5f2034..fc58a9bae9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalSort.java
@@ -81,7 +81,7 @@ public class LogicalSort<CHILD_TYPE extends Plan> extends LogicalUnary<CHILD_TYP
@Override
public String toString() {
- return "Sort (" + StringUtils.join(orderKeys, ", ") + ")";
+ return "LogicalSort (" + StringUtils.join(orderKeys, ", ") + ")";
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAggregate.java
index 4d72f13bfb..b041f79b5b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAggregate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalAggregate.java
@@ -110,7 +110,7 @@ public class PhysicalAggregate<CHILD_TYPE extends Plan> extends PhysicalUnary<CH
@Override
public String toString() {
- return "PhysicalAggregate([key=" + groupByExprList
+ return "PhysicalAggregate ([key=" + groupByExprList
+ "], [output=" + outputExpressionList + "])";
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
index 46b7150a15..0a01c689cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFilter.java
@@ -54,7 +54,7 @@ public class PhysicalFilter<CHILD_TYPE extends Plan> extends PhysicalUnary<CHILD
@Override
public String toString() {
- return "Filter (" + predicates + ")";
+ return "PhysicalFilter (" + predicates + ")";
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHeapSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHeapSort.java
index 035ac93933..78105c7466 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHeapSort.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHeapSort.java
@@ -27,6 +27,7 @@ import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
@@ -116,4 +117,11 @@ public class PhysicalHeapSort<CHILD_TYPE extends Plan> extends PhysicalUnary<CHI
public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
return new PhysicalHeapSort<>(orderKeys, limit, offset, Optional.empty(), logicalProperties.get(), child());
}
+
+ @Override
+ public String toString() {
+ return "PhysicalHeapSort ("
+ + StringUtils.join(orderKeys, ", ") + ", LIMIT " + limit + ", OFFSET " + offset
+ + ")";
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
index 03596e7d37..c2206961a1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
@@ -24,9 +24,9 @@ import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringUtils;
import java.util.List;
import java.util.Objects;
@@ -46,7 +46,7 @@ public class PhysicalOlapScan extends PhysicalRelation {
* Constructor for PhysicalOlapScan.
*
* @param olapTable OlapTable in Doris
- * @param qualifier table's name
+ * @param qualifier qualifier of table name
*/
public PhysicalOlapScan(OlapTable olapTable, List<String> qualifier,
Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties) {
@@ -78,8 +78,9 @@ public class PhysicalOlapScan extends PhysicalRelation {
@Override
public String toString() {
- return "PhysicalOlapScan([" + StringUtils.join(qualifier, ".") + "." + olapTable.getName()
- + "], [index id=" + selectedIndexId + "])";
+ return "PhysicalOlapScan (["
+ + Utils.qualifiedName(qualifier, olapTable.getName())
+ + "], [index id=" + selectedIndexId + "] )";
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index c257a80cdf..7c002371c3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -36,10 +36,6 @@ import java.util.Optional;
*/
public class ExpressionUtils {
- public static boolean isConstant(Expression expr) {
- return expr.isConstant();
- }
-
public static List<Expression> extractConjunct(Expression expr) {
return extract(ExpressionType.AND, expr);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
index 89cbac83bd..b1ef6e56ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
@@ -17,6 +17,11 @@
package org.apache.doris.nereids.util;
+import com.google.common.collect.ImmutableList;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.List;
+
/**
* Utils for Nereids.
*/
@@ -32,4 +37,18 @@ public class Utils {
return part.matches("\\w*[\\w&&[^\\d]]+\\w*")
? part : part.replace("`", "``");
}
+
+ /**
+ * Fully qualified identifier name parts, i.e., concat qualifier and name into a list.
+ */
+ public static List<String> qualifiedNameParts(List<String> qualifier, String name) {
+ return new ImmutableList.Builder<String>().addAll(qualifier).add(name).build();
+ }
+
+ /**
+ * Fully qualified identifier name, concat qualifier and name with `.` as separator.
+ */
+ public static String qualifiedName(List<String> qualifier, String name) {
+ return StringUtils.join(qualifiedNameParts(qualifier, name), ".");
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 82b7f55fda..b80df1ac4d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -198,7 +198,7 @@ public class ConnectProcessor {
boolean alreadyAddedToAuditInfoList = false;
try {
List<StatementBase> stmts = null;
- if (ctx.getSessionVariable().isEnableNereids()) {
+ if (ctx.getSessionVariable().isEnableNereidsPlanner()) {
NereidsParser nereidsParser = new NereidsParser();
try {
stmts = nereidsParser.parseSQL(originStmt);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index ae967f8b6b..5301d3ee2f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -190,7 +190,10 @@ public class SessionVariable implements Serializable, Writable {
static final String ENABLE_ARRAY_TYPE = "enable_array_type";
- public static final String ENABLE_NEREIDS = "enable_nereids";
+ public static final String ENABLE_NEREIDS_PLANNER = "enable_nereids_planner";
+
+ public static final String ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN =
+ "enable_nereids_reorder_to_eliminate_cross_join";
public static final String ENABLE_REMOVE_NO_CONJUNCTS_RUNTIME_FILTER =
"enable_remove_no_conjuncts_runtime_filter_policy";
@@ -480,8 +483,11 @@ public class SessionVariable implements Serializable, Writable {
* the new optimizer is fully developed. I hope that day
* would be coming soon.
*/
- @VariableMgr.VarAttr(name = ENABLE_NEREIDS)
- private boolean enableNereids = false;
+ @VariableMgr.VarAttr(name = ENABLE_NEREIDS_PLANNER)
+ private boolean enableNereidsPlanner = false;
+
+ @VariableMgr.VarAttr(name = ENABLE_NEREIDS_REORDER_TO_ELIMINATE_CROSS_JOIN)
+ private boolean enableNereidsReorderToEliminateCrossJoin = true;
@VariableMgr.VarAttr(name = ENABLE_REMOVE_NO_CONJUNCTS_RUNTIME_FILTER)
public boolean enableRemoveNoConjunctsRuntimeFilterPolicy = false;
@@ -990,12 +996,20 @@ public class SessionVariable implements Serializable, Writable {
*
* @return true if both nereids and vectorized engine are enabled
*/
- public boolean isEnableNereids() {
- return enableNereids && enableVectorizedEngine;
+ public boolean isEnableNereidsPlanner() {
+ return enableNereidsPlanner && enableVectorizedEngine;
+ }
+
+ public void setEnableNereidsPlanner(boolean enableNereidsPlanner) {
+ this.enableNereidsPlanner = enableNereidsPlanner;
+ }
+
+ public boolean isEnableNereidsReorderToEliminateCrossJoin() {
+ return enableNereidsReorderToEliminateCrossJoin;
}
- public void setEnableNereids(boolean enableNereids) {
- this.enableNereids = enableNereids;
+ public void setEnableNereidsReorderToEliminateCrossJoin(boolean value) {
+ enableNereidsReorderToEliminateCrossJoin = value;
}
/**
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java
index 87dd04c1f3..e28a508084 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/RewriteTopDownJobTest.java
@@ -23,12 +23,10 @@ import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.nereids.PlannerContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
-import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.properties.LogicalProperties;
-import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
@@ -72,19 +70,14 @@ public class RewriteTopDownJobTest {
new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))),
leaf
);
- Memo memo = new Memo();
- memo.initialize(project);
+ PlannerContext plannerContext = new Memo(project)
+ .newPlannerContext(new ConnectContext())
+ .setDefaultJobContext();
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
List<Rule> fakeRules = Lists.newArrayList(new FakeRule().build());
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules,
- plannerContext.getCurrentJobContext());
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
+ plannerContext.topDownRewrite(fakeRules);
- Group rootGroup = memo.getRoot();
+ Group rootGroup = plannerContext.getMemo().getRoot();
Assertions.assertEquals(1, rootGroup.getLogicalExpressions().size());
GroupExpression rootGroupExpression = rootGroup.getLogicalExpression();
List<Slot> output = rootGroup.getLogicalProperties().getOutput();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
index 1e1486dab7..d84a864fdb 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/memo/MemoTest.java
@@ -30,7 +30,7 @@ import org.junit.Test;
public class MemoTest {
@Test
- public void testInitialize() {
+ public void testCopyIn() {
UnboundRelation unboundRelation = new UnboundRelation(Lists.newArrayList("test"));
LogicalProject insideProject = new LogicalProject(
ImmutableList.of(new SlotReference("name", StringType.INSTANCE, true, ImmutableList.of("test"))),
@@ -42,8 +42,7 @@ public class MemoTest {
);
// Project -> Project -> Relation
- Memo memo = new Memo();
- memo.initialize(rootProject);
+ Memo memo = new Memo(rootProject);
Group rootGroup = memo.getRoot();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java
index 3437b78c96..7c2a2b8b34 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/pattern/GroupExpressionMatchingTest.java
@@ -18,13 +18,18 @@
package org.apache.doris.nereids.pattern;
import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.analyzer.UnboundSlot;
import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.rules.RulePromise;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -39,8 +44,7 @@ public class GroupExpressionMatchingTest {
public void testLeafNode() {
Pattern pattern = new Pattern<>(PlanType.LOGICAL_UNBOUND_RELATION);
- Memo memo = new Memo();
- memo.initialize(new UnboundRelation(Lists.newArrayList("test")));
+ Memo memo = new Memo(new UnboundRelation(Lists.newArrayList("test")));
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(pattern, memo.getRoot().getLogicalExpression());
@@ -59,8 +63,7 @@ public class GroupExpressionMatchingTest {
Plan leaf = new UnboundRelation(Lists.newArrayList("test"));
LogicalProject root = new LogicalProject(Lists.newArrayList(), leaf);
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
Plan anotherLeaf = new UnboundRelation(Lists.newArrayList("test2"));
memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false);
@@ -89,8 +92,7 @@ public class GroupExpressionMatchingTest {
Plan leaf = new UnboundRelation(Lists.newArrayList("test"));
LogicalProject root = new LogicalProject(Lists.newArrayList(), leaf);
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
Plan anotherLeaf = new UnboundRelation(Lists.newArrayList("test2"));
memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false);
@@ -112,8 +114,7 @@ public class GroupExpressionMatchingTest {
public void testLeafAny() {
Pattern pattern = Pattern.ANY;
- Memo memo = new Memo();
- memo.initialize(new UnboundRelation(Lists.newArrayList("test")));
+ Memo memo = new Memo(new UnboundRelation(Lists.newArrayList("test")));
GroupExpressionMatching groupExpressionMatching
= new GroupExpressionMatching(pattern, memo.getRoot().getLogicalExpression());
@@ -129,8 +130,7 @@ public class GroupExpressionMatchingTest {
public void testAnyWithChild() {
Plan root = new LogicalProject(Lists.newArrayList(),
new UnboundRelation(Lists.newArrayList("test")));
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
Plan anotherLeaf = new UnboundRelation(ImmutableList.of("test2"));
memo.copyIn(anotherLeaf, memo.getRoot().getLogicalExpression().child(0), false);
@@ -155,11 +155,11 @@ public class GroupExpressionMatchingTest {
new UnboundRelation(ImmutableList.of("b"))
);
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
GroupExpressionMatching groupExpressionMatching
- = new GroupExpressionMatching(patterns().innerLogicalJoin().pattern, memo.getRoot().getLogicalExpression());
+ = new GroupExpressionMatching(patterns().innerLogicalJoin().pattern,
+ memo.getRoot().getLogicalExpression());
Iterator<Plan> iterator = groupExpressionMatching.iterator();
Assertions.assertTrue(iterator.hasNext());
@@ -177,11 +177,11 @@ public class GroupExpressionMatchingTest {
new UnboundRelation(ImmutableList.of("b"))
);
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
GroupExpressionMatching groupExpressionMatching
- = new GroupExpressionMatching(patterns().innerLogicalJoin().pattern, memo.getRoot().getLogicalExpression());
+ = new GroupExpressionMatching(patterns().innerLogicalJoin().pattern,
+ memo.getRoot().getLogicalExpression());
Iterator<Plan> iterator = groupExpressionMatching.iterator();
Assertions.assertFalse(iterator.hasNext());
@@ -194,9 +194,7 @@ public class GroupExpressionMatchingTest {
new UnboundRelation(ImmutableList.of("b"))
);
-
- Memo memo = new Memo();
- memo.initialize(root);
+ Memo memo = new Memo(root);
Pattern pattern = patterns()
.innerLogicalJoin(patterns().logicalFilter(), patterns().any()).pattern;
@@ -207,7 +205,126 @@ public class GroupExpressionMatchingTest {
Assertions.assertFalse(iterator.hasNext());
}
+ @Test
+ public void testSubTreeMatch() {
+ Plan root =
+ new LogicalFilter(new EqualTo(new UnboundSlot(Lists.newArrayList("a", "id")),
+ new UnboundSlot(Lists.newArrayList("b", "id"))),
+ new LogicalJoin(JoinType.INNER_JOIN,
+ new LogicalJoin(JoinType.LEFT_OUTER_JOIN,
+ new UnboundRelation(ImmutableList.of("a")),
+ new UnboundRelation(ImmutableList.of("b"))),
+ new UnboundRelation(ImmutableList.of("c")))
+ );
+ Pattern p1 = patterns().logicalFilter(patterns().subTree(LogicalFilter.class, LogicalJoin.class)).pattern;
+ Iterator<Plan> matchResult1 = match(root, p1);
+ assertSubTreeMatch(matchResult1);
+
+ Pattern p2 = patterns().subTree(LogicalFilter.class, LogicalJoin.class).pattern;
+ Iterator<Plan> matchResult2 = match(root, p2);
+ assertSubTreeMatch(matchResult2);
+
+ Pattern p3 = patterns().subTree(LogicalProject.class).pattern;
+ Iterator<Plan> matchResult3 = match(root, p3);
+ Assertions.assertFalse(matchResult3.hasNext());
+ }
+
+ private void assertSubTreeMatch(Iterator<Plan> matchResult) {
+ Assertions.assertTrue(matchResult.hasNext());
+ Plan plan = matchResult.next();
+ System.out.println(plan.treeString());
+ new SubTreeMatchChecker().check(plan);
+ }
+
+ // TODO: add an effective approach to compare actual and expected plan tree. Maybe a DSL to generate expected plan
+ // and leverage a comparing framework to check result. We could reuse pattern match to check shape and properties.
+ private class SubTreeMatchChecker extends PlanVisitor<Void, Context> {
+ public void check(Plan plan) {
+ plan.accept(this, new Context(null));
+ }
+
+ @Override
+ public Void visit(Plan plan, Context context) {
+ notExpectedPlan(plan, context);
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<Plan> filter, Context context) {
+ Assertions.assertTrue(context.parent == null);
+ filter.child().accept(this, new Context(filter));
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalJoin(LogicalJoin<Plan, Plan> join, Context context) {
+ switch (join.getJoinType()) {
+ case INNER_JOIN:
+ Assertions.assertTrue(context.parent instanceof LogicalFilter);
+ break;
+ case LEFT_OUTER_JOIN:
+ Assertions.assertTrue(context.parent instanceof LogicalJoin);
+ LogicalJoin parent = (LogicalJoin) context.parent;
+ Assertions.assertEquals(parent.getJoinType(), JoinType.INNER_JOIN);
+ break;
+ default:
+ notExpectedPlan(join, context);
+ }
+
+ join.left().accept(this, new Context(join));
+ join.right().accept(this, new Context(join));
+ return null;
+ }
+
+ @Override
+ public Void visitGroupPlan(GroupPlan groupPlan, Context context) {
+ Plan plan = groupPlan.getGroup().logicalExpressionsAt(0).getPlan();
+ Assertions.assertTrue(plan instanceof UnboundRelation);
+ UnboundRelation relation = (UnboundRelation) plan;
+ String relationName = relation.getNameParts().get(0);
+ switch (relationName) {
+ case "a":
+ case "b": {
+ Assertions.assertTrue(context.parent instanceof LogicalJoin);
+ LogicalJoin parent = (LogicalJoin) context.parent;
+ Assertions.assertEquals(parent.getJoinType(), JoinType.LEFT_OUTER_JOIN);
+ break;
+ }
+ case "c": {
+ Assertions.assertTrue(context.parent instanceof LogicalJoin);
+ LogicalJoin parent = (LogicalJoin) context.parent;
+ Assertions.assertEquals(parent.getJoinType(), JoinType.INNER_JOIN);
+ break;
+ }
+ default:
+ notExpectedPlan(groupPlan, context);
+ }
+ return null;
+ }
+
+ private void notExpectedPlan(Plan plan, Context context) {
+ throw new RuntimeException("Not expected plan node in match result:\n"
+ + "PlanNode:\n" + plan.toString()
+ + "\nparent:\n" + context.parent);
+ }
+ }
+
+ private class Context {
+ final Plan parent;
+
+ public Context(Plan parent) {
+ this.parent = parent;
+ }
+ }
+
private org.apache.doris.nereids.pattern.GeneratedPatterns patterns() {
return () -> RulePromise.REWRITE;
}
+
+ private Iterator<Plan> match(Plan root, Pattern pattern) {
+ Memo memo = new Memo(root);
+ GroupExpressionMatching groupExpressionMatching
+ = new GroupExpressionMatching(pattern, memo.getRoot().getLogicalExpression());
+ return groupExpressionMatching.iterator();
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/plan/TestPlanOutput.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/plan/TestPlanOutput.java
index cdf325413f..aac8bca61b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/plan/TestPlanOutput.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/plan/TestPlanOutput.java
@@ -48,15 +48,15 @@ public class TestPlanOutput {
new Column("id", Type.INT, true, AggregateType.NONE, "0", ""),
new Column("name", Type.STRING, true, AggregateType.NONE, "", "")
));
- LogicalRelation relationPlan = new LogicalOlapScan(table, ImmutableList.of("a"));
+ LogicalRelation relationPlan = new LogicalOlapScan(table, ImmutableList.of("db"));
List<Slot> output = relationPlan.getOutput();
Assertions.assertEquals(2, output.size());
Assertions.assertEquals(output.get(0).getName(), "id");
- Assertions.assertEquals(output.get(0).getQualifiedName(), "a.id");
+ Assertions.assertEquals(output.get(0).getQualifiedName(), "db.a.id");
Assertions.assertEquals(output.get(0).getDataType(), IntegerType.INSTANCE);
Assertions.assertEquals(output.get(1).getName(), "name");
- Assertions.assertEquals(output.get(1).getQualifiedName(), "a.name");
+ Assertions.assertEquals(output.get(1).getQualifiedName(), "db.a.name");
Assertions.assertEquals(output.get(1).getDataType(), StringType.INSTANCE);
}
@@ -80,7 +80,7 @@ public class TestPlanOutput {
new Column("id", Type.INT, true, AggregateType.NONE, "0", ""),
new Column("name", Type.STRING, true, AggregateType.NONE, "", "")
));
- LogicalRelation relationPlan = new LogicalOlapScan(table, ImmutableList.of("a"));
+ LogicalRelation relationPlan = new LogicalOlapScan(table, ImmutableList.of("db"));
List<Slot> output = relationPlan.getOutput();
// column prune
@@ -88,7 +88,7 @@ public class TestPlanOutput {
output = newPlan.getOutput();
Assertions.assertEquals(1, output.size());
Assertions.assertEquals(output.get(0).getName(), "id");
- Assertions.assertEquals(output.get(0).getQualifiedName(), "a.id");
+ Assertions.assertEquals(output.get(0).getQualifiedName(), "db.a.id");
Assertions.assertEquals(output.get(0).getDataType(), IntegerType.INSTANCE);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
new file mode 100644
index 0000000000..c3c817f7ba
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.analysis;
+
+import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.util.PlanRewriter;
+import org.apache.doris.utframe.TestWithFeService;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+class BindRelationTest extends TestWithFeService {
+ private static final String DB1 = "db1";
+ private static final String DB2 = "db2";
+
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase(DB1);
+ createTable("CREATE TABLE db1.t ( \n"
+ + " \ta INT,\n"
+ + " \tb VARCHAR\n"
+ + ")ENGINE=OLAP\n"
+ + "DISTRIBUTED BY HASH(`a`) BUCKETS 3\n"
+ + "PROPERTIES (\"replication_num\"= \"1\");");
+ }
+
+ @Test
+ void bindInCurrentDb() {
+ connectContext.setDatabase(DEFAULT_CLUSTER_PREFIX + DB1);
+ Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(ImmutableList.of("t")),
+ connectContext, new BindRelation());
+
+ Assertions.assertTrue(plan instanceof LogicalOlapScan);
+ Assertions.assertEquals(
+ ImmutableList.of(DEFAULT_CLUSTER_PREFIX + DB1, "t"),
+ ((LogicalOlapScan) plan).qualified());
+ }
+
+ @Test
+ void bindByDbQualifier() {
+ connectContext.setDatabase(DEFAULT_CLUSTER_PREFIX + DB2);
+ Plan plan = PlanRewriter.bottomUpRewrite(new UnboundRelation(ImmutableList.of("db1", "t")),
+ connectContext, new BindRelation());
+
+ Assertions.assertTrue(plan instanceof LogicalOlapScan);
+ Assertions.assertEquals(
+ ImmutableList.of(DEFAULT_CLUSTER_PREFIX + DB1, "t"),
+ ((LogicalOlapScan) plan).qualified());
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java
index 2e88b74ccd..2b624ca73b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/implementation/LogicalProjectToPhysicalProjectTest.java
@@ -18,16 +18,12 @@
package org.apache.doris.nereids.rules.implementation;
import org.apache.doris.nereids.PlannerContext;
-import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.memo.Group;
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
-import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.Lists;
import mockit.Mocked;
@@ -38,17 +34,11 @@ import java.util.List;
public class LogicalProjectToPhysicalProjectTest {
@Test
- public void projectionImplTest(@Mocked Group group) {
+ public void projectionImplTest(@Mocked Group group, @Mocked PlannerContext plannerContext) {
Plan plan = new LogicalProject(Lists.newArrayList(), new GroupPlan(group));
-
Rule rule = new LogicalProjectToPhysicalProject().build();
-
- PlannerContext plannerContext = new PlannerContext(new Memo(), new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
List<Plan> transform = rule.transform(plan, plannerContext);
Assert.assertEquals(1, transform.size());
-
Plan implPlan = transform.get(0);
Assert.assertEquals(PlanType.PHYSICAL_PROJECT, implPlan.getType());
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateDisassembleTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateDisassembleTest.java
index a43374adb9..a618e5743a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateDisassembleTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AggregateDisassembleTest.java
@@ -21,11 +21,6 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.PlannerContext;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.rewrite.AggregateDisassemble;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -39,6 +34,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalUnary;
+import org.apache.doris.nereids.util.PlanRewriter;
import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
@@ -81,17 +77,7 @@ public class AggregateDisassembleTest {
new Alias(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
Plan root = new LogicalAggregate(groupExpressionList, outputExpressionList, rStudent);
- Memo memo = new Memo();
- memo.initialize(root);
-
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new AggregateDisassemble().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
-
- Plan after = memo.copyOut();
+ Plan after = rewrite(root);
Assertions.assertTrue(after instanceof LogicalUnary);
Assertions.assertTrue(after instanceof LogicalAggregate);
@@ -150,17 +136,7 @@ public class AggregateDisassembleTest {
new Alias(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
Plan root = new LogicalAggregate<>(groupExpressionList, outputExpressionList, rStudent);
- Memo memo = new Memo();
- memo.initialize(root);
-
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new AggregateDisassemble().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
-
- Plan after = memo.copyOut();
+ Plan after = rewrite(root);
Assertions.assertTrue(after instanceof LogicalUnary);
Assertions.assertTrue(after instanceof LogicalAggregate);
@@ -217,17 +193,7 @@ public class AggregateDisassembleTest {
new Alias(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
Plan root = new LogicalAggregate(groupExpressionList, outputExpressionList, rStudent);
- Memo memo = new Memo();
- memo.initialize(root);
-
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new AggregateDisassemble().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
-
- Plan after = memo.copyOut();
+ Plan after = rewrite(root);
Assertions.assertTrue(after instanceof LogicalUnary);
Assertions.assertTrue(after instanceof LogicalAggregate);
@@ -273,17 +239,7 @@ public class AggregateDisassembleTest {
new Alias(new Sum(rStudent.getOutput().get(0).toSlot()), "sum"));
Plan root = new LogicalAggregate(groupExpressionList, outputExpressionList, rStudent);
- Memo memo = new Memo();
- memo.initialize(root);
-
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new AggregateDisassemble().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
-
- Plan after = memo.copyOut();
+ Plan after = rewrite(root);
Assertions.assertTrue(after instanceof LogicalUnary);
Assertions.assertTrue(after instanceof LogicalAggregate);
@@ -318,4 +274,8 @@ public class AggregateDisassembleTest {
Assertions.assertEquals(outputExpressionList.get(0).getExprId(),
global.getOutputExpressionList().get(0).getExprId());
}
+
+ private Plan rewrite(Plan input) {
+ return PlanRewriter.topDownRewrite(input, new ConnectContext(), new AggregateDisassemble());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AnalyzeUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AnalyzeUtils.java
deleted file mode 100644
index 217be2aa66..0000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/AnalyzeUtils.java
+++ /dev/null
@@ -1,62 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite.logical;
-
-import org.apache.doris.nereids.PlannerContext;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob;
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.parser.NereidsParser;
-import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.analysis.BindRelation;
-import org.apache.doris.nereids.rules.analysis.BindSlotReference;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.qe.ConnectContext;
-
-/**
- * sql parse util.
- */
-public class AnalyzeUtils {
-
- private static final NereidsParser parser = new NereidsParser();
-
- /**
- * analyze sql.
- */
- public static LogicalPlan analyze(String sql, ConnectContext connectContext) {
- try {
- LogicalPlan parsed = parser.parseSingle(sql);
- return analyze(parsed, connectContext);
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- private static LogicalPlan analyze(LogicalPlan inputPlan, ConnectContext connectContext) {
- Memo memo = new Memo();
- memo.initialize(inputPlan);
- PlannerContext plannerContext = new PlannerContext(memo, connectContext);
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- plannerContext.pushJob(
- new RewriteBottomUpJob(memo.getRoot(), new BindSlotReference().buildRules(), jobContext));
- plannerContext.pushJob(
- new RewriteBottomUpJob(memo.getRoot(), new BindRelation().buildRules(), jobContext));
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
- return (LogicalPlan) memo.copyOut();
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
index 9b2c2ff397..ccaec751c3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
@@ -17,15 +17,11 @@
package org.apache.doris.nereids.rules.rewrite.logical;
-import org.apache.doris.nereids.PlannerContext;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+import org.apache.doris.nereids.util.PlanRewriter;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.TestWithFeService;
@@ -43,7 +39,6 @@ public class ColumnPruningTest extends TestWithFeService {
@Override
protected void runBeforeAll() throws Exception {
-
createDatabase("test");
createTable("create table test.student (\n" + "id int not null,\n" + "name varchar(128),\n"
@@ -59,19 +54,14 @@ public class ColumnPruningTest extends TestWithFeService {
connectContext.setDatabase("default_cluster:test");
-
}
@Test
public void testPruneColumns1() {
String sql
= "select id,name,grade from student left join score on student.id = score.sid where score.grade > 60";
- Plan plan = AnalyzeUtils.analyze(sql, connectContext);
-
- Memo memo = new Memo();
- memo.initialize(plan);
-
- Plan out = process(memo);
+ Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan out = rewrite(plan);
System.out.println(out.treeString());
Plan l1 = out.child(0).child(0);
@@ -85,16 +75,16 @@ public class ColumnPruningTest extends TestWithFeService {
List<String> target;
List<String> source;
- source = getStringList(p1);
+ source = getOutputQualifiedNames(p1);
target = Lists.newArrayList("default_cluster:test.student.name", "default_cluster:test.student.id",
"default_cluster:test.score.grade");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p20);
+ source = getOutputQualifiedNames(p20);
target = Lists.newArrayList("default_cluster:test.student.id", "default_cluster:test.student.name");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p21);
+ source = getOutputQualifiedNames(p21);
target = Lists.newArrayList("default_cluster:test.score.sid", "default_cluster:test.score.grade");
Assertions.assertTrue(source.containsAll(target));
@@ -102,16 +92,11 @@ public class ColumnPruningTest extends TestWithFeService {
@Test
public void testPruneColumns2() {
-
String sql
= "select name,sex,cid,grade from student left join score on student.id = score.sid "
+ "where score.grade > 60";
- Plan plan = AnalyzeUtils.analyze(sql, connectContext);
-
- Memo memo = new Memo();
- memo.initialize(plan);
-
- Plan out = process(memo);
+ Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
Plan l20 = l1.child(0).child(0);
@@ -124,12 +109,12 @@ public class ColumnPruningTest extends TestWithFeService {
List<String> target;
List<String> source;
- source = getStringList(p1);
+ source = getOutputQualifiedNames(p1);
target = Lists.newArrayList("default_cluster:test.student.name", "default_cluster:test.score.cid",
"default_cluster:test.score.grade", "default_cluster:test.student.sex");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p20);
+ source = getOutputQualifiedNames(p20);
target = Lists.newArrayList("default_cluster:test.student.id", "default_cluster:test.student.name",
"default_cluster:test.student.sex");
Assertions.assertTrue(source.containsAll(target));
@@ -138,14 +123,9 @@ public class ColumnPruningTest extends TestWithFeService {
@Test
public void testPruneColumns3() {
-
String sql = "select id,name from student where age > 18";
- Plan plan = AnalyzeUtils.analyze(sql, connectContext);
-
- Memo memo = new Memo();
- memo.initialize(plan);
-
- Plan out = process(memo);
+ Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
LogicalProject p1 = (LogicalProject) l1;
@@ -153,7 +133,7 @@ public class ColumnPruningTest extends TestWithFeService {
List<String> target;
List<String> source;
- source = getStringList(p1);
+ source = getOutputQualifiedNames(p1);
target = Lists.newArrayList("default_cluster:test.student.name", "default_cluster:test.student.id",
"default_cluster:test.student.age");
Assertions.assertTrue(source.containsAll(target));
@@ -162,16 +142,11 @@ public class ColumnPruningTest extends TestWithFeService {
@Test
public void testPruneColumns4() {
-
String sql
= "select name,cname,grade from student left join score on student.id = score.sid left join course "
+ "on score.cid = course.cid where score.grade > 60";
- Plan plan = AnalyzeUtils.analyze(sql, connectContext);
-
- Memo memo = new Memo();
- memo.initialize(plan);
-
- Plan out = process(memo);
+ Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
Plan l20 = l1.child(0).child(0);
@@ -193,36 +168,30 @@ public class ColumnPruningTest extends TestWithFeService {
List<String> target;
List<String> source;
- source = getStringList(p1);
+ source = getOutputQualifiedNames(p1);
target = Lists.newArrayList("default_cluster:test.student.name", "default_cluster:test.course.cname",
"default_cluster:test.score.grade");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p20);
+ source = getOutputQualifiedNames(p20);
target = Lists.newArrayList("default_cluster:test.student.name", "default_cluster:test.score.cid",
"default_cluster:test.score.grade");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p21);
+ source = getOutputQualifiedNames(p21);
target = Lists.newArrayList("default_cluster:test.course.cid", "default_cluster:test.course.cname");
Assertions.assertTrue(source.containsAll(target));
- source = getStringList(p20lo);
+ source = getOutputQualifiedNames(p20lo);
target = Lists.newArrayList("default_cluster:test.student.id", "default_cluster:test.student.name");
Assertions.assertTrue(source.containsAll(target));
}
- private Plan process(Memo memo) {
- PlannerContext plannerContext = new PlannerContext(memo, new ConnectContext());
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), 0);
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- new ColumnPruning().buildRules(), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
- return memo.copyOut();
+ private Plan rewrite(Plan plan) {
+ return PlanRewriter.topDownRewrite(plan, new ConnectContext(), new ColumnPruning());
}
- private List<String> getStringList(LogicalProject<Plan> p) {
+ private List<String> getOutputQualifiedNames(LogicalProject<Plan> p) {
return p.getProjects().stream().map(NamedExpression::getQualifiedName).collect(Collectors.toList());
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java
index 381a8c1b37..7a28782763 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushDownPredicateTest.java
@@ -21,13 +21,8 @@ import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Type;
-import org.apache.doris.nereids.PlannerContext;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.RewriteTopDownJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Between;
@@ -45,6 +40,8 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.PlanRewriter;
+import org.apache.doris.qe.ConnectContext;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
@@ -53,7 +50,6 @@ import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
-import java.util.List;
import java.util.Optional;
/**
@@ -119,26 +115,18 @@ public class PushDownPredicateTest {
filter
);
- Memo memo = new Memo();
- memo.initialize(root);
- System.out.println(memo.copyOut().treeString());
-
- PlannerContext plannerContext = new PlannerContext(memo, null);
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
+ System.out.println(root.treeString());
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
+ Memo memo = rewrite(root);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());
- System.out.println(11);
Plan op1 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(1).getLogicalExpression().getPlan();
+ Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression()
+ .getPlan();
+ Plan op3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(1).getLogicalExpression()
+ .getPlan();
Assertions.assertTrue(op1 instanceof LogicalJoin);
Assertions.assertTrue(op2 instanceof LogicalFilter);
@@ -170,25 +158,17 @@ public class PushDownPredicateTest {
filter
);
- Memo memo = new Memo();
- memo.initialize(root);
- System.out.println(memo.copyOut().treeString());
-
- PlannerContext plannerContext = new PlannerContext(memo, null);
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
-
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(),
- ImmutableList.of(new PushPredicateThroughJoin().build()), jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
+ System.out.println(root.treeString());
+ Memo memo = rewrite(root);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());
Plan op1 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(1).getLogicalExpression().getPlan();
+ Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression()
+ .getPlan();
+ Plan op3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(1).getLogicalExpression()
+ .getPlan();
Assertions.assertTrue(op1 instanceof LogicalJoin);
Assertions.assertTrue(op2 instanceof LogicalFilter);
@@ -226,7 +206,8 @@ public class PushDownPredicateTest {
// score.grade > 60
Expression whereCondition4 = new GreaterThan(rScore.getOutput().get(2), Literal.of(60));
- Expression whereCondition = ExpressionUtils.and(whereCondition1, whereCondition2, whereCondition3, whereCondition4);
+ Expression whereCondition = ExpressionUtils.and(whereCondition1, whereCondition2, whereCondition3,
+ whereCondition4);
Plan join = new LogicalJoin(JoinType.INNER_JOIN, Optional.empty(), rStudent, rScore);
Plan join1 = new LogicalJoin(JoinType.INNER_JOIN, Optional.empty(), join, rCourse);
@@ -236,27 +217,18 @@ public class PushDownPredicateTest {
Lists.newArrayList(rStudent.getOutput().get(1), rCourse.getOutput().get(1), rScore.getOutput().get(2)),
filter
);
+ System.out.println(root.treeString());
-
- Memo memo = new Memo();
- memo.initialize(root);
- System.out.println(memo.copyOut().treeString());
-
- PlannerContext plannerContext = new PlannerContext(memo, null);
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
-
- List<Rule> fakeRules = Lists.newArrayList(new PushPredicateThroughJoin().build());
- RewriteTopDownJob rewriteTopDownJob = new RewriteTopDownJob(memo.getRoot(), fakeRules, jobContext);
- plannerContext.pushJob(rewriteTopDownJob);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
-
+ Memo memo = rewrite(root);
Group rootGroup = memo.getRoot();
System.out.println(memo.copyOut().treeString());
Plan join2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan join3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op1 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().getPlan();
- Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression().child(1).getLogicalExpression().getPlan();
+ Plan join3 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression()
+ .getPlan();
+ Plan op1 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression()
+ .child(0).getLogicalExpression().getPlan();
+ Plan op2 = rootGroup.getLogicalExpression().child(0).getLogicalExpression().child(0).getLogicalExpression()
+ .child(1).getLogicalExpression().getPlan();
Assertions.assertTrue(join2 instanceof LogicalJoin);
Assertions.assertTrue(join3 instanceof LogicalJoin);
@@ -268,4 +240,8 @@ public class PushDownPredicateTest {
Assertions.assertEquals(((LogicalFilter) op1).getPredicates().toSql(), whereCondition3result.toSql());
Assertions.assertEquals(((LogicalFilter) op2).getPredicates(), whereCondition4);
}
+
+ private Memo rewrite(Plan plan) {
+ return PlanRewriter.topDownRewriteMemo(plan, new ConnectContext(), new PushPredicateThroughJoin());
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java
new file mode 100644
index 0000000000..df3d83e250
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java
@@ -0,0 +1,60 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.rules.analysis.BindFunction;
+import org.apache.doris.nereids.rules.analysis.BindRelation;
+import org.apache.doris.nereids.rules.analysis.BindSlotReference;
+import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.qe.ConnectContext;
+
+/**
+ * Analyzer for unit test.
+ * // TODO: unify the logic with ones in production files.
+ */
+public class TestAnalyzer {
+ private final ConnectContext connectContext;
+
+ public TestAnalyzer(ConnectContext connectContext) {
+ this.connectContext = connectContext;
+ }
+
+ /**
+ * Try to analyze a SQL into analyzed LogicalPlan.
+ */
+ public LogicalPlan analyze(String sql) {
+ NereidsParser parser = new NereidsParser();
+ LogicalPlan parsed = parser.parseSingle(sql);
+ return analyze(parsed);
+ }
+
+ private LogicalPlan analyze(LogicalPlan inputPlan) {
+ return (LogicalPlan) new Memo(inputPlan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext()
+ .bottomUpRewrite(new BindFunction())
+ .bottomUpRewrite(new BindRelation())
+ .bottomUpRewrite(new BindSlotReference())
+ .bottomUpRewrite(new ProjectToGlobalAggregate())
+ .getMemo()
+ .copyOut();
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/AnalyzeSSBTest.java
similarity index 53%
rename from fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java
rename to fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/AnalyzeSSBTest.java
index aa59558232..3b5c639e8f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/AnalyzeSSBTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/AnalyzeSSBTest.java
@@ -6,7 +6,7 @@
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
@@ -15,45 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids;
+package org.apache.doris.nereids.ssb;
import org.apache.doris.nereids.analyzer.Unbound;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.RewriteBottomUpJob;
-import org.apache.doris.nereids.memo.Group;
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.parser.NereidsParser;
-import org.apache.doris.nereids.properties.PhysicalProperties;
-import org.apache.doris.nereids.rules.RuleFactory;
-import org.apache.doris.nereids.rules.analysis.BindFunction;
-import org.apache.doris.nereids.rules.analysis.BindRelation;
-import org.apache.doris.nereids.rules.analysis.BindSlotReference;
-import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate;
-import org.apache.doris.nereids.ssb.SSBUtils;
+import org.apache.doris.nereids.rules.rewrite.logical.TestAnalyzer;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.utframe.TestWithFeService;
-import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.List;
-public class AnalyzeSSBTest extends TestWithFeService {
-
- private final NereidsParser parser = new NereidsParser();
-
- @Override
- protected void runBeforeAll() throws Exception {
- createDatabase("test");
- connectContext.setDatabase("default_cluster:test");
-
- SSBUtils.createTables(this);
- }
-
+public class AnalyzeSSBTest extends SSBTestBase {
/**
* TODO: check bound plan and expression details.
*/
@@ -123,43 +98,10 @@ public class AnalyzeSSBTest extends TestWithFeService {
}
private void checkAnalyze(String sql) {
- LogicalPlan analyzed = analyze(sql);
- System.out.println(analyzed.treeString());
+ LogicalPlan analyzed = new TestAnalyzer(connectContext).analyze(sql);
Assertions.assertTrue(checkBound(analyzed));
}
- private LogicalPlan analyze(String sql) {
- try {
- LogicalPlan parsed = parser.parseSingle(sql);
- return analyze(parsed, connectContext);
- } catch (Throwable t) {
- throw new IllegalStateException("Analyze failed", t);
- }
- }
-
- private LogicalPlan analyze(LogicalPlan inputPlan, ConnectContext connectContext) {
- Memo memo = new Memo();
- memo.initialize(inputPlan);
-
- PlannerContext plannerContext = new PlannerContext(memo, connectContext);
- JobContext jobContext = new JobContext(plannerContext, new PhysicalProperties(), Double.MAX_VALUE);
- plannerContext.setCurrentJobContext(jobContext);
-
- executeRewriteBottomUpJob(plannerContext, new BindFunction());
- executeRewriteBottomUpJob(plannerContext, new BindRelation());
- executeRewriteBottomUpJob(plannerContext, new BindSlotReference());
- executeRewriteBottomUpJob(plannerContext, new ProjectToGlobalAggregate());
- return (LogicalPlan) memo.copyOut();
- }
-
- private void executeRewriteBottomUpJob(PlannerContext plannerContext, RuleFactory ruleFactory) {
- Group rootGroup = plannerContext.getMemo().getRoot();
- RewriteBottomUpJob job = new RewriteBottomUpJob(rootGroup,
- plannerContext.getCurrentJobContext(), ImmutableList.of(ruleFactory));
- plannerContext.pushJob(job);
- plannerContext.getJobScheduler().executeJobPool(plannerContext);
- }
-
/**
* PlanNode and its expressions are all bound.
*/
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java
new file mode 100644
index 0000000000..efea2e0099
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java
@@ -0,0 +1,167 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.ssb;
+
+import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
+import org.apache.doris.nereids.rules.rewrite.logical.TestAnalyzer;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.nereids.util.PlanRewriter;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class SSBJoinReorderTest extends SSBTestBase {
+ @Test
+ public void q4_1() {
+ test(
+ SSBUtils.Q4_1,
+ ImmutableList.of(
+ "(lo_orderdate = d_datekey)",
+ "((lo_custkey = c_custkey) AND (c_region = 'AMERICA'))",
+ "((lo_suppkey = s_suppkey) AND (s_region = 'AMERICA'))",
+ "(lo_partkey = p_partkey)"
+ ),
+ ImmutableList.of("((p_mfgr = 'MFGR#1') OR (p_mfgr = 'MFGR#2'))")
+ );
+ }
+
+ @Test
+ public void q4_2() {
+ test(
+ SSBUtils.Q4_2,
+ ImmutableList.of(
+ "(lo_orderdate = d_datekey)",
+ "((lo_custkey = c_custkey) AND (c_region = 'AMERICA'))",
+ "((lo_suppkey = s_suppkey) AND (s_region = 'AMERICA'))",
+ "(lo_partkey = p_partkey)"
+ ),
+ ImmutableList.of(
+ "(((d_year = 1997) OR (d_year = 1998)) AND ((p_mfgr = 'MFGR#1') OR (p_mfgr = 'MFGR#2')))")
+ );
+ }
+
+ @Test
+ public void q4_3() {
+ test(
+ SSBUtils.Q4_3,
+ ImmutableList.of(
+ "(lo_orderdate = d_datekey)",
+ "(lo_custkey = c_custkey)",
+ "((lo_suppkey = s_suppkey) AND (s_nation = 'UNITED STATES'))",
+ "((lo_partkey = p_partkey) AND (p_category = 'MFGR#14'))"
+ ),
+ ImmutableList.of("((d_year = 1997) OR (d_year = 1998))")
+ );
+ }
+
+ private void test(String sql, List<String> expectJoinConditions, List<String> expectFilterPredicates) {
+ LogicalPlan analyzed = new TestAnalyzer(connectContext).analyze(sql);
+ LogicalPlan plan = testJoinReorder(analyzed);
+ new PlanChecker(expectJoinConditions, expectFilterPredicates).check(plan);
+ }
+
+ private LogicalPlan testJoinReorder(LogicalPlan plan) {
+ return (LogicalPlan) PlanRewriter.topDownRewrite(plan, connectContext, new ReorderJoin());
+ }
+
+ private static class PlanChecker extends PlanVisitor<Void, Context> {
+ private final List<LogicalRelation> joinInputs = new ArrayList<>();
+ private final List<LogicalJoin> joins = new ArrayList<>();
+ private final List<LogicalFilter> filters = new ArrayList<>();
+ // TODO: it's tricky to compare expression by string, use a graceful manner to do this in the future.
+ private final List<String> expectJoinConditions;
+ private final List<String> expectFilterPredicates;
+
+ public PlanChecker(List<String> expectJoinConditions, List<String> expectFilterPredicates) {
+ this.expectJoinConditions = expectJoinConditions;
+ this.expectFilterPredicates = expectFilterPredicates;
+ }
+
+ public void check(Plan plan) {
+ plan.accept(this, new Context(null));
+
+ // check join table orders
+ Assertions.assertEquals(
+ ImmutableList.of("dates", "lineorder", "customer", "supplier", "part"),
+ joinInputs.stream().map(p -> p.getTable().getName()).collect(Collectors.toList()));
+
+ // check join conditions
+ List<String> actualJoinConditions = joins.stream().map(j -> {
+ Optional<Expression> condition = j.getCondition();
+ return condition.map(Expression::toSql).orElse("");
+ }).collect(Collectors.toList());
+ Assertions.assertEquals(expectJoinConditions, actualJoinConditions);
+
+ // check filter predicates
+ List<String> actualFilterPredicates = filters.stream()
+ .map(f -> f.getPredicates().toSql()).collect(Collectors.toList());
+ Assertions.assertEquals(expectFilterPredicates, actualFilterPredicates);
+ }
+
+ @Override
+ public Void visit(Plan plan, Context context) {
+ for (Plan child : plan.children()) {
+ child.accept(this, new Context(plan));
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalRelation(LogicalRelation relation, Context context) {
+ if (context.parent instanceof LogicalJoin) {
+ joinInputs.add(relation);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalFilter(LogicalFilter<Plan> filter, Context context) {
+ filters.add(filter);
+ filter.child().accept(this, new Context(filter));
+ return null;
+ }
+
+ @Override
+ public Void visitLogicalJoin(LogicalJoin<Plan, Plan> join, Context context) {
+ join.left().accept(this, new Context(join));
+ join.right().accept(this, new Context(join));
+ joins.add(join);
+ return null;
+ }
+ }
+
+ private static class Context {
+ public final Plan parent;
+
+ public Context(Plan parent) {
+ this.parent = parent;
+ }
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBTestBase.java
similarity index 62%
copy from fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
copy to fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBTestBase.java
index 89cbac83bd..140ba64897 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/Utils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBTestBase.java
@@ -15,21 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-package org.apache.doris.nereids.util;
+package org.apache.doris.nereids.ssb;
-/**
- * Utils for Nereids.
- */
-public class Utils {
- /**
- * Quoted string if it contains special character or all characters are digit.
- *
- * @param part string to be quoted
- * @return quoted string
- */
- public static String quoteIfNeeded(String part) {
- // We quote strings except the ones which consist of digits only.
- return part.matches("\\w*[\\w&&[^\\d]]+\\w*")
- ? part : part.replace("`", "``");
+import org.apache.doris.utframe.TestWithFeService;
+
+public abstract class SSBTestBase extends TestWithFeService {
+ @Override
+ protected void runBeforeAll() throws Exception {
+ createDatabase("test");
+ connectContext.setDatabase("default_cluster:test");
+ SSBUtils.createTables(this);
}
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBUtils.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBUtils.java
index 074e970b31..0963688bbc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBUtils.java
@@ -162,7 +162,7 @@ public class SSBUtils {
+ " d_year,\n"
+ " c_nation,\n"
+ " SUM(lo_revenue - lo_supplycost) AS PROFIT\n"
- + "FROM lineorder, dates, customer, supplier, part\n"
+ + "FROM dates, customer, supplier, part, lineorder\n"
+ "WHERE\n"
+ " lo_custkey = c_custkey\n"
+ " AND lo_suppkey = s_suppkey\n"
@@ -182,7 +182,7 @@ public class SSBUtils {
+ " s_nation,\n"
+ " p_category,\n"
+ " SUM(lo_revenue - lo_supplycost) AS PROFIT\n"
- + "FROM lineorder, dates, customer, supplier, part\n"
+ + "FROM dates, customer, supplier, part, lineorder\n"
+ "WHERE\n"
+ " lo_custkey = c_custkey\n"
+ " AND lo_suppkey = s_suppkey\n"
@@ -206,7 +206,7 @@ public class SSBUtils {
+ " s_city,\n"
+ " p_brand,\n"
+ " SUM(lo_revenue - lo_supplycost) AS PROFIT\n"
- + "FROM lineorder, dates, customer, supplier, part\n"
+ + "FROM dates, customer, supplier, part, lineorder\n"
+ "WHERE\n"
+ " lo_custkey = c_custkey\n"
+ " AND lo_suppkey = s_suppkey\n"
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java
new file mode 100644
index 0000000000..e1eb0a7718
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/util/PlanRewriter.java
@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.util;
+
+import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleFactory;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.qe.ConnectContext;
+
+/**
+ * Utility to copy plan into {@link Memo} and apply rewrite rules.
+ */
+public class PlanRewriter {
+ public static Plan bottomUpRewrite(Plan plan, ConnectContext connectContext, RuleFactory... rules) {
+ return bottomUpRewriteMemo(plan, connectContext, rules).copyOut();
+ }
+
+ public static Plan bottomUpRewrite(Plan plan, ConnectContext connectContext, Rule... rules) {
+ return bottomUpRewriteMemo(plan, connectContext, rules).copyOut();
+ }
+
+ public static Memo bottomUpRewriteMemo(Plan plan, ConnectContext connectContext, RuleFactory... rules) {
+ return new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext()
+ .topDownRewrite(rules)
+ .getMemo();
+ }
+
+ public static Memo bottomUpRewriteMemo(Plan plan, ConnectContext connectContext, Rule... rules) {
+ return new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext()
+ .topDownRewrite(rules)
+ .getMemo();
+ }
+
+ public static Plan topDownRewrite(Plan plan, ConnectContext connectContext, RuleFactory... rules) {
+ return topDownRewriteMemo(plan, connectContext, rules).copyOut();
+ }
+
+ public static Plan topDownRewrite(Plan plan, ConnectContext connectContext, Rule... rules) {
+ return topDownRewriteMemo(plan, connectContext, rules).copyOut();
+ }
+
+ public static Memo topDownRewriteMemo(Plan plan, ConnectContext connectContext, RuleFactory... rules) {
+ return new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext()
+ .topDownRewrite(rules)
+ .getMemo();
+ }
+
+ public static Memo topDownRewriteMemo(Plan plan, ConnectContext connectContext, Rule... rules) {
+ return new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext()
+ .topDownRewrite(rules)
+ .getMemo();
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
index 2915547420..87498ed605 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java
@@ -95,6 +95,8 @@ public abstract class TestWithFeService {
protected String runningDir = "fe/mocked/" + getClass().getSimpleName() + "/" + UUID.randomUUID() + "/";
protected ConnectContext connectContext;
+ protected static final String DEFAULT_CLUSTER_PREFIX = "default_cluster:";
+
@BeforeAll
public final void beforeAll() throws Exception {
connectContext = createDefaultCtx();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org