You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by hu...@apache.org on 2022/07/26 10:32:13 UTC
[doris] branch master updated: [Refactor](Nereids) Move analysis logic to dedicated class `NereidsAnalyzer`. (#11171)
This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e99f617b21 [Refactor](Nereids) Move analysis logic to dedicated class `NereidsAnalyzer`. (#11171)
e99f617b21 is described below
commit e99f617b219a53d367339c85b885a9618d753f96
Author: Shuo Wang <wa...@gmail.com>
AuthorDate: Tue Jul 26 18:32:07 2022 +0800
[Refactor](Nereids) Move analysis logic to dedicated class `NereidsAnalyzer`. (#11171)
This PR proposes to move the analysis logic to the dedicated class NereidsAnalyzer, which has the following benefits:
Unify the analysis logic in production and test files.
Facilitate analyzing subquery plans within different scopes.
---
.../org/apache/doris/nereids/NereidsPlanner.java | 27 ++------
.../doris/nereids/analyzer/NereidsAnalyzer.java | 79 ++++++++++++++++++++++
.../rules/rewrite/logical/ColumnPruningTest.java | 9 +--
.../rules/rewrite/logical/TestAnalyzer.java | 60 ----------------
.../doris/nereids/ssb/SSBJoinReorderTest.java | 4 +-
.../doris/nereids/tpch/AnalyzeCheckTestBase.java | 4 +-
6 files changed, 95 insertions(+), 88 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 2b32416680..c5427c943c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -21,17 +21,16 @@ import org.apache.doris.analysis.DescriptorTable;
import org.apache.doris.analysis.StatementBase;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.nereids.analyzer.NereidsAnalyzer;
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator;
import org.apache.doris.nereids.glue.translator.PlanTranslatorContext;
-import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob;
import org.apache.doris.nereids.jobs.batch.DisassembleRulesJob;
import org.apache.doris.nereids.jobs.batch.JoinReorderRulesJob;
import org.apache.doris.nereids.jobs.batch.OptimizeRulesJob;
import org.apache.doris.nereids.jobs.batch.PredicatePushDownRulesJob;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
-import org.apache.doris.nereids.memo.Memo;
import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -99,29 +98,17 @@ public class NereidsPlanner extends Planner {
// TODO: refactor, just demo code here
public PhysicalPlan plan(LogicalPlan plan, PhysicalProperties outputProperties, ConnectContext connectContext)
throws AnalysisException {
- plannerContext = new Memo(plan)
- .newPlannerContext(connectContext)
+ plannerContext = new NereidsAnalyzer(connectContext)
+ .analyzeWithPlannerContext(plan)
+ // TODO: revisit this. What is the appropriate time to set physical properties? Maybe before enter
+ // cascades style optimize phase.
.setJobContext(outputProperties);
- // Get plan directly. Just for SSB.
- return doPlan();
- }
- /**
- * The actual execution of the plan, including the generation and execution of the job.
- * @return PhysicalPlan.
- */
- private PhysicalPlan doPlan() {
- analyze();
rewrite();
optimize();
- return getRoot().extractPlan();
- }
- /**
- * Analyze: bind references according to metadata in the catalog, perform semantic analysis, etc.
- */
- private void analyze() {
- new AnalyzeRulesJob(plannerContext).execute();
+ // Get plan directly. Just for SSB.
+ return getRoot().extractPlan();
}
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
new file mode 100644
index 0000000000..16daf6ec40
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/NereidsAnalyzer.java
@@ -0,0 +1,79 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.analyzer;
+
+import org.apache.doris.nereids.PlannerContext;
+import org.apache.doris.nereids.jobs.batch.AnalyzeRulesJob;
+import org.apache.doris.nereids.memo.Memo;
+import org.apache.doris.nereids.parser.NereidsParser;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.qe.ConnectContext;
+
+/**
+ * Bind symbols according to metadata in the catalog, perform semantic analysis, etc.
+ * TODO: revisit the interface after subquery analysis is supported.
+ */
+public class NereidsAnalyzer {
+ private final ConnectContext connectContext;
+
+ public NereidsAnalyzer(ConnectContext connectContext) {
+ this.connectContext = connectContext;
+ }
+
+ /**
+ * Analyze plan.
+ */
+ public LogicalPlan analyze(Plan plan) {
+ return (LogicalPlan) analyzeWithPlannerContext(plan).getMemo().copyOut();
+ }
+
+ /**
+ * Convert SQL String to analyzed plan.
+ */
+ public LogicalPlan analyze(String sql) {
+ return analyze(parse(sql));
+ }
+
+ /**
+ * Analyze plan and return {@link PlannerContext}.
+ * Thus returned {@link PlannerContext} could be reused to do
+ * further plan optimization without creating new {@link Memo} and {@link PlannerContext}.
+ */
+ public PlannerContext analyzeWithPlannerContext(Plan plan) {
+ PlannerContext plannerContext = new Memo(plan)
+ .newPlannerContext(connectContext)
+ .setDefaultJobContext();
+
+ new AnalyzeRulesJob(plannerContext).execute();
+ return plannerContext;
+ }
+
+ /**
+ * Convert SQL String to analyzed plan without copying out of {@link Memo}.
+ * Thus returned {@link PlannerContext} could be reused to do
+ * further plan optimization without creating new {@link Memo} and {@link PlannerContext}.
+ */
+ public PlannerContext analyzeWithPlannerContext(String sql) {
+ return analyzeWithPlannerContext(parse(sql));
+ }
+
+ private Plan parse(String sql) {
+ return new NereidsParser().parseSingle(sql);
+ }
+}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
index ccaec751c3..4ce915a474 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/ColumnPruningTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.nereids.rules.rewrite.logical;
+import org.apache.doris.nereids.analyzer.NereidsAnalyzer;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
@@ -60,7 +61,7 @@ public class ColumnPruningTest extends TestWithFeService {
public void testPruneColumns1() {
String sql
= "select id,name,grade from student left join score on student.id = score.sid where score.grade > 60";
- Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan plan = new NereidsAnalyzer(connectContext).analyze(sql);
Plan out = rewrite(plan);
System.out.println(out.treeString());
@@ -95,7 +96,7 @@ public class ColumnPruningTest extends TestWithFeService {
String sql
= "select name,sex,cid,grade from student left join score on student.id = score.sid "
+ "where score.grade > 60";
- Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan plan = new NereidsAnalyzer(connectContext).analyze(sql);
Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
@@ -124,7 +125,7 @@ public class ColumnPruningTest extends TestWithFeService {
@Test
public void testPruneColumns3() {
String sql = "select id,name from student where age > 18";
- Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan plan = new NereidsAnalyzer(connectContext).analyze(sql);
Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
@@ -145,7 +146,7 @@ public class ColumnPruningTest extends TestWithFeService {
String sql
= "select name,cname,grade from student left join score on student.id = score.sid left join course "
+ "on score.cid = course.cid where score.grade > 60";
- Plan plan = new TestAnalyzer(connectContext).analyze(sql);
+ Plan plan = new NereidsAnalyzer(connectContext).analyze(sql);
Plan out = rewrite(plan);
Plan l1 = out.child(0).child(0);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java
deleted file mode 100644
index df3d83e250..0000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/TestAnalyzer.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite.logical;
-
-import org.apache.doris.nereids.memo.Memo;
-import org.apache.doris.nereids.parser.NereidsParser;
-import org.apache.doris.nereids.rules.analysis.BindFunction;
-import org.apache.doris.nereids.rules.analysis.BindRelation;
-import org.apache.doris.nereids.rules.analysis.BindSlotReference;
-import org.apache.doris.nereids.rules.analysis.ProjectToGlobalAggregate;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.qe.ConnectContext;
-
-/**
- * Analyzer for unit test.
- * // TODO: unify the logic with ones in production files.
- */
-public class TestAnalyzer {
- private final ConnectContext connectContext;
-
- public TestAnalyzer(ConnectContext connectContext) {
- this.connectContext = connectContext;
- }
-
- /**
- * Try to analyze a SQL into analyzed LogicalPlan.
- */
- public LogicalPlan analyze(String sql) {
- NereidsParser parser = new NereidsParser();
- LogicalPlan parsed = parser.parseSingle(sql);
- return analyze(parsed);
- }
-
- private LogicalPlan analyze(LogicalPlan inputPlan) {
- return (LogicalPlan) new Memo(inputPlan)
- .newPlannerContext(connectContext)
- .setDefaultJobContext()
- .bottomUpRewrite(new BindFunction())
- .bottomUpRewrite(new BindRelation())
- .bottomUpRewrite(new BindSlotReference())
- .bottomUpRewrite(new ProjectToGlobalAggregate())
- .getMemo()
- .copyOut();
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java
index efea2e0099..b78de68a5a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/ssb/SSBJoinReorderTest.java
@@ -17,8 +17,8 @@
package org.apache.doris.nereids.ssb;
+import org.apache.doris.nereids.analyzer.NereidsAnalyzer;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
-import org.apache.doris.nereids.rules.rewrite.logical.TestAnalyzer;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
@@ -82,7 +82,7 @@ public class SSBJoinReorderTest extends SSBTestBase {
}
private void test(String sql, List<String> expectJoinConditions, List<String> expectFilterPredicates) {
- LogicalPlan analyzed = new TestAnalyzer(connectContext).analyze(sql);
+ LogicalPlan analyzed = new NereidsAnalyzer(connectContext).analyze(sql);
LogicalPlan plan = testJoinReorder(analyzed);
new PlanChecker(expectJoinConditions, expectFilterPredicates).check(plan);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/tpch/AnalyzeCheckTestBase.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/tpch/AnalyzeCheckTestBase.java
index 099bce18ea..bfd01077d6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/nereids/tpch/AnalyzeCheckTestBase.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/tpch/AnalyzeCheckTestBase.java
@@ -17,8 +17,8 @@
package org.apache.doris.nereids.tpch;
+import org.apache.doris.nereids.analyzer.NereidsAnalyzer;
import org.apache.doris.nereids.analyzer.Unbound;
-import org.apache.doris.nereids.rules.rewrite.logical.TestAnalyzer;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
@@ -30,7 +30,7 @@ import java.util.List;
public abstract class AnalyzeCheckTestBase extends TestWithFeService {
protected void checkAnalyze(String sql) {
- LogicalPlan analyzed = new TestAnalyzer(connectContext).analyze(sql);
+ LogicalPlan analyzed = new NereidsAnalyzer(connectContext).analyze(sql);
Assertions.assertTrue(checkBound(analyzed));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org