You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2015/10/24 01:27:26 UTC
incubator-calcite git commit: [CALCITE-890] Register all combinations
of materialization substitutions (Maryann Xue)
Repository: incubator-calcite
Updated Branches:
refs/heads/master 2c339be18 -> 9d0fef317
[CALCITE-890] Register all combinations of materialization substitutions (Maryann Xue)
Project: http://git-wip-us.apache.org/repos/asf/incubator-calcite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-calcite/commit/9d0fef31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-calcite/tree/9d0fef31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-calcite/diff/9d0fef31
Branch: refs/heads/master
Commit: 9d0fef31725e73912b37fff79037a3258bca7fa6
Parents: 2c339be
Author: Julian Hyde <jh...@apache.org>
Authored: Fri Oct 23 13:08:42 2015 -0700
Committer: Julian Hyde <jh...@apache.org>
Committed: Fri Oct 23 14:36:42 2015 -0700
----------------------------------------------------------------------
core/pom.xml | 5 ++
.../calcite/plan/volcano/VolcanoPlanner.java | 61 +++++++++++++++-----
.../calcite/test/MaterializationTest.java | 56 ++++++++++++++++++
3 files changed, 107 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
index 65adb3e..6a62d68 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -56,6 +56,11 @@ limitations under the License.
<artifactId>commons-dbcp</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index e57a760..4cc513a 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -360,7 +360,8 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
registerImpl(rel, root.set);
}
- private void useMaterialization(RelOptMaterialization materialization) {
+ private RelNode useMaterialization(RelNode root,
+ RelOptMaterialization materialization, boolean firstRun) {
// Try to rewrite the original root query in terms of the materialized
// query. If that is possible, register the remnant query as equivalent
// to the root.
@@ -368,22 +369,23 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
// This call modifies originalRoot. Doesn't look like originalRoot should be mutable though.
// Need to check.
- RelNode sub = substitute(originalRoot, materialization);
+ RelNode sub = substitute(root, materialization);
if (sub != null) {
- // TODO: try to substitute other materializations in the remnant.
- // Useful for big queries, e.g.
- // (t1 group by c1) join (t2 group by c2).
Hook.SUB.run(sub);
- registerImpl(sub, root.set);
- return;
+ registerImpl(sub, this.root.set);
+ return sub;
+ }
+
+ if (firstRun) {
+ RelSubset subset = registerImpl(materialization.queryRel, null);
+ RelNode tableRel2 =
+ RelOptUtil.createCastRel(
+ materialization.tableRel,
+ materialization.queryRel.getRowType(),
+ true);
+ registerImpl(tableRel2, subset.set);
}
- RelSubset subset = registerImpl(materialization.queryRel, null);
- RelNode tableRel2 =
- RelOptUtil.createCastRel(
- materialization.tableRel,
- materialization.queryRel.getRowType(),
- true);
- registerImpl(tableRel2, subset.set);
+ return null;
}
private RelNode substitute(
@@ -417,6 +419,33 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
.go(materialization.tableRel);
}
+ // Register all possible combinations of materialization substitution.
+ // Useful for big queries, e.g.
+ // (t1 group by c1) join (t2 group by c2).
+ private void useMaterializations(RelNode root,
+ List<RelOptMaterialization> materializations, boolean firstRun) {
+ for (RelOptMaterialization m : materializations) {
+ RelNode sub = useMaterialization(root, m, firstRun);
+ if (sub != null) {
+ useMaterializations(sub, materializations, false);
+ } else {
+ // Based on the assumption that a substitution itself won't trigger another
+ // substitution, if a materialization is not matched here it won't be useful
+ // in any subsequent matching for the current level of recursion or this level
+ // down. So we can safely remove the unmatched materialization from the remnant
+ // list for the current root.
+ List<RelOptMaterialization> newList =
+ Lists.newArrayListWithExpectedSize(materializations.size() - 1);
+ for (RelOptMaterialization elem : materializations) {
+ if (elem != m) {
+ newList.add(elem);
+ }
+ }
+ materializations = newList;
+ }
+ }
+ }
+
private void useApplicableMaterializations() {
// Avoid using materializations while populating materializations!
final CalciteConnectionConfig config =
@@ -454,6 +483,7 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
final Graphs.FrozenGraph<List<String>, DefaultEdge> frozenGraph =
Graphs.makeImmutable(usesGraph);
final Set<RelOptTable> queryTables = findTables(originalRoot);
+ final List<RelOptMaterialization> applicableMaterializations = Lists.newArrayList();
for (RelOptMaterialization materialization : materializations) {
if (materialization.starTable != null) {
// Materialization is a tile in a lattice. We will deal with it shortly.
@@ -461,10 +491,11 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
}
if (materialization.table != null) {
if (usesTable(materialization.table, queryTables, frozenGraph)) {
- useMaterialization(materialization);
+ applicableMaterializations.add(materialization);
}
}
}
+ useMaterializations(originalRoot, applicableMaterializations, true);
// Use a lattice if the query uses at least the central (fact) table of the
// lattice.
http://git-wip-us.apache.org/repos/asf/incubator-calcite/blob/9d0fef31/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index 61d49dd..07d5a44 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -29,6 +29,9 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.util.JsonBuilder;
+import org.apache.calcite.util.Util;
+
+import org.apache.commons.lang3.StringUtils;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
@@ -38,6 +41,7 @@ import org.junit.Test;
import java.math.BigDecimal;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.List;
import java.util.Map;
@@ -815,6 +819,58 @@ public class MaterializationTest {
Prepare.THREAD_TRIM.set(false);
}
}
+
+ @Test public void testSingleMaterializationMultiUsage() {
+ String q = "select *\n"
+ + "from (select * from \"emps\" where \"empid\" < 300)\n"
+ + "join (select * from \"emps\" where \"empid\" < 200) using (\"empid\")";
+ try {
+ Prepare.THREAD_TRIM.set(true);
+ MaterializationService.setThreadLocal();
+ CalciteAssert.that()
+ .withMaterializations(JdbcTest.HR_MODEL,
+ "m0", "select * from \"emps\" where \"empid\" < 500")
+ .query(q)
+ .enableMaterializations(true)
+ .explainMatches("", new Function<ResultSet, Void>() {
+ public Void apply(ResultSet s) {
+ try {
+ final String actual = Util.toLinux(CalciteAssert.toString(s));
+ final String scan = "EnumerableTableScan(table=[[hr, m0]])";
+ assertTrue(actual + " should have had two occurrences of " + scan,
+ StringUtils.countMatches(actual, scan) == 2);
+ return null;
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ })
+ .sameResultWithMaterializationsDisabled();
+ } finally {
+ Prepare.THREAD_TRIM.set(false);
+ }
+ }
+
+ @Test public void testMultiMaterializationMultiUsage() {
+ String q = "select *\n"
+ + "from (select * from \"emps\" where \"empid\" < 300)\n"
+ + "join (select \"deptno\", count(*) as c from \"emps\" group by \"deptno\") using (\"deptno\")";
+ try {
+ Prepare.THREAD_TRIM.set(true);
+ MaterializationService.setThreadLocal();
+ CalciteAssert.that()
+ .withMaterializations(JdbcTest.HR_MODEL,
+ "m0", "select \"deptno\", count(*) as c, sum(\"empid\") as s from \"emps\" group by \"deptno\"",
+ "m1", "select * from \"emps\" where \"empid\" < 500")
+ .query(q)
+ .enableMaterializations(true)
+ .explainContains("EnumerableTableScan(table=[[hr, m0]])")
+ .explainContains("EnumerableTableScan(table=[[hr, m1]])")
+ .sameResultWithMaterializationsDisabled();
+ } finally {
+ Prepare.THREAD_TRIM.set(false);
+ }
+ }
}
// End MaterializationTest.java