You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2019/03/25 17:52:27 UTC
[calcite] branch master updated (06b1894 -> 406129b)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git.
from 06b1894 Site: News item for release 1.19.0
new 79f4324 [CALCITE-2942] Materialized view rewriting logic instantiates RelMetadataQuery each time the rule is triggered
new ecc100e [CALCITE-2943] Materialized view rewriting logic calls getApplicableMaterializations each time the rule is triggered
new 2fa7fd7 [CALCITE-2946] RelBuilder wrongly skips creation of Aggregate that prunes columns if input produces one row at most
new 406129b [CALCITE-2951] Support decorrelate subquery that has aggregate with grouping sets (Haisheng Yuan)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/calcite/plan/RelOptUtil.java | 2 +-
.../calcite/plan/volcano/VolcanoPlanner.java | 2 +-
.../rel/rules/AbstractMaterializedViewRule.java | 585 +++++++++++----------
.../apache/calcite/sql2rel/RelDecorrelator.java | 16 +-
.../java/org/apache/calcite/tools/RelBuilder.java | 2 +-
.../org/apache/calcite/test/RelBuilderTest.java | 19 +
.../apache/calcite/test/SqlToRelConverterTest.java | 13 +
.../apache/calcite/test/SqlToRelConverterTest.xml | 25 +
8 files changed, 366 insertions(+), 298 deletions(-)
[calcite] 03/04: [CALCITE-2946] RelBuilder wrongly skips creation
of Aggregate that prunes columns if input produces one row at most
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 2fa7fd79b7dbb09276574784baa80b99edf37412
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Fri Mar 22 18:31:31 2019 -0700
[CALCITE-2946] RelBuilder wrongly skips creation of Aggregate that prunes columns if input produces one row at most
Close apache/calcite#1128
---
.../java/org/apache/calcite/tools/RelBuilder.java | 2 +-
.../java/org/apache/calcite/test/RelBuilderTest.java | 19 +++++++++++++++++++
2 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
index c62c42c..7f85432 100644
--- a/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
+++ b/core/src/main/java/org/apache/calcite/tools/RelBuilder.java
@@ -1431,7 +1431,7 @@ public class RelBuilder {
final Double maxRowCount = mq.getMaxRowCount(peek());
if (maxRowCount != null && maxRowCount <= 1D) {
// If there is at most one row, rel is already unique.
- return this;
+ return project(fields(groupSet.asList()));
}
}
final ImmutableList<ImmutableBitSet> groupSets;
diff --git a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
index 1a50e95..3ec2859 100644
--- a/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelBuilderTest.java
@@ -855,6 +855,25 @@ public class RelBuilderTest {
assertThat(root, hasTree(expected));
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-2946">[CALCITE-2946]
+ * RelBuilder wrongly skips creation of Aggregate that prunes columns if input
+ * produces one row at most</a>. */
+ @Test public void testAggregate5() {
+ final RelBuilder builder = RelBuilder.create(config().build());
+ RelNode root =
+ builder.scan("EMP")
+ .aggregate(builder.groupKey(), builder.count().as("C"))
+ .project(builder.literal(4), builder.literal(2), builder.field(0))
+ .aggregate(builder.groupKey(builder.field(0), builder.field(1)))
+ .build();
+ final String expected = ""
+ + "LogicalProject($f0=[4], $f1=[2])\n"
+ + " LogicalAggregate(group=[{}], C=[COUNT()])\n"
+ + " LogicalTableScan(table=[[scott, EMP]])\n";
+ assertThat(root, hasTree(expected));
+ }
+
@Test public void testAggregateFilter() {
// Equivalent SQL:
// SELECT deptno, COUNT(*) FILTER (WHERE empno > 100) AS c
[calcite] 02/04: [CALCITE-2943] Materialized view rewriting logic
calls getApplicableMaterializations each time the rule is triggered
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit ecc100ea23bdb5dd1a4c34633c3a376856b64970
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Thu Mar 21 15:21:35 2019 -0700
[CALCITE-2943] Materialized view rewriting logic calls getApplicableMaterializations each time the rule is triggered
Close apache/calcite#1125
---
.../calcite/plan/volcano/VolcanoPlanner.java | 2 +-
.../rel/rules/AbstractMaterializedViewRule.java | 583 +++++++++++----------
2 files changed, 295 insertions(+), 290 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
index cd4af85..6500110 100644
--- a/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
+++ b/core/src/main/java/org/apache/calcite/plan/volcano/VolcanoPlanner.java
@@ -331,7 +331,7 @@ public class VolcanoPlanner extends AbstractRelOptPlanner {
return latticeByName.get(table.getQualifiedName());
}
- private void registerMaterializations() {
+ protected void registerMaterializations() {
// Avoid using materializations while populating materializations!
final CalciteConnectionConfig config =
context.unwrap(CalciteConnectionConfig.class);
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
index 706dcca..01bb2e3 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
@@ -18,7 +18,6 @@ package org.apache.calcite.rel.rules;
import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.plan.RelOptMaterialization;
-import org.apache.calcite.plan.RelOptMaterializations;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptPredicateList;
import org.apache.calcite.plan.RelOptRule;
@@ -221,319 +220,325 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
return;
}
- // Obtain applicable (filtered) materializations
- // TODO: Filtering of relevant materializations needs to be
- // improved so we gather only materializations that might
- // actually generate a valid rewriting.
- final List<RelOptMaterialization> applicableMaterializations =
- RelOptMaterializations.getApplicableMaterializations(node, materializations);
+ // 2. Initialize all query related auxiliary data structures
+ // that will be used throughout query rewriting process
+ // Generate query table references
+ final Set<RelTableRef> queryTableRefs = mq.getTableReferences(node);
+ if (queryTableRefs == null) {
+ // Bail out
+ return;
+ }
- if (!applicableMaterializations.isEmpty()) {
- // 2. Initialize all query related auxiliary data structures
- // that will be used throughout query rewriting process
- // Generate query table references
- final Set<RelTableRef> queryTableRefs = mq.getTableReferences(node);
- if (queryTableRefs == null) {
- // Bail out
- return;
+ // Extract query predicates
+ final RelOptPredicateList queryPredicateList =
+ mq.getAllPredicates(node);
+ if (queryPredicateList == null) {
+ // Bail out
+ return;
+ }
+ final RexNode pred =
+ simplify.simplifyUnknownAsFalse(
+ RexUtil.composeConjunction(rexBuilder,
+ queryPredicateList.pulledUpPredicates));
+ final Triple<RexNode, RexNode, RexNode> queryPreds =
+ splitPredicates(rexBuilder, pred);
+
+ // Extract query equivalence classes. An equivalence class is a set
+ // of columns in the query output that are known to be equal.
+ final EquivalenceClasses qEC = new EquivalenceClasses();
+ for (RexNode conj : RelOptUtil.conjunctions(queryPreds.getLeft())) {
+ assert conj.isA(SqlKind.EQUALS);
+ RexCall equiCond = (RexCall) conj;
+ qEC.addEquivalenceClass(
+ (RexTableInputRef) equiCond.getOperands().get(0),
+ (RexTableInputRef) equiCond.getOperands().get(1));
+ }
+
+ // 3. We iterate through all applicable materializations trying to
+ // rewrite the given query
+ for (RelOptMaterialization materialization : materializations) {
+ RelNode view = materialization.tableRel;
+ Project topViewProject;
+ RelNode viewNode;
+ if (materialization.queryRel instanceof Project) {
+ topViewProject = (Project) materialization.queryRel;
+ viewNode = topViewProject.getInput();
+ } else {
+ topViewProject = null;
+ viewNode = materialization.queryRel;
}
- // Extract query predicates
- final RelOptPredicateList queryPredicateList =
- mq.getAllPredicates(node);
- if (queryPredicateList == null) {
- // Bail out
- return;
+ // Extract view table references
+ final Set<RelTableRef> viewTableRefs = mq.getTableReferences(viewNode);
+ if (viewTableRefs == null) {
+ // Skip it
+ continue;
}
- final RexNode pred =
- simplify.simplifyUnknownAsFalse(
- RexUtil.composeConjunction(rexBuilder,
- queryPredicateList.pulledUpPredicates));
- final Triple<RexNode, RexNode, RexNode> queryPreds =
- splitPredicates(rexBuilder, pred);
-
- // Extract query equivalence classes. An equivalence class is a set
- // of columns in the query output that are known to be equal.
- final EquivalenceClasses qEC = new EquivalenceClasses();
- for (RexNode conj : RelOptUtil.conjunctions(queryPreds.getLeft())) {
- assert conj.isA(SqlKind.EQUALS);
- RexCall equiCond = (RexCall) conj;
- qEC.addEquivalenceClass(
- (RexTableInputRef) equiCond.getOperands().get(0),
- (RexTableInputRef) equiCond.getOperands().get(1));
- }
-
- // 3. We iterate through all applicable materializations trying to
- // rewrite the given query
- for (RelOptMaterialization materialization : applicableMaterializations) {
- RelNode view = materialization.tableRel;
- Project topViewProject;
- RelNode viewNode;
- if (materialization.queryRel instanceof Project) {
- topViewProject = (Project) materialization.queryRel;
- viewNode = topViewProject.getInput();
- } else {
- topViewProject = null;
- viewNode = materialization.queryRel;
- }
- // 3.1. View checks before proceeding
- if (!isValidPlan(topViewProject, viewNode, mq)) {
- // Skip it
- continue;
+ // Filter relevant materializations. Currently, we only check whether
+ // the materialization contains any table that is used by the query
+ // TODO: Filtering of relevant materializations can be improved to be more fine-grained.
+ boolean applicable = false;
+ for (RelTableRef tableRef : viewTableRefs) {
+ if (queryTableRefs.contains(tableRef)) {
+ applicable = true;
+ break;
}
+ }
+ if (!applicable) {
+ // Skip it
+ continue;
+ }
- // 3.2. Initialize all query related auxiliary data structures
- // that will be used throughout query rewriting process
- // Extract view predicates
- final RelOptPredicateList viewPredicateList =
- mq.getAllPredicates(viewNode);
- if (viewPredicateList == null) {
- // Skip it
- continue;
- }
- final RexNode viewPred = simplify.simplifyUnknownAsFalse(
- RexUtil.composeConjunction(rexBuilder,
- viewPredicateList.pulledUpPredicates));
- final Triple<RexNode, RexNode, RexNode> viewPreds =
- splitPredicates(rexBuilder, viewPred);
+ // 3.1. View checks before proceeding
+ if (!isValidPlan(topViewProject, viewNode, mq)) {
+ // Skip it
+ continue;
+ }
- // Extract view table references
- final Set<RelTableRef> viewTableRefs = mq.getTableReferences(viewNode);
- if (viewTableRefs == null) {
+ // 3.2. Initialize all query related auxiliary data structures
+ // that will be used throughout query rewriting process
+ // Extract view predicates
+ final RelOptPredicateList viewPredicateList =
+ mq.getAllPredicates(viewNode);
+ if (viewPredicateList == null) {
+ // Skip it
+ continue;
+ }
+ final RexNode viewPred = simplify.simplifyUnknownAsFalse(
+ RexUtil.composeConjunction(rexBuilder,
+ viewPredicateList.pulledUpPredicates));
+ final Triple<RexNode, RexNode, RexNode> viewPreds =
+ splitPredicates(rexBuilder, viewPred);
+
+ // Extract view tables
+ MatchModality matchModality;
+ Multimap<RexTableInputRef, RexTableInputRef> compensationEquiColumns =
+ ArrayListMultimap.create();
+ if (!queryTableRefs.equals(viewTableRefs)) {
+ // We try to compensate, e.g., for join queries it might be
+ // possible to join missing tables with view to compute result.
+ // Two supported cases: query tables are subset of view tables (we need to
+ // check whether they are cardinality-preserving joins), or view tables are
+ // subset of query tables (add additional tables through joins if possible)
+ if (viewTableRefs.containsAll(queryTableRefs)) {
+ matchModality = MatchModality.QUERY_PARTIAL;
+ final EquivalenceClasses vEC = new EquivalenceClasses();
+ for (RexNode conj : RelOptUtil.conjunctions(viewPreds.getLeft())) {
+ assert conj.isA(SqlKind.EQUALS);
+ RexCall equiCond = (RexCall) conj;
+ vEC.addEquivalenceClass(
+ (RexTableInputRef) equiCond.getOperands().get(0),
+ (RexTableInputRef) equiCond.getOperands().get(1));
+ }
+ if (!compensatePartial(viewTableRefs, vEC, queryTableRefs,
+ compensationEquiColumns)) {
+ // Cannot rewrite, skip it
+ continue;
+ }
+ } else if (queryTableRefs.containsAll(viewTableRefs)) {
+ matchModality = MatchModality.VIEW_PARTIAL;
+ ViewPartialRewriting partialRewritingResult = compensateViewPartial(
+ call.builder(), rexBuilder, mq, view,
+ topProject, node, queryTableRefs, qEC,
+ topViewProject, viewNode, viewTableRefs);
+ if (partialRewritingResult == null) {
+ // Cannot rewrite, skip it
+ continue;
+ }
+ // Rewrite succeeded
+ view = partialRewritingResult.newView;
+ topViewProject = partialRewritingResult.newTopViewProject;
+ viewNode = partialRewritingResult.newViewNode;
+ } else {
// Skip it
continue;
}
+ } else {
+ matchModality = MatchModality.COMPLETE;
+ }
- // Extract view tables
- MatchModality matchModality;
- Multimap<RexTableInputRef, RexTableInputRef> compensationEquiColumns =
- ArrayListMultimap.create();
- if (!queryTableRefs.equals(viewTableRefs)) {
- // We try to compensate, e.g., for join queries it might be
- // possible to join missing tables with view to compute result.
- // Two supported cases: query tables are subset of view tables (we need to
- // check whether they are cardinality-preserving joins), or view tables are
- // subset of query tables (add additional tables through joins if possible)
- if (viewTableRefs.containsAll(queryTableRefs)) {
- matchModality = MatchModality.QUERY_PARTIAL;
- final EquivalenceClasses vEC = new EquivalenceClasses();
- for (RexNode conj : RelOptUtil.conjunctions(viewPreds.getLeft())) {
- assert conj.isA(SqlKind.EQUALS);
- RexCall equiCond = (RexCall) conj;
- vEC.addEquivalenceClass(
- (RexTableInputRef) equiCond.getOperands().get(0),
- (RexTableInputRef) equiCond.getOperands().get(1));
- }
- if (!compensatePartial(viewTableRefs, vEC, queryTableRefs,
- compensationEquiColumns)) {
- // Cannot rewrite, skip it
- continue;
- }
- } else if (queryTableRefs.containsAll(viewTableRefs)) {
- matchModality = MatchModality.VIEW_PARTIAL;
- ViewPartialRewriting partialRewritingResult = compensateViewPartial(
- call.builder(), rexBuilder, mq, view,
- topProject, node, queryTableRefs, qEC,
- topViewProject, viewNode, viewTableRefs);
- if (partialRewritingResult == null) {
- // Cannot rewrite, skip it
- continue;
- }
- // Rewrite succeeded
- view = partialRewritingResult.newView;
- topViewProject = partialRewritingResult.newTopViewProject;
- viewNode = partialRewritingResult.newViewNode;
- } else {
- // Skip it
- continue;
+ // 4. We map every table in the query to a table with the same qualified
+ // name (all query tables are contained in the view, thus this is equivalent
+ // to mapping every table in the query to a view table).
+ final Multimap<RelTableRef, RelTableRef> multiMapTables = ArrayListMultimap.create();
+ for (RelTableRef queryTableRef1 : queryTableRefs) {
+ for (RelTableRef queryTableRef2 : queryTableRefs) {
+ if (queryTableRef1.getQualifiedName().equals(
+ queryTableRef2.getQualifiedName())) {
+ multiMapTables.put(queryTableRef1, queryTableRef2);
}
- } else {
- matchModality = MatchModality.COMPLETE;
}
+ }
- // 4. We map every table in the query to a table with the same qualified
- // name (all query tables are contained in the view, thus this is equivalent
- // to mapping every table in the query to a view table).
- final Multimap<RelTableRef, RelTableRef> multiMapTables = ArrayListMultimap.create();
- for (RelTableRef queryTableRef1 : queryTableRefs) {
- for (RelTableRef queryTableRef2 : queryTableRefs) {
- if (queryTableRef1.getQualifiedName().equals(
- queryTableRef2.getQualifiedName())) {
- multiMapTables.put(queryTableRef1, queryTableRef2);
- }
+ // If a table is used multiple times, we will create multiple mappings,
+ // and we will try to rewrite the query using each of the mappings.
+ // Then, we will try to map every source table (query) to a target
+ // table (view), and if we are successful, we will try to create
+ // compensation predicates to filter the view results further
+ // (if needed).
+ final List<BiMap<RelTableRef, RelTableRef>> flatListMappings =
+ generateTableMappings(multiMapTables);
+ for (BiMap<RelTableRef, RelTableRef> queryToViewTableMapping : flatListMappings) {
+ // TableMapping : mapping query tables -> view tables
+ // 4.0. If compensation equivalence classes exist, we need to add
+ // the mapping to the query mapping
+ final EquivalenceClasses currQEC = EquivalenceClasses.copy(qEC);
+ if (matchModality == MatchModality.QUERY_PARTIAL) {
+ for (Entry<RexTableInputRef, RexTableInputRef> e
+ : compensationEquiColumns.entries()) {
+ // Copy origin
+ RelTableRef queryTableRef = queryToViewTableMapping.inverse().get(
+ e.getKey().getTableRef());
+ RexTableInputRef queryColumnRef = RexTableInputRef.of(queryTableRef,
+ e.getKey().getIndex(), e.getKey().getType());
+ // Add to query equivalence classes and table mapping
+ currQEC.addEquivalenceClass(queryColumnRef, e.getValue());
+ queryToViewTableMapping.put(e.getValue().getTableRef(),
+ e.getValue().getTableRef()); // identity
}
}
- // If a table is used multiple times, we will create multiple mappings,
- // and we will try to rewrite the query using each of the mappings.
- // Then, we will try to map every source table (query) to a target
- // table (view), and if we are successful, we will try to create
- // compensation predicates to filter the view results further
- // (if needed).
- final List<BiMap<RelTableRef, RelTableRef>> flatListMappings =
- generateTableMappings(multiMapTables);
- for (BiMap<RelTableRef, RelTableRef> queryToViewTableMapping : flatListMappings) {
- // TableMapping : mapping query tables -> view tables
- // 4.0. If compensation equivalence classes exist, we need to add
- // the mapping to the query mapping
- final EquivalenceClasses currQEC = EquivalenceClasses.copy(qEC);
- if (matchModality == MatchModality.QUERY_PARTIAL) {
- for (Entry<RexTableInputRef, RexTableInputRef> e
- : compensationEquiColumns.entries()) {
- // Copy origin
- RelTableRef queryTableRef = queryToViewTableMapping.inverse().get(
- e.getKey().getTableRef());
- RexTableInputRef queryColumnRef = RexTableInputRef.of(queryTableRef,
- e.getKey().getIndex(), e.getKey().getType());
- // Add to query equivalence classes and table mapping
- currQEC.addEquivalenceClass(queryColumnRef, e.getValue());
- queryToViewTableMapping.put(e.getValue().getTableRef(),
- e.getValue().getTableRef()); // identity
- }
+ // 4.1. Compute compensation predicates, i.e., predicates that need to be
+ // enforced over the view to retain query semantics. The resulting predicates
+ // are expressed using {@link RexTableInputRef} over the query.
+ // First, to establish relationship, we swap column references of the view
+ // predicates to point to query tables and compute equivalence classes.
+ final RexNode viewColumnsEquiPred = RexUtil.swapTableReferences(
+ rexBuilder, viewPreds.getLeft(), queryToViewTableMapping.inverse());
+ final EquivalenceClasses queryBasedVEC = new EquivalenceClasses();
+ for (RexNode conj : RelOptUtil.conjunctions(viewColumnsEquiPred)) {
+ assert conj.isA(SqlKind.EQUALS);
+ RexCall equiCond = (RexCall) conj;
+ queryBasedVEC.addEquivalenceClass(
+ (RexTableInputRef) equiCond.getOperands().get(0),
+ (RexTableInputRef) equiCond.getOperands().get(1));
+ }
+ Triple<RexNode, RexNode, RexNode> compensationPreds =
+ computeCompensationPredicates(rexBuilder, simplify,
+ currQEC, queryPreds, queryBasedVEC, viewPreds,
+ queryToViewTableMapping);
+ if (compensationPreds == null && generateUnionRewriting) {
+ // Attempt partial rewriting using union operator. This rewriting
+ // will read some data from the view and the rest of the data from
+ // the query computation. The resulting predicates are expressed
+ // using {@link RexTableInputRef} over the view.
+ compensationPreds = computeCompensationPredicates(rexBuilder, simplify,
+ queryBasedVEC, viewPreds, currQEC, queryPreds,
+ queryToViewTableMapping.inverse());
+ if (compensationPreds == null) {
+ // This was our last chance to use the view, skip it
+ continue;
}
-
- // 4.1. Compute compensation predicates, i.e., predicates that need to be
- // enforced over the view to retain query semantics. The resulting predicates
- // are expressed using {@link RexTableInputRef} over the query.
- // First, to establish relationship, we swap column references of the view
- // predicates to point to query tables and compute equivalence classes.
- final RexNode viewColumnsEquiPred = RexUtil.swapTableReferences(
- rexBuilder, viewPreds.getLeft(), queryToViewTableMapping.inverse());
- final EquivalenceClasses queryBasedVEC = new EquivalenceClasses();
- for (RexNode conj : RelOptUtil.conjunctions(viewColumnsEquiPred)) {
- assert conj.isA(SqlKind.EQUALS);
- RexCall equiCond = (RexCall) conj;
- queryBasedVEC.addEquivalenceClass(
- (RexTableInputRef) equiCond.getOperands().get(0),
- (RexTableInputRef) equiCond.getOperands().get(1));
+ RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
+ RexNode otherCompensationPred =
+ RexUtil.composeConjunction(rexBuilder,
+ ImmutableList.of(compensationPreds.getMiddle(),
+ compensationPreds.getRight()));
+ assert !compensationColumnsEquiPred.isAlwaysTrue()
+ || !otherCompensationPred.isAlwaysTrue();
+
+ // b. Generate union branch (query).
+ final RelNode unionInputQuery = rewriteQuery(call.builder(), rexBuilder,
+ simplify, mq, compensationColumnsEquiPred, otherCompensationPred,
+ topProject, node, queryToViewTableMapping, queryBasedVEC, currQEC);
+ if (unionInputQuery == null) {
+ // Skip it
+ continue;
}
- Triple<RexNode, RexNode, RexNode> compensationPreds =
- computeCompensationPredicates(rexBuilder, simplify,
- currQEC, queryPreds, queryBasedVEC, viewPreds,
- queryToViewTableMapping);
- if (compensationPreds == null && generateUnionRewriting) {
- // Attempt partial rewriting using union operator. This rewriting
- // will read some data from the view and the rest of the data from
- // the query computation. The resulting predicates are expressed
- // using {@link RexTableInputRef} over the view.
- compensationPreds = computeCompensationPredicates(rexBuilder, simplify,
- queryBasedVEC, viewPreds, currQEC, queryPreds,
- queryToViewTableMapping.inverse());
- if (compensationPreds == null) {
- // This was our last chance to use the view, skip it
- continue;
- }
- RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
- RexNode otherCompensationPred =
- RexUtil.composeConjunction(rexBuilder,
- ImmutableList.of(compensationPreds.getMiddle(),
- compensationPreds.getRight()));
- assert !compensationColumnsEquiPred.isAlwaysTrue()
- || !otherCompensationPred.isAlwaysTrue();
-
- // b. Generate union branch (query).
- final RelNode unionInputQuery = rewriteQuery(call.builder(), rexBuilder,
- simplify, mq, compensationColumnsEquiPred, otherCompensationPred,
- topProject, node, queryToViewTableMapping, queryBasedVEC, currQEC);
- if (unionInputQuery == null) {
- // Skip it
- continue;
- }
- // c. Generate union branch (view).
- // We trigger the unifying method. This method will either create a Project
- // or an Aggregate operator on top of the view. It will also compute the
- // output expressions for the query.
- final RelNode unionInputView = rewriteView(call.builder(), rexBuilder, simplify, mq,
- matchModality, true, view, topProject, node, topViewProject, viewNode,
- queryToViewTableMapping, currQEC);
- if (unionInputView == null) {
- // Skip it
- continue;
- }
+ // c. Generate union branch (view).
+ // We trigger the unifying method. This method will either create a Project
+ // or an Aggregate operator on top of the view. It will also compute the
+ // output expressions for the query.
+ final RelNode unionInputView = rewriteView(call.builder(), rexBuilder, simplify, mq,
+ matchModality, true, view, topProject, node, topViewProject, viewNode,
+ queryToViewTableMapping, currQEC);
+ if (unionInputView == null) {
+ // Skip it
+ continue;
+ }
- // d. Generate final rewriting (union).
- final RelNode result = createUnion(call.builder(), rexBuilder,
- topProject, unionInputQuery, unionInputView);
- if (result == null) {
- // Skip it
- continue;
- }
- call.transformTo(result);
- } else if (compensationPreds != null) {
- RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
- RexNode otherCompensationPred =
- RexUtil.composeConjunction(rexBuilder,
- ImmutableList.of(compensationPreds.getMiddle(),
- compensationPreds.getRight()));
-
- // a. Compute final compensation predicate.
- if (!compensationColumnsEquiPred.isAlwaysTrue()
- || !otherCompensationPred.isAlwaysTrue()) {
- // All columns required by compensating predicates must be contained
- // in the view output (condition 2).
- List<RexNode> viewExprs = topViewProject == null
- ? extractReferences(rexBuilder, view)
- : topViewProject.getChildExps();
- // For compensationColumnsEquiPred, we use the view equivalence classes,
- // since we want to enforce the rest
- if (!compensationColumnsEquiPred.isAlwaysTrue()) {
- compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
- view, viewNode, viewExprs, queryToViewTableMapping.inverse(), queryBasedVEC,
- false, compensationColumnsEquiPred);
- if (compensationColumnsEquiPred == null) {
- // Skip it
- continue;
- }
- }
- // For the rest, we use the query equivalence classes
- if (!otherCompensationPred.isAlwaysTrue()) {
- otherCompensationPred = rewriteExpression(rexBuilder, mq,
- view, viewNode, viewExprs, queryToViewTableMapping.inverse(), currQEC,
- true, otherCompensationPred);
- if (otherCompensationPred == null) {
- // Skip it
- continue;
- }
+ // d. Generate final rewriting (union).
+ final RelNode result = createUnion(call.builder(), rexBuilder,
+ topProject, unionInputQuery, unionInputView);
+ if (result == null) {
+ // Skip it
+ continue;
+ }
+ call.transformTo(result);
+ } else if (compensationPreds != null) {
+ RexNode compensationColumnsEquiPred = compensationPreds.getLeft();
+ RexNode otherCompensationPred =
+ RexUtil.composeConjunction(rexBuilder,
+ ImmutableList.of(compensationPreds.getMiddle(),
+ compensationPreds.getRight()));
+
+ // a. Compute final compensation predicate.
+ if (!compensationColumnsEquiPred.isAlwaysTrue()
+ || !otherCompensationPred.isAlwaysTrue()) {
+ // All columns required by compensating predicates must be contained
+ // in the view output (condition 2).
+ List<RexNode> viewExprs = topViewProject == null
+ ? extractReferences(rexBuilder, view)
+ : topViewProject.getChildExps();
+ // For compensationColumnsEquiPred, we use the view equivalence classes,
+ // since we want to enforce the rest
+ if (!compensationColumnsEquiPred.isAlwaysTrue()) {
+ compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
+ view, viewNode, viewExprs, queryToViewTableMapping.inverse(), queryBasedVEC,
+ false, compensationColumnsEquiPred);
+ if (compensationColumnsEquiPred == null) {
+ // Skip it
+ continue;
}
}
- final RexNode viewCompensationPred =
- RexUtil.composeConjunction(rexBuilder,
- ImmutableList.of(compensationColumnsEquiPred,
- otherCompensationPred));
-
- // b. Generate final rewriting if possible.
- // First, we add the compensation predicate (if any) on top of the view.
- // Then, we trigger the unifying method. This method will either create a
- // Project or an Aggregate operator on top of the view. It will also compute
- // the output expressions for the query.
- RelBuilder builder = call.builder();
- RelNode viewWithFilter;
- if (!viewCompensationPred.isAlwaysTrue()) {
- RexNode newPred =
- simplify.simplifyUnknownAsFalse(viewCompensationPred);
- viewWithFilter = builder.push(view).filter(newPred).build();
- // We add (and push) the filter to the view plan before triggering the rewriting.
- // This is useful in case some of the columns can be folded to same value after
- // filter is added.
- Pair<RelNode, RelNode> pushedNodes =
- pushFilterToOriginalViewPlan(builder, topViewProject, viewNode, newPred);
- topViewProject = (Project) pushedNodes.left;
- viewNode = pushedNodes.right;
- } else {
- viewWithFilter = builder.push(view).build();
- }
- final RelNode result = rewriteView(builder, rexBuilder, simplify, mq, matchModality,
- false, viewWithFilter, topProject, node, topViewProject, viewNode,
- queryToViewTableMapping, currQEC);
- if (result == null) {
- // Skip it
- continue;
+ // For the rest, we use the query equivalence classes
+ if (!otherCompensationPred.isAlwaysTrue()) {
+ otherCompensationPred = rewriteExpression(rexBuilder, mq,
+ view, viewNode, viewExprs, queryToViewTableMapping.inverse(), currQEC,
+ true, otherCompensationPred);
+ if (otherCompensationPred == null) {
+ // Skip it
+ continue;
+ }
}
- call.transformTo(result);
- } // end else
- }
+ }
+ final RexNode viewCompensationPred =
+ RexUtil.composeConjunction(rexBuilder,
+ ImmutableList.of(compensationColumnsEquiPred,
+ otherCompensationPred));
+
+ // b. Generate final rewriting if possible.
+ // First, we add the compensation predicate (if any) on top of the view.
+ // Then, we trigger the unifying method. This method will either create a
+ // Project or an Aggregate operator on top of the view. It will also compute
+ // the output expressions for the query.
+ RelBuilder builder = call.builder();
+ RelNode viewWithFilter;
+ if (!viewCompensationPred.isAlwaysTrue()) {
+ RexNode newPred =
+ simplify.simplifyUnknownAsFalse(viewCompensationPred);
+ viewWithFilter = builder.push(view).filter(newPred).build();
+ // We add (and push) the filter to the view plan before triggering the rewriting.
+ // This is useful in case some of the columns can be folded to same value after
+ // filter is added.
+ Pair<RelNode, RelNode> pushedNodes =
+ pushFilterToOriginalViewPlan(builder, topViewProject, viewNode, newPred);
+ topViewProject = (Project) pushedNodes.left;
+ viewNode = pushedNodes.right;
+ } else {
+ viewWithFilter = builder.push(view).build();
+ }
+ final RelNode result = rewriteView(builder, rexBuilder, simplify, mq, matchModality,
+ false, viewWithFilter, topProject, node, topViewProject, viewNode,
+ queryToViewTableMapping, currQEC);
+ if (result == null) {
+ // Skip it
+ continue;
+ }
+ call.transformTo(result);
+ } // end else
}
}
}
[calcite] 01/04: [CALCITE-2942] Materialized view rewriting logic
instantiates RelMetadataQuery each time the rule is triggered
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 79f432457f843060a76177c4d80f3c5d72d8047d
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Thu Mar 21 09:25:06 2019 -0700
[CALCITE-2942] Materialized view rewriting logic instantiates RelMetadataQuery each time the rule is triggered
Close apache/calcite#1124
---
core/src/main/java/org/apache/calcite/plan/RelOptUtil.java | 2 +-
.../java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
index 1c69c15..c7f9048 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptUtil.java
@@ -201,7 +201,7 @@ public abstract class RelOptUtil {
*/
public static List<RelOptTable> findAllTables(RelNode rel) {
final Multimap<Class<? extends RelNode>, RelNode> nodes =
- RelMetadataQuery.instance().getNodeTypes(rel);
+ rel.getCluster().getMetadataQuery().getNodeTypes(rel);
final List<RelOptTable> usedTables = new ArrayList<>();
for (Entry<Class<? extends RelNode>, Collection<RelNode>> e : nodes.asMap().entrySet()) {
if (TableScan.class.isAssignableFrom(e.getKey())) {
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
index 94c456f..706dcca 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
@@ -203,7 +203,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
*/
protected void perform(RelOptRuleCall call, Project topProject, RelNode node) {
final RexBuilder rexBuilder = node.getCluster().getRexBuilder();
- final RelMetadataQuery mq = RelMetadataQuery.instance();
+ final RelMetadataQuery mq = call.getMetadataQuery();
final RelOptPlanner planner = call.getPlanner();
final RexExecutor executor =
Util.first(planner.getExecutor(), RexUtil.EXECUTOR);
[calcite] 04/04: [CALCITE-2951] Support decorrelate subquery that
has aggregate with grouping sets (Haisheng Yuan)
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 406129b97a47262f7307de64bf24c26b2eba70eb
Author: Haisheng Yuan <h....@alibaba-inc.com>
AuthorDate: Tue Mar 19 18:30:36 2019 -0700
[CALCITE-2951] Support decorrelate subquery that has aggregate with grouping sets (Haisheng Yuan)
Close apache/calcite#1118
---
.../apache/calcite/sql2rel/RelDecorrelator.java | 16 +++++++++-----
.../apache/calcite/test/SqlToRelConverterTest.java | 13 +++++++++++
.../apache/calcite/test/SqlToRelConverterTest.xml | 25 ++++++++++++++++++++++
3 files changed, 49 insertions(+), 5 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
index 10cbfa1..da674d9 100644
--- a/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
+++ b/core/src/main/java/org/apache/calcite/sql2rel/RelDecorrelator.java
@@ -76,7 +76,6 @@ import org.apache.calcite.sql.fun.SqlSingleValueAggFunction;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.Bug;
import org.apache.calcite.util.Holder;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Litmus;
@@ -447,9 +446,6 @@ public class RelDecorrelator implements ReflectiveVisitor {
* @param rel Aggregate to rewrite
*/
public Frame decorrelateRel(LogicalAggregate rel) {
- if (rel.getGroupType() != Aggregate.Group.SIMPLE) {
- throw new AssertionError(Bug.CALCITE_461_FIXED);
- }
//
// Rewrite logic:
//
@@ -560,6 +556,16 @@ public class RelDecorrelator implements ReflectiveVisitor {
List<AggregateCall> newAggCalls = new ArrayList<>();
List<AggregateCall> oldAggCalls = rel.getAggCallList();
+ ImmutableList<ImmutableBitSet> newGroupSets = null;
+ if (rel.getGroupType() != Aggregate.Group.SIMPLE) {
+ final ImmutableBitSet addedGroupSet =
+ ImmutableBitSet.range(oldGroupKeyCount, newGroupKeyCount);
+ final Iterable<ImmutableBitSet> tmpGroupSets =
+ Iterables.transform(rel.getGroupSets(),
+ bitSet -> bitSet.union(addedGroupSet));
+ newGroupSets = ImmutableBitSet.ORDERING.immutableSortedCopy(tmpGroupSets);
+ }
+
int oldInputOutputFieldCount = rel.getGroupSet().cardinality();
int newInputOutputFieldCount = newGroupSet.cardinality();
@@ -592,7 +598,7 @@ public class RelDecorrelator implements ReflectiveVisitor {
}
relBuilder.push(
- LogicalAggregate.create(newProject, newGroupSet, null, newAggCalls));
+ LogicalAggregate.create(newProject, newGroupSet, newGroupSets, newAggCalls));
if (!omittedConstants.isEmpty()) {
final List<RexNode> postProjects = new ArrayList<>(relBuilder.fields());
diff --git a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
index 9976321..d416eb8 100644
--- a/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
+++ b/core/src/test/java/org/apache/calcite/test/SqlToRelConverterTest.java
@@ -2457,6 +2457,19 @@ public class SqlToRelConverterTest extends SqlToRelTestBase {
sql(sql).decorrelate(true).ok();
}
+ /**
+ * Test case for decorrelating sub-query that has aggregate with
+ * grouping sets.
+ */
+ @Test public void testCorrelationAggregateGroupSets() {
+ final String sql = "select sum(e1.empno)\n"
+ + "from emp e1, dept d1\n"
+ + "where e1.deptno = d1.deptno\n"
+ + "and e1.sal > (select avg(e2.sal) from emp e2\n"
+ + " where e2.deptno = d1.deptno group by cube(comm, mgr))";
+ sql(sql).decorrelate(true).ok();
+ }
+
@Test public void testCustomColumnResolving() {
final String sql = "select k0 from struct.t";
sql(sql).ok();
diff --git a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
index c24e21b..24289d3 100644
--- a/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/SqlToRelConverterTest.xml
@@ -3972,6 +3972,31 @@ LogicalProject(EMPNO=[$0])
]]>
</Resource>
</TestCase>
+ <TestCase name="testCorrelationAggregateGroupSets">
+ <Resource name="sql">
+ <![CDATA[select sum(e1.empno)
+from emp e1, dept d1
+where e1.deptno = d1.deptno
+and e1.sal > (select avg(e2.sal) from emp e2
+where e2.deptno = d1.deptno group by cube(comm, mgr))]]>
+ </Resource>
+ <Resource name="plan">
+ <![CDATA[
+LogicalAggregate(group=[{}], EXPR$0=[SUM($0)])
+ LogicalProject(EMPNO=[$0])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5], COMM=[$6], DEPTNO=[$7], SLACKER=[$8], DEPTNO0=[$9], NAME=[$10], DEPTNO1=[CAST($11):INTEGER], $f1=[CAST($12):INTEGER])
+ LogicalJoin(condition=[AND(=($9, $11), >($5, $12))], joinType=[inner])
+ LogicalJoin(condition=[=($7, $9)], joinType=[inner])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ LogicalAggregate(group=[{0}], agg#0=[SINGLE_VALUE($1)])
+ LogicalProject(DEPTNO=[$2], EXPR$0=[$3])
+ LogicalAggregate(group=[{0, 1, 2}], groups=[[{0, 1, 2}, {0, 2}, {1, 2}, {2}]], EXPR$0=[AVG($3)])
+ LogicalProject(COMM=[$6], MGR=[$3], DEPTNO=[$7], SAL=[$5])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
<TestCase name="testFakeStar">
<Resource name="sql">
<![CDATA[SELECT * FROM (VALUES (0, 0)) AS T(A, "*")]]>