You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2020/02/27 04:52:23 UTC
[calcite] 01/02: [CALCITE-3825] Split AbstractMaterializedViewRule
into multiple classes
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit efa5cb7ba6a754ef98281f4f142c16d895a49df4
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Mon Sep 9 15:04:05 2019 -0700
[CALCITE-3825] Split AbstractMaterializedViewRule into multiple classes
Close apache/calcite#1831
---
.../java/org/apache/calcite/plan/RelOptRules.java | 19 +-
.../materialize/MaterializedViewAggregateRule.java | 933 ++++++++++++++
.../materialize/MaterializedViewJoinRule.java | 283 +++++
.../MaterializedViewOnlyAggregateRule.java | 65 +
.../MaterializedViewOnlyFilterRule.java | 46 +
.../materialize/MaterializedViewOnlyJoinRule.java | 46 +
.../MaterializedViewProjectAggregateRule.java | 67 +
.../MaterializedViewProjectFilterRule.java | 49 +
.../MaterializedViewProjectJoinRule.java | 49 +
.../MaterializedViewRule.java} | 1293 +-------------------
.../rel/rules/materialize/package-info.java | 21 +
.../java/org/apache/calcite/test/LatticeTest.java | 19 +-
site/_docs/materialized_views.md | 4 +-
13 files changed, 1609 insertions(+), 1285 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java
index 94c395f..475163b 100644
--- a/core/src/main/java/org/apache/calcite/plan/RelOptRules.java
+++ b/core/src/main/java/org/apache/calcite/plan/RelOptRules.java
@@ -21,7 +21,6 @@ import org.apache.calcite.config.CalciteSystemProperty;
import org.apache.calcite.interpreter.NoneToBindableConverterRule;
import org.apache.calcite.linq4j.function.Experimental;
import org.apache.calcite.plan.volcano.AbstractConverter;
-import org.apache.calcite.rel.rules.AbstractMaterializedViewRule;
import org.apache.calcite.rel.rules.AggregateCaseToFilterRule;
import org.apache.calcite.rel.rules.AggregateExpandDistinctAggregatesRule;
import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
@@ -69,6 +68,12 @@ import org.apache.calcite.rel.rules.UnionMergeRule;
import org.apache.calcite.rel.rules.UnionPullUpConstantsRule;
import org.apache.calcite.rel.rules.UnionToDistinctRule;
import org.apache.calcite.rel.rules.ValuesReduceRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyAggregateRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyFilterRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyJoinRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectAggregateRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectFilterRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectJoinRule;
import com.google.common.collect.ImmutableList;
@@ -186,10 +191,10 @@ public class RelOptRules {
public static final List<RelOptRule> MATERIALIZATION_RULES = ImmutableList.of(
MaterializedViewFilterScanRule.INSTANCE,
- AbstractMaterializedViewRule.INSTANCE_PROJECT_FILTER,
- AbstractMaterializedViewRule.INSTANCE_FILTER,
- AbstractMaterializedViewRule.INSTANCE_PROJECT_JOIN,
- AbstractMaterializedViewRule.INSTANCE_JOIN,
- AbstractMaterializedViewRule.INSTANCE_PROJECT_AGGREGATE,
- AbstractMaterializedViewRule.INSTANCE_AGGREGATE);
+ MaterializedViewProjectFilterRule.INSTANCE,
+ MaterializedViewOnlyFilterRule.INSTANCE,
+ MaterializedViewProjectJoinRule.INSTANCE,
+ MaterializedViewOnlyJoinRule.INSTANCE,
+ MaterializedViewProjectAggregateRule.INSTANCE,
+ MaterializedViewOnlyAggregateRule.INSTANCE);
}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java
new file mode 100644
index 0000000..de48ccd
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewAggregateRule.java
@@ -0,0 +1,933 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.avatica.util.TimeUnitRange;
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.plan.hep.HepProgramBuilder;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule;
+import org.apache.calcite.rel.rules.FilterAggregateTransposeRule;
+import org.apache.calcite.rel.rules.FilterProjectTransposeRule;
+import org.apache.calcite.rel.rules.ProjectMergeRule;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexPermuteInputsShuttle;
+import org.apache.calcite.rex.RexSimplify;
+import org.apache.calcite.rex.RexTableInputRef;
+import org.apache.calcite.rex.RexTableInputRef.RelTableRef;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilder.AggCall;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.ImmutableBitSet;
+import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.mapping.Mapping;
+import org.apache.calcite.util.mapping.MappingType;
+import org.apache.calcite.util.mapping.Mappings;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/** Materialized view rewriting for aggregate */
+public abstract class MaterializedViewAggregateRule extends MaterializedViewRule {
+
+ protected static final ImmutableList<TimeUnitRange> SUPPORTED_DATE_TIME_ROLLUP_UNITS =
+ ImmutableList.of(TimeUnitRange.YEAR, TimeUnitRange.QUARTER, TimeUnitRange.MONTH,
+ TimeUnitRange.DAY, TimeUnitRange.HOUR, TimeUnitRange.MINUTE,
+ TimeUnitRange.SECOND, TimeUnitRange.MILLISECOND, TimeUnitRange.MICROSECOND);
+
+ //~ Instance fields --------------------------------------------------------
+
+ /** Instance of rule to push filter through project. */
+ protected final RelOptRule filterProjectTransposeRule;
+
+ /** Instance of rule to push filter through aggregate. */
+ protected final RelOptRule filterAggregateTransposeRule;
+
+ /** Instance of rule to pull up constants into aggregate. */
+ protected final RelOptRule aggregateProjectPullUpConstantsRule;
+
+ /** Instance of rule to merge project operators. */
+ protected final RelOptRule projectMergeRule;
+
+
+ /** Creates a MaterializedViewAggregateRule. */
+ protected MaterializedViewAggregateRule(RelOptRuleOperand operand,
+ RelBuilderFactory relBuilderFactory, String description,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
+ this(operand, relBuilderFactory, description, generateUnionRewriting,
+ unionRewritingPullProgram,
+ new FilterProjectTransposeRule(
+ Filter.class, Project.class, true, true, relBuilderFactory),
+ new FilterAggregateTransposeRule(
+ Filter.class, relBuilderFactory, Aggregate.class),
+ new AggregateProjectPullUpConstantsRule(
+ Aggregate.class, Filter.class, relBuilderFactory, "AggFilterPullUpConstants"),
+ new ProjectMergeRule(true, ProjectMergeRule.DEFAULT_BLOAT,
+ relBuilderFactory));
+ }
+
+ /** Creates a MaterializedViewAggregateRule. */
+ protected MaterializedViewAggregateRule(RelOptRuleOperand operand,
+ RelBuilderFactory relBuilderFactory, String description,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ RelOptRule filterProjectTransposeRule,
+ RelOptRule filterAggregateTransposeRule,
+ RelOptRule aggregateProjectPullUpConstantsRule,
+ RelOptRule projectMergeRule) {
+ super(operand, relBuilderFactory, description, generateUnionRewriting,
+ unionRewritingPullProgram, false);
+ this.filterProjectTransposeRule = filterProjectTransposeRule;
+ this.filterAggregateTransposeRule = filterAggregateTransposeRule;
+ this.aggregateProjectPullUpConstantsRule = aggregateProjectPullUpConstantsRule;
+ this.projectMergeRule = projectMergeRule;
+ }
+
+ @Override protected boolean isValidPlan(Project topProject, RelNode node,
+ RelMetadataQuery mq) {
+ if (!(node instanceof Aggregate)) {
+ return false;
+ }
+ Aggregate aggregate = (Aggregate) node;
+ if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
+ // TODO: Rewriting with grouping sets not supported yet
+ return false;
+ }
+ return isValidRelNodePlan(aggregate.getInput(), mq);
+ }
+
+ @Override protected ViewPartialRewriting compensateViewPartial(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RelMetadataQuery mq,
+ RelNode input,
+ Project topProject,
+ RelNode node,
+ Set<RelTableRef> queryTableRefs,
+ EquivalenceClasses queryEC,
+ Project topViewProject,
+ RelNode viewNode,
+ Set<RelTableRef> viewTableRefs) {
+ // Modify view to join with missing tables and add Project on top to reorder columns.
+ // In turn, modify view plan to join with missing tables before Aggregate operator,
+ // change Aggregate operator to group by previous grouping columns and columns in
+ // attached tables, and add a final Project on top.
+ // We only need to add the missing tables on top of the view and view plan using
+ // a cartesian product.
+ // Then the rest of the rewriting algorithm can be executed in the same
+ // fashion, and if there are predicates between the existing and missing
+ // tables, the rewriting algorithm will enforce them.
+ final Set<RelTableRef> extraTableRefs = new HashSet<>();
+ for (RelTableRef tRef : queryTableRefs) {
+ if (!viewTableRefs.contains(tRef)) {
+ // Add to extra tables if table is not part of the view
+ extraTableRefs.add(tRef);
+ }
+ }
+ Collection<RelNode> tableScanNodes = mq.getNodeTypes(node).get(TableScan.class);
+ List<RelNode> newRels = new ArrayList<>();
+ for (RelTableRef tRef : extraTableRefs) {
+ int i = 0;
+ for (RelNode relNode : tableScanNodes) {
+ if (tRef.getQualifiedName().equals(relNode.getTable().getQualifiedName())) {
+ if (tRef.getEntityNumber() == i++) {
+ newRels.add(relNode);
+ break;
+ }
+ }
+ }
+ }
+ assert extraTableRefs.size() == newRels.size();
+
+ relBuilder.push(input);
+ for (RelNode newRel : newRels) {
+ // Add to the view
+ relBuilder.push(newRel);
+ relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
+ }
+ final RelNode newView = relBuilder.build();
+
+ final Aggregate aggregateViewNode = (Aggregate) viewNode;
+ relBuilder.push(aggregateViewNode.getInput());
+ int offset = 0;
+ for (RelNode newRel : newRels) {
+ // Add to the view plan
+ relBuilder.push(newRel);
+ relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
+ offset += newRel.getRowType().getFieldCount();
+ }
+ // Modify aggregate: add grouping columns
+ ImmutableBitSet.Builder groupSet = ImmutableBitSet.builder();
+ groupSet.addAll(aggregateViewNode.getGroupSet());
+ groupSet.addAll(
+ ImmutableBitSet.range(
+ aggregateViewNode.getInput().getRowType().getFieldCount(),
+ aggregateViewNode.getInput().getRowType().getFieldCount() + offset));
+ final Aggregate newViewNode = aggregateViewNode.copy(
+ aggregateViewNode.getTraitSet(), relBuilder.build(),
+ groupSet.build(), null, aggregateViewNode.getAggCallList());
+
+ relBuilder.push(newViewNode);
+ List<RexNode> nodes = new ArrayList<>();
+ List<String> fieldNames = new ArrayList<>();
+ if (topViewProject != null) {
+ // Insert existing expressions (and shift aggregation arguments),
+ // then append rest of columns
+ Mappings.TargetMapping shiftMapping = Mappings.createShiftMapping(
+ newViewNode.getRowType().getFieldCount(),
+ 0, 0, aggregateViewNode.getGroupCount(),
+ newViewNode.getGroupCount(), aggregateViewNode.getGroupCount(),
+ aggregateViewNode.getAggCallList().size());
+ for (int i = 0; i < topViewProject.getChildExps().size(); i++) {
+ nodes.add(
+ topViewProject.getChildExps().get(i).accept(
+ new RexPermuteInputsShuttle(shiftMapping, newViewNode)));
+ fieldNames.add(topViewProject.getRowType().getFieldNames().get(i));
+ }
+ for (int i = aggregateViewNode.getRowType().getFieldCount();
+ i < newViewNode.getRowType().getFieldCount(); i++) {
+ int idx = i - aggregateViewNode.getAggCallList().size();
+ nodes.add(rexBuilder.makeInputRef(newViewNode, idx));
+ fieldNames.add(newViewNode.getRowType().getFieldNames().get(idx));
+ }
+ } else {
+ // Original grouping columns, aggregation columns, then new grouping columns
+ for (int i = 0; i < newViewNode.getRowType().getFieldCount(); i++) {
+ int idx;
+ if (i < aggregateViewNode.getGroupCount()) {
+ idx = i;
+ } else if (i < aggregateViewNode.getRowType().getFieldCount()) {
+ idx = i + offset;
+ } else {
+ idx = i - aggregateViewNode.getAggCallList().size();
+ }
+ nodes.add(rexBuilder.makeInputRef(newViewNode, idx));
+ fieldNames.add(newViewNode.getRowType().getFieldNames().get(idx));
+ }
+ }
+ relBuilder.project(nodes, fieldNames, true);
+ final Project newTopViewProject = (Project) relBuilder.build();
+
+ return ViewPartialRewriting.of(newView, newTopViewProject, newViewNode);
+ }
+
+ @Override protected RelNode rewriteQuery(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RexSimplify simplify,
+ RelMetadataQuery mq,
+ RexNode compensationColumnsEquiPred,
+ RexNode otherCompensationPred,
+ Project topProject,
+ RelNode node,
+ BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
+ EquivalenceClasses viewEC, EquivalenceClasses queryEC) {
+ Aggregate aggregate = (Aggregate) node;
+
+ // Our target node is the node below the root, which should have the maximum
+ // number of available expressions in the tree in order to maximize our
+ // number of rewritings.
+ // If the program is available, we execute it to maximize rewriting opportunities.
+ // For instance, a program might pull up all the expressions that are below the
+ // aggregate so we can introduce compensation filters easily. This is important
+ // depending on the planner strategy.
+ RelNode newAggregateInput = aggregate.getInput(0);
+ RelNode target = aggregate.getInput(0);
+ if (unionRewritingPullProgram != null) {
+ final HepPlanner tmpPlanner = new HepPlanner(unionRewritingPullProgram);
+ tmpPlanner.setRoot(newAggregateInput);
+ newAggregateInput = tmpPlanner.findBestExp();
+ target = newAggregateInput.getInput(0);
+ }
+
+ // We need to check that all columns required by compensating predicates
+ // are contained in the query.
+ List<RexNode> queryExprs = extractReferences(rexBuilder, target);
+ if (!compensationColumnsEquiPred.isAlwaysTrue()) {
+ compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
+ target, target, queryExprs, queryToViewTableMapping, queryEC, false,
+ compensationColumnsEquiPred);
+ if (compensationColumnsEquiPred == null) {
+ // Skip it
+ return null;
+ }
+ }
+ // For the rest, we use the query equivalence classes
+ if (!otherCompensationPred.isAlwaysTrue()) {
+ otherCompensationPred = rewriteExpression(rexBuilder, mq,
+ target, target, queryExprs, queryToViewTableMapping, viewEC, true,
+ otherCompensationPred);
+ if (otherCompensationPred == null) {
+ // Skip it
+ return null;
+ }
+ }
+ final RexNode queryCompensationPred = RexUtil.not(
+ RexUtil.composeConjunction(rexBuilder,
+ ImmutableList.of(compensationColumnsEquiPred,
+ otherCompensationPred)));
+
+ // Generate query rewriting.
+ RelNode rewrittenPlan = relBuilder
+ .push(target)
+ .filter(simplify.simplifyUnknownAsFalse(queryCompensationPred))
+ .build();
+ if (unionRewritingPullProgram != null) {
+ return aggregate.copy(aggregate.getTraitSet(),
+ ImmutableList.of(
+ newAggregateInput.copy(newAggregateInput.getTraitSet(),
+ ImmutableList.of(rewrittenPlan))));
+ }
+ return aggregate.copy(aggregate.getTraitSet(), ImmutableList.of(rewrittenPlan));
+ }
+
+ @Override protected RelNode createUnion(RelBuilder relBuilder, RexBuilder rexBuilder,
+ RelNode topProject, RelNode unionInputQuery, RelNode unionInputView) {
+ // Union
+ relBuilder.push(unionInputQuery);
+ relBuilder.push(unionInputView);
+ relBuilder.union(true);
+ List<RexNode> exprList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
+ List<String> nameList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
+ for (int i = 0; i < relBuilder.peek().getRowType().getFieldCount(); i++) {
+ // We can take unionInputQuery as it is query based.
+ RelDataTypeField field = unionInputQuery.getRowType().getFieldList().get(i);
+ exprList.add(
+ rexBuilder.ensureType(
+ field.getType(),
+ rexBuilder.makeInputRef(relBuilder.peek(), i),
+ true));
+ nameList.add(field.getName());
+ }
+ relBuilder.project(exprList, nameList);
+ // Rollup aggregate
+ Aggregate aggregate = (Aggregate) unionInputQuery;
+ final ImmutableBitSet groupSet = ImmutableBitSet.range(aggregate.getGroupCount());
+ final List<AggCall> aggregateCalls = new ArrayList<>();
+ for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
+ AggregateCall aggCall = aggregate.getAggCallList().get(i);
+ if (aggCall.isDistinct()) {
+ // Cannot ROLLUP distinct
+ return null;
+ }
+ SqlAggFunction rollupAgg =
+ getRollup(aggCall.getAggregation());
+ if (rollupAgg == null) {
+ // Cannot rollup this aggregate, bail out
+ return null;
+ }
+ final RexInputRef operand =
+ rexBuilder.makeInputRef(relBuilder.peek(),
+ aggregate.getGroupCount() + i);
+ aggregateCalls.add(
+ // TODO: handle aggregate ordering
+ relBuilder.aggregateCall(rollupAgg, operand)
+ .distinct(aggCall.isDistinct())
+ .approximate(aggCall.isApproximate())
+ .as(aggCall.name));
+ }
+ RelNode prevNode = relBuilder.peek();
+ RelNode result = relBuilder
+ .aggregate(relBuilder.groupKey(groupSet), aggregateCalls)
+ .build();
+ if (prevNode == result && groupSet.cardinality() != result.getRowType().getFieldCount()) {
+ // Aggregate was not inserted but we need to prune columns
+ result = relBuilder
+ .push(result)
+ .project(relBuilder.fields(groupSet))
+ .build();
+ }
+ if (topProject != null) {
+ // Top project
+ return topProject.copy(topProject.getTraitSet(), ImmutableList.of(result));
+ }
+ // Result
+ return result;
+ }
+
+ @Override protected RelNode rewriteView(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RexSimplify simplify,
+ RelMetadataQuery mq,
+ MatchModality matchModality,
+ boolean unionRewriting,
+ RelNode input,
+ Project topProject,
+ RelNode node,
+ Project topViewProject,
+ RelNode viewNode,
+ BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
+ EquivalenceClasses queryEC) {
+ final Aggregate queryAggregate = (Aggregate) node;
+ final Aggregate viewAggregate = (Aggregate) viewNode;
+ // Get group by references and aggregate call input references needed
+ ImmutableBitSet.Builder indexes = ImmutableBitSet.builder();
+ ImmutableBitSet references = null;
+ if (topProject != null && !unionRewriting) {
+ // We have a Project on top, gather only what is needed
+ final RelOptUtil.InputFinder inputFinder =
+ new RelOptUtil.InputFinder(new LinkedHashSet<>());
+ for (RexNode e : topProject.getChildExps()) {
+ e.accept(inputFinder);
+ }
+ references = inputFinder.inputBitSet.build();
+ for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
+ indexes.set(queryAggregate.getGroupSet().nth(i));
+ }
+ for (int i = 0; i < queryAggregate.getAggCallList().size(); i++) {
+ if (references.get(queryAggregate.getGroupCount() + i)) {
+ for (int inputIdx : queryAggregate.getAggCallList().get(i).getArgList()) {
+ indexes.set(inputIdx);
+ }
+ }
+ }
+ } else {
+ // No project on top, all of them are needed
+ for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
+ indexes.set(queryAggregate.getGroupSet().nth(i));
+ }
+ for (AggregateCall queryAggCall : queryAggregate.getAggCallList()) {
+ for (int inputIdx : queryAggCall.getArgList()) {
+ indexes.set(inputIdx);
+ }
+ }
+ }
+
+ // Create mapping from query columns to view columns
+ List<RexNode> rollupNodes = new ArrayList<>();
+ Multimap<Integer, Integer> m = generateMapping(rexBuilder, simplify, mq,
+ queryAggregate.getInput(), viewAggregate.getInput(), indexes.build(),
+ queryToViewTableMapping, queryEC, rollupNodes);
+ if (m == null) {
+ // Bail out
+ return null;
+ }
+
+ // We could map all expressions. Create aggregate mapping.
+ int viewAggregateAdditionalFieldCount = rollupNodes.size();
+ int viewInputFieldCount = viewAggregate.getInput().getRowType().getFieldCount();
+ int viewInputDifferenceViewFieldCount =
+ viewAggregate.getRowType().getFieldCount() - viewInputFieldCount;
+ int viewAggregateTotalFieldCount =
+ viewAggregate.getRowType().getFieldCount() + rollupNodes.size();
+ boolean forceRollup = false;
+ Mapping aggregateMapping = Mappings.create(MappingType.FUNCTION,
+ queryAggregate.getRowType().getFieldCount(), viewAggregateTotalFieldCount);
+ for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
+ Collection<Integer> c = m.get(queryAggregate.getGroupSet().nth(i));
+ for (int j : c) {
+ if (j >= viewAggregate.getInput().getRowType().getFieldCount()) {
+ // This is one of the rollup columns
+ aggregateMapping.set(i, j + viewInputDifferenceViewFieldCount);
+ forceRollup = true;
+ } else {
+ int targetIdx = viewAggregate.getGroupSet().indexOf(j);
+ if (targetIdx == -1) {
+ continue;
+ }
+ aggregateMapping.set(i, targetIdx);
+ }
+ break;
+ }
+ if (aggregateMapping.getTargetOpt(i) == -1) {
+ // It is not part of group by, we bail out
+ return null;
+ }
+ }
+ boolean containsDistinctAgg = false;
+ for (int idx = 0; idx < queryAggregate.getAggCallList().size(); idx++) {
+ if (references != null && !references.get(queryAggregate.getGroupCount() + idx)) {
+ // Ignore
+ continue;
+ }
+ AggregateCall queryAggCall = queryAggregate.getAggCallList().get(idx);
+ if (queryAggCall.filterArg >= 0) {
+ // Not supported currently
+ return null;
+ }
+ List<Integer> queryAggCallIndexes = new ArrayList<>();
+ for (int aggCallIdx : queryAggCall.getArgList()) {
+ queryAggCallIndexes.add(m.get(aggCallIdx).iterator().next());
+ }
+ for (int j = 0; j < viewAggregate.getAggCallList().size(); j++) {
+ AggregateCall viewAggCall = viewAggregate.getAggCallList().get(j);
+ if (queryAggCall.getAggregation().getKind() != viewAggCall.getAggregation().getKind()
+ || queryAggCall.isDistinct() != viewAggCall.isDistinct()
+ || queryAggCall.getArgList().size() != viewAggCall.getArgList().size()
+ || queryAggCall.getType() != viewAggCall.getType()
+ || viewAggCall.filterArg >= 0) {
+ // Continue
+ continue;
+ }
+ if (!queryAggCallIndexes.equals(viewAggCall.getArgList())) {
+ // Continue
+ continue;
+ }
+ aggregateMapping.set(queryAggregate.getGroupCount() + idx,
+ viewAggregate.getGroupCount() + j);
+ if (queryAggCall.isDistinct()) {
+ containsDistinctAgg = true;
+ }
+ break;
+ }
+ }
+
+ // If we reach here, to simplify things, we create an identity topViewProject
+ // if not present
+ if (topViewProject == null) {
+ topViewProject = (Project) relBuilder.push(viewNode)
+ .project(relBuilder.fields(), ImmutableList.of(), true).build();
+ }
+
+ // Generate result rewriting
+ final List<RexNode> additionalViewExprs = new ArrayList<>();
+ Mapping rewritingMapping = null;
+ RelNode result = relBuilder.push(input).build();
+ // We create view expressions that will be used in a Project on top of the
+ // view in case we need to rollup the expression
+ final List<RexNode> inputViewExprs = new ArrayList<>();
+ inputViewExprs.addAll(relBuilder.push(result).fields());
+ relBuilder.clear();
+ if (forceRollup || queryAggregate.getGroupCount() != viewAggregate.getGroupCount()
+ || matchModality == MatchModality.VIEW_PARTIAL) {
+ if (containsDistinctAgg) {
+ // Cannot rollup DISTINCT aggregate
+ return null;
+ }
+ // Target is coarser level of aggregation. Generate an aggregate.
+ rewritingMapping = Mappings.create(MappingType.FUNCTION,
+ topViewProject.getRowType().getFieldCount() + viewAggregateAdditionalFieldCount,
+ queryAggregate.getRowType().getFieldCount());
+ final ImmutableBitSet.Builder groupSetB = ImmutableBitSet.builder();
+ for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
+ int targetIdx = aggregateMapping.getTargetOpt(i);
+ if (targetIdx == -1) {
+ // No matching group by column, we bail out
+ return null;
+ }
+ boolean added = false;
+ if (targetIdx >= viewAggregate.getRowType().getFieldCount()) {
+ RexNode targetNode = rollupNodes.get(
+ targetIdx - viewInputFieldCount - viewInputDifferenceViewFieldCount);
+ // We need to rollup this expression
+ final Multimap<RexNode, Integer> exprsLineage = ArrayListMultimap.create();
+ final ImmutableBitSet refs = RelOptUtil.InputFinder.bits(targetNode);
+ for (int childTargetIdx : refs) {
+ added = false;
+ for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
+ RexNode n = topViewProject.getChildExps().get(k);
+ if (!n.isA(SqlKind.INPUT_REF)) {
+ continue;
+ }
+ final int ref = ((RexInputRef) n).getIndex();
+ if (ref == childTargetIdx) {
+ exprsLineage.put(
+ new RexInputRef(ref, targetNode.getType()), k);
+ added = true;
+ }
+ }
+ if (!added) {
+ // No matching column needed for computed expression, bail out
+ return null;
+ }
+ }
+ // We create the new node pointing to the index
+ groupSetB.set(inputViewExprs.size());
+ rewritingMapping.set(inputViewExprs.size(), i);
+ additionalViewExprs.add(
+ new RexInputRef(targetIdx, targetNode.getType()));
+ // We need to create the rollup expression
+ inputViewExprs.add(
+ shuttleReferences(rexBuilder, targetNode, exprsLineage));
+ added = true;
+ } else {
+ // This expression should be referenced directly
+ for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
+ RexNode n = topViewProject.getChildExps().get(k);
+ if (!n.isA(SqlKind.INPUT_REF)) {
+ continue;
+ }
+ int ref = ((RexInputRef) n).getIndex();
+ if (ref == targetIdx) {
+ groupSetB.set(k);
+ rewritingMapping.set(k, i);
+ added = true;
+ }
+ }
+ }
+ if (!added) {
+ // No matching group by column, we bail out
+ return null;
+ }
+ }
+ final ImmutableBitSet groupSet = groupSetB.build();
+ final List<AggCall> aggregateCalls = new ArrayList<>();
+ for (int i = 0; i < queryAggregate.getAggCallList().size(); i++) {
+ if (references != null && !references.get(queryAggregate.getGroupCount() + i)) {
+ // Ignore
+ continue;
+ }
+ int sourceIdx = queryAggregate.getGroupCount() + i;
+ int targetIdx =
+ aggregateMapping.getTargetOpt(sourceIdx);
+ if (targetIdx < 0) {
+ // No matching aggregation column, we bail out
+ return null;
+ }
+ AggregateCall queryAggCall = queryAggregate.getAggCallList().get(i);
+ boolean added = false;
+ for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
+ RexNode n = topViewProject.getChildExps().get(k);
+ if (!n.isA(SqlKind.INPUT_REF)) {
+ continue;
+ }
+ int ref = ((RexInputRef) n).getIndex();
+ if (ref == targetIdx) {
+ SqlAggFunction rollupAgg =
+ getRollup(queryAggCall.getAggregation());
+ if (rollupAgg == null) {
+ // Cannot rollup this aggregate, bail out
+ return null;
+ }
+ rewritingMapping.set(k, queryAggregate.getGroupCount() + aggregateCalls.size());
+ final RexInputRef operand = rexBuilder.makeInputRef(input, k);
+ aggregateCalls.add(
+ // TODO: handle aggregate ordering
+ relBuilder.aggregateCall(rollupAgg, operand)
+ .approximate(queryAggCall.isApproximate())
+ .distinct(queryAggCall.isDistinct())
+ .as(queryAggCall.name));
+ added = true;
+ }
+ }
+ if (!added) {
+ // No matching aggregation column, we bail out
+ return null;
+ }
+ }
+ // Create aggregate on top of input
+ RelNode prevNode = result;
+ relBuilder.push(result);
+ if (inputViewExprs.size() != result.getRowType().getFieldCount()) {
+ relBuilder.project(inputViewExprs);
+ }
+ result = relBuilder
+ .aggregate(relBuilder.groupKey(groupSet), aggregateCalls)
+ .build();
+ if (prevNode == result && groupSet.cardinality() != result.getRowType().getFieldCount()) {
+ // Aggregate was not inserted but we need to prune columns
+ result = relBuilder
+ .push(result)
+ .project(relBuilder.fields(groupSet))
+ .build();
+ }
+ // We introduce a project on top, as group by columns order is lost
+ List<RexNode> projects = new ArrayList<>();
+ Mapping inverseMapping = rewritingMapping.inverse();
+ for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
+ projects.add(
+ rexBuilder.makeInputRef(result,
+ groupSet.indexOf(inverseMapping.getTarget(i))));
+ }
+ // We add aggregate functions that are present in result to projection list
+ for (int i = queryAggregate.getGroupCount(); i < result.getRowType().getFieldCount(); i++) {
+ projects.add(
+ rexBuilder.makeInputRef(result, i));
+ }
+ result = relBuilder
+ .push(result)
+ .project(projects)
+ .build();
+ } // end if queryAggregate.getGroupCount() != viewAggregate.getGroupCount()
+
+ // Add query expressions on top. We first map query expressions to view
+ // expressions. Once we have done that, if the expression is contained
+ // and we have introduced already an operator on top of the input node,
+ // we use the mapping to resolve the position of the expression in the
+ // node.
+ final RelDataType topRowType;
+ final List<RexNode> topExprs = new ArrayList<>();
+ if (topProject != null && !unionRewriting) {
+ topExprs.addAll(topProject.getChildExps());
+ topRowType = topProject.getRowType();
+ } else {
+ // Add all
+ for (int pos = 0; pos < queryAggregate.getRowType().getFieldCount(); pos++) {
+ topExprs.add(rexBuilder.makeInputRef(queryAggregate, pos));
+ }
+ topRowType = queryAggregate.getRowType();
+ }
+ // Available in view.
+ final Multimap<RexNode, Integer> viewExprs = ArrayListMultimap.create();
+ int numberViewExprs = 0;
+ for (RexNode viewExpr : topViewProject.getChildExps()) {
+ viewExprs.put(viewExpr, numberViewExprs++);
+ }
+ for (RexNode additionalViewExpr : additionalViewExprs) {
+ viewExprs.put(additionalViewExpr, numberViewExprs++);
+ }
+ final List<RexNode> rewrittenExprs = new ArrayList<>(topExprs.size());
+ for (RexNode expr : topExprs) {
+ // First map through the aggregate
+ RexNode rewrittenExpr = shuttleReferences(rexBuilder, expr, aggregateMapping);
+ if (rewrittenExpr == null) {
+ // Cannot map expression
+ return null;
+ }
+ // Next map through the last project
+ rewrittenExpr =
+ shuttleReferences(rexBuilder, rewrittenExpr, viewExprs, result, rewritingMapping);
+ if (rewrittenExpr == null) {
+ // Cannot map expression
+ return null;
+ }
+ rewrittenExprs.add(rewrittenExpr);
+ }
+ return relBuilder
+ .push(result)
+ .project(rewrittenExprs)
+ .convert(topRowType, false)
+ .build();
+ }
+
+ /**
+ * Mapping from node expressions to target expressions.
+ *
+ * <p>If any of the expressions cannot be mapped, we return null.
+ */
+ protected Multimap<Integer, Integer> generateMapping(
+ RexBuilder rexBuilder,
+ RexSimplify simplify,
+ RelMetadataQuery mq,
+ RelNode node,
+ RelNode target,
+ ImmutableBitSet positions,
+ BiMap<RelTableRef, RelTableRef> tableMapping,
+ EquivalenceClasses sourceEC,
+ List<RexNode> additionalExprs) {
+ Preconditions.checkArgument(additionalExprs.isEmpty());
+ Multimap<Integer, Integer> m = ArrayListMultimap.create();
+ Map<RexTableInputRef, Set<RexTableInputRef>> equivalenceClassesMap =
+ sourceEC.getEquivalenceClassesMap();
+ Multimap<RexNode, Integer> exprsLineage = ArrayListMultimap.create();
+ List<RexNode> timestampExprs = new ArrayList<>();
+ for (int i = 0; i < target.getRowType().getFieldCount(); i++) {
+ Set<RexNode> s = mq.getExpressionLineage(target, rexBuilder.makeInputRef(target, i));
+ if (s == null) {
+ // Bail out
+ continue;
+ }
+ // We only support project - filter - join, thus it should map to
+ // a single expression
+ final RexNode e = Iterables.getOnlyElement(s);
+ // Rewrite expr to be expressed on query tables
+ final RexNode simplified = simplify.simplifyUnknownAsFalse(e);
+ RexNode expr = RexUtil.swapTableColumnReferences(rexBuilder,
+ simplified,
+ tableMapping.inverse(),
+ equivalenceClassesMap);
+ exprsLineage.put(expr, i);
+ SqlTypeName sqlTypeName = expr.getType().getSqlTypeName();
+ if (sqlTypeName == SqlTypeName.TIMESTAMP
+ || sqlTypeName == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
+ timestampExprs.add(expr);
+ }
+ }
+
+ // If this is a column of TIMESTAMP (WITH LOCAL TIME ZONE)
+ // type, we add the possible rollup columns too.
+ // This way we will be able to match FLOOR(ts to HOUR) to
+ // FLOOR(ts to DAY) via FLOOR(FLOOR(ts to HOUR) to DAY)
+ for (RexNode timestampExpr : timestampExprs) {
+ for (TimeUnitRange value : SUPPORTED_DATE_TIME_ROLLUP_UNITS) {
+ // CEIL
+ RexNode ceilExpr =
+ rexBuilder.makeCall(getCeilSqlFunction(value),
+ timestampExpr, rexBuilder.makeFlag(value));
+ // References self-row
+ RexNode rewrittenCeilExpr =
+ shuttleReferences(rexBuilder, ceilExpr, exprsLineage);
+ if (rewrittenCeilExpr != null) {
+ // We add the CEIL expression to the additional expressions, replacing the child
+ // expression by the position that it references
+ additionalExprs.add(rewrittenCeilExpr);
+ // Then we simplify the expression and we add it to the expressions lineage so we
+ // can try to find a match
+ final RexNode simplified =
+ simplify.simplifyUnknownAsFalse(ceilExpr);
+ exprsLineage.put(simplified,
+ target.getRowType().getFieldCount() + additionalExprs.size() - 1);
+ }
+ // FLOOR
+ RexNode floorExpr =
+ rexBuilder.makeCall(getFloorSqlFunction(value),
+ timestampExpr, rexBuilder.makeFlag(value));
+ // References self-row
+ RexNode rewrittenFloorExpr =
+ shuttleReferences(rexBuilder, floorExpr, exprsLineage);
+ if (rewrittenFloorExpr != null) {
+ // We add the FLOOR expression to the additional expressions, replacing the child
+ // expression by the position that it references
+ additionalExprs.add(rewrittenFloorExpr);
+ // Then we simplify the expression and we add it to the expressions lineage so we
+ // can try to find a match
+ final RexNode simplified =
+ simplify.simplifyUnknownAsFalse(floorExpr);
+ exprsLineage.put(simplified,
+ target.getRowType().getFieldCount() + additionalExprs.size() - 1);
+ }
+ }
+ }
+
+ for (int i : positions) {
+ Set<RexNode> s = mq.getExpressionLineage(node, rexBuilder.makeInputRef(node, i));
+ if (s == null) {
+ // Bail out
+ return null;
+ }
+ // We only support project - filter - join, thus it should map to
+ // a single expression
+ final RexNode e = Iterables.getOnlyElement(s);
+ // Rewrite expr to be expressed on query tables
+ final RexNode simplified = simplify.simplifyUnknownAsFalse(e);
+ RexNode targetExpr = RexUtil.swapColumnReferences(rexBuilder,
+ simplified, equivalenceClassesMap);
+ final Collection<Integer> c = exprsLineage.get(targetExpr);
+ if (!c.isEmpty()) {
+ for (Integer j : c) {
+ m.put(i, j);
+ }
+ } else {
+ // If we did not find the expression, try to navigate it
+ RexNode rewrittenTargetExpr =
+ shuttleReferences(rexBuilder, targetExpr, exprsLineage);
+ if (rewrittenTargetExpr == null) {
+ // Some expressions were not present
+ return null;
+ }
+ m.put(i, target.getRowType().getFieldCount() + additionalExprs.size());
+ additionalExprs.add(rewrittenTargetExpr);
+ }
+ }
+ return m;
+ }
+
+ /**
+ * Get ceil function datetime.
+ */
+ protected SqlFunction getCeilSqlFunction(TimeUnitRange flag) {
+ return SqlStdOperatorTable.CEIL;
+ }
+
+ /**
+ * Get floor function datetime.
+ */
+ protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) {
+ return SqlStdOperatorTable.FLOOR;
+ }
+
+ /**
+ * Get rollup aggregation function.
+ */
+ protected SqlAggFunction getRollup(SqlAggFunction aggregation) {
+ if (aggregation == SqlStdOperatorTable.SUM
+ || aggregation == SqlStdOperatorTable.MIN
+ || aggregation == SqlStdOperatorTable.MAX
+ || aggregation == SqlStdOperatorTable.SUM0
+ || aggregation == SqlStdOperatorTable.ANY_VALUE) {
+ return aggregation;
+ } else if (aggregation == SqlStdOperatorTable.COUNT) {
+ return SqlStdOperatorTable.SUM0;
+ } else {
+ return null;
+ }
+ }
+
+ @Override public Pair<RelNode, RelNode> pushFilterToOriginalViewPlan(RelBuilder builder,
+ RelNode topViewProject, RelNode viewNode, RexNode cond) {
+ // We add (and push) the filter to the view plan before triggering the rewriting.
+ // This is useful in case some of the columns can be folded to same value after
+ // filter is added.
+ HepProgramBuilder pushFiltersProgram = new HepProgramBuilder();
+ if (topViewProject != null) {
+ pushFiltersProgram.addRuleInstance(filterProjectTransposeRule);
+ }
+ pushFiltersProgram
+ .addRuleInstance(this.filterAggregateTransposeRule)
+ .addRuleInstance(this.aggregateProjectPullUpConstantsRule)
+ .addRuleInstance(this.projectMergeRule);
+ final HepPlanner tmpPlanner = new HepPlanner(pushFiltersProgram.build());
+ // Now that the planner is created, push the node
+ RelNode topNode = builder
+ .push(topViewProject != null ? topViewProject : viewNode)
+ .filter(cond).build();
+ tmpPlanner.setRoot(topNode);
+ topNode = tmpPlanner.findBestExp();
+ RelNode resultTopViewProject = null;
+ RelNode resultViewNode = null;
+ while (topNode != null) {
+ if (topNode instanceof Project) {
+ if (resultTopViewProject != null) {
+ // Both projects could not be merged, we will bail out
+ return Pair.of(topViewProject, viewNode);
+ }
+ resultTopViewProject = topNode;
+ topNode = topNode.getInput(0);
+ } else if (topNode instanceof Aggregate) {
+ resultViewNode = topNode;
+ topNode = null;
+ } else {
+ // We move to the child
+ topNode = topNode.getInput(0);
+ }
+ }
+ return Pair.of(resultTopViewProject, resultViewNode);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java
new file mode 100644
index 0000000..7b470fb
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewJoinRule.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.hep.HepPlanner;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.JoinRelType;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.TableScan;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rel.type.RelDataTypeField;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexSimplify;
+import org.apache.calcite.rex.RexTableInputRef.RelTableRef;
+import org.apache.calcite.rex.RexUtil;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.tools.RelBuilderFactory;
+import org.apache.calcite.util.Pair;
+
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/** Materialized view rewriting for join */
+public abstract class MaterializedViewJoinRule extends MaterializedViewRule {
+
+ /** Creates a MaterializedViewJoinRule. */
+ protected MaterializedViewJoinRule(RelOptRuleOperand operand,
+ RelBuilderFactory relBuilderFactory, String description,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ boolean fastBailOut) {
+ super(operand, relBuilderFactory, description, generateUnionRewriting,
+ unionRewritingPullProgram, fastBailOut);
+ }
+
+ @Override protected boolean isValidPlan(Project topProject, RelNode node,
+ RelMetadataQuery mq) {
+ return isValidRelNodePlan(node, mq);
+ }
+
+ @Override protected ViewPartialRewriting compensateViewPartial(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RelMetadataQuery mq,
+ RelNode input,
+ Project topProject,
+ RelNode node,
+ Set<RelTableRef> queryTableRefs,
+ EquivalenceClasses queryEC,
+ Project topViewProject,
+ RelNode viewNode,
+ Set<RelTableRef> viewTableRefs) {
+ // We only create the rewriting in the minimal subtree of plan operators.
+ // Otherwise we will produce many EQUAL rewritings at different levels of
+ // the plan.
+ // View: (A JOIN B) JOIN C
+ // Query: (((A JOIN B) JOIN D) JOIN C) JOIN E
+ // We produce it at:
+ // ((A JOIN B) JOIN D) JOIN C
+ // But not at:
+ // (((A JOIN B) JOIN D) JOIN C) JOIN E
+ if (fastBailOut) {
+ for (RelNode joinInput : node.getInputs()) {
+ if (mq.getTableReferences(joinInput).containsAll(viewTableRefs)) {
+ return null;
+ }
+ }
+ }
+
+ // Extract tables that are in the query and not in the view
+ final Set<RelTableRef> extraTableRefs = new HashSet<>();
+ for (RelTableRef tRef : queryTableRefs) {
+ if (!viewTableRefs.contains(tRef)) {
+ // Add to extra tables if table is not part of the view
+ extraTableRefs.add(tRef);
+ }
+ }
+
+ // Rewrite the view and the view plan. We only need to add the missing
+ // tables on top of the view and view plan using a cartesian product.
+ // Then the rest of the rewriting algorithm can be executed in the same
+ // fashion, and if there are predicates between the existing and missing
+ // tables, the rewriting algorithm will enforce them.
+ Collection<RelNode> tableScanNodes = mq.getNodeTypes(node).get(TableScan.class);
+ List<RelNode> newRels = new ArrayList<>();
+ for (RelTableRef tRef : extraTableRefs) {
+ int i = 0;
+ for (RelNode relNode : tableScanNodes) {
+ if (tRef.getQualifiedName().equals(relNode.getTable().getQualifiedName())) {
+ if (tRef.getEntityNumber() == i++) {
+ newRels.add(relNode);
+ break;
+ }
+ }
+ }
+ }
+ assert extraTableRefs.size() == newRels.size();
+
+ relBuilder.push(input);
+ for (RelNode newRel : newRels) {
+ // Add to the view
+ relBuilder.push(newRel);
+ relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
+ }
+ final RelNode newView = relBuilder.build();
+
+ relBuilder.push(topViewProject != null ? topViewProject : viewNode);
+ for (RelNode newRel : newRels) {
+ // Add to the view plan
+ relBuilder.push(newRel);
+ relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
+ }
+ final RelNode newViewNode = relBuilder.build();
+
+ return ViewPartialRewriting.of(newView, null, newViewNode);
+ }
+
+ @Override protected RelNode rewriteQuery(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RexSimplify simplify,
+ RelMetadataQuery mq,
+ RexNode compensationColumnsEquiPred,
+ RexNode otherCompensationPred,
+ Project topProject,
+ RelNode node,
+ BiMap<RelTableRef, RelTableRef> viewToQueryTableMapping,
+ EquivalenceClasses viewEC, EquivalenceClasses queryEC) {
+ // Our target node is the node below the root, which should have the maximum
+ // number of available expressions in the tree in order to maximize our
+ // number of rewritings.
+ // We create a project on top. If the program is available, we execute
+ // it to maximize rewriting opportunities. For instance, a program might
+ // pull up all the expressions that are below the aggregate so we can
+ // introduce compensation filters easily. This is important depending on
+ // the planner strategy.
+ RelNode newNode = node;
+ RelNode target = node;
+ if (unionRewritingPullProgram != null) {
+ final HepPlanner tmpPlanner = new HepPlanner(unionRewritingPullProgram);
+ tmpPlanner.setRoot(newNode);
+ newNode = tmpPlanner.findBestExp();
+ target = newNode.getInput(0);
+ }
+
+ // All columns required by compensating predicates must be contained
+ // in the query.
+ List<RexNode> queryExprs = extractReferences(rexBuilder, target);
+
+ if (!compensationColumnsEquiPred.isAlwaysTrue()) {
+ compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
+ target, target, queryExprs, viewToQueryTableMapping.inverse(), queryEC, false,
+ compensationColumnsEquiPred);
+ if (compensationColumnsEquiPred == null) {
+ // Skip it
+ return null;
+ }
+ }
+ // For the rest, we use the query equivalence classes
+ if (!otherCompensationPred.isAlwaysTrue()) {
+ otherCompensationPred = rewriteExpression(rexBuilder, mq,
+ target, target, queryExprs, viewToQueryTableMapping.inverse(), viewEC, true,
+ otherCompensationPred);
+ if (otherCompensationPred == null) {
+ // Skip it
+ return null;
+ }
+ }
+ final RexNode queryCompensationPred = RexUtil.not(
+ RexUtil.composeConjunction(rexBuilder,
+ ImmutableList.of(compensationColumnsEquiPred,
+ otherCompensationPred)));
+
+ // Generate query rewriting.
+ RelNode rewrittenPlan = relBuilder
+ .push(target)
+ .filter(simplify.simplifyUnknownAsFalse(queryCompensationPred))
+ .build();
+ if (unionRewritingPullProgram != null) {
+ rewrittenPlan = newNode.copy(
+ newNode.getTraitSet(), ImmutableList.of(rewrittenPlan));
+ }
+ if (topProject != null) {
+ return topProject.copy(topProject.getTraitSet(), ImmutableList.of(rewrittenPlan));
+ }
+ return rewrittenPlan;
+ }
+
+ @Override protected RelNode createUnion(RelBuilder relBuilder, RexBuilder rexBuilder,
+ RelNode topProject, RelNode unionInputQuery, RelNode unionInputView) {
+ relBuilder.push(unionInputQuery);
+ relBuilder.push(unionInputView);
+ relBuilder.union(true);
+ List<RexNode> exprList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
+ List<String> nameList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
+ for (int i = 0; i < relBuilder.peek().getRowType().getFieldCount(); i++) {
+ // We can take unionInputQuery as it is query based.
+ RelDataTypeField field = unionInputQuery.getRowType().getFieldList().get(i);
+ exprList.add(
+ rexBuilder.ensureType(
+ field.getType(),
+ rexBuilder.makeInputRef(relBuilder.peek(), i),
+ true));
+ nameList.add(field.getName());
+ }
+ relBuilder.project(exprList, nameList);
+ return relBuilder.build();
+ }
+
+ @Override protected RelNode rewriteView(
+ RelBuilder relBuilder,
+ RexBuilder rexBuilder,
+ RexSimplify simplify,
+ RelMetadataQuery mq,
+ MatchModality matchModality,
+ boolean unionRewriting,
+ RelNode input,
+ Project topProject,
+ RelNode node,
+ Project topViewProject,
+ RelNode viewNode,
+ BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
+ EquivalenceClasses queryEC) {
+ List<RexNode> exprs = topProject == null
+ ? extractReferences(rexBuilder, node)
+ : topProject.getChildExps();
+ List<RexNode> exprsLineage = new ArrayList<>(exprs.size());
+ for (RexNode expr : exprs) {
+ Set<RexNode> s = mq.getExpressionLineage(node, expr);
+ if (s == null) {
+ // Bail out
+ return null;
+ }
+ assert s.size() == 1;
+ // Rewrite expr. Take first element from the corresponding equivalence class
+ // (no need to swap the table references following the table mapping)
+ exprsLineage.add(
+ RexUtil.swapColumnReferences(rexBuilder,
+ s.iterator().next(), queryEC.getEquivalenceClassesMap()));
+ }
+ List<RexNode> viewExprs = topViewProject == null
+ ? extractReferences(rexBuilder, viewNode)
+ : topViewProject.getChildExps();
+ List<RexNode> rewrittenExprs = rewriteExpressions(rexBuilder, mq, input, viewNode, viewExprs,
+ queryToViewTableMapping.inverse(), queryEC, true, exprsLineage);
+ if (rewrittenExprs == null) {
+ return null;
+ }
+ return relBuilder
+ .push(input)
+ .project(rewrittenExprs)
+ .convert(topProject != null ? topProject.getRowType() : node.getRowType(), false)
+ .build();
+ }
+
+ @Override public Pair<RelNode, RelNode> pushFilterToOriginalViewPlan(RelBuilder builder,
+ RelNode topViewProject, RelNode viewNode, RexNode cond) {
+ // Nothing to do
+ return Pair.of(topViewProject, viewNode);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyAggregateRule.java
new file mode 100644
index 0000000..fbcd82f
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyAggregateRule.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelOptRuleOperand;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Aggregate. */
+public class MaterializedViewOnlyAggregateRule extends MaterializedViewAggregateRule {
+
+ public static final MaterializedViewOnlyAggregateRule INSTANCE =
+ new MaterializedViewOnlyAggregateRule(RelFactories.LOGICAL_BUILDER,
+ true, null);
+
+ public MaterializedViewOnlyAggregateRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
+ super(
+ operand(Aggregate.class, any()),
+ relBuilderFactory,
+ "MaterializedViewAggregateRule(Aggregate)",
+ generateUnionRewriting, unionRewritingPullProgram);
+ }
+
+ public MaterializedViewOnlyAggregateRule(RelOptRuleOperand operand,
+ RelBuilderFactory relBuilderFactory, String description,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ RelOptRule filterProjectTransposeRule,
+ RelOptRule filterAggregateTransposeRule,
+ RelOptRule aggregateProjectPullUpConstantsRule,
+ RelOptRule projectMergeRule) {
+ super(
+ operand(Aggregate.class, any()),
+ relBuilderFactory,
+ "MaterializedViewAggregateRule(Aggregate)",
+ generateUnionRewriting, unionRewritingPullProgram,
+ filterProjectTransposeRule,
+ filterAggregateTransposeRule,
+ aggregateProjectPullUpConstantsRule,
+ projectMergeRule);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Aggregate aggregate = call.rel(0);
+ perform(call, null, aggregate);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyFilterRule.java
new file mode 100644
index 0000000..2b5926f
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyFilterRule.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Filter. */
+public class MaterializedViewOnlyFilterRule extends MaterializedViewJoinRule {
+
+ public static final MaterializedViewOnlyFilterRule INSTANCE =
+ new MaterializedViewOnlyFilterRule(RelFactories.LOGICAL_BUILDER,
+ true, null, true);
+
+ public MaterializedViewOnlyFilterRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ boolean fastBailOut) {
+ super(
+ operand(Filter.class, any()),
+ relBuilderFactory,
+ "MaterializedViewJoinRule(Filter)",
+ generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Filter filter = call.rel(0);
+ perform(call, null, filter);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyJoinRule.java
new file mode 100644
index 0000000..ec8426e
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewOnlyJoinRule.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Join. */
+public class MaterializedViewOnlyJoinRule extends MaterializedViewJoinRule {
+
+ public static final MaterializedViewOnlyJoinRule INSTANCE =
+ new MaterializedViewOnlyJoinRule(RelFactories.LOGICAL_BUILDER,
+ true, null, true);
+
+ public MaterializedViewOnlyJoinRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ boolean fastBailOut) {
+ super(
+ operand(Join.class, any()),
+ relBuilderFactory,
+ "MaterializedViewJoinRule(Join)",
+ generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Join join = call.rel(0);
+ perform(call, null, join);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectAggregateRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectAggregateRule.java
new file mode 100644
index 0000000..a4c21a8
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectAggregateRule.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Project on Aggregate. */
+public class MaterializedViewProjectAggregateRule extends MaterializedViewAggregateRule {
+
+ public static final MaterializedViewProjectAggregateRule INSTANCE =
+ new MaterializedViewProjectAggregateRule(RelFactories.LOGICAL_BUILDER,
+ true, null);
+
+ public MaterializedViewProjectAggregateRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
+ super(
+ operand(Project.class,
+ operand(Aggregate.class, any())),
+ relBuilderFactory,
+ "MaterializedViewAggregateRule(Project-Aggregate)",
+ generateUnionRewriting, unionRewritingPullProgram);
+ }
+
+ public MaterializedViewProjectAggregateRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ RelOptRule filterProjectTransposeRule,
+ RelOptRule filterAggregateTransposeRule,
+ RelOptRule aggregateProjectPullUpConstantsRule,
+ RelOptRule projectMergeRule) {
+ super(
+ operand(Project.class,
+ operand(Aggregate.class, any())),
+ relBuilderFactory,
+ "MaterializedViewAggregateRule(Project-Aggregate)",
+ generateUnionRewriting, unionRewritingPullProgram,
+ filterProjectTransposeRule,
+ filterAggregateTransposeRule,
+ aggregateProjectPullUpConstantsRule,
+ projectMergeRule);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ final Aggregate aggregate = call.rel(1);
+ perform(call, project, aggregate);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectFilterRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectFilterRule.java
new file mode 100644
index 0000000..abe768b
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectFilterRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Filter;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Project on Filter. */
+public class MaterializedViewProjectFilterRule extends MaterializedViewJoinRule {
+
+ public static final MaterializedViewProjectFilterRule INSTANCE =
+ new MaterializedViewProjectFilterRule(RelFactories.LOGICAL_BUILDER,
+ true, null, true);
+
+ public MaterializedViewProjectFilterRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ boolean fastBailOut) {
+ super(
+ operand(Project.class,
+ operand(Filter.class, any())),
+ relBuilderFactory,
+ "MaterializedViewJoinRule(Project-Filter)",
+ generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ final Filter filter = call.rel(1);
+ perform(call, project, filter);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectJoinRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectJoinRule.java
new file mode 100644
index 0000000..7c54ac9
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewProjectJoinRule.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules.materialize;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.hep.HepProgram;
+import org.apache.calcite.rel.core.Join;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.tools.RelBuilderFactory;
+
+/** Rule that matches Project on Join. */
+public class MaterializedViewProjectJoinRule extends MaterializedViewJoinRule {
+
+ public static final MaterializedViewProjectJoinRule INSTANCE =
+ new MaterializedViewProjectJoinRule(RelFactories.LOGICAL_BUILDER,
+ true, null, true);
+
+ public MaterializedViewProjectJoinRule(RelBuilderFactory relBuilderFactory,
+ boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
+ boolean fastBailOut) {
+ super(
+ operand(Project.class,
+ operand(Join.class, any())),
+ relBuilderFactory,
+ "MaterializedViewJoinRule(Project-Join)",
+ generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Project project = call.rel(0);
+ final Join join = call.rel(1);
+ perform(call, project, join);
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
similarity index 51%
rename from core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
rename to core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
index d1338a9..de53001 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AbstractMaterializedViewRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/MaterializedViewRule.java
@@ -14,9 +14,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.calcite.rel.rules;
+package org.apache.calcite.rel.rules.materialize;
-import org.apache.calcite.avatica.util.TimeUnitRange;
import org.apache.calcite.plan.RelOptMaterialization;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptPredicateList;
@@ -25,53 +24,38 @@ import org.apache.calcite.plan.RelOptRuleCall;
import org.apache.calcite.plan.RelOptRuleOperand;
import org.apache.calcite.plan.RelOptUtil;
import org.apache.calcite.plan.SubstitutionVisitor;
-import org.apache.calcite.plan.hep.HepPlanner;
import org.apache.calcite.plan.hep.HepProgram;
-import org.apache.calcite.plan.hep.HepProgramBuilder;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelReferentialConstraint;
import org.apache.calcite.rel.core.Aggregate;
-import org.apache.calcite.rel.core.AggregateCall;
import org.apache.calcite.rel.core.Filter;
import org.apache.calcite.rel.core.Join;
import org.apache.calcite.rel.core.JoinRelType;
import org.apache.calcite.rel.core.Project;
-import org.apache.calcite.rel.core.RelFactories;
import org.apache.calcite.rel.core.TableScan;
import org.apache.calcite.rel.metadata.RelMetadataQuery;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexExecutor;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
-import org.apache.calcite.rex.RexPermuteInputsShuttle;
import org.apache.calcite.rex.RexShuttle;
import org.apache.calcite.rex.RexSimplify;
import org.apache.calcite.rex.RexTableInputRef;
import org.apache.calcite.rex.RexTableInputRef.RelTableRef;
import org.apache.calcite.rex.RexUtil;
-import org.apache.calcite.sql.SqlAggFunction;
-import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
-import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.tools.RelBuilder;
-import org.apache.calcite.tools.RelBuilder.AggCall;
import org.apache.calcite.tools.RelBuilderFactory;
-import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
import org.apache.calcite.util.Util;
import org.apache.calcite.util.graph.DefaultDirectedGraph;
import org.apache.calcite.util.graph.DefaultEdge;
import org.apache.calcite.util.graph.DirectedGraph;
import org.apache.calcite.util.mapping.Mapping;
-import org.apache.calcite.util.mapping.MappingType;
-import org.apache.calcite.util.mapping.Mappings;
-import org.apache.calcite.util.trace.CalciteLogger;
-import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
@@ -81,8 +65,6 @@ import com.google.common.collect.Iterables;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
-import org.slf4j.LoggerFactory;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -101,34 +83,7 @@ import java.util.Set;
* {@link org.apache.calcite.rel.core.Aggregate} to a scan (and possibly
* other operations) over a materialized view.
*/
-public abstract class AbstractMaterializedViewRule extends RelOptRule {
-
- private static final CalciteLogger LOGGER =
- new CalciteLogger(LoggerFactory.getLogger(AbstractMaterializedViewRule.class));
-
- public static final MaterializedViewProjectFilterRule INSTANCE_PROJECT_FILTER =
- new MaterializedViewProjectFilterRule(RelFactories.LOGICAL_BUILDER,
- true, null, true);
-
- public static final MaterializedViewOnlyFilterRule INSTANCE_FILTER =
- new MaterializedViewOnlyFilterRule(RelFactories.LOGICAL_BUILDER,
- true, null, true);
-
- public static final MaterializedViewProjectJoinRule INSTANCE_PROJECT_JOIN =
- new MaterializedViewProjectJoinRule(RelFactories.LOGICAL_BUILDER,
- true, null, true);
-
- public static final MaterializedViewOnlyJoinRule INSTANCE_JOIN =
- new MaterializedViewOnlyJoinRule(RelFactories.LOGICAL_BUILDER,
- true, null, true);
-
- public static final MaterializedViewProjectAggregateRule INSTANCE_PROJECT_AGGREGATE =
- new MaterializedViewProjectAggregateRule(RelFactories.LOGICAL_BUILDER,
- true, null);
-
- public static final MaterializedViewOnlyAggregateRule INSTANCE_AGGREGATE =
- new MaterializedViewOnlyAggregateRule(RelFactories.LOGICAL_BUILDER,
- true, null);
+public abstract class MaterializedViewRule extends RelOptRule {
//~ Instance fields --------------------------------------------------------
@@ -147,7 +102,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
//~ Constructors -----------------------------------------------------------
/** Creates a AbstractMaterializedViewRule. */
- protected AbstractMaterializedViewRule(RelOptRuleOperand operand,
+ protected MaterializedViewRule(RelOptRuleOperand operand,
RelBuilderFactory relBuilderFactory, String description,
boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
boolean fastBailOut) {
@@ -605,1206 +560,6 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
protected abstract Pair<RelNode, RelNode> pushFilterToOriginalViewPlan(RelBuilder builder,
RelNode topViewProject, RelNode viewNode, RexNode cond);
- //~ Instances Join ---------------------------------------------------------
-
- /** Materialized view rewriting for join */
- private abstract static class MaterializedViewJoinRule
- extends AbstractMaterializedViewRule {
- /** Creates a MaterializedViewJoinRule. */
- protected MaterializedViewJoinRule(RelOptRuleOperand operand,
- RelBuilderFactory relBuilderFactory, String description,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
- boolean fastBailOut) {
- super(operand, relBuilderFactory, description, generateUnionRewriting,
- unionRewritingPullProgram, fastBailOut);
- }
-
- @Override protected boolean isValidPlan(Project topProject, RelNode node,
- RelMetadataQuery mq) {
- return isValidRelNodePlan(node, mq);
- }
-
- @Override protected ViewPartialRewriting compensateViewPartial(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RelMetadataQuery mq,
- RelNode input,
- Project topProject,
- RelNode node,
- Set<RelTableRef> queryTableRefs,
- EquivalenceClasses queryEC,
- Project topViewProject,
- RelNode viewNode,
- Set<RelTableRef> viewTableRefs) {
- // We only create the rewriting in the minimal subtree of plan operators.
- // Otherwise we will produce many EQUAL rewritings at different levels of
- // the plan.
- // View: (A JOIN B) JOIN C
- // Query: (((A JOIN B) JOIN D) JOIN C) JOIN E
- // We produce it at:
- // ((A JOIN B) JOIN D) JOIN C
- // But not at:
- // (((A JOIN B) JOIN D) JOIN C) JOIN E
- if (fastBailOut) {
- for (RelNode joinInput : node.getInputs()) {
- if (mq.getTableReferences(joinInput).containsAll(viewTableRefs)) {
- return null;
- }
- }
- }
-
- // Extract tables that are in the query and not in the view
- final Set<RelTableRef> extraTableRefs = new HashSet<>();
- for (RelTableRef tRef : queryTableRefs) {
- if (!viewTableRefs.contains(tRef)) {
- // Add to extra tables if table is not part of the view
- extraTableRefs.add(tRef);
- }
- }
-
- // Rewrite the view and the view plan. We only need to add the missing
- // tables on top of the view and view plan using a cartesian product.
- // Then the rest of the rewriting algorithm can be executed in the same
- // fashion, and if there are predicates between the existing and missing
- // tables, the rewriting algorithm will enforce them.
- Collection<RelNode> tableScanNodes = mq.getNodeTypes(node).get(TableScan.class);
- List<RelNode> newRels = new ArrayList<>();
- for (RelTableRef tRef : extraTableRefs) {
- int i = 0;
- for (RelNode relNode : tableScanNodes) {
- if (tRef.getQualifiedName().equals(relNode.getTable().getQualifiedName())) {
- if (tRef.getEntityNumber() == i++) {
- newRels.add(relNode);
- break;
- }
- }
- }
- }
- assert extraTableRefs.size() == newRels.size();
-
- relBuilder.push(input);
- for (RelNode newRel : newRels) {
- // Add to the view
- relBuilder.push(newRel);
- relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
- }
- final RelNode newView = relBuilder.build();
-
- relBuilder.push(topViewProject != null ? topViewProject : viewNode);
- for (RelNode newRel : newRels) {
- // Add to the view plan
- relBuilder.push(newRel);
- relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
- }
- final RelNode newViewNode = relBuilder.build();
-
- return ViewPartialRewriting.of(newView, null, newViewNode);
- }
-
- @Override protected RelNode rewriteQuery(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RexSimplify simplify,
- RelMetadataQuery mq,
- RexNode compensationColumnsEquiPred,
- RexNode otherCompensationPred,
- Project topProject,
- RelNode node,
- BiMap<RelTableRef, RelTableRef> viewToQueryTableMapping,
- EquivalenceClasses viewEC, EquivalenceClasses queryEC) {
- // Our target node is the node below the root, which should have the maximum
- // number of available expressions in the tree in order to maximize our
- // number of rewritings.
- // We create a project on top. If the program is available, we execute
- // it to maximize rewriting opportunities. For instance, a program might
- // pull up all the expressions that are below the aggregate so we can
- // introduce compensation filters easily. This is important depending on
- // the planner strategy.
- RelNode newNode = node;
- RelNode target = node;
- if (unionRewritingPullProgram != null) {
- final HepPlanner tmpPlanner = new HepPlanner(unionRewritingPullProgram);
- tmpPlanner.setRoot(newNode);
- newNode = tmpPlanner.findBestExp();
- target = newNode.getInput(0);
- }
-
- // All columns required by compensating predicates must be contained
- // in the query.
- List<RexNode> queryExprs = extractReferences(rexBuilder, target);
-
- if (!compensationColumnsEquiPred.isAlwaysTrue()) {
- compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
- target, target, queryExprs, viewToQueryTableMapping.inverse(), queryEC, false,
- compensationColumnsEquiPred);
- if (compensationColumnsEquiPred == null) {
- // Skip it
- return null;
- }
- }
- // For the rest, we use the query equivalence classes
- if (!otherCompensationPred.isAlwaysTrue()) {
- otherCompensationPred = rewriteExpression(rexBuilder, mq,
- target, target, queryExprs, viewToQueryTableMapping.inverse(), viewEC, true,
- otherCompensationPred);
- if (otherCompensationPred == null) {
- // Skip it
- return null;
- }
- }
- final RexNode queryCompensationPred = RexUtil.not(
- RexUtil.composeConjunction(rexBuilder,
- ImmutableList.of(compensationColumnsEquiPred,
- otherCompensationPred)));
-
- // Generate query rewriting.
- RelNode rewrittenPlan = relBuilder
- .push(target)
- .filter(simplify.simplifyUnknownAsFalse(queryCompensationPred))
- .build();
- if (unionRewritingPullProgram != null) {
- rewrittenPlan = newNode.copy(
- newNode.getTraitSet(), ImmutableList.of(rewrittenPlan));
- }
- if (topProject != null) {
- return topProject.copy(topProject.getTraitSet(), ImmutableList.of(rewrittenPlan));
- }
- return rewrittenPlan;
- }
-
- @Override protected RelNode createUnion(RelBuilder relBuilder, RexBuilder rexBuilder,
- RelNode topProject, RelNode unionInputQuery, RelNode unionInputView) {
- relBuilder.push(unionInputQuery);
- relBuilder.push(unionInputView);
- relBuilder.union(true);
- List<RexNode> exprList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
- List<String> nameList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
- for (int i = 0; i < relBuilder.peek().getRowType().getFieldCount(); i++) {
- // We can take unionInputQuery as it is query based.
- RelDataTypeField field = unionInputQuery.getRowType().getFieldList().get(i);
- exprList.add(
- rexBuilder.ensureType(
- field.getType(),
- rexBuilder.makeInputRef(relBuilder.peek(), i),
- true));
- nameList.add(field.getName());
- }
- relBuilder.project(exprList, nameList);
- return relBuilder.build();
- }
-
- @Override protected RelNode rewriteView(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RexSimplify simplify,
- RelMetadataQuery mq,
- MatchModality matchModality,
- boolean unionRewriting,
- RelNode input,
- Project topProject,
- RelNode node,
- Project topViewProject,
- RelNode viewNode,
- BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
- EquivalenceClasses queryEC) {
- List<RexNode> exprs = topProject == null
- ? extractReferences(rexBuilder, node)
- : topProject.getChildExps();
- List<RexNode> exprsLineage = new ArrayList<>(exprs.size());
- for (RexNode expr : exprs) {
- Set<RexNode> s = mq.getExpressionLineage(node, expr);
- if (s == null) {
- // Bail out
- return null;
- }
- assert s.size() == 1;
- // Rewrite expr. Take first element from the corresponding equivalence class
- // (no need to swap the table references following the table mapping)
- exprsLineage.add(
- RexUtil.swapColumnReferences(rexBuilder,
- s.iterator().next(), queryEC.getEquivalenceClassesMap()));
- }
- List<RexNode> viewExprs = topViewProject == null
- ? extractReferences(rexBuilder, viewNode)
- : topViewProject.getChildExps();
- List<RexNode> rewrittenExprs = rewriteExpressions(rexBuilder, mq, input, viewNode, viewExprs,
- queryToViewTableMapping.inverse(), queryEC, true, exprsLineage);
- if (rewrittenExprs == null) {
- return null;
- }
- return relBuilder
- .push(input)
- .project(rewrittenExprs)
- .convert(topProject != null ? topProject.getRowType() : node.getRowType(), false)
- .build();
- }
-
- @Override public Pair<RelNode, RelNode> pushFilterToOriginalViewPlan(RelBuilder builder,
- RelNode topViewProject, RelNode viewNode, RexNode cond) {
- // Nothing to do
- return Pair.of(topViewProject, viewNode);
- }
- }
-
- /** Rule that matches Project on Join. */
- public static class MaterializedViewProjectJoinRule extends MaterializedViewJoinRule {
- public MaterializedViewProjectJoinRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
- boolean fastBailOut) {
- super(
- operand(Project.class,
- operand(Join.class, any())),
- relBuilderFactory,
- "MaterializedViewJoinRule(Project-Join)",
- generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Project project = call.rel(0);
- final Join join = call.rel(1);
- perform(call, project, join);
- }
- }
-
- /** Rule that matches Project on Filter. */
- public static class MaterializedViewProjectFilterRule extends MaterializedViewJoinRule {
- public MaterializedViewProjectFilterRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
- boolean fastBailOut) {
- super(
- operand(Project.class,
- operand(Filter.class, any())),
- relBuilderFactory,
- "MaterializedViewJoinRule(Project-Filter)",
- generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Project project = call.rel(0);
- final Filter filter = call.rel(1);
- perform(call, project, filter);
- }
- }
-
- /** Rule that matches Join. */
- public static class MaterializedViewOnlyJoinRule extends MaterializedViewJoinRule {
- public MaterializedViewOnlyJoinRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
- boolean fastBailOut) {
- super(
- operand(Join.class, any()),
- relBuilderFactory,
- "MaterializedViewJoinRule(Join)",
- generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Join join = call.rel(0);
- perform(call, null, join);
- }
- }
-
- /** Rule that matches Filter. */
- public static class MaterializedViewOnlyFilterRule extends MaterializedViewJoinRule {
- public MaterializedViewOnlyFilterRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram,
- boolean fastBailOut) {
- super(
- operand(Filter.class, any()),
- relBuilderFactory,
- "MaterializedViewJoinRule(Filter)",
- generateUnionRewriting, unionRewritingPullProgram, fastBailOut);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Filter filter = call.rel(0);
- perform(call, null, filter);
- }
- }
-
- //~ Instances Aggregate ----------------------------------------------------
-
- /** Materialized view rewriting for aggregate */
- private abstract static class MaterializedViewAggregateRule
- extends AbstractMaterializedViewRule {
-
- private static final ImmutableList<TimeUnitRange> SUPPORTED_DATE_TIME_ROLLUP_UNITS =
- ImmutableList.of(TimeUnitRange.YEAR, TimeUnitRange.QUARTER, TimeUnitRange.MONTH,
- TimeUnitRange.DAY, TimeUnitRange.HOUR, TimeUnitRange.MINUTE,
- TimeUnitRange.SECOND, TimeUnitRange.MILLISECOND, TimeUnitRange.MICROSECOND);
-
- //~ Instance fields --------------------------------------------------------
-
- /** Instance of rule to push filter through project. */
- protected final RelOptRule filterProjectTransposeRule;
-
- /** Instance of rule to push filter through aggregate. */
- protected final RelOptRule filterAggregateTransposeRule;
-
- /** Instance of rule to pull up constants into aggregate. */
- protected final RelOptRule aggregateProjectPullUpConstantsRule;
-
- /** Instance of rule to merge project operators. */
- protected final RelOptRule projectMergeRule;
-
-
- /** Creates a MaterializedViewAggregateRule. */
- protected MaterializedViewAggregateRule(RelOptRuleOperand operand,
- RelBuilderFactory relBuilderFactory, String description,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
- super(operand, relBuilderFactory, description, generateUnionRewriting,
- unionRewritingPullProgram, false);
- this.filterProjectTransposeRule = new FilterProjectTransposeRule(
- Filter.class, Project.class, true, true, relBuilderFactory);
- this.filterAggregateTransposeRule = new FilterAggregateTransposeRule(
- Filter.class, relBuilderFactory, Aggregate.class);
- this.aggregateProjectPullUpConstantsRule = new AggregateProjectPullUpConstantsRule(
- Aggregate.class, Filter.class, relBuilderFactory, "AggFilterPullUpConstants");
- this.projectMergeRule =
- new ProjectMergeRule(true, ProjectMergeRule.DEFAULT_BLOAT,
- relBuilderFactory);
- }
-
- @Override protected boolean isValidPlan(Project topProject, RelNode node,
- RelMetadataQuery mq) {
- if (!(node instanceof Aggregate)) {
- return false;
- }
- Aggregate aggregate = (Aggregate) node;
- if (aggregate.getGroupType() != Aggregate.Group.SIMPLE) {
- // TODO: Rewriting with grouping sets not supported yet
- return false;
- }
- return isValidRelNodePlan(aggregate.getInput(), mq);
- }
-
- @Override protected ViewPartialRewriting compensateViewPartial(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RelMetadataQuery mq,
- RelNode input,
- Project topProject,
- RelNode node,
- Set<RelTableRef> queryTableRefs,
- EquivalenceClasses queryEC,
- Project topViewProject,
- RelNode viewNode,
- Set<RelTableRef> viewTableRefs) {
- // Modify view to join with missing tables and add Project on top to reorder columns.
- // In turn, modify view plan to join with missing tables before Aggregate operator,
- // change Aggregate operator to group by previous grouping columns and columns in
- // attached tables, and add a final Project on top.
- // We only need to add the missing tables on top of the view and view plan using
- // a cartesian product.
- // Then the rest of the rewriting algorithm can be executed in the same
- // fashion, and if there are predicates between the existing and missing
- // tables, the rewriting algorithm will enforce them.
- final Set<RelTableRef> extraTableRefs = new HashSet<>();
- for (RelTableRef tRef : queryTableRefs) {
- if (!viewTableRefs.contains(tRef)) {
- // Add to extra tables if table is not part of the view
- extraTableRefs.add(tRef);
- }
- }
- Collection<RelNode> tableScanNodes = mq.getNodeTypes(node).get(TableScan.class);
- List<RelNode> newRels = new ArrayList<>();
- for (RelTableRef tRef : extraTableRefs) {
- int i = 0;
- for (RelNode relNode : tableScanNodes) {
- if (tRef.getQualifiedName().equals(relNode.getTable().getQualifiedName())) {
- if (tRef.getEntityNumber() == i++) {
- newRels.add(relNode);
- break;
- }
- }
- }
- }
- assert extraTableRefs.size() == newRels.size();
-
- relBuilder.push(input);
- for (RelNode newRel : newRels) {
- // Add to the view
- relBuilder.push(newRel);
- relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
- }
- final RelNode newView = relBuilder.build();
-
- final Aggregate aggregateViewNode = (Aggregate) viewNode;
- relBuilder.push(aggregateViewNode.getInput());
- int offset = 0;
- for (RelNode newRel : newRels) {
- // Add to the view plan
- relBuilder.push(newRel);
- relBuilder.join(JoinRelType.INNER, rexBuilder.makeLiteral(true));
- offset += newRel.getRowType().getFieldCount();
- }
- // Modify aggregate: add grouping columns
- ImmutableBitSet.Builder groupSet = ImmutableBitSet.builder();
- groupSet.addAll(aggregateViewNode.getGroupSet());
- groupSet.addAll(
- ImmutableBitSet.range(
- aggregateViewNode.getInput().getRowType().getFieldCount(),
- aggregateViewNode.getInput().getRowType().getFieldCount() + offset));
- final Aggregate newViewNode = aggregateViewNode.copy(
- aggregateViewNode.getTraitSet(), relBuilder.build(),
- groupSet.build(), null, aggregateViewNode.getAggCallList());
-
- relBuilder.push(newViewNode);
- List<RexNode> nodes = new ArrayList<>();
- List<String> fieldNames = new ArrayList<>();
- if (topViewProject != null) {
- // Insert existing expressions (and shift aggregation arguments),
- // then append rest of columns
- Mappings.TargetMapping shiftMapping = Mappings.createShiftMapping(
- newViewNode.getRowType().getFieldCount(),
- 0, 0, aggregateViewNode.getGroupCount(),
- newViewNode.getGroupCount(), aggregateViewNode.getGroupCount(),
- aggregateViewNode.getAggCallList().size());
- for (int i = 0; i < topViewProject.getChildExps().size(); i++) {
- nodes.add(
- topViewProject.getChildExps().get(i).accept(
- new RexPermuteInputsShuttle(shiftMapping, newViewNode)));
- fieldNames.add(topViewProject.getRowType().getFieldNames().get(i));
- }
- for (int i = aggregateViewNode.getRowType().getFieldCount();
- i < newViewNode.getRowType().getFieldCount(); i++) {
- int idx = i - aggregateViewNode.getAggCallList().size();
- nodes.add(rexBuilder.makeInputRef(newViewNode, idx));
- fieldNames.add(newViewNode.getRowType().getFieldNames().get(idx));
- }
- } else {
- // Original grouping columns, aggregation columns, then new grouping columns
- for (int i = 0; i < newViewNode.getRowType().getFieldCount(); i++) {
- int idx;
- if (i < aggregateViewNode.getGroupCount()) {
- idx = i;
- } else if (i < aggregateViewNode.getRowType().getFieldCount()) {
- idx = i + offset;
- } else {
- idx = i - aggregateViewNode.getAggCallList().size();
- }
- nodes.add(rexBuilder.makeInputRef(newViewNode, idx));
- fieldNames.add(newViewNode.getRowType().getFieldNames().get(idx));
- }
- }
- relBuilder.project(nodes, fieldNames, true);
- final Project newTopViewProject = (Project) relBuilder.build();
-
- return ViewPartialRewriting.of(newView, newTopViewProject, newViewNode);
- }
-
- @Override protected RelNode rewriteQuery(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RexSimplify simplify,
- RelMetadataQuery mq,
- RexNode compensationColumnsEquiPred,
- RexNode otherCompensationPred,
- Project topProject,
- RelNode node,
- BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
- EquivalenceClasses viewEC, EquivalenceClasses queryEC) {
- Aggregate aggregate = (Aggregate) node;
-
- // Our target node is the node below the root, which should have the maximum
- // number of available expressions in the tree in order to maximize our
- // number of rewritings.
- // If the program is available, we execute it to maximize rewriting opportunities.
- // For instance, a program might pull up all the expressions that are below the
- // aggregate so we can introduce compensation filters easily. This is important
- // depending on the planner strategy.
- RelNode newAggregateInput = aggregate.getInput(0);
- RelNode target = aggregate.getInput(0);
- if (unionRewritingPullProgram != null) {
- final HepPlanner tmpPlanner = new HepPlanner(unionRewritingPullProgram);
- tmpPlanner.setRoot(newAggregateInput);
- newAggregateInput = tmpPlanner.findBestExp();
- target = newAggregateInput.getInput(0);
- }
-
- // We need to check that all columns required by compensating predicates
- // are contained in the query.
- List<RexNode> queryExprs = extractReferences(rexBuilder, target);
- if (!compensationColumnsEquiPred.isAlwaysTrue()) {
- compensationColumnsEquiPred = rewriteExpression(rexBuilder, mq,
- target, target, queryExprs, queryToViewTableMapping, queryEC, false,
- compensationColumnsEquiPred);
- if (compensationColumnsEquiPred == null) {
- // Skip it
- return null;
- }
- }
- // For the rest, we use the query equivalence classes
- if (!otherCompensationPred.isAlwaysTrue()) {
- otherCompensationPred = rewriteExpression(rexBuilder, mq,
- target, target, queryExprs, queryToViewTableMapping, viewEC, true,
- otherCompensationPred);
- if (otherCompensationPred == null) {
- // Skip it
- return null;
- }
- }
- final RexNode queryCompensationPred = RexUtil.not(
- RexUtil.composeConjunction(rexBuilder,
- ImmutableList.of(compensationColumnsEquiPred,
- otherCompensationPred)));
-
- // Generate query rewriting.
- RelNode rewrittenPlan = relBuilder
- .push(target)
- .filter(simplify.simplifyUnknownAsFalse(queryCompensationPred))
- .build();
- if (unionRewritingPullProgram != null) {
- return aggregate.copy(aggregate.getTraitSet(),
- ImmutableList.of(
- newAggregateInput.copy(newAggregateInput.getTraitSet(),
- ImmutableList.of(rewrittenPlan))));
- }
- return aggregate.copy(aggregate.getTraitSet(), ImmutableList.of(rewrittenPlan));
- }
-
- @Override protected RelNode createUnion(RelBuilder relBuilder, RexBuilder rexBuilder,
- RelNode topProject, RelNode unionInputQuery, RelNode unionInputView) {
- // Union
- relBuilder.push(unionInputQuery);
- relBuilder.push(unionInputView);
- relBuilder.union(true);
- List<RexNode> exprList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
- List<String> nameList = new ArrayList<>(relBuilder.peek().getRowType().getFieldCount());
- for (int i = 0; i < relBuilder.peek().getRowType().getFieldCount(); i++) {
- // We can take unionInputQuery as it is query based.
- RelDataTypeField field = unionInputQuery.getRowType().getFieldList().get(i);
- exprList.add(
- rexBuilder.ensureType(
- field.getType(),
- rexBuilder.makeInputRef(relBuilder.peek(), i),
- true));
- nameList.add(field.getName());
- }
- relBuilder.project(exprList, nameList);
- // Rollup aggregate
- Aggregate aggregate = (Aggregate) unionInputQuery;
- final ImmutableBitSet groupSet = ImmutableBitSet.range(aggregate.getGroupCount());
- final List<AggCall> aggregateCalls = new ArrayList<>();
- for (int i = 0; i < aggregate.getAggCallList().size(); i++) {
- AggregateCall aggCall = aggregate.getAggCallList().get(i);
- if (aggCall.isDistinct()) {
- // Cannot ROLLUP distinct
- return null;
- }
- SqlAggFunction rollupAgg =
- getRollup(aggCall.getAggregation());
- if (rollupAgg == null) {
- // Cannot rollup this aggregate, bail out
- return null;
- }
- final RexInputRef operand =
- rexBuilder.makeInputRef(relBuilder.peek(),
- aggregate.getGroupCount() + i);
- aggregateCalls.add(
- // TODO: handle aggregate ordering
- relBuilder.aggregateCall(rollupAgg, operand)
- .distinct(aggCall.isDistinct())
- .approximate(aggCall.isApproximate())
- .as(aggCall.name));
- }
- RelNode prevNode = relBuilder.peek();
- RelNode result = relBuilder
- .aggregate(relBuilder.groupKey(groupSet), aggregateCalls)
- .build();
- if (prevNode == result && groupSet.cardinality() != result.getRowType().getFieldCount()) {
- // Aggregate was not inserted but we need to prune columns
- result = relBuilder
- .push(result)
- .project(relBuilder.fields(groupSet))
- .build();
- }
- if (topProject != null) {
- // Top project
- return topProject.copy(topProject.getTraitSet(), ImmutableList.of(result));
- }
- // Result
- return result;
- }
-
- @Override protected RelNode rewriteView(
- RelBuilder relBuilder,
- RexBuilder rexBuilder,
- RexSimplify simplify,
- RelMetadataQuery mq,
- MatchModality matchModality,
- boolean unionRewriting,
- RelNode input,
- Project topProject,
- RelNode node,
- Project topViewProject,
- RelNode viewNode,
- BiMap<RelTableRef, RelTableRef> queryToViewTableMapping,
- EquivalenceClasses queryEC) {
- final Aggregate queryAggregate = (Aggregate) node;
- final Aggregate viewAggregate = (Aggregate) viewNode;
- // Get group by references and aggregate call input references needed
- ImmutableBitSet.Builder indexes = ImmutableBitSet.builder();
- ImmutableBitSet references = null;
- if (topProject != null && !unionRewriting) {
- // We have a Project on top, gather only what is needed
- final RelOptUtil.InputFinder inputFinder =
- new RelOptUtil.InputFinder(new LinkedHashSet<>());
- for (RexNode e : topProject.getChildExps()) {
- e.accept(inputFinder);
- }
- references = inputFinder.inputBitSet.build();
- for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
- indexes.set(queryAggregate.getGroupSet().nth(i));
- }
- for (int i = 0; i < queryAggregate.getAggCallList().size(); i++) {
- if (references.get(queryAggregate.getGroupCount() + i)) {
- for (int inputIdx : queryAggregate.getAggCallList().get(i).getArgList()) {
- indexes.set(inputIdx);
- }
- }
- }
- } else {
- // No project on top, all of them are needed
- for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
- indexes.set(queryAggregate.getGroupSet().nth(i));
- }
- for (AggregateCall queryAggCall : queryAggregate.getAggCallList()) {
- for (int inputIdx : queryAggCall.getArgList()) {
- indexes.set(inputIdx);
- }
- }
- }
-
- // Create mapping from query columns to view columns
- List<RexNode> rollupNodes = new ArrayList<>();
- Multimap<Integer, Integer> m = generateMapping(rexBuilder, simplify, mq,
- queryAggregate.getInput(), viewAggregate.getInput(), indexes.build(),
- queryToViewTableMapping, queryEC, rollupNodes);
- if (m == null) {
- // Bail out
- return null;
- }
-
- // We could map all expressions. Create aggregate mapping.
- int viewAggregateAdditionalFieldCount = rollupNodes.size();
- int viewInputFieldCount = viewAggregate.getInput().getRowType().getFieldCount();
- int viewInputDifferenceViewFieldCount =
- viewAggregate.getRowType().getFieldCount() - viewInputFieldCount;
- int viewAggregateTotalFieldCount =
- viewAggregate.getRowType().getFieldCount() + rollupNodes.size();
- boolean forceRollup = false;
- Mapping aggregateMapping = Mappings.create(MappingType.FUNCTION,
- queryAggregate.getRowType().getFieldCount(), viewAggregateTotalFieldCount);
- for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
- Collection<Integer> c = m.get(queryAggregate.getGroupSet().nth(i));
- for (int j : c) {
- if (j >= viewAggregate.getInput().getRowType().getFieldCount()) {
- // This is one of the rollup columns
- aggregateMapping.set(i, j + viewInputDifferenceViewFieldCount);
- forceRollup = true;
- } else {
- int targetIdx = viewAggregate.getGroupSet().indexOf(j);
- if (targetIdx == -1) {
- continue;
- }
- aggregateMapping.set(i, targetIdx);
- }
- break;
- }
- if (aggregateMapping.getTargetOpt(i) == -1) {
- // It is not part of group by, we bail out
- return null;
- }
- }
- boolean containsDistinctAgg = false;
- for (int idx = 0; idx < queryAggregate.getAggCallList().size(); idx++) {
- if (references != null && !references.get(queryAggregate.getGroupCount() + idx)) {
- // Ignore
- continue;
- }
- AggregateCall queryAggCall = queryAggregate.getAggCallList().get(idx);
- if (queryAggCall.filterArg >= 0) {
- // Not supported currently
- return null;
- }
- List<Integer> queryAggCallIndexes = new ArrayList<>();
- for (int aggCallIdx : queryAggCall.getArgList()) {
- queryAggCallIndexes.add(m.get(aggCallIdx).iterator().next());
- }
- for (int j = 0; j < viewAggregate.getAggCallList().size(); j++) {
- AggregateCall viewAggCall = viewAggregate.getAggCallList().get(j);
- if (queryAggCall.getAggregation().getKind() != viewAggCall.getAggregation().getKind()
- || queryAggCall.isDistinct() != viewAggCall.isDistinct()
- || queryAggCall.getArgList().size() != viewAggCall.getArgList().size()
- || queryAggCall.getType() != viewAggCall.getType()
- || viewAggCall.filterArg >= 0) {
- // Continue
- continue;
- }
- if (!queryAggCallIndexes.equals(viewAggCall.getArgList())) {
- // Continue
- continue;
- }
- aggregateMapping.set(queryAggregate.getGroupCount() + idx,
- viewAggregate.getGroupCount() + j);
- if (queryAggCall.isDistinct()) {
- containsDistinctAgg = true;
- }
- break;
- }
- }
-
- // If we reach here, to simplify things, we create an identity topViewProject
- // if not present
- if (topViewProject == null) {
- topViewProject = (Project) relBuilder.push(viewNode)
- .project(relBuilder.fields(), ImmutableList.of(), true).build();
- }
-
- // Generate result rewriting
- final List<RexNode> additionalViewExprs = new ArrayList<>();
- Mapping rewritingMapping = null;
- RelNode result = relBuilder.push(input).build();
- // We create view expressions that will be used in a Project on top of the
- // view in case we need to rollup the expression
- final List<RexNode> inputViewExprs = new ArrayList<>();
- inputViewExprs.addAll(relBuilder.push(result).fields());
- relBuilder.clear();
- if (forceRollup || queryAggregate.getGroupCount() != viewAggregate.getGroupCount()
- || matchModality == MatchModality.VIEW_PARTIAL) {
- if (containsDistinctAgg) {
- // Cannot rollup DISTINCT aggregate
- return null;
- }
- // Target is coarser level of aggregation. Generate an aggregate.
- rewritingMapping = Mappings.create(MappingType.FUNCTION,
- topViewProject.getRowType().getFieldCount() + viewAggregateAdditionalFieldCount,
- queryAggregate.getRowType().getFieldCount());
- final ImmutableBitSet.Builder groupSetB = ImmutableBitSet.builder();
- for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
- int targetIdx = aggregateMapping.getTargetOpt(i);
- if (targetIdx == -1) {
- // No matching group by column, we bail out
- return null;
- }
- boolean added = false;
- if (targetIdx >= viewAggregate.getRowType().getFieldCount()) {
- RexNode targetNode = rollupNodes.get(
- targetIdx - viewInputFieldCount - viewInputDifferenceViewFieldCount);
- // We need to rollup this expression
- final Multimap<RexNode, Integer> exprsLineage = ArrayListMultimap.create();
- final ImmutableBitSet refs = RelOptUtil.InputFinder.bits(targetNode);
- for (int childTargetIdx : refs) {
- added = false;
- for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
- RexNode n = topViewProject.getChildExps().get(k);
- if (!n.isA(SqlKind.INPUT_REF)) {
- continue;
- }
- final int ref = ((RexInputRef) n).getIndex();
- if (ref == childTargetIdx) {
- exprsLineage.put(
- new RexInputRef(ref, targetNode.getType()), k);
- added = true;
- }
- }
- if (!added) {
- // No matching column needed for computed expression, bail out
- return null;
- }
- }
- // We create the new node pointing to the index
- groupSetB.set(inputViewExprs.size());
- rewritingMapping.set(inputViewExprs.size(), i);
- additionalViewExprs.add(
- new RexInputRef(targetIdx, targetNode.getType()));
- // We need to create the rollup expression
- inputViewExprs.add(
- shuttleReferences(rexBuilder, targetNode, exprsLineage));
- added = true;
- } else {
- // This expression should be referenced directly
- for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
- RexNode n = topViewProject.getChildExps().get(k);
- if (!n.isA(SqlKind.INPUT_REF)) {
- continue;
- }
- int ref = ((RexInputRef) n).getIndex();
- if (ref == targetIdx) {
- groupSetB.set(k);
- rewritingMapping.set(k, i);
- added = true;
- }
- }
- }
- if (!added) {
- // No matching group by column, we bail out
- return null;
- }
- }
- final ImmutableBitSet groupSet = groupSetB.build();
- final List<AggCall> aggregateCalls = new ArrayList<>();
- for (int i = 0; i < queryAggregate.getAggCallList().size(); i++) {
- if (references != null && !references.get(queryAggregate.getGroupCount() + i)) {
- // Ignore
- continue;
- }
- int sourceIdx = queryAggregate.getGroupCount() + i;
- int targetIdx =
- aggregateMapping.getTargetOpt(sourceIdx);
- if (targetIdx < 0) {
- // No matching aggregation column, we bail out
- return null;
- }
- AggregateCall queryAggCall = queryAggregate.getAggCallList().get(i);
- boolean added = false;
- for (int k = 0; k < topViewProject.getChildExps().size() && !added; k++) {
- RexNode n = topViewProject.getChildExps().get(k);
- if (!n.isA(SqlKind.INPUT_REF)) {
- continue;
- }
- int ref = ((RexInputRef) n).getIndex();
- if (ref == targetIdx) {
- SqlAggFunction rollupAgg =
- getRollup(queryAggCall.getAggregation());
- if (rollupAgg == null) {
- // Cannot rollup this aggregate, bail out
- return null;
- }
- rewritingMapping.set(k, queryAggregate.getGroupCount() + aggregateCalls.size());
- final RexInputRef operand = rexBuilder.makeInputRef(input, k);
- aggregateCalls.add(
- // TODO: handle aggregate ordering
- relBuilder.aggregateCall(rollupAgg, operand)
- .approximate(queryAggCall.isApproximate())
- .distinct(queryAggCall.isDistinct())
- .as(queryAggCall.name));
- added = true;
- }
- }
- if (!added) {
- // No matching aggregation column, we bail out
- return null;
- }
- }
- // Create aggregate on top of input
- RelNode prevNode = result;
- relBuilder.push(result);
- if (inputViewExprs.size() != result.getRowType().getFieldCount()) {
- relBuilder.project(inputViewExprs);
- }
- result = relBuilder
- .aggregate(relBuilder.groupKey(groupSet), aggregateCalls)
- .build();
- if (prevNode == result && groupSet.cardinality() != result.getRowType().getFieldCount()) {
- // Aggregate was not inserted but we need to prune columns
- result = relBuilder
- .push(result)
- .project(relBuilder.fields(groupSet))
- .build();
- }
- // We introduce a project on top, as group by columns order is lost
- List<RexNode> projects = new ArrayList<>();
- Mapping inverseMapping = rewritingMapping.inverse();
- for (int i = 0; i < queryAggregate.getGroupCount(); i++) {
- projects.add(
- rexBuilder.makeInputRef(result,
- groupSet.indexOf(inverseMapping.getTarget(i))));
- }
- // We add aggregate functions that are present in result to projection list
- for (int i = queryAggregate.getGroupCount(); i < result.getRowType().getFieldCount(); i++) {
- projects.add(
- rexBuilder.makeInputRef(result, i));
- }
- result = relBuilder
- .push(result)
- .project(projects)
- .build();
- } // end if queryAggregate.getGroupCount() != viewAggregate.getGroupCount()
-
- // Add query expressions on top. We first map query expressions to view
- // expressions. Once we have done that, if the expression is contained
- // and we have introduced already an operator on top of the input node,
- // we use the mapping to resolve the position of the expression in the
- // node.
- final RelDataType topRowType;
- final List<RexNode> topExprs = new ArrayList<>();
- if (topProject != null && !unionRewriting) {
- topExprs.addAll(topProject.getChildExps());
- topRowType = topProject.getRowType();
- } else {
- // Add all
- for (int pos = 0; pos < queryAggregate.getRowType().getFieldCount(); pos++) {
- topExprs.add(rexBuilder.makeInputRef(queryAggregate, pos));
- }
- topRowType = queryAggregate.getRowType();
- }
- // Available in view.
- final Multimap<RexNode, Integer> viewExprs = ArrayListMultimap.create();
- int numberViewExprs = 0;
- for (RexNode viewExpr : topViewProject.getChildExps()) {
- viewExprs.put(viewExpr, numberViewExprs++);
- }
- for (RexNode additionalViewExpr : additionalViewExprs) {
- viewExprs.put(additionalViewExpr, numberViewExprs++);
- }
- final List<RexNode> rewrittenExprs = new ArrayList<>(topExprs.size());
- for (RexNode expr : topExprs) {
- // First map through the aggregate
- RexNode rewrittenExpr = shuttleReferences(rexBuilder, expr, aggregateMapping);
- if (rewrittenExpr == null) {
- // Cannot map expression
- return null;
- }
- // Next map through the last project
- rewrittenExpr =
- shuttleReferences(rexBuilder, rewrittenExpr, viewExprs, result, rewritingMapping);
- if (rewrittenExpr == null) {
- // Cannot map expression
- return null;
- }
- rewrittenExprs.add(rewrittenExpr);
- }
- return relBuilder
- .push(result)
- .project(rewrittenExprs)
- .convert(topRowType, false)
- .build();
- }
-
- /**
- * Mapping from node expressions to target expressions.
- *
- * <p>If any of the expressions cannot be mapped, we return null.
- */
- protected Multimap<Integer, Integer> generateMapping(
- RexBuilder rexBuilder,
- RexSimplify simplify,
- RelMetadataQuery mq,
- RelNode node,
- RelNode target,
- ImmutableBitSet positions,
- BiMap<RelTableRef, RelTableRef> tableMapping,
- EquivalenceClasses sourceEC,
- List<RexNode> additionalExprs) {
- Preconditions.checkArgument(additionalExprs.isEmpty());
- Multimap<Integer, Integer> m = ArrayListMultimap.create();
- Map<RexTableInputRef, Set<RexTableInputRef>> equivalenceClassesMap =
- sourceEC.getEquivalenceClassesMap();
- Multimap<RexNode, Integer> exprsLineage = ArrayListMultimap.create();
- List<RexNode> timestampExprs = new ArrayList<>();
- for (int i = 0; i < target.getRowType().getFieldCount(); i++) {
- Set<RexNode> s = mq.getExpressionLineage(target, rexBuilder.makeInputRef(target, i));
- if (s == null) {
- // Bail out
- continue;
- }
- // We only support project - filter - join, thus it should map to
- // a single expression
- final RexNode e = Iterables.getOnlyElement(s);
- // Rewrite expr to be expressed on query tables
- final RexNode simplified = simplify.simplifyUnknownAsFalse(e);
- RexNode expr = RexUtil.swapTableColumnReferences(rexBuilder,
- simplified,
- tableMapping.inverse(),
- equivalenceClassesMap);
- exprsLineage.put(expr, i);
- SqlTypeName sqlTypeName = expr.getType().getSqlTypeName();
- if (sqlTypeName == SqlTypeName.TIMESTAMP
- || sqlTypeName == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
- timestampExprs.add(expr);
- }
- }
-
- // If this is a column of TIMESTAMP (WITH LOCAL TIME ZONE)
- // type, we add the possible rollup columns too.
- // This way we will be able to match FLOOR(ts to HOUR) to
- // FLOOR(ts to DAY) via FLOOR(FLOOR(ts to HOUR) to DAY)
- for (RexNode timestampExpr : timestampExprs) {
- for (TimeUnitRange value : SUPPORTED_DATE_TIME_ROLLUP_UNITS) {
- // CEIL
- RexNode ceilExpr =
- rexBuilder.makeCall(getCeilSqlFunction(value),
- timestampExpr, rexBuilder.makeFlag(value));
- // References self-row
- RexNode rewrittenCeilExpr =
- shuttleReferences(rexBuilder, ceilExpr, exprsLineage);
- if (rewrittenCeilExpr != null) {
- // We add the CEIL expression to the additional expressions, replacing the child
- // expression by the position that it references
- additionalExprs.add(rewrittenCeilExpr);
- // Then we simplify the expression and we add it to the expressions lineage so we
- // can try to find a match
- final RexNode simplified =
- simplify.simplifyUnknownAsFalse(ceilExpr);
- exprsLineage.put(simplified,
- target.getRowType().getFieldCount() + additionalExprs.size() - 1);
- }
- // FLOOR
- RexNode floorExpr =
- rexBuilder.makeCall(getFloorSqlFunction(value),
- timestampExpr, rexBuilder.makeFlag(value));
- // References self-row
- RexNode rewrittenFloorExpr =
- shuttleReferences(rexBuilder, floorExpr, exprsLineage);
- if (rewrittenFloorExpr != null) {
- // We add the FLOOR expression to the additional expressions, replacing the child
- // expression by the position that it references
- additionalExprs.add(rewrittenFloorExpr);
- // Then we simplify the expression and we add it to the expressions lineage so we
- // can try to find a match
- final RexNode simplified =
- simplify.simplifyUnknownAsFalse(floorExpr);
- exprsLineage.put(simplified,
- target.getRowType().getFieldCount() + additionalExprs.size() - 1);
- }
- }
- }
-
- for (int i : positions) {
- Set<RexNode> s = mq.getExpressionLineage(node, rexBuilder.makeInputRef(node, i));
- if (s == null) {
- // Bail out
- return null;
- }
- // We only support project - filter - join, thus it should map to
- // a single expression
- final RexNode e = Iterables.getOnlyElement(s);
- // Rewrite expr to be expressed on query tables
- final RexNode simplified = simplify.simplifyUnknownAsFalse(e);
- RexNode targetExpr = RexUtil.swapColumnReferences(rexBuilder,
- simplified, equivalenceClassesMap);
- final Collection<Integer> c = exprsLineage.get(targetExpr);
- if (!c.isEmpty()) {
- for (Integer j : c) {
- m.put(i, j);
- }
- } else {
- // If we did not find the expression, try to navigate it
- RexNode rewrittenTargetExpr =
- shuttleReferences(rexBuilder, targetExpr, exprsLineage);
- if (rewrittenTargetExpr == null) {
- // Some expressions were not present
- return null;
- }
- m.put(i, target.getRowType().getFieldCount() + additionalExprs.size());
- additionalExprs.add(rewrittenTargetExpr);
- }
- }
- return m;
- }
-
- /**
- * Get ceil function datetime.
- */
- protected SqlFunction getCeilSqlFunction(TimeUnitRange flag) {
- return SqlStdOperatorTable.CEIL;
- }
-
- /**
- * Get floor function datetime.
- */
- protected SqlFunction getFloorSqlFunction(TimeUnitRange flag) {
- return SqlStdOperatorTable.FLOOR;
- }
-
- /**
- * Get rollup aggregation function.
- */
- protected SqlAggFunction getRollup(SqlAggFunction aggregation) {
- if (aggregation == SqlStdOperatorTable.SUM
- || aggregation == SqlStdOperatorTable.MIN
- || aggregation == SqlStdOperatorTable.MAX
- || aggregation == SqlStdOperatorTable.SUM0
- || aggregation == SqlStdOperatorTable.ANY_VALUE) {
- return aggregation;
- } else if (aggregation == SqlStdOperatorTable.COUNT) {
- return SqlStdOperatorTable.SUM0;
- } else {
- return null;
- }
- }
-
- @Override public Pair<RelNode, RelNode> pushFilterToOriginalViewPlan(RelBuilder builder,
- RelNode topViewProject, RelNode viewNode, RexNode cond) {
- // We add (and push) the filter to the view plan before triggering the rewriting.
- // This is useful in case some of the columns can be folded to same value after
- // filter is added.
- HepProgramBuilder pushFiltersProgram = new HepProgramBuilder();
- if (topViewProject != null) {
- pushFiltersProgram.addRuleInstance(filterProjectTransposeRule);
- }
- pushFiltersProgram
- .addRuleInstance(this.filterAggregateTransposeRule)
- .addRuleInstance(this.aggregateProjectPullUpConstantsRule)
- .addRuleInstance(this.projectMergeRule);
- final HepPlanner tmpPlanner = new HepPlanner(pushFiltersProgram.build());
- // Now that the planner is created, push the node
- RelNode topNode = builder
- .push(topViewProject != null ? topViewProject : viewNode)
- .filter(cond).build();
- tmpPlanner.setRoot(topNode);
- topNode = tmpPlanner.findBestExp();
- RelNode resultTopViewProject = null;
- RelNode resultViewNode = null;
- while (topNode != null) {
- if (topNode instanceof Project) {
- if (resultTopViewProject != null) {
- // Both projects could not be merged, we will bail out
- return Pair.of(topViewProject, viewNode);
- }
- resultTopViewProject = topNode;
- topNode = topNode.getInput(0);
- } else if (topNode instanceof Aggregate) {
- resultViewNode = topNode;
- topNode = null;
- } else {
- // We move to the child
- topNode = topNode.getInput(0);
- }
- }
- return Pair.of(resultTopViewProject, resultViewNode);
- }
- }
-
- /** Rule that matches Project on Aggregate. */
- public static class MaterializedViewProjectAggregateRule extends MaterializedViewAggregateRule {
-
- public MaterializedViewProjectAggregateRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
- super(
- operand(Project.class,
- operand(Aggregate.class, any())),
- relBuilderFactory,
- "MaterializedViewAggregateRule(Project-Aggregate)",
- generateUnionRewriting, unionRewritingPullProgram);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Project project = call.rel(0);
- final Aggregate aggregate = call.rel(1);
- perform(call, project, aggregate);
- }
- }
-
- /** Rule that matches Aggregate. */
- public static class MaterializedViewOnlyAggregateRule extends MaterializedViewAggregateRule {
-
- public MaterializedViewOnlyAggregateRule(RelBuilderFactory relBuilderFactory,
- boolean generateUnionRewriting, HepProgram unionRewritingPullProgram) {
- super(
- operand(Aggregate.class, any()),
- relBuilderFactory,
- "MaterializedViewAggregateRule(Aggregate)",
- generateUnionRewriting, unionRewritingPullProgram);
- }
-
- @Override public void onMatch(RelOptRuleCall call) {
- final Aggregate aggregate = call.rel(0);
- perform(call, null, aggregate);
- }
- }
//~ Methods ----------------------------------------------------------------
@@ -1813,7 +568,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* Otherwise, it returns a list of references to all columns in the node.
* The returned list is immutable.
*/
- private static List<RexNode> extractReferences(RexBuilder rexBuilder, RelNode node) {
+ protected List<RexNode> extractReferences(RexBuilder rexBuilder, RelNode node) {
ImmutableList.Builder<RexNode> exprs = ImmutableList.builder();
if (node instanceof Aggregate) {
Aggregate aggregate = (Aggregate) node;
@@ -1833,7 +588,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* producing all possible combinations of mappings. Each of the mappings will
* be bi-directional.
*/
- private static List<BiMap<RelTableRef, RelTableRef>> generateTableMappings(
+ protected List<BiMap<RelTableRef, RelTableRef>> generateTableMappings(
Multimap<RelTableRef, RelTableRef> multiMapTables) {
if (multiMapTables.isEmpty()) {
return ImmutableList.of();
@@ -1869,7 +624,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
}
/** Currently we only support TableScan - Project - Filter - Inner Join */
- private static boolean isValidRelNodePlan(RelNode node, RelMetadataQuery mq) {
+ protected boolean isValidRelNodePlan(RelNode node, RelMetadataQuery mq) {
final Multimap<Class<? extends RelNode>, RelNode> m =
mq.getNodeTypes(node);
if (m == null) {
@@ -1910,7 +665,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* <p>For each category, it creates the conjunction of the predicates. The
* result is an pair of RexNode objects corresponding to each category.
*/
- private static Pair<RexNode, RexNode> splitPredicates(
+ protected Pair<RexNode, RexNode> splitPredicates(
RexBuilder rexBuilder, RexNode pred) {
List<RexNode> equiColumnsPreds = new ArrayList<>();
List<RexNode> residualPreds = new ArrayList<>();
@@ -1952,7 +707,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* predicates in the input {@code compensationEquiColumns} multimap if it is provided.
* If it cannot be rewritten, it returns false.
*/
- private static boolean compensatePartial(
+ protected boolean compensatePartial(
Set<RelTableRef> sourceTableRefs,
EquivalenceClasses sourceEC,
Set<RelTableRef> targetTableRefs,
@@ -2057,7 +812,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
*
* <p>In turn, if containment cannot be confirmed, the method returns null.
*/
- private static Pair<RexNode, RexNode> computeCompensationPredicates(
+ protected Pair<RexNode, RexNode> computeCompensationPredicates(
RexBuilder rexBuilder,
RexSimplify simplify,
EquivalenceClasses sourceEC,
@@ -2106,7 +861,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* the target to make the equivalences classes match, it returns that
* compensation predicate.
*/
- private static RexNode generateEquivalenceClasses(RexBuilder rexBuilder,
+ protected RexNode generateEquivalenceClasses(RexBuilder rexBuilder,
EquivalenceClasses sourceEC, EquivalenceClasses targetEC) {
if (sourceEC.getEquivalenceClasses().isEmpty() && targetEC.getEquivalenceClasses().isEmpty()) {
// No column equality predicates in query and view
@@ -2166,7 +921,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* <p>If any of the source equivalence classes cannot be mapped to a target equivalence
* class, it returns null.
*/
- private static Multimap<Integer, Integer> extractPossibleMapping(
+ protected Multimap<Integer, Integer> extractPossibleMapping(
List<Set<RexTableInputRef>> sourceEquivalenceClasses,
List<Set<RexTableInputRef>> targetEquivalenceClasses) {
Multimap<Integer, Integer> mapping = ArrayListMultimap.create();
@@ -2203,7 +958,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* <p>The method will return the rewritten expression. If any of the expressions in the input
* expression cannot be mapped, it will return null.
*/
- private static RexNode rewriteExpression(
+ protected RexNode rewriteExpression(
RexBuilder rexBuilder,
RelMetadataQuery mq,
RelNode targetNode,
@@ -2234,7 +989,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* <p>The method will return the rewritten expressions. If any of the subexpressions in the input
* expressions cannot be mapped, it will return null.
*/
- private static List<RexNode> rewriteExpressions(
+ protected List<RexNode> rewriteExpressions(
RexBuilder rexBuilder,
RelMetadataQuery mq,
RelNode targetNode,
@@ -2270,7 +1025,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* It swaps the table references and then the column references of the input
* expressions using the table mapping and the equivalence classes.
*/
- private static NodeLineage generateSwapTableColumnReferencesLineage(
+ protected NodeLineage generateSwapTableColumnReferencesLineage(
RexBuilder rexBuilder,
RelMetadataQuery mq,
RelNode node,
@@ -2304,7 +1059,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* It swaps the column references and then the table references of the input
* expressions using the equivalence classes and the table mapping.
*/
- private static NodeLineage generateSwapColumnTableReferencesLineage(
+ protected NodeLineage generateSwapColumnTableReferencesLineage(
RexBuilder rexBuilder,
RelMetadataQuery mq,
RelNode node,
@@ -2340,7 +1095,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* digest of the expression and the index that the replacement input ref should
* point to.
*/
- private static RexNode replaceWithOriginalReferences(final RexBuilder rexBuilder,
+ protected RexNode replaceWithOriginalReferences(final RexBuilder rexBuilder,
final RelNode node, final NodeLineage nodeLineage, final RexNode exprToRewrite) {
// Currently we allow the following:
// 1) compensation pred can be directly map to expression
@@ -2381,7 +1136,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* input column set. If a reference index cannot be found in
* the input set, then we return null.
*/
- private static RexNode shuttleReferences(final RexBuilder rexBuilder,
+ protected RexNode shuttleReferences(final RexBuilder rexBuilder,
final RexNode node, final Mapping mapping) {
try {
RexShuttle visitor =
@@ -2406,7 +1161,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* Replaces all the possible sub-expressions by input references
* to the input node.
*/
- private static RexNode shuttleReferences(final RexBuilder rexBuilder,
+ protected RexNode shuttleReferences(final RexBuilder rexBuilder,
final RexNode expr, final Multimap<RexNode, Integer> exprsLineage) {
return shuttleReferences(rexBuilder, expr,
exprsLineage, null, null);
@@ -2418,7 +1173,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
* to change the position to reference. Takes the reference type
* from the input node.
*/
- private static RexNode shuttleReferences(final RexBuilder rexBuilder,
+ protected RexNode shuttleReferences(final RexBuilder rexBuilder,
final RexNode expr, final Multimap<RexNode, Integer> exprsLineage,
final RelNode node, final Mapping rewritingMapping) {
try {
@@ -2494,7 +1249,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
/**
* Class representing an equivalence class, i.e., a set of equivalent columns
*/
- private static class EquivalenceClasses {
+ protected static class EquivalenceClasses {
private final Map<RexTableInputRef, Set<RexTableInputRef>> nodeToEquivalenceClass;
private Map<RexTableInputRef, Set<RexTableInputRef>> cacheEquivalenceClassesMap;
@@ -2580,7 +1335,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
}
/** Expression lineage details. */
- private static class NodeLineage {
+ protected static class NodeLineage {
private final Map<RexNode, Integer> exprsLineage;
private final Map<RexNode, Integer> exprsLineageLosslessCasts;
@@ -2593,7 +1348,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
}
/** Edge for graph */
- private static class Edge extends DefaultEdge {
+ protected static class Edge extends DefaultEdge {
final Multimap<RexTableInputRef, RexTableInputRef> equiColumns =
ArrayListMultimap.create();
@@ -2607,7 +1362,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
}
/** View partitioning result */
- private static class ViewPartialRewriting {
+ protected static class ViewPartialRewriting {
private final RelNode newView;
private final Project newTopViewProject;
private final RelNode newViewNode;
@@ -2625,7 +1380,7 @@ public abstract class AbstractMaterializedViewRule extends RelOptRule {
}
/** Complete, view partial, or query partial. */
- private enum MatchModality {
+ protected enum MatchModality {
COMPLETE,
VIEW_PARTIAL,
QUERY_PARTIAL
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java b/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java
new file mode 100644
index 0000000..5432e29
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/materialize/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides a materialized rewriting algorithm encapsulated within a planner rule.
+ */
+package org.apache.calcite.rel.rules.materialize;
diff --git a/core/src/test/java/org/apache/calcite/test/LatticeTest.java b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
index b781d8a..6da06e9 100644
--- a/core/src/test/java/org/apache/calcite/test/LatticeTest.java
+++ b/core/src/test/java/org/apache/calcite/test/LatticeTest.java
@@ -23,7 +23,12 @@ import org.apache.calcite.materialize.MaterializationService;
import org.apache.calcite.plan.RelOptPlanner;
import org.apache.calcite.plan.RelOptRule;
import org.apache.calcite.plan.RelOptUtil;
-import org.apache.calcite.rel.rules.AbstractMaterializedViewRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyAggregateRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyFilterRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewOnlyJoinRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectAggregateRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectFilterRule;
+import org.apache.calcite.rel.rules.materialize.MaterializedViewProjectJoinRule;
import org.apache.calcite.runtime.Hook;
import org.apache.calcite.schema.SchemaPlus;
import org.apache.calcite.util.ImmutableBitSet;
@@ -502,12 +507,12 @@ public class LatticeTest {
private void checkTileAlgorithm(String statisticProvider,
String expectedExplain) {
final RelOptRule[] rules = {
- AbstractMaterializedViewRule.INSTANCE_PROJECT_FILTER,
- AbstractMaterializedViewRule.INSTANCE_FILTER,
- AbstractMaterializedViewRule.INSTANCE_PROJECT_JOIN,
- AbstractMaterializedViewRule.INSTANCE_JOIN,
- AbstractMaterializedViewRule.INSTANCE_PROJECT_AGGREGATE,
- AbstractMaterializedViewRule.INSTANCE_AGGREGATE
+ MaterializedViewProjectFilterRule.INSTANCE,
+ MaterializedViewOnlyFilterRule.INSTANCE,
+ MaterializedViewProjectJoinRule.INSTANCE,
+ MaterializedViewOnlyJoinRule.INSTANCE,
+ MaterializedViewProjectAggregateRule.INSTANCE,
+ MaterializedViewOnlyAggregateRule.INSTANCE
};
MaterializationService.setThreadLocal();
MaterializationService.instance().clear();
diff --git a/site/_docs/materialized_views.md b/site/_docs/materialized_views.md
index 8fdbb28..2da6799 100644
--- a/site/_docs/materialized_views.md
+++ b/site/_docs/materialized_views.md
@@ -67,7 +67,7 @@ views, e.g., views with an arbitrary number of join operators.
In turn, an alternative rule that attempts to match queries to views by extracting some structural information about the expression to replace has been proposed.
-`AbstractMaterializedViewRule` builds on the ideas presented in [<a href="#ref-gl01">GL01</a>] and introduces some additional extensions.
+`MaterializedViewRule` builds on the ideas presented in [<a href="#ref-gl01">GL01</a>] and introduces some additional extensions.
The rule can rewrite expressions containing arbitrary chains of Join, Filter, and Project operators.
Additionally, the rule can rewrite expressions rooted at an Aggregate operator, rolling aggregations up if necessary. In turn, it can also produce rewritings using Union operators if the query can be partially answered from a view.
@@ -75,7 +75,7 @@ To produce a larger number of rewritings, the rule relies on information exposed
##### Rewriting coverage
-Let us illustrate with some examples the coverage of the view rewriting algorithm implemented in `AbstractMaterializedViewRule`. The examples are based on the following database schema.
+Let us illustrate with some examples the coverage of the view rewriting algorithm implemented in `MaterializedViewRule`. The examples are based on the following database schema.
```
CREATE TABLE depts(