You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jc...@apache.org on 2019/03/12 22:54:03 UTC
[calcite] branch CALCITE-2912 created (now 1e81f95)
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a change to branch CALCITE-2912
in repository https://gitbox.apache.org/repos/asf/calcite.git.
at 1e81f95 [CALCITE-2912]
This branch includes the following new commits:
new 1e81f95 [CALCITE-2912]
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[calcite] 01/01: [CALCITE-2912]
Posted by jc...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jcamacho pushed a commit to branch CALCITE-2912
in repository https://gitbox.apache.org/repos/asf/calcite.git
commit 1e81f956a4736b70c258b5594850236a4100e2a6
Author: Jesus Camacho Rodriguez <jc...@apache.org>
AuthorDate: Tue Mar 12 15:53:52 2019 -0700
[CALCITE-2912]
---
.../rules/AggregateProjectPullUpConstantsRule.java | 35 ++++--
.../calcite/rel/rules/AggregateReduceRule.java | 127 +++++++++++++++++++++
.../apache/calcite/test/MaterializationTest.java | 16 +++
.../org/apache/calcite/test/RelOptRulesTest.java | 10 +-
.../org/apache/calcite/test/RelOptRulesTest.xml | 31 ++---
5 files changed, 192 insertions(+), 27 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectPullUpConstantsRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectPullUpConstantsRule.java
index a12e6d1..9a65488 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectPullUpConstantsRule.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateProjectPullUpConstantsRule.java
@@ -31,11 +31,13 @@ import org.apache.calcite.rel.type.RelDataTypeField;
import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.tools.RelBuilder;
import org.apache.calcite.tools.RelBuilderFactory;
import org.apache.calcite.util.ImmutableBitSet;
import org.apache.calcite.util.Pair;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.NavigableMap;
@@ -100,7 +102,7 @@ public class AggregateProjectPullUpConstantsRule extends RelOptRule {
assert !aggregate.indicator : "predicate ensured no grouping sets";
final int groupCount = aggregate.getGroupCount();
- if (groupCount == 1) {
+ if (groupCount < 1) {
// No room for optimization since we cannot convert from non-empty
// GROUP BY list to the empty one.
return;
@@ -127,14 +129,7 @@ public class AggregateProjectPullUpConstantsRule extends RelOptRule {
return;
}
- if (groupCount == map.size()) {
- // At least a single item in group by is required.
- // Otherwise "GROUP BY 1, 2" might be altered to "GROUP BY ()".
- // Removing of the first element is not optimal here,
- // however it will allow us to use fast path below (just trim
- // groupCount).
- map.remove(map.navigableKeySet().first());
- }
+ final boolean empty = groupCount == map.size();
ImmutableBitSet newGroupSet = aggregate.getGroupSet();
for (int key : map.keySet()) {
@@ -154,7 +149,25 @@ public class AggregateProjectPullUpConstantsRule extends RelOptRule {
aggCall.adaptTo(input, aggCall.getArgList(), aggCall.filterArg,
groupCount, newGroupCount));
}
- relBuilder.aggregate(relBuilder.groupKey(newGroupSet), newAggCalls);
+
+ // Create aggregate operator.
+ if (empty) {
+ // If empty, create an additional count(*) field
+ Aggregate tmpAggregate = (Aggregate) relBuilder
+ .aggregate(relBuilder.groupKey(), relBuilder.countStar(null))
+ .build();
+ newAggCalls.add(tmpAggregate.getAggCallList().get(0));
+ // Reset stack and create new aggregate call
+ relBuilder.push(tmpAggregate.getInput());
+ relBuilder.aggregate(relBuilder.groupKey(), newAggCalls);
+ // Add a filter on the new count(*) != 0
+ relBuilder.filter(
+ rexBuilder.makeCall(SqlStdOperatorTable.NOT_EQUALS,
+ relBuilder.field(relBuilder.peek().getRowType().getFieldCount() - 1),
+ rexBuilder.makeBigintLiteral(BigDecimal.ZERO)));
+ } else {
+ relBuilder.aggregate(relBuilder.groupKey(newGroupSet), newAggCalls);
+ }
// Create a projection back again.
List<Pair<RexNode, String>> projects = new ArrayList<>();
@@ -186,6 +199,8 @@ public class AggregateProjectPullUpConstantsRule extends RelOptRule {
projects.add(Pair.of(expr, field.getName()));
}
relBuilder.project(Pair.left(projects), Pair.right(projects)); // inverse
+ // Create top Project fixing nullability of fields
+ relBuilder.convert(aggregate.getRowType(), false);
call.transformTo(relBuilder.build());
}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceRule.java
new file mode 100644
index 0000000..4d237de
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateReduceRule.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.AggregateCall;
+import org.apache.calcite.rel.core.RelFactories;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.tools.RelBuilder;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Planner rule that reduces aggregate functions in
+ * {@link org.apache.calcite.rel.core.Aggregate}s to simpler forms.
+ *
+ * <p>Rewrites:
+ * <ul>
+ *
+ * <li>COUNT(x) → COUNT(*) if x is not nullable
+ * </ul>
+ *
+ * It also removes duplicate aggregate calls.
+ */
+public class AggregateReduceRule extends RelOptRule {
+
+ /** The singleton. */
+ public static final AggregateReduceRule INSTANCE =
+ new AggregateReduceRule();
+
+ /** Private constructor. */
+ private AggregateReduceRule() {
+ super(operand(LogicalAggregate.class, any()),
+ RelFactories.LOGICAL_BUILDER, null);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final RelBuilder relBuilder = call.builder();
+ final Aggregate aggRel = call.rel(0);
+ final RexBuilder rexBuilder = aggRel.getCluster().getRexBuilder();
+
+ // We try to rewrite COUNT(x) into COUNT(*) if x is not nullable.
+ // We remove duplicate aggregate calls as well.
+ boolean rewrite = false;
+ boolean identity = true;
+ final Map<AggregateCall, Integer> mapping = new HashMap<>();
+ final List<Integer> indexes = new ArrayList<>();
+ final List<AggregateCall> aggCalls = aggRel.getAggCallList();
+ final List<AggregateCall> newAggCalls = new ArrayList<>(aggCalls.size());
+ int nextIdx = aggRel.getGroupCount() + aggRel.getIndicatorCount();
+ for (int i = 0; i < aggCalls.size(); i++) {
+ AggregateCall aggCall = aggCalls.get(i);
+ if (aggCall.getAggregation().getKind() == SqlKind.COUNT && !aggCall.isDistinct()) {
+ final List<Integer> args = aggCall.getArgList();
+ final List<Integer> nullableArgs = new ArrayList<>(args.size());
+ for (int arg : args) {
+ if (aggRel.getInput().getRowType().getFieldList().get(arg).getType().isNullable()) {
+ nullableArgs.add(arg);
+ }
+ }
+ if (nullableArgs.size() != args.size()) {
+ aggCall = aggCall.copy(nullableArgs, aggCall.filterArg, aggCall.collation);
+ rewrite = true;
+ }
+ }
+ Integer idx = mapping.get(aggCall);
+ if (idx == null) {
+ newAggCalls.add(aggCall);
+ idx = nextIdx++;
+ mapping.put(aggCall, idx);
+ } else {
+ rewrite = true;
+ identity = false;
+ }
+ indexes.add(idx);
+ }
+
+ if (rewrite) {
+ // We trigger the transform
+ final Aggregate newAggregate = aggRel.copy(aggRel.getTraitSet(), aggRel.getInput(),
+ aggRel.indicator, aggRel.getGroupSet(), aggRel.getGroupSets(),
+ newAggCalls);
+ if (identity) {
+ call.transformTo(newAggregate);
+ } else {
+ final int offset = aggRel.getGroupCount() + aggRel.getIndicatorCount();
+ final List<RexNode> projList = new ArrayList<>();
+ for (int i = 0; i < offset; ++i) {
+ projList.add(
+ rexBuilder.makeInputRef(
+ aggRel.getRowType().getFieldList().get(i).getType(), i));
+ }
+ for (int i = offset; i < aggRel.getRowType().getFieldCount(); ++i) {
+ projList.add(
+ rexBuilder.makeInputRef(
+ aggRel.getRowType().getFieldList().get(i).getType(), indexes.get(i - offset)));
+ }
+ call.transformTo(relBuilder.push(newAggregate).project(projList).build());
+ }
+ }
+ }
+
+}
+
+// End AggregateReduceRule.java
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index 3126db2..5192bbe 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -2193,6 +2193,22 @@ public class MaterializationTest {
+ " EnumerableTableScan(table=[[hr, m0]]"));
}
+// @Test public void testAggregateMaterializationWithConstantFilter() {
+// checkMaterialize(
+// "select \"deptno\", \"name\", count(*) as c\n"
+// + "from \"emps\" group by \"deptno\", \"name\"",
+// "select \"name\", count(*) as c\n"
+// + "from \"emps\" where \"name\" = 'a_name' group by \"name\"");
+// }
+//
+// @Test public void testAggregateMaterializationWithConstantFilter2() {
+// checkMaterialize(
+// "select \"deptno\", \"name\", \"salary\", count(*) as c\n"
+// + "from \"emps\" group by \"deptno\", \"name\", \"salary\"",
+// "select \"deptno\", \"name\", count(*) as c\n"
+// + "from \"emps\" where \"name\" = 'a_name' group by \"deptno\", \"name\"");
+// }
+
@Test public void testMaterializationSubstitution() {
String q = "select *\n"
+ "from (select * from \"emps\" where \"empid\" < 300)\n"
diff --git a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
index ef2b705..a8ec4fd 100644
--- a/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
+++ b/core/src/test/java/org/apache/calcite/test/RelOptRulesTest.java
@@ -56,6 +56,7 @@ import org.apache.calcite.rel.rules.AggregateJoinTransposeRule;
import org.apache.calcite.rel.rules.AggregateProjectMergeRule;
import org.apache.calcite.rel.rules.AggregateProjectPullUpConstantsRule;
import org.apache.calcite.rel.rules.AggregateReduceFunctionsRule;
+import org.apache.calcite.rel.rules.AggregateReduceRule;
import org.apache.calcite.rel.rules.AggregateUnionAggregateRule;
import org.apache.calcite.rel.rules.AggregateUnionTransposeRule;
import org.apache.calcite.rel.rules.AggregateValuesRule;
@@ -3822,17 +3823,20 @@ public class RelOptRulesTest extends RelOptTestBase {
checkPlanning(new HepPlanner(program), sql);
}
- /** Tests {@link AggregateProjectPullUpConstantsRule} where reduction is not
- * possible because "deptno" is the only key. */
+ /** Tests {@link AggregateProjectPullUpConstantsRule} where all columns can be
+ * reduced. */
@Test public void testAggregateConstantKeyRule2() {
final HepProgram program = new HepProgramBuilder()
.addRuleInstance(AggregateProjectPullUpConstantsRule.INSTANCE2)
+ .addRuleInstance(AggregateReduceRule.INSTANCE)
+ .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
+ .addRuleInstance(ProjectMergeRule.INSTANCE)
.build();
final String sql = "select count(*) as c\n"
+ "from sales.emp\n"
+ "where deptno = 10\n"
+ "group by deptno";
- checkPlanUnchanged(new HepPlanner(program), sql);
+ checkPlanning(new HepPlanner(program), sql);
}
/** Tests {@link AggregateProjectPullUpConstantsRule} where both keys are
diff --git a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
index c94842c..fb03df2 100644
--- a/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
+++ b/core/src/test/resources/org/apache/calcite/test/RelOptRulesTest.xml
@@ -4047,10 +4047,11 @@ LogicalAggregate(group=[{0, 1}], EXPR$2=[MAX($2)])
</Resource>
<Resource name="planAfter">
<![CDATA[
-LogicalProject(EXPR$0=[$0], EXPR$1=[+(2, 3)], EXPR$2=[$1])
- LogicalAggregate(group=[{0}], EXPR$2=[MAX($2)])
- LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], MGR=[$3])
- LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], EXPR$2=[$0])
+ LogicalFilter(condition=[<>($1, 0)])
+ LogicalAggregate(group=[{}], EXPR$2=[MAX($2)], agg#1=[COUNT()])
+ LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], MGR=[$3])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
]]>
</Resource>
</TestCase>
@@ -4069,10 +4070,11 @@ LogicalAggregate(group=[{0, 1}], EXPR$2=[MAX($2)])
</Resource>
<Resource name="planAfter">
<![CDATA[
-LogicalProject(EXPR$0=[$0], EXPR$1=[+(2, 3)], EXPR$2=[$1])
- LogicalAggregate(group=[{0}], EXPR$2=[MAX($2)])
- LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], FIVE=[5])
- LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], EXPR$2=[CAST($0):INTEGER NOT NULL])
+ LogicalFilter(condition=[<>($1, 0)])
+ LogicalAggregate(group=[{}], EXPR$2=[MAX($2)], agg#1=[COUNT()])
+ LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], FIVE=[5])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
]]>
</Resource>
</TestCase>
@@ -4091,10 +4093,11 @@ LogicalAggregate(group=[{0, 1}], EXPR$2=[MAX($2)])
</Resource>
<Resource name="planAfter">
<![CDATA[
-LogicalProject(EXPR$0=[$0], EXPR$1=[+(2, 3)], EXPR$2=[$1])
- LogicalAggregate(group=[{0}], EXPR$2=[MAX($2)])
- LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], $f2=[5])
- LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], EXPR$2=[CAST($0):INTEGER NOT NULL])
+ LogicalFilter(condition=[<>($1, 0)])
+ LogicalAggregate(group=[{}], EXPR$2=[MAX($2)], agg#1=[COUNT()])
+ LogicalProject(EXPR$0=[4], EXPR$1=[+(2, 3)], $f2=[5])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
]]>
</Resource>
</TestCase>
@@ -7977,8 +7980,8 @@ LogicalProject(C=[$1])
</Resource>
<Resource name="planAfter">
<![CDATA[
-LogicalProject(C=[$1])
- LogicalAggregate(group=[{0}], C=[COUNT()])
+LogicalFilter(condition=[<>($0, 0)])
+ LogicalAggregate(group=[{}], C=[COUNT()])
LogicalProject(DEPTNO=[$7])
LogicalFilter(condition=[=($7, 10)])
LogicalTableScan(table=[[CATALOG, SALES, EMP]])