You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by hy...@apache.org on 2019/06/11 17:44:51 UTC
[calcite] branch master updated: [CALCITE-3087]
AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its
Mapping breaks ordering (DonnyZone)
This is an automated email from the ASF dual-hosted git repository.
hyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/master by this push:
new a3c56be [CALCITE-3087] AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its Mapping breaks ordering (DonnyZone)
a3c56be is described below
commit a3c56be7bccc58859524ba39e5b30b7078f97d00
Author: wellfengzhu <we...@gmail.com>
AuthorDate: Mon Jun 3 21:15:27 2019 +0800
[CALCITE-3087] AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its Mapping breaks ordering (DonnyZone)
---
.../apache/calcite/plan/SubstitutionVisitor.java | 25 +++++++++--
.../calcite/rel/mutable/MutableAggregate.java | 5 +++
.../apache/calcite/rel/mutable/MutableRels.java | 2 +-
.../org/apache/calcite/util/mapping/Mappings.java | 23 +++++++++-
.../apache/calcite/test/MaterializationTest.java | 50 ++++++++++++++++++++++
5 files changed, 99 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index 3ddf3c3..9dc77ec 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -1340,10 +1340,29 @@ public class SubstitutionVisitor {
if (mapping == null) {
return null;
}
+ Mapping inverseMapping = mapping.inverse();
final MutableAggregate aggregate2 =
- permute(query, project.getInput(), mapping.inverse());
- final MutableRel result = unifyAggregates(aggregate2, target);
- return result == null ? null : call.result(result);
+ permute(query, project.getInput(), inverseMapping);
+ final MutableRel unifiedAggregate = unifyAggregates(aggregate2, target);
+ if (unifiedAggregate == null) {
+ return null;
+ }
+ MutableRel result = unifiedAggregate;
+ // Add Project if the mapping breaks order of fields in GroupSet
+ if (!Mappings.keepsOrdering(mapping)) {
+ final List<Integer> posList = new ArrayList<>();
+ final int fieldCount = aggregate2.rowType.getFieldCount();
+ for (int group: aggregate2.groupSet) {
+ if (inverseMapping.getTargetOpt(group) != -1) {
+ posList.add(inverseMapping.getTarget(group));
+ }
+ }
+ for (int i = posList.size(); i < fieldCount; i++) {
+ posList.add(i);
+ }
+ result = MutableRels.createProject(unifiedAggregate, posList);
+ }
+ return call.result(result);
}
}
diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
index 0a743e3..886714e 100644
--- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
+++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
@@ -40,6 +40,11 @@ public class MutableAggregate extends MutableSingleRel {
this.groupSets = groupSets == null
? ImmutableList.of(groupSet)
: ImmutableList.copyOf(groupSets);
+ assert ImmutableBitSet.ORDERING.isStrictlyOrdered(
+ this.groupSets) : this.groupSets;
+ for (ImmutableBitSet set : this.groupSets) {
+ assert groupSet.contains(set);
+ }
this.aggCalls = aggCalls;
}
diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
index 8d52c98..0604c9a 100644
--- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
+++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
@@ -138,7 +138,7 @@ public abstract class MutableRels {
}
return MutableProject.of(
RelOptUtil.permute(child.cluster.getTypeFactory(), rowType,
- Mappings.bijection(posList)),
+ Mappings.bijection(posList).inverse()),
child,
new AbstractList<RexNode>() {
public int size() {
diff --git a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
index e952b60..5e16f6d 100644
--- a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
+++ b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
@@ -214,12 +214,13 @@ public abstract class Mappings {
*
* @param mapping Mapping
* @param bitSets Collection of bit sets
- * @return Bit sets with mapping applied
+ * @return Sorted bit sets with mapping applied
*/
public static ImmutableList<ImmutableBitSet> apply2(final Mapping mapping,
Iterable<ImmutableBitSet> bitSets) {
return ImmutableList.copyOf(
- Iterables.transform(bitSets, input1 -> apply(mapping, input1)));
+ ImmutableBitSet.ORDERING.sortedCopy(
+ Iterables.transform(bitSets, input1 -> apply(mapping, input1))));
}
/**
@@ -421,6 +422,24 @@ public abstract class Mappings {
}
/**
+ * Returns whether a mapping keeps order.
+ *
+ * <p>For example, {0:0, 1:1} and {0:1, 1:1} keeps order,
+ * and {0:1, 1:0} breaks the initial order.
+ */
+ public static boolean keepsOrdering(TargetMapping mapping) {
+ int prevTarget = -1;
+ for (int i = 0; i < mapping.getSourceCount(); i++) {
+ int target = mapping.getTargetOpt(i);
+ if (target != -1 && target < prevTarget) {
+ return false;
+ }
+ prevTarget = target;
+ }
+ return true;
+ }
+
+ /**
* Creates a mapping that consists of a set of contiguous ranges.
*
* <p>For example,</p>
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index fc5c610..9e9507c 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -598,6 +598,56 @@ public class MaterializationTest {
+ " EnumerableTableScan(table=[[hr, m0]])"));
}
+ /** Test case for
+ * <a href="https://issues.apache.org/jira/browse/CALCITE-3087">[CALCITE-3087]
+ * AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its
+ * Mapping breaks ordering</a>. */
+ @Test public void testAggregateOnProject1() {
+ checkMaterialize(
+ "select \"empid\", \"deptno\", count(*) as c, sum(\"empid\") as s from \"emps\" "
+ + "group by \"empid\", \"deptno\"",
+ "select count(*) + 1 as c, \"deptno\" from \"emps\" group by \"deptno\", \"empid\"");
+ }
+
+ @Test public void testAggregateOnProject2() {
+ checkMaterialize(
+ "select \"empid\", \"deptno\", count(*) as c, sum(\"salary\") as s from \"emps\" "
+ + "group by \"empid\", \"deptno\"",
+ "select count(*) + 1 as c, \"deptno\" from \"emps\" group by cube(\"deptno\", \"empid\")",
+ HR_FKUK_MODEL,
+ CalciteAssert.checkResultContains(
+ "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], "
+ + "expr#4=[+($t2, $t3)], C=[$t4], deptno=[$t1])\n"
+ + " EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {1}, {}]], agg#0=[$SUM0($2)])\n"
+ + " EnumerableTableScan(table=[[hr, m0]])"));
+ }
+
+ @Test public void testAggregateOnProject3() {
+ checkMaterialize(
+ "select \"empid\", \"deptno\", count(*) as c, sum(\"salary\") as s from \"emps\" "
+ + "group by \"empid\", \"deptno\"",
+ "select count(*) + 1 as c, \"deptno\" from \"emps\" group by rollup(\"deptno\", \"empid\")",
+ HR_FKUK_MODEL,
+ CalciteAssert.checkResultContains(
+ "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], "
+ + "expr#4=[+($t2, $t3)], C=[$t4], deptno=[$t1])\n"
+ + " EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {1}, {}]], agg#0=[$SUM0($2)])\n"
+ + " EnumerableTableScan(table=[[hr, m0]])"));
+ }
+
+ @Test public void testAggregateOnProject4() {
+ checkMaterialize(
+ "select \"salary\", \"empid\", \"deptno\", count(*) as c, sum(\"commission\") as s from \"emps\" "
+ + "group by \"salary\", \"empid\", \"deptno\"",
+ "select count(*) + 1 as c, \"deptno\" from \"emps\" group by rollup(\"empid\", \"deptno\", \"salary\")",
+ HR_FKUK_MODEL,
+ CalciteAssert.checkResultContains(
+ "EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], "
+ + "expr#5=[+($t3, $t4)], C=[$t5], deptno=[$t2])\n"
+ + " EnumerableAggregate(group=[{0, 1, 2}], groups=[[{0, 1, 2}, {1, 2}, {1}, {}]], agg#0=[$SUM0($3)])\n"
+ + " EnumerableTableScan(table=[[hr, m0]])"));
+ }
+
@Test public void testSwapJoin() {
checkMaterialize(
"select count(*) as c from \"foodmart\".\"sales_fact_1997\" as s join \"foodmart\".\"time_by_day\" as t on s.\"time_id\" = t.\"time_id\"",