You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by hy...@apache.org on 2019/06/11 17:44:51 UTC

[calcite] branch master updated: [CALCITE-3087] AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its Mapping breaks ordering (DonnyZone)

This is an automated email from the ASF dual-hosted git repository.

hyuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/calcite.git


The following commit(s) were added to refs/heads/master by this push:
     new a3c56be  [CALCITE-3087] AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its Mapping breaks ordering (DonnyZone)
a3c56be is described below

commit a3c56be7bccc58859524ba39e5b30b7078f97d00
Author: wellfengzhu <we...@gmail.com>
AuthorDate: Mon Jun 3 21:15:27 2019 +0800

    [CALCITE-3087] AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its Mapping breaks ordering (DonnyZone)
---
 .../apache/calcite/plan/SubstitutionVisitor.java   | 25 +++++++++--
 .../calcite/rel/mutable/MutableAggregate.java      |  5 +++
 .../apache/calcite/rel/mutable/MutableRels.java    |  2 +-
 .../org/apache/calcite/util/mapping/Mappings.java  | 23 +++++++++-
 .../apache/calcite/test/MaterializationTest.java   | 50 ++++++++++++++++++++++
 5 files changed, 99 insertions(+), 6 deletions(-)

diff --git a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
index 3ddf3c3..9dc77ec 100644
--- a/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
+++ b/core/src/main/java/org/apache/calcite/plan/SubstitutionVisitor.java
@@ -1340,10 +1340,29 @@ public class SubstitutionVisitor {
       if (mapping == null) {
         return null;
       }
+      Mapping inverseMapping = mapping.inverse();
       final MutableAggregate aggregate2 =
-          permute(query, project.getInput(), mapping.inverse());
-      final MutableRel result = unifyAggregates(aggregate2, target);
-      return result == null ? null : call.result(result);
+          permute(query, project.getInput(), inverseMapping);
+      final MutableRel unifiedAggregate = unifyAggregates(aggregate2, target);
+      if (unifiedAggregate == null) {
+        return null;
+      }
+      MutableRel result = unifiedAggregate;
+      // Add Project if the mapping breaks order of fields in GroupSet
+      if (!Mappings.keepsOrdering(mapping)) {
+        final List<Integer> posList = new ArrayList<>();
+        final int fieldCount = aggregate2.rowType.getFieldCount();
+        for (int group: aggregate2.groupSet) {
+          if (inverseMapping.getTargetOpt(group) != -1) {
+            posList.add(inverseMapping.getTarget(group));
+          }
+        }
+        for (int i = posList.size(); i < fieldCount; i++) {
+          posList.add(i);
+        }
+        result = MutableRels.createProject(unifiedAggregate, posList);
+      }
+      return call.result(result);
     }
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
index 0a743e3..886714e 100644
--- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
+++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableAggregate.java
@@ -40,6 +40,11 @@ public class MutableAggregate extends MutableSingleRel {
     this.groupSets = groupSets == null
         ? ImmutableList.of(groupSet)
         : ImmutableList.copyOf(groupSets);
+    assert ImmutableBitSet.ORDERING.isStrictlyOrdered(
+        this.groupSets) : this.groupSets;
+    for (ImmutableBitSet set : this.groupSets) {
+      assert groupSet.contains(set);
+    }
     this.aggCalls = aggCalls;
   }
 
diff --git a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
index 8d52c98..0604c9a 100644
--- a/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
+++ b/core/src/main/java/org/apache/calcite/rel/mutable/MutableRels.java
@@ -138,7 +138,7 @@ public abstract class MutableRels {
     }
     return MutableProject.of(
         RelOptUtil.permute(child.cluster.getTypeFactory(), rowType,
-            Mappings.bijection(posList)),
+            Mappings.bijection(posList).inverse()),
         child,
         new AbstractList<RexNode>() {
           public int size() {
diff --git a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
index e952b60..5e16f6d 100644
--- a/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
+++ b/core/src/main/java/org/apache/calcite/util/mapping/Mappings.java
@@ -214,12 +214,13 @@ public abstract class Mappings {
    *
    * @param mapping Mapping
    * @param bitSets Collection of bit sets
-   * @return Bit sets with mapping applied
+   * @return Sorted bit sets with mapping applied
    */
   public static ImmutableList<ImmutableBitSet> apply2(final Mapping mapping,
       Iterable<ImmutableBitSet> bitSets) {
     return ImmutableList.copyOf(
-        Iterables.transform(bitSets, input1 -> apply(mapping, input1)));
+        ImmutableBitSet.ORDERING.sortedCopy(
+            Iterables.transform(bitSets, input1 -> apply(mapping, input1))));
   }
 
   /**
@@ -421,6 +422,24 @@ public abstract class Mappings {
   }
 
   /**
+   * Returns whether a mapping keeps order.
+   *
+   * <p>For example, {0:0, 1:1} and {0:1, 1:1} keeps order,
+   * and {0:1, 1:0} breaks the initial order.
+   */
+  public static boolean keepsOrdering(TargetMapping mapping) {
+    int prevTarget = -1;
+    for (int i = 0; i < mapping.getSourceCount(); i++) {
+      int target = mapping.getTargetOpt(i);
+      if (target != -1 && target < prevTarget) {
+        return false;
+      }
+      prevTarget = target;
+    }
+    return true;
+  }
+
+  /**
    * Creates a mapping that consists of a set of contiguous ranges.
    *
    * <p>For example,</p>
diff --git a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
index fc5c610..9e9507c 100644
--- a/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
+++ b/core/src/test/java/org/apache/calcite/test/MaterializationTest.java
@@ -598,6 +598,56 @@ public class MaterializationTest {
                 + "    EnumerableTableScan(table=[[hr, m0]])"));
   }
 
+  /** Test case for
+   * <a href="https://issues.apache.org/jira/browse/CALCITE-3087">[CALCITE-3087]
+   * AggregateOnProjectToAggregateUnifyRule ignores Project incorrectly when its
+   * Mapping breaks ordering</a>. */
+  @Test public void testAggregateOnProject1() {
+    checkMaterialize(
+        "select \"empid\", \"deptno\", count(*) as c, sum(\"empid\") as s from \"emps\" "
+            + "group by \"empid\", \"deptno\"",
+        "select count(*) + 1 as c, \"deptno\" from \"emps\" group by \"deptno\", \"empid\"");
+  }
+
+  @Test public void testAggregateOnProject2() {
+    checkMaterialize(
+        "select \"empid\", \"deptno\", count(*) as c, sum(\"salary\") as s from \"emps\" "
+            + "group by \"empid\", \"deptno\"",
+        "select count(*) + 1 as c,  \"deptno\" from \"emps\" group by cube(\"deptno\", \"empid\")",
+        HR_FKUK_MODEL,
+        CalciteAssert.checkResultContains(
+            "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], "
+                + "expr#4=[+($t2, $t3)], C=[$t4], deptno=[$t1])\n"
+                + "  EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {0}, {1}, {}]], agg#0=[$SUM0($2)])\n"
+                + "    EnumerableTableScan(table=[[hr, m0]])"));
+  }
+
+  @Test public void testAggregateOnProject3() {
+    checkMaterialize(
+        "select \"empid\", \"deptno\", count(*) as c, sum(\"salary\") as s from \"emps\" "
+            + "group by \"empid\", \"deptno\"",
+        "select count(*) + 1 as c,  \"deptno\" from \"emps\" group by rollup(\"deptno\", \"empid\")",
+        HR_FKUK_MODEL,
+        CalciteAssert.checkResultContains(
+            "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], "
+                + "expr#4=[+($t2, $t3)], C=[$t4], deptno=[$t1])\n"
+                + "  EnumerableAggregate(group=[{0, 1}], groups=[[{0, 1}, {1}, {}]], agg#0=[$SUM0($2)])\n"
+                + "    EnumerableTableScan(table=[[hr, m0]])"));
+  }
+
+  @Test public void testAggregateOnProject4() {
+    checkMaterialize(
+        "select \"salary\", \"empid\", \"deptno\", count(*) as c, sum(\"commission\") as s from \"emps\" "
+            + "group by \"salary\", \"empid\", \"deptno\"",
+        "select count(*) + 1 as c,  \"deptno\" from \"emps\" group by rollup(\"empid\", \"deptno\", \"salary\")",
+        HR_FKUK_MODEL,
+        CalciteAssert.checkResultContains(
+            "EnumerableCalc(expr#0..3=[{inputs}], expr#4=[1], "
+                + "expr#5=[+($t3, $t4)], C=[$t5], deptno=[$t2])\n"
+                + "  EnumerableAggregate(group=[{0, 1, 2}], groups=[[{0, 1, 2}, {1, 2}, {1}, {}]], agg#0=[$SUM0($3)])\n"
+                + "    EnumerableTableScan(table=[[hr, m0]])"));
+  }
+
   @Test public void testSwapJoin() {
     checkMaterialize(
         "select count(*) as c from \"foodmart\".\"sales_fact_1997\" as s join \"foodmart\".\"time_by_day\" as t on s.\"time_id\" = t.\"time_id\"",