You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2024/04/01 16:59:31 UTC

(ignite-3) branch ignite-21897 created (now 5746d704f3)

This is an automated email from the ASF dual-hosted git repository.

amashenkov pushed a change to branch ignite-21897
in repository https://gitbox.apache.org/repos/asf/ignite-3.git


      at 5746d704f3 Fix Collation pass-through for sorted map aggregate

This branch includes the following new commits:

     new 8a09204ea8 Fix test
     new 5746d704f3 Fix Collation pass-through for sorted map aggregate

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(ignite-3) 02/02: Fix Collation pass-through for sorted map aggregate

Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

amashenkov pushed a commit to branch ignite-21897
in repository https://gitbox.apache.org/repos/asf/ignite-3.git

commit 5746d704f3d5adef638ee9d7543c13489745616f
Author: amashenkov <an...@gmail.com>
AuthorDate: Wed Mar 27 21:09:35 2024 +0300

    Fix Collation pass-through for sorted map aggregate
---
 .../engine/rel/agg/IgniteSortAggregateBase.java    | 49 ++++++++++++++++------
 1 file changed, 36 insertions(+), 13 deletions(-)

diff --git a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/agg/IgniteSortAggregateBase.java b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/agg/IgniteSortAggregateBase.java
index d763c49b95..e5e88d9e18 100644
--- a/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/agg/IgniteSortAggregateBase.java
+++ b/modules/sql-engine/src/main/java/org/apache/ignite/internal/sql/engine/rel/agg/IgniteSortAggregateBase.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.internal.sql.engine.rel.agg;
 
+import static java.util.function.Predicate.not;
 import static org.apache.ignite.internal.sql.engine.util.Commons.maxPrefix;
 
 import it.unimi.dsi.fastutil.ints.IntList;
@@ -29,9 +30,11 @@ import org.apache.calcite.rel.RelCollations;
 import org.apache.calcite.rel.RelFieldCollation;
 import org.apache.calcite.util.ImmutableBitSet;
 import org.apache.calcite.util.Pair;
+import org.apache.calcite.util.mapping.Mapping;
 import org.apache.ignite.internal.sql.engine.rel.IgniteRel;
 import org.apache.ignite.internal.sql.engine.trait.TraitUtils;
 import org.apache.ignite.internal.sql.engine.trait.TraitsAwareIgniteRel;
+import org.apache.ignite.internal.sql.engine.util.Commons;
 
 /**
  * Defines common methods for {@link IgniteRel relational nodes} that implement sort-base aggregates.
@@ -77,24 +80,35 @@ interface IgniteSortAggregateBase extends TraitsAwareIgniteRel {
             RelTraitSet nodeTraits, List<RelTraitSet> inputTraits
     ) {
         RelCollation required = TraitUtils.collation(nodeTraits);
-        ImmutableBitSet requiredKeys = ImmutableBitSet.of(required.getKeys());
-        RelCollation collation;
 
-        if (getGroupSet().contains(requiredKeys)) {
-            List<RelFieldCollation> newCollationFields = new ArrayList<>(getGroupSet().cardinality());
-            newCollationFields.addAll(required.getFieldCollations());
-
-            ImmutableBitSet keysLeft = getGroupSet().except(requiredKeys);
+        if (getGroupSet().isEmpty()) {
+            return passThroughCollation(nodeTraits, inputTraits, RelCollations.EMPTY); // Erase collation for a single group.
+        } else if (required.getFieldCollations().isEmpty()) {
+            return passThroughCollation(nodeTraits, inputTraits, TraitUtils.createCollation(getGroupSet().asList())); // No match.
+        }
 
-            keysLeft.forEach(fieldIdx -> newCollationFields.add(TraitUtils.createFieldCollation(fieldIdx)));
+        ImmutableBitSet groupingColumns = ImmutableBitSet.range(getGroupSet().cardinality());
+        IntList prefix = maxPrefix(required.getKeys(), groupingColumns.asSet());
 
-            collation = RelCollations.of(newCollationFields);
-        } else {
-            collation = TraitUtils.createCollation(getGroupSet().toList());
+        if (prefix.isEmpty()) {
+            return passThroughCollation(nodeTraits, inputTraits, TraitUtils.createCollation(getGroupSet().asList())); // No match.
         }
 
-        return Pair.of(nodeTraits.replace(collation),
-                List.of(inputTraits.get(0).replace(collation)));
+        // Rearrange grouping columns to satisfy required collation (as much as possible).
+        List<RelFieldCollation> newCollationColumns = new ArrayList<>(getGroupSet().cardinality());
+        Mapping mapping = Commons.trimmingMapping(groupingColumns.cardinality(), groupingColumns);
+
+        // Add required columns first.
+        prefix.intStream().map(mapping::getTarget)
+                .forEach(fieldIdx -> newCollationColumns.add(TraitUtils.createFieldCollation(fieldIdx)));
+
+        // Then add missed grouping columns.
+        groupingColumns.asList().stream()
+                .filter(not(prefix::contains))
+                .map(mapping::getTarget)
+                .forEach(fieldIdx -> newCollationColumns.add(TraitUtils.createFieldCollation(fieldIdx)));
+
+        return passThroughCollation(nodeTraits, inputTraits, RelCollations.of(newCollationColumns));
     }
 
     /** {@inheritDoc} */
@@ -118,4 +132,13 @@ interface IgniteSortAggregateBase extends TraitsAwareIgniteRel {
                 inputTraits
         ));
     }
+
+    private static Pair<RelTraitSet, List<RelTraitSet>> passThroughCollation(
+            RelTraitSet nodeTraits,
+            List<RelTraitSet> inputTraits,
+            RelCollation collation
+    ) {
+        return Pair.of(nodeTraits.replace(collation),
+                List.of(inputTraits.get(0).replace(collation)));
+    }
 }


(ignite-3) 01/02: Fix test

Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

amashenkov pushed a commit to branch ignite-21897
in repository https://gitbox.apache.org/repos/asf/ignite-3.git

commit 8a09204ea8a320549f70f4454de44da47a02de2d
Author: amashenkov <an...@gmail.com>
AuthorDate: Mon Apr 1 16:06:39 2024 +0300

    Fix test
---
 .../engine/planner/MapReduceSortAggregatePlannerTest.java   | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/MapReduceSortAggregatePlannerTest.java b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/MapReduceSortAggregatePlannerTest.java
index 5329cccbc0..642cb903fc 100644
--- a/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/MapReduceSortAggregatePlannerTest.java
+++ b/modules/sql-engine/src/test/java/org/apache/ignite/internal/sql/engine/planner/MapReduceSortAggregatePlannerTest.java
@@ -520,13 +520,10 @@ public class MapReduceSortAggregatePlannerTest extends AbstractAggregatePlannerT
         Predicate<IgniteReduceSortAggregate> inputAgg = isInstanceOf(IgniteReduceSortAggregate.class)
                 .and(hasGroupSets(IgniteReduceSortAggregate::getGroupSets, 0))
                 .and(hasCollation(RelCollations.of(0)))
-                .and(input(isInstanceOf(IgniteExchange.class)
-                        .and(hasCollation(RelCollations.of(0)))
-                        .and(hasDistribution(single()))
-                        .and(input(isInstanceOf(IgniteMapSortAggregate.class)
-                                .and(hasCollation(RelCollations.of(1)))
-                                .and(hasGroupSets(Aggregate::getGroupSets, 1))
-                ))));
+                .and(input(isInstanceOf(IgniteMapSortAggregate.class)
+                        .and(hasCollation(RelCollations.of(1)))
+                        .and(hasGroupSets(Aggregate::getGroupSets, 1))
+                ));
 
         assertPlan(TestCase.CASE_24_1, nodeOrAnyChild(isInstanceOf(IgniteReduceSortAggregate.class)
                         .and(hasNoGroupSets(IgniteReduceSortAggregate::getGroupSets))
@@ -815,8 +812,6 @@ public class MapReduceSortAggregatePlannerTest extends AbstractAggregatePlannerT
         assertPlan(testCase,
                 nodeOrAnyChild(isInstanceOf(IgniteReduceSortAggregate.class)
                         .and(input(isInstanceOf(IgniteMapSortAggregate.class)
-                                // TODO: https://issues.apache.org/jira/browse/IGNITE-20095
-                                // Why can't Map be pushed down to under 'exchange'.
                                 .and(input(isInstanceOf(IgniteSort.class)
                                         .and(s -> s.collation().equals(collation))
                                         .and(input(isTableScan("TEST")))