You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by zh...@apache.org on 2026/04/27 05:44:24 UTC
(calcite) branch main updated: [CALCITE-7479] Remove redundant aggregate group keys with FD
This is an automated email from the ASF dual-hosted git repository.
xiedeyantu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/calcite.git
The following commit(s) were added to refs/heads/main by this push:
new 0344850d54 [CALCITE-7479] Remove redundant aggregate group keys with FD
0344850d54 is described below
commit 0344850d5465bb4ce9af0640b1950920e5114ee5
Author: Zhen Chen <cz...@163.com>
AuthorDate: Thu Apr 23 08:41:18 2026 +0800
[CALCITE-7479] Remove redundant aggregate group keys with FD
---
.../rules/AggregateRemoveDuplicateKeysRule.java | 181 +++++++++++++++++++++
.../org/apache/calcite/rel/rules/CoreRules.java | 6 +
.../test/AggregateRemoveDuplicateKeysRuleTest.java | 87 ++++++++++
.../test/AggregateRemoveDuplicateKeysRuleTest.xml | 94 +++++++++++
4 files changed, 368 insertions(+)
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java
new file mode 100644
index 0000000000..2254a0094d
--- /dev/null
+++ b/core/src/main/java/org/apache/calcite/rel/rules/AggregateRemoveDuplicateKeysRule.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.rel.rules;
+
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.plan.RelRule;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.logical.LogicalAggregate;
+import org.apache.calcite.rel.metadata.RelMetadataQuery;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.sql.fun.SqlStdOperatorTable;
+import org.apache.calcite.tools.RelBuilder;
+import org.apache.calcite.util.ImmutableBitSet;
+
+import org.immutables.value.Value;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Planner rule that removes redundant grouping keys from an
+ * {@link Aggregate} when the later keys are functionally determined by the
+ * earlier retained keys.
+ *
+ * <p>The original output schema is preserved by adding {@code ANY_VALUE}
+ * aggregate calls for removed grouping keys and then projecting the row back
+ * into the original field order.
+ *
+ * <p>The original SQL:
+ * <pre>{@code
+ * SELECT deptno, name, count(*) AS c
+ * FROM sales.dept
+ * GROUP BY deptno, name
+ * }</pre>
+ *
+ * <p>The original logical plan:
+ * <pre>
+ * LogicalAggregate(group=[{0, 1}], C=[COUNT()])
+ * LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ * </pre>
+ *
+ * <p>After optimization:
+ * <pre>
+ * LogicalAggregate(group=[{0}], NAME=[ANY_VALUE($1)], C=[COUNT()])
+ * LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+ * </pre>
+ */
+@Value.Enclosing
+public class AggregateRemoveDuplicateKeysRule
+ extends RelRule<AggregateRemoveDuplicateKeysRule.Config>
+ implements TransformationRule {
+
+ /** Creates an AggregateRemoveDuplicateKeysRule. */
+ protected AggregateRemoveDuplicateKeysRule(Config config) {
+ super(config);
+ }
+
+ @Override public void onMatch(RelOptRuleCall call) {
+ final Aggregate aggregate = call.rel(0);
+ if (!Aggregate.isSimple(aggregate) || aggregate.getGroupCount() <= 1) {
+ return;
+ }
+
+ final RelMetadataQuery mq = call.getMetadataQuery();
+ final List<Integer> groupKeys = aggregate.getGroupSet().asList();
+ final int keyCount = groupKeys.size();
+
+ // Classify each group-key position (index into groupKeys) as retained or redundant.
+ // retainedBits tracks *positions* (not input field indices) already retained, so that
+ // mq.determinesSet can check whether they functionally determine the candidate position
+ // on the aggregate's own output schema.
+ final List<Integer> retainedPos = new ArrayList<>();
+ final List<Integer> removedPos = new ArrayList<>();
+ final ImmutableBitSet.Builder newGroupSetBuilder = ImmutableBitSet.builder();
+ ImmutableBitSet retainedBits = ImmutableBitSet.of();
+
+ for (int i = 0; i < keyCount; i++) {
+ if (!retainedBits.isEmpty()
+ && mq.determinesSet(aggregate, retainedBits, ImmutableBitSet.of(i))) {
+ removedPos.add(i);
+ } else {
+ retainedPos.add(i);
+ retainedBits = retainedBits.union(ImmutableBitSet.of(i));
+ newGroupSetBuilder.set(groupKeys.get(i));
+ }
+ }
+
+ if (removedPos.isEmpty()) {
+ return;
+ }
+
+ final RelBuilder relBuilder = call.builder();
+ relBuilder.push(aggregate.getInput());
+
+ // Build new agg calls. ANY_VALUE calls for removed keys are placed first so
+ // their output indices are contiguous with the retained group-key columns:
+ //
+ // new aggregate output layout:
+ // [0 .. retainedCount-1] retained group keys
+ // [retainedCount .. retainedCount+removedCount-1] ANY_VALUE calls
+ // [retainedCount + removedCount .. ...] original aggregate calls
+ //
+ final List<String> inputFieldNames = aggregate.getInput().getRowType().getFieldNames();
+ final List<RelBuilder.AggCall> newAggCalls = new ArrayList<>();
+ for (int removedKeyPos : removedPos) {
+ final int inputIdx = groupKeys.get(removedKeyPos);
+ newAggCalls.add(
+ relBuilder.aggregateCall(SqlStdOperatorTable.ANY_VALUE,
+ relBuilder.field(inputIdx))
+ .as(inputFieldNames.get(inputIdx)));
+ }
+ for (org.apache.calcite.rel.core.AggregateCall aggCall : aggregate.getAggCallList()) {
+ newAggCalls.add(relBuilder.aggregateCall(aggCall));
+ }
+
+ relBuilder.aggregate(relBuilder.groupKey(newGroupSetBuilder.build()), newAggCalls);
+
+ // Build a project to restore the original field order.
+ // Precompute position-in-groupKeys → new-output-column-index to avoid O(n) indexOf.
+ final int retainedCount = retainedPos.size();
+ final int removedCount = removedPos.size();
+ final int origAggCount = aggregate.getAggCallList().size();
+ final int[] posToCol = new int[keyCount];
+ for (int ri = 0; ri < retainedCount; ri++) {
+ posToCol[retainedPos.get(ri)] = ri;
+ }
+ for (int di = 0; di < removedCount; di++) {
+ posToCol[removedPos.get(di)] = retainedCount + di;
+ }
+
+ final List<RexNode> projects = new ArrayList<>(aggregate.getRowType().getFieldCount());
+ for (int pos = 0; pos < keyCount; pos++) {
+ projects.add(relBuilder.field(posToCol[pos]));
+ }
+ for (int i = 0; i < origAggCount; i++) {
+ projects.add(relBuilder.field(retainedCount + removedCount + i));
+ }
+
+ relBuilder.project(projects, aggregate.getRowType().getFieldNames());
+
+ call.getPlanner().prune(aggregate);
+ call.transformTo(relBuilder.build());
+ }
+
+ /** Rule configuration. */
+ @Value.Immutable
+ public interface Config extends RelRule.Config {
+ Config DEFAULT = ImmutableAggregateRemoveDuplicateKeysRule.Config.of()
+ .withOperandSupplier(b0 ->
+ b0.operand(LogicalAggregate.class)
+ .predicate(Aggregate::isSimple)
+ .anyInputs());
+
+ @Override default AggregateRemoveDuplicateKeysRule toRule() {
+ return new AggregateRemoveDuplicateKeysRule(this);
+ }
+
+ /** Defines an operand tree for the given aggregate class. */
+ default Config withOperandFor(Class<? extends Aggregate> aggregateClass) {
+ return withOperandSupplier(b0 ->
+ b0.operand(aggregateClass)
+ .predicate(Aggregate::isSimple)
+ .anyInputs())
+ .as(Config.class);
+ }
+ }
+}
diff --git a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java
index 0337b1d5ab..d7068dcc7c 100644
--- a/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java
+++ b/core/src/main/java/org/apache/calcite/rel/rules/CoreRules.java
@@ -66,6 +66,12 @@ private CoreRules() {}
public static final AggregateProjectMergeRule AGGREGATE_PROJECT_MERGE =
AggregateProjectMergeRule.Config.DEFAULT.toRule();
+ /** Rule that removes redundant grouping keys from an {@link Aggregate}
+ * when they are functionally determined by earlier grouping keys. */
+ public static final AggregateRemoveDuplicateKeysRule
+ AGGREGATE_REMOVE_DUPLICATE_KEYS =
+ AggregateRemoveDuplicateKeysRule.Config.DEFAULT.toRule();
+
/** Rule that removes constant keys from an {@link Aggregate}. */
public static final AggregateProjectPullUpConstantsRule
AGGREGATE_PROJECT_PULL_UP_CONSTANTS =
diff --git a/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java b/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java
new file mode 100644
index 0000000000..3f65724a68
--- /dev/null
+++ b/core/src/test/java/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.test;
+
+import org.apache.calcite.rel.rules.AggregateRemoveDuplicateKeysRule;
+import org.apache.calcite.rel.rules.CoreRules;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link AggregateRemoveDuplicateKeysRule}.
+ *
+ * <p>Relevant tickets:
+ * <ul>
+ * <li><a href="https://issues.apache.org/jira/browse/CALCITE-7479">
+ * [CALCITE-7479] Remove redundant aggregate group keys with FD
+ * </a></li>
+ * </ul>
+ */
+class AggregateRemoveDuplicateKeysRuleTest {
+
+ private static RelOptFixture fixture() {
+ return RelOptFixture.DEFAULT.withDiffRepos(
+ DiffRepository.lookup(AggregateRemoveDuplicateKeysRuleTest.class));
+ }
+
+ private static RelOptFixture sql(String sql) {
+ return fixture().sql(sql);
+ }
+
+ @Test void testRemoveOneRedundantGroupKey() {
+ final String sql = "select deptno, name, count(*) as c\n"
+ + "from sales.dept\n"
+ + "group by deptno, name";
+
+ sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS)
+ .check();
+ }
+
+ @Test void testRemoveMultipleRedundantGroupKeys() {
+ final String sql = "select empno, ename, job, count(*) as c\n"
+ + "from emp\n"
+ + "group by empno, ename, job";
+
+ sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS)
+ .check();
+ }
+
+ @Test void testRemoveRedundantComputedGroupKey() {
+ // deptno + 2 is a deterministic function of deptno, so it is determined
+ // by deptno and can be removed from the GROUP BY.
+ final String sql = "select deptno, deptno + 2\n"
+ + "from emp\n"
+ + "group by deptno, deptno + 2";
+
+ sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS)
+ .check();
+ }
+
+ @Test void testKeepsNonRedundantGroupKeys() {
+ final String sql = "select deptno, job, count(*) as c\n"
+ + "from emp\n"
+ + "group by deptno, job";
+
+ sql(sql).withRule(CoreRules.AGGREGATE_REMOVE_DUPLICATE_KEYS)
+ .checkUnchanged();
+ }
+
+ @AfterAll static void checkActualAndReferenceFiles() {
+ fixture().diffRepos.checkActualAndReferenceFiles();
+ }
+}
diff --git a/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml b/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml
new file mode 100644
index 0000000000..f01bb8e675
--- /dev/null
+++ b/core/src/test/resources/org/apache/calcite/test/AggregateRemoveDuplicateKeysRuleTest.xml
@@ -0,0 +1,94 @@
+<?xml version="1.0" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to you under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<Root>
+ <TestCase name="testKeepsNonRedundantGroupKeys">
+ <Resource name="sql">
+ <![CDATA[select deptno, job, count(*) as c
+from emp
+group by deptno, job]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1}], C=[COUNT()])
+ LogicalProject(DEPTNO=[$7], JOB=[$2])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testRemoveMultipleRedundantGroupKeys">
+ <Resource name="sql">
+ <![CDATA[select empno, ename, job, count(*) as c
+from emp
+group by empno, ename, job]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1, 2}], C=[COUNT()])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalAggregate(group=[{0}], ENAME=[ANY_VALUE($1)], JOB=[ANY_VALUE($2)], C=[COUNT()])
+ LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testRemoveOneRedundantGroupKey">
+ <Resource name="sql">
+ <![CDATA[select deptno, name, count(*) as c
+from sales.dept
+group by deptno, name]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1}], C=[COUNT()])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalAggregate(group=[{0}], NAME=[ANY_VALUE($1)], C=[COUNT()])
+ LogicalTableScan(table=[[CATALOG, SALES, DEPT]])
+]]>
+ </Resource>
+ </TestCase>
+ <TestCase name="testRemoveRedundantComputedGroupKey">
+ <Resource name="sql">
+ <![CDATA[select deptno, deptno + 2
+from emp
+group by deptno, deptno + 2]]>
+ </Resource>
+ <Resource name="planBefore">
+ <![CDATA[
+LogicalAggregate(group=[{0, 1}])
+ LogicalProject(DEPTNO=[$7], EXPR$1=[+($7, 2)])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ <Resource name="planAfter">
+ <![CDATA[
+LogicalAggregate(group=[{0}], EXPR$1=[ANY_VALUE($1)])
+ LogicalProject(DEPTNO=[$7], EXPR$1=[+($7, 2)])
+ LogicalTableScan(table=[[CATALOG, SALES, EMP]])
+]]>
+ </Resource>
+ </TestCase>
+</Root>