You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/09/01 08:11:04 UTC

(spark) branch branch-4.x updated: [SPARK-58847][SQL] Collation-aware hash aggregation

This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 87627ffa2058 [SPARK-58847][SQL] Collation-aware hash aggregation
87627ffa2058 is described below

commit 87627ffa2058efccbb15507f7d9c78eee7368250
Author: Stefan Kandic <st...@databricks.com>
AuthorDate: Tue Sep 1 08:10:38 2026 +0000

    [SPARK-58847][SQL] Collation-aware hash aggregation
    
    ### What changes were proposed in this pull request?
    Implementing support for binary unstable types (eg collated strings) in hash aggregators (both `HashAggregate` and `ObjectHashAggregate`). Binary unstable types are types for whom we can't derive equality / ordering by their bytes representations alone (eg `a` and `A` are equal in any case-insensitive collation).
    
    ### Why are the changes needed?
    Hash aggregation is the default aggregation mode in spark, and we've seen regressions in queries after moving from `UTF8_BINARY` to other collations as they would have to fall back to `SortAggregate`.
    
    ## Benchmark results
    
    4.2M rows, three runs per case:
    
    | Collation | Groups | HashAggregate | SortAggregate | Speedup |
    |---|---:|---:|---:|---:|
    | `UTF8_LCASE` | 4K | 1.30s | 6.35s | 4.9x |
    | `UNICODE_CI` | 4K | 2.89s | 9.23s | 3.2x |
    | `UTF8_LCASE` | 1M | 5.72s | 9.08s | 1.6x |
    | `UNICODE_CI` | 1M | 9.46s | 12.49s | 1.3x |
    
    ### Does this PR introduce _any_ user-facing change?
    No.
    
    ### How was this patch tested?
    Unit and query-level tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Generated-by: gpt-5.6-sol
    
    Closes #58083 from stefankandic/collation-hash-aggregate.
    
    Authored-by: Stefan Kandic <st...@databricks.com>
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
    (cherry picked from commit c7190edd2617a128f03050eed3ed0c6e0227421b)
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
 .../apache/spark/unsafe/map/BytesToBytesMap.java   | 176 +++++++++-
 .../unsafe/map/AbstractBytesToBytesMapSuite.java   | 130 +++++++
 .../plans/logical/basicLogicalOperators.scala      |  13 +-
 .../sql/catalyst/util/UnsafeRowKeyOperations.scala | 383 +++++++++++++++++++++
 .../sql/catalyst/util/UnsafeRowUtilsSuite.scala    | 237 ++++++++++++-
 .../execution/UnsafeFixedWidthAggregationMap.java  |  28 +-
 .../spark/sql/execution/aggregate/AggUtils.scala   |  13 +-
 .../execution/aggregate/HashAggregateExec.scala    |  25 +-
 .../aggregate/ObjectAggregationIterator.scala      |   3 +-
 .../execution/aggregate/ObjectAggregationMap.scala |  60 +++-
 .../aggregate/TungstenAggregationIterator.scala    |   2 +-
 .../sql/collation/CollationAggregationSuite.scala  | 351 ++++++++++++++++---
 .../spark/sql/collation/CollationSuite.scala       |   4 +-
 .../AdaptivePartialAggregationSuite.scala          |  21 ++
 14 files changed, 1352 insertions(+), 94 deletions(-)

diff --git a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index fc1bcd8bb336..8c86389b4c33 100644
--- a/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -68,6 +68,34 @@ import org.apache.spark.util.collection.unsafe.sort.UnsafeSorterSpillWriter;
  */
 public final class BytesToBytesMap extends MemoryConsumer {
 
+  /**
+   * Hashes and compares keys whose semantics cannot be determined from their raw bytes.
+   * Implementations must assign the same hash code to equal keys, and byte-identical keys must
+   * compare equal.
+   */
+  public interface KeyOperations {
+    int hash(Object base, long offset, int length);
+
+    boolean equals(
+      Object leftBase,
+      long leftOffset,
+      int leftLength,
+      Object rightBase,
+      long rightOffset,
+      int rightLength);
+  }
+
+  /**
+   * Creates independent key operations for a single {@link Location}, allowing each instance to
+   * reuse mutable state. All instances must implement the same key semantics. Maps configured
+   * with this factory must use the key-operations lookup methods exclusively. Callers may invoke
+   * {@link #create()} concurrently, so factory implementations must be thread-safe.
+   */
+  @FunctionalInterface
+  public interface KeyOperationsFactory {
+    KeyOperations create();
+  }
+
   private static final SparkLogger logger = SparkLoggerFactory.getLogger(BytesToBytesMap.class);
 
   private static final HashMapGrowthStrategy growthStrategy = HashMapGrowthStrategy.DOUBLING;
@@ -127,6 +155,8 @@ public final class BytesToBytesMap extends MemoryConsumer {
 
   private final double loadFactor;
 
+  @Nullable private final KeyOperationsFactory keyOperationsFactory;
+
   /**
    * The size of the data pages that hold key and value data. Map entries cannot span multiple
    * pages, so this limits the maximum entry size.
@@ -180,11 +210,30 @@ public final class BytesToBytesMap extends MemoryConsumer {
       int initialCapacity,
       double loadFactor,
       long pageSizeBytes) {
+    this(
+      taskMemoryManager,
+      blockManager,
+      serializerManager,
+      initialCapacity,
+      loadFactor,
+      pageSizeBytes,
+      null);
+  }
+
+  public BytesToBytesMap(
+      TaskMemoryManager taskMemoryManager,
+      BlockManager blockManager,
+      SerializerManager serializerManager,
+      int initialCapacity,
+      double loadFactor,
+      long pageSizeBytes,
+      @Nullable KeyOperationsFactory keyOperationsFactory) {
     super(taskMemoryManager, pageSizeBytes, taskMemoryManager.getTungstenMemoryMode());
     this.taskMemoryManager = taskMemoryManager;
     this.blockManager = blockManager;
     this.serializerManager = serializerManager;
     this.loadFactor = loadFactor;
+    this.keyOperationsFactory = keyOperationsFactory;
     this.loc = new Location();
     this.pageSizeBytes = pageSizeBytes;
     if (initialCapacity <= 0) {
@@ -206,6 +255,14 @@ public final class BytesToBytesMap extends MemoryConsumer {
       TaskMemoryManager taskMemoryManager,
       int initialCapacity,
       long pageSizeBytes) {
+    this(taskMemoryManager, initialCapacity, pageSizeBytes, null);
+  }
+
+  public BytesToBytesMap(
+      TaskMemoryManager taskMemoryManager,
+      int initialCapacity,
+      long pageSizeBytes,
+      @Nullable KeyOperationsFactory keyOperationsFactory) {
     this(
       taskMemoryManager,
       SparkEnv.get() != null ? SparkEnv.get().blockManager() :  null,
@@ -213,7 +270,8 @@ public final class BytesToBytesMap extends MemoryConsumer {
       initialCapacity,
       // In order to re-use the longArray for sorting, the load factor cannot be larger than 0.5.
       0.5,
-      pageSizeBytes);
+      pageSizeBytes,
+      keyOperationsFactory);
   }
 
   /**
@@ -498,8 +556,7 @@ public final class BytesToBytesMap extends MemoryConsumer {
    * This function is not thread-safe.
    */
   public Location lookup(Object keyBase, long keyOffset, int keyLength) {
-    safeLookup(keyBase, keyOffset, keyLength, loc,
-      Murmur3_x86_32.hashUnsafeWords(keyBase, keyOffset, keyLength, 42));
+    safeLookup(keyBase, keyOffset, keyLength, loc);
     return loc;
   }
 
@@ -507,6 +564,8 @@ public final class BytesToBytesMap extends MemoryConsumer {
    * Looks up a key, and return a {@link Location} handle that can be used to test existence
    * and read/write values.
    *
+   * The provided hash is ignored when this map has configured key operations.
+   *
    * This function always returns the same {@link Location} instance to avoid object allocation.
    * This function is not thread-safe.
    */
@@ -516,12 +575,38 @@ public final class BytesToBytesMap extends MemoryConsumer {
   }
 
   /**
-   * Looks up a key, and saves the result in provided `loc`.
+   * Looks up a key and saves the result in the provided `loc`.
+   *
+   * This is a thread-safe version of `lookup`, provided that each thread supplies its own
+   * {@link Location}. This guarantee excludes probe statistics, which may be inaccurate under
+   * concurrent lookup. The map must not be modified concurrently.
+   */
+  public void safeLookup(Object keyBase, long keyOffset, int keyLength, Location loc) {
+    if (keyOperationsFactory == null) {
+      safeLookup(
+        keyBase,
+        keyOffset,
+        keyLength,
+        loc,
+        Murmur3_x86_32.hashUnsafeWords(keyBase, keyOffset, keyLength, 42));
+    } else {
+      safeLookupWithKeyOperations(keyBase, keyOffset, keyLength, loc);
+    }
+  }
+
+  /**
+   * Looks up a key with a precomputed hash and saves the result in the provided `loc`.
    *
-   * This is a thread-safe version of `lookup`, could be used by multiple threads.
+   * The provided hash is ignored when this map has configured key operations. Each thread must
+   * supply its own {@link Location}. Probe statistics may be inaccurate under concurrent lookup,
+   * and the map must not be modified concurrently.
    */
   public void safeLookup(Object keyBase, long keyOffset, int keyLength, Location loc, int hash) {
     assert(longArray != null);
+    if (keyOperationsFactory != null) {
+      safeLookupWithKeyOperations(keyBase, keyOffset, keyLength, loc);
+      return;
+    }
 
     numKeyLookups++;
 
@@ -557,19 +642,62 @@ public final class BytesToBytesMap extends MemoryConsumer {
     }
   }
 
-  /**
-   * Handle returned by {@link BytesToBytesMap#lookup(Object, long, int)} function.
-   */
+  private void safeLookupWithKeyOperations(
+      Object keyBase, long keyOffset, int keyLength, Location loc) {
+    assert(longArray != null);
+    assert(keyOperationsFactory != null);
+
+    final KeyOperations keyOperations = loc.getKeyOperations();
+    assert(keyOperations != null);
+    final int hash = keyOperations.hash(keyBase, keyOffset, keyLength);
+
+    numKeyLookups++;
+
+    int pos = hash & mask;
+    int step = 1;
+    while (true) {
+      numProbes++;
+      if (longArray.get(pos * 2) == 0) {
+        // This is a new key.
+        loc.with(pos, hash, false);
+        return;
+      } else {
+        long stored = longArray.get(pos * 2 + 1);
+        if ((int) (stored) == hash) {
+          // Full hash code matches. Let's compare the keys for equality.
+          loc.with(pos, hash, true);
+          if (loc.getKeyLength() == keyLength &&
+              ByteArrayMethods.arrayEquals(
+                keyBase,
+                keyOffset,
+                loc.getKeyBase(),
+                loc.getKeyOffset(),
+                keyLength)) {
+            return;
+          }
+          if (keyOperations.equals(
+              keyBase,
+              keyOffset,
+              keyLength,
+              loc.getKeyBase(),
+              loc.getKeyOffset(),
+              loc.getKeyLength())) {
+            return;
+          }
+        }
+      }
+      pos = (pos + step) & mask;
+      step++;
+    }
+  }
+
+  /** Handle returned by this map's lookup methods. */
   public final class Location {
     /** An index into the hash map's Long array */
     private int pos;
     /** True if this location points to a position where a key is defined, false otherwise */
     private boolean isDefined;
-    /**
-     * The hashcode of the most recent key passed to
-     * {@link BytesToBytesMap#lookup(Object, long, int, int)}. Caching this hashcode here allows us
-     * to avoid re-hashing the key when storing a value for that key.
-     */
+    /** The hash code used by the most recent lookup. */
     private int keyHashcode;
     private Object baseObject;  // the base object for key and value
     private long keyOffset;
@@ -577,11 +705,21 @@ public final class BytesToBytesMap extends MemoryConsumer {
     private long valueOffset;
     private int valueLength;
 
+    @Nullable private KeyOperations keyOperations;
+
     /**
      * Memory page containing the record. Only set if created by {@link BytesToBytesMap#iterator()}.
      */
     @Nullable private MemoryBlock memoryPage;
 
+    @Nullable
+    private KeyOperations getKeyOperations() {
+      if (keyOperations == null && keyOperationsFactory != null) {
+        keyOperations = keyOperationsFactory.create();
+      }
+      return keyOperations;
+    }
+
     private void updateAddressesAndSizes(long fullKeyAddress) {
       updateAddressesAndSizes(
         taskMemoryManager.getPage(fullKeyAddress),
@@ -723,10 +861,10 @@ public final class BytesToBytesMap extends MemoryConsumer {
 
     /**
      * Append a new value for the key. This method could be called multiple times for a given key.
-     * The return value indicates whether the put succeeded or whether it failed because additional
-     * memory could not be acquired.
+     * The return value indicates whether the put succeeded or whether it failed because the map
+     * reached its capacity or additional memory could not be acquired.
      * <p>
-     * It is only valid to call this method immediately after calling `lookup()` using the same key.
+     * It is only valid to call this method immediately after looking up the same key.
      * </p>
      * <p>
      * The key and value must be word-aligned (that is, their sizes must be a multiple of 8).
@@ -750,8 +888,8 @@ public final class BytesToBytesMap extends MemoryConsumer {
      * Unspecified behavior if the key is not defined.
      * </p>
      *
-     * @return true if the put() was successful and false if the put() failed because memory could
-     *         not be acquired.
+     * @return true if the put() was successful and false if the map reached its capacity or memory
+     *         could not be acquired.
      */
     public boolean append(Object kbase, long koff, int klen, Object vbase, long voff, int vlen) {
       assert (klen % 8 == 0);
@@ -947,6 +1085,8 @@ public final class BytesToBytesMap extends MemoryConsumer {
 
   /**
    * Returns the average number of probes per key lookup.
+   *
+   * This statistic may be inaccurate if lookups are performed concurrently.
    */
   public double getAvgHashProbesPerKey() {
     return (1.0 * numProbes) / numKeyLookups;
diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index d5ca07a302d4..12ec5143c842 100644
--- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -21,6 +21,13 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.*;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
 
 import scala.Tuple2$;
 
@@ -128,6 +135,129 @@ public abstract class AbstractBytesToBytesMapSuite {
 
   protected abstract boolean useOffHeapMemoryAllocator();
 
+  @Test
+  public void supportsSemanticKeyOperations() throws Exception {
+    AtomicInteger operationsCreated = new AtomicInteger();
+    BytesToBytesMap.KeyOperationsFactory caseInsensitiveWithConstantHash = () -> {
+      operationsCreated.incrementAndGet();
+      return new BytesToBytesMap.KeyOperations() {
+        private final AtomicReference<Thread> owner = new AtomicReference<>();
+
+        private void assertThreadOwnership() {
+          final Thread currentThread = Thread.currentThread();
+          owner.compareAndSet(null, currentThread);
+          assertSame(currentThread, owner.get());
+        }
+
+        @Override
+        public int hash(Object base, long offset, int length) {
+          assertThreadOwnership();
+          return 0;
+        }
+
+        @Override
+        public boolean equals(
+            Object leftBase,
+            long leftOffset,
+            int leftLength,
+            Object rightBase,
+            long rightOffset,
+            int rightLength) {
+          assertThreadOwnership();
+          return Character.toLowerCase(Platform.getByte(leftBase, leftOffset)) ==
+            Character.toLowerCase(Platform.getByte(rightBase, rightOffset));
+        }
+      };
+    };
+    BytesToBytesMap map = new BytesToBytesMap(
+      taskMemoryManager, 64, PAGE_SIZE_BYTES, caseInsensitiveWithConstantHash);
+    ExecutorService executor = Executors.newFixedThreadPool(2);
+
+    byte[] firstStoredKey = new byte[8];
+    firstStoredKey[0] = 'A';
+    byte[] firstEquivalentKey = new byte[16];
+    firstEquivalentKey[0] = 'a';
+    byte[] firstValue = new byte[8];
+    Platform.putLong(firstValue, Platform.BYTE_ARRAY_OFFSET, 1L);
+    byte[] secondStoredKey = new byte[8];
+    secondStoredKey[0] = 'B';
+    byte[] secondEquivalentKey = new byte[16];
+    secondEquivalentKey[0] = 'b';
+    byte[] secondValue = new byte[8];
+    Platform.putLong(secondValue, Platform.BYTE_ARRAY_OFFSET, 2L);
+
+    try {
+      BytesToBytesMap.Location location = map.lookup(
+        firstStoredKey, Platform.BYTE_ARRAY_OFFSET, firstStoredKey.length);
+      assertFalse(location.isDefined());
+      assertTrue(location.append(
+        firstStoredKey,
+        Platform.BYTE_ARRAY_OFFSET,
+        firstStoredKey.length,
+        firstValue,
+        Platform.BYTE_ARRAY_OFFSET,
+        firstValue.length));
+
+      location = map.lookup(
+        secondStoredKey, Platform.BYTE_ARRAY_OFFSET, secondStoredKey.length);
+      assertFalse(location.isDefined());
+      assertTrue(location.append(
+        secondStoredKey,
+        Platform.BYTE_ARRAY_OFFSET,
+        secondStoredKey.length,
+        secondValue,
+        Platform.BYTE_ARRAY_OFFSET,
+        secondValue.length));
+
+      location = map.lookup(
+        firstEquivalentKey, Platform.BYTE_ARRAY_OFFSET, firstEquivalentKey.length);
+      assertTrue(location.isDefined());
+      assertEquals(firstStoredKey.length, location.getKeyLength());
+      assertEquals(1L, Platform.getLong(location.getValueBase(), location.getValueOffset()));
+
+      location = map.lookup(
+        secondEquivalentKey, Platform.BYTE_ARRAY_OFFSET, secondEquivalentKey.length);
+      assertTrue(location.isDefined());
+      assertEquals(secondStoredKey.length, location.getKeyLength());
+      assertEquals(2L, Platform.getLong(location.getValueBase(), location.getValueOffset()));
+
+      location = map.lookup(
+        firstEquivalentKey,
+        Platform.BYTE_ARRAY_OFFSET,
+        firstEquivalentKey.length,
+        Integer.MAX_VALUE);
+      assertTrue(location.isDefined());
+      assertEquals(2, map.numKeys());
+
+      CountDownLatch ready = new CountDownLatch(2);
+      CountDownLatch start = new CountDownLatch(1);
+      Callable<Void> concurrentLookup = () -> {
+        BytesToBytesMap.Location threadLocation = map.new Location();
+        ready.countDown();
+        start.await();
+        for (int i = 0; i < 100; i++) {
+          map.safeLookup(
+            firstEquivalentKey,
+            Platform.BYTE_ARRAY_OFFSET,
+            firstEquivalentKey.length,
+            threadLocation);
+          assertTrue(threadLocation.isDefined());
+        }
+        return null;
+      };
+      Future<Void> firstLookup = executor.submit(concurrentLookup);
+      Future<Void> secondLookup = executor.submit(concurrentLookup);
+      ready.await();
+      start.countDown();
+      firstLookup.get();
+      secondLookup.get();
+      assertEquals(3, operationsCreated.get());
+    } finally {
+      executor.shutdownNow();
+      map.free();
+    }
+  }
+
   private static byte[] getByteArray(Object base, long offset, int size) {
     final byte[] arr = new byte[size];
     Platform.copyMemory(base, offset, arr, Platform.BYTE_ARRAY_OFFSET, size);
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 5d56eca1c1f2..8d1125f91f8d 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -1323,18 +1323,25 @@ object Aggregate {
     schema.forall(f => UnsafeRow.isMutable(f.dataType))
   }
 
+  /**
+   * Returns whether grouping keys of this type can use raw binary equality or schema-aware key
+   * operations in hash aggregation.
+   */
+  def supportsHashAggregateGroupingKey(dataType: DataType): Boolean = {
+    UnsafeRowKeyOperations.supportsDataType(dataType)
+  }
+
   def supportsHashAggregate(
       aggregateBufferAttributes: Seq[Attribute], groupingExpression: Seq[Expression]): Boolean = {
     val aggregationBufferSchema = DataTypeUtils.fromAttributes(aggregateBufferAttributes)
     isAggregateBufferMutable(aggregationBufferSchema) &&
-      groupingExpression.forall(e => UnsafeRowUtils.isBinaryStable(e.dataType))
+      groupingExpression.forall(e => supportsHashAggregateGroupingKey(e.dataType))
   }
 
   def supportsObjectHashAggregate(
       aggregateExpressions: Seq[AggregateExpression],
       groupingExpressions: Seq[Expression]): Boolean = {
-    // We should not use hash aggregation on binary unstable types.
-    if (groupingExpressions.exists(e => !UnsafeRowUtils.isBinaryStable(e.dataType))) {
+    if (groupingExpressions.exists(e => !supportsHashAggregateGroupingKey(e.dataType))) {
       return false
     }
 
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowKeyOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowKeyOperations.scala
new file mode 100644
index 000000000000..591fba9556ed
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/UnsafeRowKeyOperations.scala
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.util
+
+import com.ibm.icu.text.RawCollationKey
+
+import org.apache.spark.SparkException
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{SpecializedGetters, UnsafeArrayData, UnsafeRow}
+import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.Platform
+import org.apache.spark.unsafe.array.ByteArrayMethods
+import org.apache.spark.unsafe.hash.Murmur3_x86_32
+import org.apache.spark.unsafe.map.BytesToBytesMap
+import org.apache.spark.unsafe.types.UTF8String
+
+/** Hashes and compares unsafe grouping keys containing non-binary collated strings. */
+final class UnsafeRowKeyOperations(val schema: StructType)
+    extends BytesToBytesMap.KeyOperationsFactory {
+  import UnsafeRowKeyOperations._
+
+  require(
+    schema.forall(field => supportsDataType(field.dataType)),
+    s"Unsupported grouping key schema: $schema")
+
+  private val operations = new RowOperations(schema, new HashScratch)
+
+  def hash(row: UnsafeRow): Int = operations.hash(row, HASH_SEED)
+
+  def areEqual(left: UnsafeRow, right: UnsafeRow): Boolean = {
+    operations.areEqual(left, right)
+  }
+
+  override def create(): BytesToBytesMap.KeyOperations = new BytesToBytesMap.KeyOperations {
+    private val localOperations = new RowOperations(schema, new HashScratch)
+    private val left = new UnsafeRow(schema.length)
+    private val right = new UnsafeRow(schema.length)
+
+    override def hash(base: AnyRef, offset: Long, length: Int): Int = {
+      left.pointTo(base, offset, length)
+      localOperations.hash(left, HASH_SEED)
+    }
+
+    override def equals(
+        leftBase: AnyRef,
+        leftOffset: Long,
+        leftLength: Int,
+        rightBase: AnyRef,
+        rightOffset: Long,
+        rightLength: Int): Boolean = {
+      left.pointTo(leftBase, leftOffset, leftLength)
+      right.pointTo(rightBase, rightOffset, rightLength)
+      localOperations.areEqual(left, right)
+    }
+  }
+}
+
+object UnsafeRowKeyOperations {
+  private val HASH_SEED = 42
+
+  /** Returns whether this type can be hashed and compared as part of an unsafe row key. */
+  def supportsDataType(dataType: DataType): Boolean = {
+    if (UnsafeRowUtils.isBinaryStable(dataType)) {
+      true
+    } else {
+      dataType match {
+        case st: StringType => !st.supportsBinaryEquality
+        case ArrayType(elementType, _) => supportsDataType(elementType)
+        case StructType(fields) => fields.forall(field => supportsDataType(field.dataType))
+        case _ => false
+      }
+    }
+  }
+
+  private final class BinaryRegion {
+    var base: AnyRef = _
+    var offset: Long = 0L
+    var length: Int = 0
+  }
+
+  private final class HashScratch {
+    private var reusableRawCollationKey: RawCollationKey = _
+
+    def rawCollationKey: RawCollationKey = {
+      if (reusableRawCollationKey == null) {
+        reusableRawCollationKey = new RawCollationKey()
+      }
+      reusableRawCollationKey
+    }
+  }
+
+  private sealed trait FieldOperations {
+    final def hash(input: SpecializedGetters, ordinal: Int, seed: Int): Int = {
+      if (input.isNullAt(ordinal)) seed else hashNonNull(input, ordinal, seed)
+    }
+
+    final def areEqual(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean = {
+      val leftIsNull = left.isNullAt(leftOrdinal)
+      val rightIsNull = right.isNullAt(rightOrdinal)
+      if (leftIsNull || rightIsNull) {
+        leftIsNull == rightIsNull
+      } else {
+        areEqualNonNull(left, leftOrdinal, right, rightOrdinal)
+      }
+    }
+
+    protected def hashNonNull(input: SpecializedGetters, ordinal: Int, seed: Int): Int
+
+    protected def areEqualNonNull(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean
+  }
+
+  private final class BinaryStableOperations(dataType: DataType) extends FieldOperations {
+    private val leftRegion = new BinaryRegion
+    private val rightRegion = new BinaryRegion
+
+    override protected def hashNonNull(
+        input: SpecializedGetters, ordinal: Int, seed: Int): Int = {
+      setRegion(input, ordinal, leftRegion)
+      Murmur3_x86_32.hashUnsafeBytes(
+        leftRegion.base, leftRegion.offset, leftRegion.length, seed)
+    }
+
+    override protected def areEqualNonNull(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean = {
+      setRegion(left, leftOrdinal, leftRegion)
+      setRegion(right, rightOrdinal, rightRegion)
+      leftRegion.length == rightRegion.length && ByteArrayMethods.arrayEquals(
+        leftRegion.base,
+        leftRegion.offset,
+        rightRegion.base,
+        rightRegion.offset,
+        leftRegion.length)
+    }
+
+    private def setRegion(
+        input: SpecializedGetters, ordinal: Int, region: BinaryRegion): Unit = input match {
+      case row: UnsafeRow =>
+        region.base = row.getBaseObject
+        if (UnsafeRow.isFixedLength(dataType)) {
+          region.offset = row.getBaseOffset +
+            UnsafeRow.calculateBitSetWidthInBytes(row.numFields()) + ordinal * 8L
+          region.length = 8
+        } else {
+          setVariableLengthRegion(row.getBaseOffset, row.getLong(ordinal), region)
+        }
+
+      case other =>
+        throw SparkException.internalError(
+          s"Expected unsafe grouping-key storage, found ${other.getClass.getName}")
+    }
+  }
+
+  private final class CollatedStringOperations(
+      dataType: StringType,
+      hashScratch: HashScratch) extends FieldOperations {
+    private val collation = CollationFactory.fetchCollation(dataType.collationId)
+    private val useRawCollationKey = collation.provider == CollationFactory.PROVIDER_ICU
+
+    override protected def hashNonNull(
+        input: SpecializedGetters, ordinal: Int, seed: Int): Int = {
+      val value = input.getUTF8String(ordinal)
+      if (useRawCollationKey) {
+        hashICUCollationKey(value, seed)
+      } else {
+        val key = collation.sortKeyFunction.apply(value)
+        Murmur3_x86_32.hashUnsafeBytes(key, Platform.BYTE_ARRAY_OFFSET, key.length, seed)
+      }
+    }
+
+    /**
+     * Hashes an ICU sort key from a reusable buffer after applying configured space trimming.
+     * `toValidString` applies Spark's replacement policy for malformed UTF-8 before calling ICU.
+     */
+    private def hashICUCollationKey(value: UTF8String, seed: Int): Int = {
+      val normalizedValue = if (collation.supportsSpaceTrimming) {
+        CollationFactory.applyTrimmingPolicy(value, dataType.collationId)
+      } else {
+        value
+      }
+      val key = collation.getCollator.getRawCollationKey(
+        normalizedValue.toValidString, hashScratch.rawCollationKey)
+      Murmur3_x86_32.hashUnsafeBytes(
+        key.bytes, Platform.BYTE_ARRAY_OFFSET, key.size, seed)
+    }
+
+    override protected def areEqualNonNull(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean = {
+      collation.equalsFunction.apply(
+        left.getUTF8String(leftOrdinal), right.getUTF8String(rightOrdinal))
+    }
+  }
+
+  private final class ArrayOperations(
+      elementType: DataType,
+      hashScratch: HashScratch) extends FieldOperations {
+    private val elementOperations = createFieldOperations(elementType, hashScratch)
+    private val hashArray = new UnsafeArrayData
+    private val leftArray = new UnsafeArrayData
+    private val rightArray = new UnsafeArrayData
+
+    override protected def hashNonNull(
+        input: SpecializedGetters, ordinal: Int, seed: Int): Int = {
+      pointToArray(input, ordinal, hashArray)
+      var result = seed
+      var index = 0
+      while (index < hashArray.numElements()) {
+        result = elementOperations.hash(hashArray, index, result)
+        index += 1
+      }
+      result
+    }
+
+    override protected def areEqualNonNull(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean = {
+      pointToArray(left, leftOrdinal, leftArray)
+      pointToArray(right, rightOrdinal, rightArray)
+      if (leftArray.numElements() != rightArray.numElements()) {
+        return false
+      }
+      var index = 0
+      while (index < leftArray.numElements()) {
+        if (!elementOperations.areEqual(leftArray, index, rightArray, index)) {
+          return false
+        }
+        index += 1
+      }
+      true
+    }
+  }
+
+  private final class StructOperations(
+      dataType: StructType,
+      hashScratch: HashScratch) extends FieldOperations {
+    private val operations = new RowOperations(dataType, hashScratch)
+    private val hashRow = new UnsafeRow(dataType.length)
+    private val leftRow = new UnsafeRow(dataType.length)
+    private val rightRow = new UnsafeRow(dataType.length)
+
+    override protected def hashNonNull(
+        input: SpecializedGetters, ordinal: Int, seed: Int): Int = {
+      pointToStruct(input, ordinal, hashRow)
+      operations.hash(hashRow, seed)
+    }
+
+    override protected def areEqualNonNull(
+        left: SpecializedGetters,
+        leftOrdinal: Int,
+        right: SpecializedGetters,
+        rightOrdinal: Int): Boolean = {
+      pointToStruct(left, leftOrdinal, leftRow)
+      pointToStruct(right, rightOrdinal, rightRow)
+      operations.areEqual(leftRow, rightRow)
+    }
+  }
+
+  private final class RowOperations(dataType: StructType, hashScratch: HashScratch) {
+    private val fieldOperations =
+      dataType.fields.map(field => createFieldOperations(field.dataType, hashScratch))
+
+    def hash(row: InternalRow, seed: Int): Int = {
+      var result = seed
+      var ordinal = 0
+      while (ordinal < fieldOperations.length) {
+        result = fieldOperations(ordinal).hash(row, ordinal, result)
+        ordinal += 1
+      }
+      result
+    }
+
+    def areEqual(left: InternalRow, right: InternalRow): Boolean = {
+      var ordinal = 0
+      while (ordinal < fieldOperations.length) {
+        if (!fieldOperations(ordinal).areEqual(left, ordinal, right, ordinal)) {
+          return false
+        }
+        ordinal += 1
+      }
+      true
+    }
+  }
+
+  private def createFieldOperations(
+      dataType: DataType,
+      hashScratch: HashScratch): FieldOperations = {
+    if (UnsafeRowUtils.isBinaryStable(dataType)) {
+      new BinaryStableOperations(dataType)
+    } else {
+      dataType match {
+        case stringType: StringType => new CollatedStringOperations(stringType, hashScratch)
+        case ArrayType(elementType, _) => new ArrayOperations(elementType, hashScratch)
+        case structType: StructType => new StructOperations(structType, hashScratch)
+        case _ =>
+          throw SparkException.internalError(s"Unsupported grouping key type: $dataType")
+      }
+    }
+  }
+
+  private def setVariableLengthRegion(
+      baseOffset: Long, offsetAndSize: Long, region: BinaryRegion): Unit = {
+    region.offset = baseOffset + (offsetAndSize >> 32).toInt
+    region.length = offsetAndSize.toInt
+  }
+
+  private def pointToArray(
+      input: SpecializedGetters,
+      ordinal: Int,
+      target: UnsafeArrayData): Unit = input match {
+    case row: UnsafeRow =>
+      pointToArray(row.getBaseObject, row.getBaseOffset, row.getLong(ordinal), target)
+    case array: UnsafeArrayData =>
+      pointToArray(array.getBaseObject, array.getBaseOffset, array.getLong(ordinal), target)
+    case other =>
+      throw SparkException.internalError(
+        s"Expected unsafe grouping-key storage, found ${other.getClass.getName}")
+  }
+
+  private def pointToArray(
+      base: AnyRef,
+      baseOffset: Long,
+      offsetAndSize: Long,
+      target: UnsafeArrayData): Unit = {
+    target.pointTo(
+      base,
+      baseOffset + (offsetAndSize >> 32).toInt,
+      offsetAndSize.toInt)
+  }
+
+  private def pointToStruct(
+      input: SpecializedGetters,
+      ordinal: Int,
+      target: UnsafeRow): Unit = input match {
+    case row: UnsafeRow =>
+      pointToStruct(row.getBaseObject, row.getBaseOffset, row.getLong(ordinal), target)
+    case array: UnsafeArrayData =>
+      pointToStruct(array.getBaseObject, array.getBaseOffset, array.getLong(ordinal), target)
+    case other =>
+      throw SparkException.internalError(
+        s"Expected unsafe grouping-key storage, found ${other.getClass.getName}")
+  }
+
+  private def pointToStruct(
+      base: AnyRef,
+      baseOffset: Long,
+      offsetAndSize: Long,
+      target: UnsafeRow): Unit = {
+    target.pointTo(
+      base,
+      baseOffset + (offsetAndSize >> 32).toInt,
+      offsetAndSize.toInt)
+  }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala
index 9f405818598b..faa4c6cf3238 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/UnsafeRowUtilsSuite.scala
@@ -21,10 +21,16 @@ import java.math.{BigDecimal => JavaBigDecimal}
 import java.nio.ByteOrder.{nativeOrder, BIG_ENDIAN}
 
 import org.apache.spark.SparkFunSuite
-import org.apache.spark.sql.catalyst.expressions.{SpecificInternalRow, UnsafeProjection, UnsafeRow}
-import org.apache.spark.sql.types.{ArrayType, Decimal, DecimalType, IntegerType, MapType, StringType, StructField, StructType}
+import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper}
+import org.apache.spark.sql.catalyst.expressions.{CodegenObjectFactoryMode, SpecificInternalRow, UnsafeProjection, UnsafeRow}
+import org.apache.spark.sql.catalyst.plans.logical.Aggregate
+import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.types.{ArrayType, BinaryType, BooleanType, CalendarIntervalType, Decimal, DecimalType, GeographyType, GeometryType, IntegerType, LongType, MapType, StringType, StructField, StructType}
+import org.apache.spark.unsafe.Platform
+import org.apache.spark.unsafe.hash.Murmur3_x86_32
+import org.apache.spark.unsafe.types.{BinaryView, CalendarInterval, UTF8String}
 
-class UnsafeRowUtilsSuite extends SparkFunSuite {
+class UnsafeRowUtilsSuite extends SparkFunSuite with SQLConfHelper {
 
   val testKeys: Seq[String] = Seq("key1", "key2")
   val testValues: Seq[String] = Seq("sum(key1)", "sum(key2)")
@@ -159,6 +165,231 @@ class UnsafeRowUtilsSuite extends SparkFunSuite {
         StructType(StructField("sub", nonBinaryStringType) :: Nil)) :: Nil)))
   }
 
+  test("hash aggregate grouping key support for binary-unstable types") {
+    val collatedString = StringType(CollationFactory.collationNameToId("UTF8_LCASE"))
+
+    assert(Aggregate.supportsHashAggregateGroupingKey(collatedString))
+    assert(Aggregate.supportsHashAggregateGroupingKey(ArrayType(collatedString)))
+    assert(Aggregate.supportsHashAggregateGroupingKey(
+      StructType(StructField("s", collatedString) :: Nil)))
+    assert(!Aggregate.supportsHashAggregateGroupingKey(MapType(collatedString, IntegerType)))
+    assert(Aggregate.supportsHashAggregateGroupingKey(StructType(Seq(
+      StructField("s", collatedString),
+      StructField("m", MapType(StringType, IntegerType))))))
+    assert(!Aggregate.supportsHashAggregateGroupingKey(StructType(Seq(
+      StructField("s", collatedString),
+      StructField("m", MapType(collatedString, IntegerType))))))
+  }
+
+  test("UnsafeRowKeyOperations equality and hash contract") {
+    case class KeyContractCase(
+        name: String,
+        schema: StructType,
+        left: InternalRow,
+        right: InternalRow,
+        different: InternalRow)
+
+    def collatedString(collationName: String): StringType = {
+      StringType(CollationFactory.collationNameToId(collationName))
+    }
+
+    def utf8(value: String): UTF8String = UTF8String.fromString(value)
+
+    def array(values: Any*): GenericArrayData = new GenericArrayData(values.toArray)
+
+    val lcase = collatedString("UTF8_LCASE")
+    val unicodeCI = collatedString("UNICODE_CI")
+    val binaryRtrim = collatedString("UTF8_BINARY_RTRIM")
+    val lcaseRtrim = collatedString("UTF8_LCASE_RTRIM")
+    val geometry = GeometryType(4326)
+    val geography = GeographyType(4326)
+    val nestedType = StructType(Seq(
+      StructField("id", IntegerType),
+      StructField("label", lcaseRtrim)))
+
+    val testCases = Seq(
+      KeyContractCase(
+        "scalar LCASE string",
+        StructType(StructField("s", lcase) :: Nil),
+        InternalRow(utf8("AAA")),
+        InternalRow(utf8("aaa")),
+        InternalRow(utf8("bbb"))),
+      KeyContractCase(
+        "mixed primitive fields",
+        StructType(Seq(
+          StructField("id", IntegerType),
+          StructField("s", lcase),
+          StructField("flag", BooleanType),
+          StructField("count", LongType))),
+        InternalRow(1, utf8("HELLO"), true, 10L),
+        InternalRow(1, utf8("hello"), true, 10L),
+        InternalRow(2, utf8("hello"), true, 10L)),
+      KeyContractCase(
+        "array with ICU collation and null element",
+        StructType(StructField("a", ArrayType(unicodeCI)) :: Nil),
+        InternalRow(array(utf8("HELLO"), null, utf8("WORLD"))),
+        InternalRow(array(utf8("hello"), null, utf8("world"))),
+        InternalRow(array(utf8("hello"), null, utf8("other")))),
+      KeyContractCase(
+        "nested array with ICU collation",
+        StructType(StructField("a", ArrayType(ArrayType(unicodeCI))) :: Nil),
+        InternalRow(array(array(utf8("HELLO")), array(utf8("WORLD")))),
+        InternalRow(array(array(utf8("hello")), array(utf8("world")))),
+        InternalRow(array(array(utf8("hello")), array(utf8("other"))))),
+      KeyContractCase(
+        "nested array with unequal lengths",
+        StructType(StructField("a", ArrayType(ArrayType(unicodeCI))) :: Nil),
+        InternalRow(array(array(utf8("HELLO")))),
+        InternalRow(array(array(utf8("hello")))),
+        InternalRow(array(array(utf8("hello"), utf8("world"))))),
+      KeyContractCase(
+        "nested empty and non-empty arrays",
+        StructType(StructField("a", ArrayType(ArrayType(unicodeCI))) :: Nil),
+        InternalRow(array(array())),
+        InternalRow(array(array())),
+        InternalRow(array(array(utf8("value"))))),
+      KeyContractCase(
+        "nested struct with LCASE RTRIM collation",
+        StructType(StructField("nested", nestedType) :: Nil),
+        InternalRow(InternalRow(7, utf8("VALUE"))),
+        InternalRow(InternalRow(7, utf8("value   "))),
+        InternalRow(InternalRow(8, utf8("value")))),
+      KeyContractCase(
+        "array of structs with LCASE RTRIM collation",
+        StructType(StructField("a", ArrayType(nestedType)) :: Nil),
+        InternalRow(array(InternalRow(7, utf8("VALUE")))),
+        InternalRow(array(InternalRow(7, utf8("value   ")))),
+        InternalRow(array(InternalRow(8, utf8("value"))))),
+      KeyContractCase(
+        "binary RTRIM string with different lengths",
+        StructType(StructField("s", binaryRtrim) :: Nil),
+        InternalRow(utf8("value")),
+        InternalRow(utf8("value   ")),
+        InternalRow(utf8("VALUE"))),
+      KeyContractCase(
+        "null collated field",
+        StructType(Seq(StructField("s", unicodeCI), StructField("id", IntegerType))),
+        InternalRow(null, 1),
+        InternalRow(null, 1),
+        InternalRow(utf8("value"), 1)),
+      KeyContractCase(
+        "collated string with opaque binary-stable fields",
+        StructType(Seq(
+          StructField("s", lcase),
+          StructField("interval", CalendarIntervalType),
+          StructField("geometry", geometry),
+          StructField("geography", geography),
+          StructField("payload", BinaryType))),
+        InternalRow(
+          utf8("HELLO"),
+          new CalendarInterval(1, 2, 3),
+          BinaryView.fromBytes(Array[Byte](1, 2, 3)),
+          BinaryView.fromBytes(Array[Byte](4, 5, 6)),
+          Array[Byte](7, 8, 9)),
+        InternalRow(
+          utf8("hello"),
+          new CalendarInterval(1, 2, 3),
+          BinaryView.fromBytes(Array[Byte](1, 2, 3)),
+          BinaryView.fromBytes(Array[Byte](4, 5, 6)),
+          Array[Byte](7, 8, 9)),
+        InternalRow(
+          utf8("hello"),
+          new CalendarInterval(1, 2, 4),
+          BinaryView.fromBytes(Array[Byte](1, 2, 3)),
+          BinaryView.fromBytes(Array[Byte](4, 5, 6)),
+          Array[Byte](7, 8, 9))))
+
+    Seq(CodegenObjectFactoryMode.CODEGEN_ONLY, CodegenObjectFactoryMode.NO_CODEGEN).foreach {
+      codegenMode =>
+        withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode.toString) {
+          testCases.foreach { testCase =>
+            withClue(s"$codegenMode: ${testCase.name}: ") {
+              val projection = UnsafeProjection.create(testCase.schema)
+              val left = projection(testCase.left).copy()
+              val right = projection(testCase.right).copy()
+              val different = projection(testCase.different).copy()
+              val keyOperations = new UnsafeRowKeyOperations(testCase.schema)
+
+              assert(keyOperations.areEqual(left, right))
+              assert(keyOperations.areEqual(right, left))
+              assert(!keyOperations.areEqual(left, different))
+              assert(!keyOperations.areEqual(different, left))
+              assert(keyOperations.hash(left) == keyOperations.hash(right))
+
+              val serializedOperations = keyOperations.create()
+              assert(serializedOperations.equals(
+                left.getBaseObject,
+                left.getBaseOffset,
+                left.getSizeInBytes,
+                right.getBaseObject,
+                right.getBaseOffset,
+                right.getSizeInBytes))
+              assert(serializedOperations.equals(
+                right.getBaseObject,
+                right.getBaseOffset,
+                right.getSizeInBytes,
+                left.getBaseObject,
+                left.getBaseOffset,
+                left.getSizeInBytes))
+              assert(!serializedOperations.equals(
+                left.getBaseObject,
+                left.getBaseOffset,
+                left.getSizeInBytes,
+                different.getBaseObject,
+                different.getBaseOffset,
+                different.getSizeInBytes))
+              assert(serializedOperations.hash(
+                left.getBaseObject,
+                left.getBaseOffset,
+                left.getSizeInBytes) == serializedOperations.hash(
+                right.getBaseObject,
+                right.getBaseOffset,
+                right.getSizeInBytes))
+            }
+          }
+        }
+    }
+  }
+
+  test("UnsafeRowKeyOperations raw ICU keys match materialized sort-key hashes") {
+    // scalastyle:off nonascii
+    val values = Seq(
+      "a" * 4096,
+      "short",
+      "Résumé",
+      "resume   ",
+      "")
+    // scalastyle:on nonascii
+
+    Seq("UNICODE", "UNICODE_CI", "UNICODE_RTRIM", "UNICODE_CI_RTRIM").foreach {
+      collationName =>
+        val stringType = StringType(CollationFactory.collationNameToId(collationName))
+        val schema = StructType(StructField("value", stringType) :: Nil)
+        val projection = UnsafeProjection.create(schema)
+        val keyOperations = new UnsafeRowKeyOperations(schema)
+        val serializedOperations = keyOperations.create()
+        val collation = CollationFactory.fetchCollation(stringType.collationId)
+
+        values.foreach { value =>
+          val row = projection(InternalRow(UTF8String.fromString(value))).copy()
+          val materializedKey = collation.sortKeyFunction.apply(row.getUTF8String(0))
+          val expectedHash = Murmur3_x86_32.hashUnsafeBytes(
+            materializedKey,
+            Platform.BYTE_ARRAY_OFFSET,
+            materializedKey.length,
+            42)
+
+          withClue(s"$collationName: $value: ") {
+            assert(keyOperations.hash(row) == expectedHash)
+            assert(serializedOperations.hash(
+              row.getBaseObject,
+              row.getBaseOffset,
+              row.getSizeInBytes) == expectedHash)
+          }
+        }
+    }
+  }
+
   test("PaddingProvider handles endianness") {
     // The following arrays contain the same 8 byte field values as represented
     // in memory on little endian and big endian platforms. They are
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java
index d850d0d18bef..c6adcad32d96 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/UnsafeFixedWidthAggregationMap.java
@@ -26,6 +26,8 @@ import org.apache.spark.sql.catalyst.InternalRow;
 import org.apache.spark.sql.catalyst.expressions.UnsafeProjection;
 import org.apache.spark.sql.catalyst.expressions.UnsafeRow;
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate$;
+import org.apache.spark.sql.catalyst.util.UnsafeRowKeyOperations;
+import org.apache.spark.sql.catalyst.util.UnsafeRowUtils$;
 import org.apache.spark.sql.types.StructType;
 import org.apache.spark.unsafe.KVIterator;
 import org.apache.spark.unsafe.Platform;
@@ -53,6 +55,8 @@ public final class UnsafeFixedWidthAggregationMap {
    */
   private final UnsafeProjection groupingKeyProjection;
 
+  private final UnsafeRowKeyOperations keyOperations;
+
   /**
    * A hashmap which maps from opaque bytearray keys to bytearray values.
    */
@@ -92,8 +96,13 @@ public final class UnsafeFixedWidthAggregationMap {
     this.currentAggregationBuffer = new UnsafeRow(aggregationBufferSchema.length());
     this.groupingKeyProjection = UnsafeProjection.create(groupingKeySchema);
     this.groupingKeySchema = groupingKeySchema;
+    this.keyOperations = UnsafeRowUtils$.MODULE$.isBinaryStable(groupingKeySchema) ?
+      null : new UnsafeRowKeyOperations(groupingKeySchema);
     this.map = new BytesToBytesMap(
-      taskContext.taskMemoryManager(), initialCapacity, pageSizeBytes);
+      taskContext.taskMemoryManager(),
+      initialCapacity,
+      pageSizeBytes,
+      keyOperations);
 
     // Initialize the buffer for aggregation value
     final UnsafeProjection valueProjection = UnsafeProjection.create(aggregationBufferSchema);
@@ -119,7 +128,12 @@ public final class UnsafeFixedWidthAggregationMap {
   }
 
   public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key) {
-    return getAggregationBufferFromUnsafeRow(key, key.hashCode());
+    return keyOperations == null ?
+      getAggregationBufferFromUnsafeRow(key, key.hashCode()) :
+      getAggregationBufferFromLocation(key, map.lookup(
+        key.getBaseObject(),
+        key.getBaseOffset(),
+        key.getSizeInBytes()));
   }
 
   public UnsafeRow getAggregationBufferFromUnsafeRow(UnsafeRow key, int hash) {
@@ -129,6 +143,11 @@ public final class UnsafeFixedWidthAggregationMap {
       key.getBaseOffset(),
       key.getSizeInBytes(),
       hash);
+    return getAggregationBufferFromLocation(key, loc);
+  }
+
+  private UnsafeRow getAggregationBufferFromLocation(
+      UnsafeRow key, BytesToBytesMap.Location loc) {
     if (!loc.isDefined()) {
       // This is the first time that we've seen this grouping key, so we'll insert a copy of the
       // empty aggregation buffer into the map:
@@ -235,6 +254,11 @@ public final class UnsafeFixedWidthAggregationMap {
     return map.numKeys();
   }
 
+  /** Returns whether two grouping keys are semantically equal. */
+  public boolean keysEqual(UnsafeRow left, UnsafeRow right) {
+    return keyOperations == null ? left.equals(right) : keyOperations.areEqual(left, right);
+  }
+
   /**
    * Sorts the map's records in place, spill them to disk, and returns an [[UnsafeKVExternalSorter]]
    *
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
index 16c34b404de6..1bb5b0e916b3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/AggUtils.scala
@@ -21,6 +21,7 @@ import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.aggregate._
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate
+import org.apache.spark.sql.catalyst.util.UnsafeRowUtils
 import org.apache.spark.sql.execution.SparkPlan
 import org.apache.spark.sql.execution.streaming.{ProjectAggregationBufferExec, StatefulStreamlineAggregateExec}
 import org.apache.spark.sql.execution.streaming.operators.stateful._
@@ -76,8 +77,12 @@ object AggUtils {
       initialInputBufferOffset: Int = 0,
       resultExpressions: Seq[NamedExpression] = Nil,
       child: SparkPlan): SparkPlan = {
-    val useHash = child.conf.useHashAggregation && Aggregate.supportsHashAggregate(
-      aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes), groupingExpressions)
+    val supportsStreamingGroupingKeys = !isStreaming ||
+      groupingExpressions.forall(e => UnsafeRowUtils.isBinaryStable(e.dataType))
+    val useHash = child.conf.useHashAggregation && supportsStreamingGroupingKeys &&
+      Aggregate.supportsHashAggregate(
+        aggregateExpressions.flatMap(_.aggregateFunction.aggBufferAttributes),
+        groupingExpressions)
 
     val forceObjHashAggregate = forceApplyObjectHashAggregate(child.conf)
 
@@ -94,8 +99,8 @@ object AggUtils {
         child = child)
     } else {
       val objectHashEnabled = child.conf.useObjectHashAggregation
-      val useObjectHash = Aggregate.supportsObjectHashAggregate(
-        aggregateExpressions, groupingExpressions)
+      val useObjectHash = supportsStreamingGroupingKeys &&
+        Aggregate.supportsObjectHashAggregate(aggregateExpressions, groupingExpressions)
 
       if (forceObjHashAggregate || (objectHashEnabled && useObjectHash)) {
         ObjectHashAggregateExec(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index f3561fb91394..7844ca6bced4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -33,8 +33,8 @@ import org.apache.spark.sql.catalyst.expressions.codegen.Block._
 import org.apache.spark.sql.catalyst.plans.logical.Aggregate
 import org.apache.spark.sql.catalyst.types.DataTypeUtils
 import org.apache.spark.sql.catalyst.types.DataTypeUtils.toAttributes
+import org.apache.spark.sql.catalyst.util.{truncatedString, UnsafeRowUtils}
 import org.apache.spark.sql.catalyst.util.DateTimeConstants.NANOS_PER_MILLIS
-import org.apache.spark.sql.catalyst.util.truncatedString
 import org.apache.spark.sql.execution._
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.execution.vectorized.MutableColumnarRow
@@ -60,6 +60,8 @@ case class HashAggregateExec(
   extends AggregateCodegenSupport {
 
   require(Aggregate.supportsHashAggregate(aggregateBufferAttributes, groupingExpressions))
+  require(!isStreaming ||
+    groupingExpressions.forall(e => UnsafeRowUtils.isBinaryStable(e.dataType)))
 
   override def allAttributes: AttributeSeq =
     child.output ++ aggregateBufferAttributes ++ aggregateAttributes ++
@@ -373,7 +375,7 @@ case class HashAggregateExec(
           var findNextGroup = false
           while (!findNextGroup && sortedIter.next()) {
             val key = sortedIter.getKey
-            if (currentKey.equals(key)) {
+            if (hashMap.keysEqual(currentKey, key)) {
               mergeProjection(joinedRow(currentRow, sortedIter.getValue))
             } else {
               // We find a new group.
@@ -504,6 +506,7 @@ case class HashAggregateExec(
    */
   private def checkIfFastHashMapSupported(): Boolean = {
     val isSupported =
+      groupingExpressions.forall(e => UnsafeRowUtils.isBinaryStable(e.dataType)) &&
       (groupingKeySchema ++ bufferSchema).forall(f => CodeGenerator.isPrimitiveType(f.dataType) ||
         f.dataType.isInstanceOf[DecimalType] || f.dataType.isInstanceOf[StringType] ||
         f.dataType.isInstanceOf[CalendarIntervalType])
@@ -908,7 +911,6 @@ case class HashAggregateExec(
     val fastRowKeys = ctx.generateExpressions(
       bindReferences[Expression](groupingExpressions, child.output))
     val unsafeRowKeys = unsafeRowKeyCode.value
-    val unsafeRowKeyHash = ctx.freshName("unsafeRowKeyHash")
     val unsafeRowBuffer = ctx.freshName("unsafeRowAggBuffer")
     val fastRowBuffer = ctx.freshName("fastAggBuffer")
 
@@ -967,13 +969,21 @@ case class HashAggregateExec(
       // per regular-map row (see below); emitting it in more than one runtime branch is unsafe
       // because the projection's subexpression/writer state assigned in one branch would be read
       // stale from another (e.g. the adaptive pass-through path would reuse the last probed key).
+      val (computeKeyHash, lookupBuffer) =
+        if (UnsafeRowUtils.isBinaryStable(groupingKeySchema)) {
+          val unsafeRowKeyHash = ctx.freshName("unsafeRowKeyHash")
+          (s"int $unsafeRowKeyHash = ${unsafeRowKeyCode.value}.hashCode();",
+            s"$hashMapTerm.getAggregationBufferFromUnsafeRow(" +
+              s"$unsafeRowKeys, $unsafeRowKeyHash)")
+        } else {
+          ("", s"$hashMapTerm.getAggregationBufferFromUnsafeRow($unsafeRowKeys)")
+        }
       val probeRegularMap =
         s"""
-           |int $unsafeRowKeyHash = ${unsafeRowKeyCode.value}.hashCode();
+           |$computeKeyHash
            |if ($checkFallbackForBytesToBytesMap) {
            |  // try to get the buffer from hash map
-           |  $unsafeRowBuffer =
-           |    $hashMapTerm.getAggregationBufferFromUnsafeRow($unsafeRowKeys, $unsafeRowKeyHash);
+           |  $unsafeRowBuffer = $lookupBuffer;
            |}
          """.stripMargin
 
@@ -984,8 +994,7 @@ case class HashAggregateExec(
            |$resetCounter
            |// the hash map had been spilled, so it should have enough memory now,
            |// try to allocate buffer again.
-           |$unsafeRowBuffer = $hashMapTerm.getAggregationBufferFromUnsafeRow(
-           |  $unsafeRowKeys, $unsafeRowKeyHash);
+           |$unsafeRowBuffer = $lookupBuffer;
            |if ($unsafeRowBuffer == null) {
            |  // failed to allocate the first page
            |  throw QueryExecutionErrors.aggregateOutOfMemoryError();
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
index fbd3ce040e48..65ce202e978e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationIterator.scala
@@ -152,7 +152,8 @@ class ObjectAggregationIterator(
   // spills are merged together for sort-based aggregation.
   private def processInputs(): Unit = {
     // In-memory map to store aggregation buffer for hash-based aggregation.
-    val hashMap = new ObjectAggregationMap()
+    val groupingSchema = DataTypeUtils.fromAttributes(groupingExpressions.map(_.toAttribute))
+    val hashMap = ObjectAggregationMap.create(groupingSchema)
 
     // If in-memory map is unable to stores all aggregation buffer, fallback to sort-based
     // aggregation backed by sorted physical storage.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
index 5384f939c31a..b12f3e273847 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/ObjectAggregationMap.scala
@@ -25,7 +25,65 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, UnsafeProjection, UnsafeRow}
 import org.apache.spark.sql.catalyst.expressions.aggregate.{AggregateFunction, TypedImperativeAggregate}
 import org.apache.spark.sql.catalyst.types.DataTypeUtils
+import org.apache.spark.sql.catalyst.util.{UnsafeRowKeyOperations, UnsafeRowUtils}
 import org.apache.spark.sql.execution.UnsafeKVExternalSorter
+import org.apache.spark.sql.types.StructType
+
+private[aggregate] object ObjectAggregationMap {
+  private final class SemanticKey(
+      val keyOperations: UnsafeRowKeyOperations,
+      var row: UnsafeRow) {
+    override def hashCode(): Int = keyOperations.hash(row)
+
+    override def equals(other: Any): Boolean = other match {
+      case that: SemanticKey => row.equals(that.row) || keyOperations.areEqual(row, that.row)
+      case _ => false
+    }
+  }
+
+  def create(groupingSchema: StructType): ObjectAggregationMap = {
+    if (UnsafeRowUtils.isBinaryStable(groupingSchema)) {
+      new ObjectAggregationMap()
+    } else {
+      new SemanticObjectAggregationMap(groupingSchema)
+    }
+  }
+
+  private final class SemanticObjectAggregationMap(groupingSchema: StructType)
+      extends ObjectAggregationMap {
+    private val keyOperations = new UnsafeRowKeyOperations(groupingSchema)
+    private val hashMap = new ju.LinkedHashMap[SemanticKey, InternalRow]
+    private val reusableLookupKey = new SemanticKey(keyOperations, null)
+
+    override def getAggregationBuffer(groupingKey: UnsafeRow): InternalRow = {
+      reusableLookupKey.row = groupingKey
+      hashMap.get(reusableLookupKey)
+    }
+
+    override def putAggregationBuffer(groupingKey: UnsafeRow, aggBuffer: InternalRow): Unit = {
+      hashMap.put(new SemanticKey(keyOperations, groupingKey), aggBuffer)
+    }
+
+    override def size: Int = hashMap.size()
+
+    override def destructiveIterator(): Iterator[AggregationBufferEntry] = {
+      val iter = hashMap.entrySet().iterator()
+      new Iterator[AggregationBufferEntry] {
+        override def hasNext: Boolean = iter.hasNext
+
+        override def next(): AggregationBufferEntry = {
+          val entry = iter.next()
+          iter.remove()
+          new AggregationBufferEntry(entry.getKey.row, entry.getValue)
+        }
+      }
+    }
+
+    override def clear(): Unit = {
+      hashMap.clear()
+    }
+  }
+}
 
 /**
  * An aggregation map that supports using safe `SpecificInternalRow`s aggregation buffers, so that
@@ -101,7 +159,7 @@ class ObjectAggregationMap() {
       )
     }
 
-    hashMap.clear()
+    clear()
     sorter
   }
 
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TungstenAggregationIterator.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TungstenAggregationIterator.scala
index 9521a68ae4b2..779d3a80ad8a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TungstenAggregationIterator.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/TungstenAggregationIterator.scala
@@ -396,7 +396,7 @@ class TungstenAggregationIterator(
       val inputAggregationBuffer = sortedKVIterator.getValue
 
       // Check if the current row belongs the current input row.
-      if (currentGroupingKey.equals(groupingKey)) {
+      if (hashMap.keysEqual(currentGroupingKey, groupingKey)) {
         sortBasedProcessRow(sortBasedAggregationBuffer, inputAggregationBuffer)
 
         hasNext = sortedKVIterator.next()
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationAggregationSuite.scala
index b894abe61476..cdaadeb728b3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationAggregationSuite.scala
@@ -17,73 +17,322 @@
 
 package org.apache.spark.sql.collation
 
-import org.apache.spark.sql.Row
+import org.apache.spark.sql.{DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
+import org.apache.spark.sql.execution.WholeStageCodegenExec
 import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
 import org.apache.spark.sql.execution.aggregate.{HashAggregateExec, ObjectHashAggregateExec, SortAggregateExec}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.unsafe.types.CalendarInterval
 
 class CollationAggregationSuite
-  extends SharedSparkSession
+  extends QueryTest
+  with SharedSparkSession
   with AdaptiveSparkPlanHelper {
 
-  test("group by collated column doesn't work with obj hash aggregate") {
-    val tblName = "grp_by_tbl"
-    withTable(tblName) {
-      sql(s"CREATE TABLE $tblName (c1 STRING COLLATE UTF8_LCASE, c2 INT) USING PARQUET")
-      sql(s"INSERT INTO $tblName VALUES ('hello', 1), ('HELLO', 2), ('HeLlO', 3)")
-
-      // Result is correct without forcing object hash aggregate.
-      checkAnswer(
-        sql(s"SELECT COUNT(*) FROM $tblName GROUP BY c1"),
-        Seq(Row(3)))
-
-      withSQLConf("spark.sql.test.forceApplyObjectHashAggregate" -> true.toString) {
-        checkAnswer(
-          sql(s"SELECT COUNT(*) FROM $tblName GROUP BY c1"),
-          Seq(Row(1), Row(1), Row(1)))
-
-        checkAnswer(
-          sql(s"SELECT COLLECT_LIST(c2) AS c3 FROM $tblName GROUP BY c1 ORDER BY c3"),
-          Seq(Row(Seq(1)), Row(Seq(2)), Row(Seq(3))))
+  import testImplicits._
+
+  private def assertUsesHashAggregate(df: DataFrame): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(exists(plan)(_.isInstanceOf[HashAggregateExec]), plan.toString)
+    assert(!exists(plan)(_.isInstanceOf[SortAggregateExec]), plan.toString)
+  }
+
+  private def assertUsesObjectHashAggregate(df: DataFrame): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(exists(plan)(_.isInstanceOf[ObjectHashAggregateExec]), plan.toString)
+    assert(!exists(plan)(_.isInstanceOf[SortAggregateExec]), plan.toString)
+  }
+
+  private def assertUsesSortAggregate(df: DataFrame): Unit = {
+    val plan = df.queryExecution.executedPlan
+    assert(exists(plan)(_.isInstanceOf[SortAggregateExec]), plan.toString)
+    assert(!exists(plan)(_.isInstanceOf[HashAggregateExec]), plan.toString)
+    assert(!exists(plan)(_.isInstanceOf[ObjectHashAggregateExec]), plan.toString)
+  }
+
+  private def generatedCode(df: DataFrame): String = {
+    flatMap(df.queryExecution.executedPlan) {
+      case stage: WholeStageCodegenExec => Seq(stage.doCodeGen()._2.body)
+      case _ => Nil
+    }.mkString("\n")
+  }
+
+  test("HashAggregateExec groups collated string keys") {
+    val testCases = Seq(
+      "UTF8_LCASE" -> Seq("hello", "HELLO", "HeLlO"),
+      "UNICODE_CI" -> Seq("hello", "HELLO", "HeLlO"),
+      "UTF8_BINARY_RTRIM" -> Seq("hello", "hello ", "hello  "))
+
+    testCases.foreach { case (collation, spellings) =>
+      withTempView("collated_keys") {
+        val values = spellings.zipWithIndex
+          .map { case (key, index) => s"('$key', ${index + 1})" }
+          .mkString(", ")
+        sql(
+          s"""
+             |SELECT CAST(key AS STRING COLLATE $collation) AS key, value
+             |FROM VALUES $values, (NULL, 4) AS data(key, value)
+             |""".stripMargin).createOrReplaceTempView("collated_keys")
+
+        val result = sql(
+          """
+            |SELECT LOWER(RTRIM(key)) AS normalized_key, SUM(value) AS total
+            |FROM collated_keys
+            |GROUP BY key
+            |ORDER BY normalized_key NULLS LAST
+            |""".stripMargin)
+        assertUsesHashAggregate(result)
+        checkAnswer(result, Seq(Row("hello", 6L), Row(null, 4L)))
+      }
+    }
+  }
+
+  test("HashAggregateExec preserves binary semantics for stable sibling keys") {
+    withTempView("interval_keys") {
+      Seq(
+        ("hello", new CalendarInterval(1, 2, 3), 1),
+        ("HELLO", new CalendarInterval(1, 2, 3), 2))
+        .toDF("key", "calendar_interval", "value")
+        .selectExpr(
+          "CAST(key AS STRING COLLATE UTF8_LCASE) AS key",
+          "calendar_interval",
+          "value")
+        .repartition(1)
+        .createOrReplaceTempView("interval_keys")
+
+      withSQLConf(
+        SQLConf.CODEGEN_FACTORY_MODE.key -> CodegenObjectFactoryMode.NO_CODEGEN.toString) {
+        val result = sql(
+          """
+            |SELECT LOWER(key) AS normalized_key, SUM(value) AS total
+            |FROM interval_keys
+            |GROUP BY key, calendar_interval
+            |""".stripMargin)
+        assertUsesHashAggregate(result)
+        checkAnswer(result, Row("hello", 3L))
+
+        val objectResult = sql(
+          """
+            |SELECT LOWER(key) AS normalized_key, ARRAY_SORT(COLLECT_LIST(value)) AS values
+            |FROM interval_keys
+            |GROUP BY key, calendar_interval
+            |""".stripMargin)
+        assertUsesObjectHashAggregate(objectResult)
+        checkAnswer(objectResult, Row("hello", Seq(1, 2)))
       }
     }
+
+    withTempView("geometry_keys") {
+      sql(
+        """
+          |SELECT
+          |  CAST(key AS STRING COLLATE UTF8_LCASE) AS key,
+          |  ST_GeomFromWKB(wkb) AS geometry,
+          |  value
+          |FROM VALUES
+          |  ('hello', X'0101000000000000000000F03F0000000000000040', 1),
+          |  ('HELLO', X'0101000000000000000000F03F0000000000000040', 2)
+          |AS data(key, wkb, value)
+          |""".stripMargin)
+        .repartition(1)
+        .createOrReplaceTempView("geometry_keys")
+
+      val result = sql(
+        """
+          |SELECT LOWER(key) AS normalized_key, SUM(value) AS total
+          |FROM geometry_keys
+          |GROUP BY key, geometry
+          |""".stripMargin)
+      assertUsesHashAggregate(result)
+      checkAnswer(result, Row("hello", 3L))
+    }
   }
 
-  test("imperative aggregate fn does not use objectHashAggregate when group by collated column") {
-    val tblName = "imp_agg"
-    Seq(true, false).foreach { useObjHashAgg =>
-      withTable(tblName) {
-        withSQLConf("spark.sql.execution.useObjectHashAggregateExec" -> useObjHashAgg.toString) {
-          sql(
-            s"""
-               |CREATE TABLE $tblName (
-               |  c1 STRING COLLATE UTF8_LCASE,
-               |  c2 INT
-               |) USING PARQUET
-               |""".stripMargin)
-          sql(s"INSERT INTO $tblName VALUES ('HELLO', 1), ('hello', 2), ('HeLlO', 3)")
-
-          val df = sql(s"SELECT COLLECT_LIST(c2) as list FROM $tblName GROUP BY c1")
-          val executedPlan = df.queryExecution.executedPlan
-
-          // Plan should not have any hash aggregate nodes.
-          collectFirst(executedPlan) {
-            case _: ObjectHashAggregateExec => fail("ObjectHashAggregateExec should not be used.")
-            case _: HashAggregateExec => fail("HashAggregateExec should not be used.")
+  test("nested and distinct collated grouping works across partial and final aggregation") {
+    withTempView("nested_keys") {
+      sql(
+        """
+          |SELECT id, CAST(key AS STRING COLLATE UTF8_LCASE) AS key, bucket, value
+          |FROM VALUES
+          |  (1, 'alpha', 0, 10),
+          |  (2, 'ALPHA', 0, 20),
+          |  (3, 'beta', 1, 30),
+          |  (4, 'BETA', 1, 40),
+          |  (5, NULL, 2, 50),
+          |  (6, NULL, 2, 60)
+          |AS data(id, key, bucket, value)
+          |""".stripMargin)
+        .repartition(4)
+        .createOrReplaceTempView("nested_keys")
+
+      val distinctKeys = sql(
+        """
+          |SELECT LOWER(key) AS normalized_key
+          |FROM (SELECT DISTINCT key FROM nested_keys)
+          |ORDER BY normalized_key NULLS LAST
+          |""".stripMargin)
+      assertUsesHashAggregate(distinctKeys)
+      checkAnswer(distinctKeys, Seq(Row("alpha"), Row("beta"), Row(null)))
+
+      val result = sql(
+        """
+          |SELECT
+          |  LOWER(array_key[0]) AS array_key,
+          |  LOWER(struct_key.s) AS struct_key,
+          |  bucket,
+          |  distinct_ids,
+          |  total
+          |FROM (
+          |  SELECT
+          |    ARRAY(key) AS array_key,
+          |    NAMED_STRUCT('s', key) AS struct_key,
+          |    bucket,
+          |    COUNT(DISTINCT id) AS distinct_ids,
+          |    SUM(value) AS total
+          |  FROM nested_keys
+          |  GROUP BY ARRAY(key), NAMED_STRUCT('s', key), bucket
+          |)
+          |ORDER BY bucket
+          |""".stripMargin)
+      assertUsesHashAggregate(result)
+      checkAnswer(result, Seq(
+        Row("alpha", "alpha", 0, 2L, 30L),
+        Row("beta", "beta", 1, 2L, 70L),
+        Row(null, null, 2, 2L, 110L)))
+    }
+  }
+
+  test("ObjectHashAggregateExec groups collated keys before and after sort fallback") {
+    withTempView("object_keys") {
+      sql(
+        """
+          |SELECT CAST(key AS STRING COLLATE UTF8_LCASE) AS key, value
+          |FROM VALUES
+          |  ('hello', 1), ('world', 4),
+          |  ('HELLO', 2), ('WORLD', 5),
+          |  ('HeLlO', 3), ('WoRlD', 6)
+          |AS data(key, value)
+          |""".stripMargin).createOrReplaceTempView("object_keys")
+
+      Seq(Int.MaxValue.toString, "1").foreach { fallbackThreshold =>
+        withSQLConf(
+          SQLConf.OBJECT_AGG_SORT_BASED_FALLBACK_THRESHOLD.key -> fallbackThreshold) {
+          val result = sql(
+            """
+              |SELECT
+              |  LOWER(key) AS normalized_key,
+              |  ARRAY_SORT(COLLECT_LIST(value)) AS collected_list,
+              |  ARRAY_SORT(COLLECT_SET(value)) AS collected_set
+              |FROM object_keys
+              |GROUP BY key
+              |ORDER BY normalized_key
+              |""".stripMargin)
+          assertUsesObjectHashAggregate(result)
+          checkAnswer(result, Seq(
+            Row("hello", Seq(1, 2, 3), Seq(1, 2, 3)),
+            Row("world", Seq(4, 5, 6), Seq(4, 5, 6))))
+
+          if (fallbackThreshold == "1") {
+            assert(exists(result.queryExecution.executedPlan) {
+              case aggregate: ObjectHashAggregateExec =>
+                aggregate.metrics("numTasksFallBacked").value > 0
+              case _ => false
+            })
           }
+        }
+      }
+    }
+  }
 
-          // Plan should have a [[SortAggregateExec]] node.
-          assert(collectFirst(executedPlan) {
-            case _: SortAggregateExec => true
-          }.nonEmpty)
+  test("HashAggregateExec spill merges collation-equivalent keys") {
+    withTempView("spill_keys") {
+      sql(
+        """
+          |SELECT CAST(key AS STRING COLLATE UTF8_LCASE_RTRIM) AS key, value
+          |FROM VALUES
+          |  ('hello', 1), ('world', 4),
+          |  ('HELLO ', 2), ('WORLD ', 5),
+          |  ('HeLlO  ', 3), ('WoRlD  ', 6)
+          |AS data(key, value)
+          |""".stripMargin).createOrReplaceTempView("spill_keys")
 
-          checkAnswer(
-            // Sort the values to get deterministic output.
-            df.selectExpr("array_sort(list)"),
-            Seq(Row(Seq(1, 2, 3)))
-          )
+      Seq(true, false).foreach { wholeStage =>
+        withSQLConf(
+          SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> wholeStage.toString,
+          "spark.sql.TungstenAggregate.testFallbackStartsAt" -> "0, 1") {
+          withClue(s"wholeStage=$wholeStage: ") {
+            val result = sql(
+              """
+                |SELECT LOWER(RTRIM(key)) AS normalized_key, SUM(value) AS total
+                |FROM spill_keys
+                |GROUP BY key
+                |ORDER BY normalized_key
+                |""".stripMargin)
+            assertUsesHashAggregate(result)
+            checkAnswer(result, Seq(Row("hello", 6L), Row("world", 15L)))
+            assert(exists(result.queryExecution.executedPlan) {
+              case aggregate: HashAggregateExec =>
+                aggregate.metrics("numTasksFallBacked").value > 0
+              case _ => false
+            })
+          }
         }
       }
     }
   }
+
+  test("two-level aggregation remains enabled only for binary-stable grouping keys") {
+    withSQLConf(SQLConf.ENABLE_TWOLEVEL_AGG_MAP.key -> "true") {
+      val binary = sql(
+        """
+          |SELECT key, SUM(value)
+          |FROM VALUES ('a', 1), ('a', 2), ('b', 3) AS data(key, value)
+          |GROUP BY key
+          |ORDER BY key
+          |""".stripMargin)
+      assertUsesHashAggregate(binary)
+      checkAnswer(binary, Seq(Row("a", 3L), Row("b", 3L)))
+      val binaryCode = generatedCode(binary)
+      assert(binaryCode.contains("FastHashMap"))
+
+      val collated = sql(
+        """
+          |SELECT LOWER(key) AS normalized_key, SUM(value)
+          |FROM (
+          |  SELECT CAST(key AS STRING COLLATE UTF8_LCASE) AS key, value
+          |  FROM VALUES ('a', 1), ('A', 2) AS data(key, value)
+          |)
+          |GROUP BY key
+          |""".stripMargin)
+      assertUsesHashAggregate(collated)
+      checkAnswer(collated, Row("a", 3L))
+      val collatedCode = generatedCode(collated)
+      assert(!collatedCode.contains("FastHashMap"))
+      assert(collatedCode.contains("getAggregationBufferFromUnsafeRow"))
+    }
+  }
+
+  test("SortAggregateExec groups collated keys when hash aggregation is disabled") {
+    withSQLConf(
+      SQLConf.USE_HASH_AGG.key -> "false",
+      SQLConf.USE_OBJECT_HASH_AGG.key -> "false") {
+      val result = sql(
+        """
+          |SELECT LOWER(key) AS normalized_key, SUM(value) AS total
+          |FROM (
+          |  SELECT CAST(key AS STRING COLLATE UTF8_LCASE) AS key, value
+          |  FROM VALUES
+          |    ('hello', 1), ('HELLO', 2),
+          |    ('world', 4), ('WORLD', 5)
+          |  AS data(key, value)
+          |)
+          |GROUP BY key
+          |ORDER BY normalized_key
+          |""".stripMargin)
+      assertUsesSortAggregate(result)
+      checkAnswer(result, Seq(Row("hello", 3L), Row("world", 9L)))
+    }
+  }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
index 76a28e1af51e..8cadcb759d9b 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/CollationSuite.scala
@@ -455,7 +455,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
     }
   }
 
-  test("hash agg is not used for non binary collations") {
+  test("hash agg is used for binary and non-binary collations") {
     val tableNameNonBinary = "T_NON_BINARY"
     val tableNameBinary = "T_BINARY"
     withTable(tableNameNonBinary) {
@@ -468,7 +468,7 @@ class CollationSuite extends DatasourceV2SQLBase with AdaptiveSparkPlanHelper {
         val dfNonBinary = sql(s"SELECT COUNT(*), c FROM $tableNameNonBinary GROUP BY c")
         assert(collectFirst(dfNonBinary.queryExecution.executedPlan) {
           case _: HashAggregateExec | _: ObjectHashAggregateExec => ()
-        }.isEmpty)
+        }.nonEmpty)
 
         val dfBinary = sql(s"SELECT COUNT(*), c FROM $tableNameBinary GROUP BY c")
         assert(collectFirst(dfBinary.queryExecution.executedPlan) {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/aggregate/AdaptivePartialAggregationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/aggregate/AdaptivePartialAggregationSuite.scala
index 5eb9b4a8af6f..5af3793c9f88 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/aggregate/AdaptivePartialAggregationSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/aggregate/AdaptivePartialAggregationSuite.scala
@@ -340,6 +340,27 @@ class AdaptivePartialAggregationSuite extends QueryTest with SharedSparkSession
     }
   }
 
+  test("collation-equivalent keys across the bypass cutoff stay in one group") {
+    forEachCodegenAndMap() { clue =>
+      // The first eight rows are distinct, so the eighth row flips pass-through on. Its key is
+      // already in the frozen map when the equivalent spelling on the ninth row bypasses it.
+      val df = () => spark.range(0, 200, 1, 1)
+        .selectExpr(
+          """CAST(CASE
+            |  WHEN id = 7 THEN 'COLLATED'
+            |  WHEN id = 8 THEN 'collated'
+            |  ELSE CONCAT('key-', id)
+            |END AS STRING COLLATE UTF8_LCASE) AS k""".stripMargin,
+          "id AS v")
+        .groupBy($"k")
+        .agg(sum($"v") as "s", count(lit(1)) as "c")
+      withClue(clue) {
+        assert(numBypassingRows(df) > 0,
+          "expected the equivalent spelling after the cutoff to bypass partial aggregation")
+      }
+    }
+  }
+
   test("results unchanged with nullable grouping keys") {
     // Nulls are sparse enough (1 in 40) that the keys stay close to unique and the input really
     // does bypass; a denser null key would lift the compaction ratio above the threshold and the


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org