(spark) branch master updated: Revert "[SPARK-59261][SQL] Memoize outputPartitioning on the projection and SPJ grouping nodes"

[pt...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

peter-toth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 8ec12b12bca7 Revert "[SPARK-59261][SQL] Memoize outputPartitioning on the projection and SPJ grouping nodes"
8ec12b12bca7 is described below

commit 8ec12b12bca79ec5a00ac144a8b80ecc30743021
Author: Peter Toth <pe...@gmail.com>
AuthorDate: Mon Sep 7 11:57:31 2026 +0200

    Revert "[SPARK-59261][SQL] Memoize outputPartitioning on the projection and SPJ grouping nodes"
    
    This reverts commit 9373486f98324a8954a74375edc4b771001d0f11.
---
 .../spark/sql/execution/AliasAwareOutputExpression.scala      | 11 +++--------
 .../sql/execution/datasources/v2/GroupPartitionsExec.scala    |  5 +----
 2 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
index 8c8c37d6d9d0..d6003b7b49e7 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/AliasAwareOutputExpression.scala
@@ -29,14 +29,9 @@ import org.apache.spark.sql.catalyst.trees.MultiTransform
  */
 trait PartitioningPreservingUnaryExecNode extends UnaryExecNode
   with AliasAwareOutputExpression {
-  // A `lazy val` because the planner asks a node for its partitioning many times, and this body
-  // projects every partitioning expression through the output aliases, and for a
-  // `KeyedPartitioning` child also builds an `ExpressionSet` per key position and cross-products
-  // the per-position alternatives. Read no live config here, or memoizing would freeze it.
-  @transient final override lazy val outputPartitioning: Partitioning = {
-    val childPartitioning = child.outputPartitioning
+  final override def outputPartitioning: Partitioning = {
     val (keyedPartitionings, otherPartitionings) =
-      PartitioningCollection.flatten(childPartitioning)
+      PartitioningCollection.flatten(child.outputPartitioning)
         .partition(_.isInstanceOf[KeyedPartitioning])
 
     val projectedKPs =
@@ -49,7 +44,7 @@ trait PartitioningPreservingUnaryExecNode extends UnaryExecNode
     // deep projection chain that nesting overflows the stack when the partitioning is later
     // serialized or deeply traversed.
     (projectedKPs ++ projectedOthers).take(aliasCandidateLimit).toList match {
-      case Seq() => UnknownPartitioning(childPartitioning.numPartitions)
+      case Seq() => UnknownPartitioning(child.outputPartitioning.numPartitions)
       case Seq(p) => p
       case ps => PartitioningCollection.fromPartitionings(ps)
     }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/GroupPartitionsExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/GroupPartitionsExec.scala
index 5d2d3d4773f1..61f57fd5a2cd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/GroupPartitionsExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/GroupPartitionsExec.scala
@@ -67,10 +67,7 @@ case class GroupPartitionsExec(
     @transient enableSortedMerge: Boolean = false
   ) extends UnaryExecNode {
 
-  // A `lazy val` because the planner asks a node for its partitioning many times, and this body
-  // rebuilds every `KeyedPartitioning` in the child's on top of the already memoized `grouping`.
-  // Read no live config here, or memoizing would freeze it.
-  @transient override lazy val outputPartitioning: Partitioning = {
+  override def outputPartitioning: Partitioning = {
     child.outputPartitioning match {
       case p: Partitioning with Expression =>
         // There can be multiple `KeyedPartitioning`s in an output partitioning of a join, but they


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org