You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/20 19:04:43 UTC

(spark) branch master updated: [SPARK-51820][SQL][FOLLOWUP][CONNECT] Don't add `UnresolvedOrdinal` when appending grouping to aggregate expressions

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 4b3a65307681 [SPARK-51820][SQL][FOLLOWUP][CONNECT] Don't add `UnresolvedOrdinal` when appending grouping to aggregate expressions
4b3a65307681 is described below

commit 4b3a65307681ff653e7f6ee90957976817057226
Author: Mihailo Timotic <mi...@databricks.com>
AuthorDate: Wed May 21 03:04:24 2025 +0800

    [SPARK-51820][SQL][FOLLOWUP][CONNECT] Don't add `UnresolvedOrdinal` when appending grouping to aggregate expressions
    
    ### What changes were proposed in this pull request?
    Don't add `UnresolvedOrdinal` when appending grouping to aggregate expressions in Spark Connect.
    
    ### Why are the changes needed?
    Change is needed to fix a regression caused by https://github.com/apache/spark/pull/50606 where `UnresolvedOrdinal` would end up in aggregate expression and propagate all the way to `CheckAnalysis` where it would throw an error
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Added a test case for the correct behavior.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No.
    
    Closes #50958 from mihailotim-db/mihailotim-db/fix_unresolved_ordinal.
    
    Authored-by: Mihailo Timotic <mi...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../sql/connect/planner/SparkConnectPlanner.scala  | 26 +++++++++--------
 .../connect/planner/SparkConnectPlannerSuite.scala | 33 +++++++++++++++++++++-
 2 files changed, 46 insertions(+), 13 deletions(-)

diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index d821046d0831..1bdf408664e4 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -2457,11 +2457,13 @@ class SparkConnectPlanner(
         input
       }
 
-    val groupingExpressionsWithOrdinals = rel.getGroupingExpressionsList.asScala.toSeq
-      .map(transformGroupingExpressionAndReplaceOrdinals)
+    val groupingExpressions =
+      rel.getGroupingExpressionsList.asScala.toSeq.map(transformExpression)
+    val groupingExpressionsWithOrdinals =
+      groupingExpressions.map(replaceOrdinalsInGroupingExpressions)
     val aggExprs = rel.getAggregateExpressionsList.asScala.toSeq
       .map(expr => transformExpressionWithTypedReduceExpression(expr, logicalPlan))
-    val aliasedAgg = (groupingExpressionsWithOrdinals ++ aggExprs).map(toNamedExpression)
+    val aliasedAgg = (groupingExpressions ++ aggExprs).map(toNamedExpression)
 
     rel.getGroupType match {
       case proto.Aggregate.GroupType.GROUP_TYPE_GROUPBY =>
@@ -2506,7 +2508,10 @@ class SparkConnectPlanner(
         val groupingSetsExpressionsWithOrdinals =
           rel.getGroupingSetsList.asScala.toSeq.map { getGroupingSets =>
             getGroupingSets.getGroupingSetList.asScala.toSeq
-              .map(transformGroupingExpressionAndReplaceOrdinals)
+              .map(groupingExpressions => {
+                val transformedGroupingExpression = transformExpression(groupingExpressions)
+                replaceOrdinalsInGroupingExpressions(transformedGroupingExpression)
+              })
           }
         logical.Aggregate(
           groupingExpressions = Seq(
@@ -2521,18 +2526,15 @@ class SparkConnectPlanner(
   }
 
   /**
-   * Transforms an input protobuf grouping expression into the Catalyst expression and converts
-   * top-level integer [[Literal]]s to [[UnresolvedOrdinal]]s, if `groupByOrdinal` is enabled.
+   * Replaces top-level integer [[Literal]]s to [[UnresolvedOrdinal]]s, if `groupByOrdinal` is
+   * enabled.
    */
-  private def transformGroupingExpressionAndReplaceOrdinals(
-      groupingExpression: proto.Expression) = {
-    val transformedGroupingExpression = transformExpression(groupingExpression)
+  private def replaceOrdinalsInGroupingExpressions(groupingExpression: Expression) =
     if (session.sessionState.conf.groupByOrdinal) {
-      replaceIntegerLiteralWithOrdinal(transformedGroupingExpression)
+      replaceIntegerLiteralWithOrdinal(groupingExpression)
     } else {
-      transformedGroupingExpression
+      groupingExpression
     }
-  }
 
   @deprecated("TypedReduce is now implemented using a normal UDAF aggregator.", "4.0.0")
   private def transformTypedReduceExpression(
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
index 72f7065b4424..9a69151ccfc2 100644
--- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
+++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/planner/SparkConnectPlannerSuite.scala
@@ -29,7 +29,8 @@ import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{UnresolvedAlias, UnresolvedFunction, UnresolvedRelation}
 import org.apache.spark.sql.catalyst.expressions.{AttributeReference, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical
-import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
+import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, LogicalPlan}
+import org.apache.spark.sql.catalyst.trees.TreePattern
 import org.apache.spark.sql.catalyst.types.DataTypeUtils
 import org.apache.spark.sql.classic.Dataset
 import org.apache.spark.sql.connect.SparkConnectTestUtils
@@ -922,4 +923,34 @@ class SparkConnectPlannerSuite extends SparkFunSuite with SparkConnectPlanTest {
     assert(fn3.nameParts.head == "abcde")
     assert(fn3.isInternal)
   }
+
+  test("SPARK-51820 aggregate list should not contain UnresolvedOrdinal") {
+    val ordinal = proto.Expression
+      .newBuilder()
+      .setLiteral(proto.Expression.Literal.newBuilder().setInteger(1).build())
+      .build()
+
+    val sum =
+      proto.Expression
+        .newBuilder()
+        .setUnresolvedFunction(
+          proto.Expression.UnresolvedFunction
+            .newBuilder()
+            .setFunctionName("sum")
+            .addArguments(ordinal))
+        .build()
+
+    val aggregate = proto.Aggregate.newBuilder
+      .setInput(readRel)
+      .addAggregateExpressions(sum)
+      .addGroupingExpressions(ordinal)
+      .setGroupType(proto.Aggregate.GroupType.GROUP_TYPE_ROLLUP)
+      .build()
+
+    val plan =
+      transform(proto.Relation.newBuilder.setAggregate(aggregate).build()).asInstanceOf[Aggregate]
+
+    assert(plan.aggregateExpressions.forall(aggregateExpression =>
+      !aggregateExpression.containsPattern(TreePattern.UNRESOLVED_ORDINAL)))
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org