(spark) branch master updated: [SPARK-51624][SQL] Propagate GetStructField metadata in CreateNamedStruct.dataType

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 0f36edc2b985 [SPARK-51624][SQL] Propagate GetStructField metadata in CreateNamedStruct.dataType
0f36edc2b985 is described below

commit 0f36edc2b9856a66eb184540eacd847fb1348220
Author: Andy Lam <an...@databricks.com>
AuthorDate: Thu Mar 27 15:44:23 2025 +0800

    [SPARK-51624][SQL] Propagate GetStructField metadata in CreateNamedStruct.dataType
    
    ### What changes were proposed in this pull request?
    
    This change intends to propagate GetStructField metadata in CreateNamedStruct.dataType.
    
    ### Why are the changes needed?
    
    This is important because dataType comparisons are important for optimizer rules such as SimplifyCasts, which can cascade down to more expression optimizations.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added unit test.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #50412 from andylam-db/simplify-casts-comp.
    
    Lead-authored-by: Andy Lam <an...@databricks.com>
    Co-authored-by: Wenchen Fan <cl...@gmail.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../sql/catalyst/expressions/complexTypeCreator.scala  |  1 +
 .../scala/org/apache/spark/sql/ComplexTypesSuite.scala | 18 ++++++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 1fc8fe8f247b..81484f8dd7da 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -457,6 +457,7 @@ case class CreateNamedStruct(children: Seq[Expression]) extends Expression with
       case (name, expr) =>
         val metadata = expr match {
           case ne: NamedExpression => ne.metadata
+          case gsf: GetStructField => gsf.metadata
           case _ => Metadata.empty
         }
         StructField(name.toString, expr.dataType, expr.nullable, metadata)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
index 5dbb4b598b8c..f6e4a9824eff 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ComplexTypesSuite.scala
@@ -19,10 +19,10 @@ package org.apache.spark.sql
 
 import scala.jdk.CollectionConverters._
 
-import org.apache.spark.sql.catalyst.expressions.CreateNamedStruct
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, CreateNamedStruct, GetStructField, Literal}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.test.SharedSparkSession
-import org.apache.spark.sql.types.{ArrayType, IntegerType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, IntegerType, MetadataBuilder, StringType, StructField, StructType}
 
 class ComplexTypesSuite extends QueryTest with SharedSparkSession {
   import testImplicits._
@@ -171,4 +171,18 @@ class ComplexTypesSuite extends QueryTest with SharedSparkSession {
     assert(df.schema == expectedSchema)
     checkAnswer(df, Seq(Row(Row(1, 2))))
   }
+
+  test("SPARK-51624: Propagate StructField metadata in CreateNamedStruct.dataType") {
+    val metadata = new MetadataBuilder().putString("comment", "hello").build()
+    val structRef = AttributeReference("s1",
+      StructType(StructField("col1", StringType, false, metadata) :: Nil))()
+    val createNamedStruct = CreateNamedStruct(
+      Seq(
+        Literal("a"),
+        GetStructField(structRef, 0)
+      )
+    )
+    val dataType = createNamedStruct.dataType
+    assert(dataType.asInstanceOf[StructType].fields.head.metadata == metadata)
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org