You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/02/14 05:57:11 UTC

(spark) branch master updated: [SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 83ee2a072227 [SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution
83ee2a072227 is described below

commit 83ee2a072227d0c3ec8dbe9ec0f4022cb46d04fa
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Fri Feb 14 13:56:58 2025 +0800

    [SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution
    
    Small follow up for: https://github.com/apache/spark/pull/49942
    
    ### What changes were proposed in this pull request?
    Restrict the logic of https://github.com/apache/spark/pull/49942 to only resolved and folded existsDefaults.
    
    ### Why are the changes needed?
    Previous part of PR may have unexpected behavior if expression is not resolved/folded.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Existing unit test
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #49947 from szehon-ho/SPARK-51119-follow.
    
    Authored-by: Szehon Ho <sz...@gmail.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/catalyst/plans/logical/ColumnDefinition.scala    |  6 +++++-
 .../scala/org/apache/spark/sql/types/StructTypeSuite.scala     | 10 +++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
index 7bc523839168..ec7df606632a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
@@ -75,7 +75,11 @@ case class ColumnDefinition(
       // For v1 CREATE TABLE command, we will resolve and execute the default value expression later
       // in the rule `DataSourceAnalysis`. We just need to put the default value SQL string here.
       metadataBuilder.putString(CURRENT_DEFAULT_COLUMN_METADATA_KEY, default.originalSQL)
-      metadataBuilder.putString(EXISTS_DEFAULT_COLUMN_METADATA_KEY, default.child.sql)
+      val existsSQL = default.child match {
+        case l: Literal => l.sql
+        case _ => default.originalSQL
+      }
+      metadataBuilder.putString(EXISTS_DEFAULT_COLUMN_METADATA_KEY, existsSQL)
     }
     generationExpression.foreach { generationExpr =>
       metadataBuilder.putString(GeneratedColumn.GENERATION_EXPRESSION_METADATA_KEY, generationExpr)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index f247806fb413..269c76093291 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -804,10 +804,10 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
   test("SPARK-51208: ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution") {
 
     def validateConvertedDefaults(
-      colName: String,
-      dataType: DataType,
-      defaultSQL: String,
-      expectedExists: String): Unit = {
+        colName: String,
+        dataType: DataType,
+        defaultSQL: String,
+        expectedExists: String): Unit = {
       val existsDefault = ResolveDefaultColumns.analyze(colName, dataType, defaultSQL, "")
       val col =
         ColumnDefinition(colName, dataType, true, None,
@@ -829,6 +829,6 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
     validateConvertedDefaults("c3", VariantType, "parse_json('{\"k\": \"v\"}')",
       "PARSE_JSON('{\"k\":\"v\"}')")
     validateConvertedDefaults("c4", VariantType, "parse_json(null)", "CAST(NULL AS VARIANT)")
-
+    validateConvertedDefaults("c5", IntegerType, "1 + 1", "2")
   }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org