You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/02/14 05:57:11 UTC
(spark) branch master updated: [SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 83ee2a072227 [SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution
83ee2a072227 is described below
commit 83ee2a072227d0c3ec8dbe9ec0f4022cb46d04fa
Author: Szehon Ho <sz...@gmail.com>
AuthorDate: Fri Feb 14 13:56:58 2025 +0800
[SPARK-51119][SQL][FOLLOW-UP] ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution
Small follow up for: https://github.com/apache/spark/pull/49942
### What changes were proposed in this pull request?
Restrict the logic of https://github.com/apache/spark/pull/49942 to only resolved and folded existsDefaults.
### Why are the changes needed?
Previous part of PR may have unexpected behavior if expression is not resolved/folded.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing unit test
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #49947 from szehon-ho/SPARK-51119-follow.
Authored-by: Szehon Ho <sz...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/plans/logical/ColumnDefinition.scala | 6 +++++-
.../scala/org/apache/spark/sql/types/StructTypeSuite.scala | 10 +++++-----
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
index 7bc523839168..ec7df606632a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ColumnDefinition.scala
@@ -75,7 +75,11 @@ case class ColumnDefinition(
// For v1 CREATE TABLE command, we will resolve and execute the default value expression later
// in the rule `DataSourceAnalysis`. We just need to put the default value SQL string here.
metadataBuilder.putString(CURRENT_DEFAULT_COLUMN_METADATA_KEY, default.originalSQL)
- metadataBuilder.putString(EXISTS_DEFAULT_COLUMN_METADATA_KEY, default.child.sql)
+ val existsSQL = default.child match {
+ case l: Literal => l.sql
+ case _ => default.originalSQL
+ }
+ metadataBuilder.putString(EXISTS_DEFAULT_COLUMN_METADATA_KEY, existsSQL)
}
generationExpression.foreach { generationExpr =>
metadataBuilder.putString(GeneratedColumn.GENERATION_EXPRESSION_METADATA_KEY, generationExpr)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
index f247806fb413..269c76093291 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
@@ -804,10 +804,10 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
test("SPARK-51208: ColumnDefinition.toV1Column should preserve EXISTS_DEFAULT resolution") {
def validateConvertedDefaults(
- colName: String,
- dataType: DataType,
- defaultSQL: String,
- expectedExists: String): Unit = {
+ colName: String,
+ dataType: DataType,
+ defaultSQL: String,
+ expectedExists: String): Unit = {
val existsDefault = ResolveDefaultColumns.analyze(colName, dataType, defaultSQL, "")
val col =
ColumnDefinition(colName, dataType, true, None,
@@ -829,6 +829,6 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
validateConvertedDefaults("c3", VariantType, "parse_json('{\"k\": \"v\"}')",
"PARSE_JSON('{\"k\":\"v\"}')")
validateConvertedDefaults("c4", VariantType, "parse_json(null)", "CAST(NULL AS VARIANT)")
-
+ validateConvertedDefaults("c5", IntegerType, "1 + 1", "2")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org