You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/02/27 10:19:05 UTC
(spark) branch branch-4.0 updated: [SPARK-51310][SQL] Resolve the type of default string producing expressions
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 6599fe4a36b0 [SPARK-51310][SQL] Resolve the type of default string producing expressions
6599fe4a36b0 is described below
commit 6599fe4a36b0c16d4254c7f9dd80aa93068ae67f
Author: Stefan Kandic <st...@databricks.com>
AuthorDate: Thu Feb 27 18:18:41 2025 +0800
[SPARK-51310][SQL] Resolve the type of default string producing expressions
### What changes were proposed in this pull request?
Add casts to `DefaultStringProducingExpression` to have the correct type based on the default string type.
Currently, we can only test this on views, until we implement schema level collations when we will be able to test the behavior on newly created tables as well.
### Why are the changes needed?
These expressions should return the default string type, which could be different in some DDL commands.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added unit tests for view creation.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50053 from stefankandic/resolveDefaultStringExprs.
Authored-by: Stefan Kandic <st...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 412da42ab2e91a643980634d032fec0738a59590)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../analysis/ResolveDDLCommandStringTypes.scala | 30 ++++++++++-
.../sql/collation/DefaultCollationTestSuite.scala | 59 ++++++++++++++++++++++
2 files changed, 87 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveDDLCommandStringTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveDDLCommandStringTypes.scala
index ebca92ab8dc2..9ac04236a1b1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveDDLCommandStringTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveDDLCommandStringTypes.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.analysis
-import org.apache.spark.sql.catalyst.expressions.{Cast, Expression, Literal}
+import org.apache.spark.sql.catalyst.expressions.{Cast, DefaultStringProducingExpression, Expression, Literal}
import org.apache.spark.sql.catalyst.plans.logical.{AddColumns, AlterColumns, AlterColumnSpec, AlterTableCommand, AlterViewAs, ColumnDefinition, CreateTable, CreateView, LogicalPlan, QualifiedColType, ReplaceColumns, V2CreateTablePlan}
import org.apache.spark.sql.catalyst.rules.Rule
import org.apache.spark.sql.connector.catalog.TableCatalog
@@ -105,11 +105,13 @@ object ResolveDDLCommandStringTypes extends Rule[LogicalPlan] {
* new type instead of the default string type.
*/
private def transformPlan(plan: LogicalPlan, newType: StringType): LogicalPlan = {
- plan resolveExpressionsUp { expression =>
+ val transformedPlan = plan resolveExpressionsUp { expression =>
transformExpression
.andThen(_.apply(newType))
.applyOrElse(expression, identity[Expression])
}
+
+ castDefaultStringExpressions(transformedPlan, newType)
}
/**
@@ -126,6 +128,30 @@ object ResolveDDLCommandStringTypes extends Rule[LogicalPlan] {
newType => Literal(value, replaceDefaultStringType(dt, newType))
}
+ /**
+ * Casts [[DefaultStringProducingExpression]] in the plan to the `newType`.
+ */
+ private def castDefaultStringExpressions(plan: LogicalPlan, newType: StringType): LogicalPlan = {
+ if (newType == StringType) return plan
+
+ def inner(ex: Expression): Expression = ex match {
+ // Skip if we already added a cast in the previous pass.
+ case cast @ Cast(e: DefaultStringProducingExpression, dt, _, _) if newType == dt =>
+ cast.copy(child = e.withNewChildren(e.children.map(inner)))
+
+ // Add cast on top of [[DefaultStringProducingExpression]].
+ case e: DefaultStringProducingExpression =>
+ Cast(e.withNewChildren(e.children.map(inner)), newType)
+
+ case other =>
+ other.withNewChildren(other.children.map(inner))
+ }
+
+ plan resolveOperators { operator =>
+ operator.mapExpressions(inner)
+ }
+ }
+
private def hasDefaultStringType(dataType: DataType): Boolean =
dataType.existsRecursively(isDefaultStringType)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
index 9e1968022c74..9679ecb55438 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
@@ -25,6 +25,13 @@ import org.apache.spark.sql.types.StringType
abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSession {
+ val defaultStringProducingExpressions: Seq[String] = Seq(
+ "current_timezone()", "current_database()", "md5('Spark' collate unicode)",
+ "soundex('Spark' collate unicode)", "url_encode('https://spark.apache.org' collate unicode)",
+ "url_decode('https%3A%2F%2Fspark.apache.org')", "uuid()", "chr(65)", "collation('UNICODE')",
+ "version()", "space(5)", "randstr(5, 123)"
+ )
+
def dataSource: String = "parquet"
def testTable: String = "test_tbl"
def testView: String = "test_view"
@@ -329,6 +336,58 @@ class DefaultCollationTestSuiteV1 extends DefaultCollationTestSuite {
}
}
}
+
+ test("view has utf8 binary collation by default") {
+ withView(testTable) {
+ sql(s"CREATE VIEW $testTable AS SELECT current_database() AS db")
+ assertTableColumnCollation(testTable, "db", "UTF8_BINARY")
+ }
+ }
+
+ test("default string producing expressions in view definition") {
+ val viewDefaultCollation = Seq(
+ "UTF8_BINARY", "UNICODE"
+ )
+
+ viewDefaultCollation.foreach { collation =>
+ withView(testTable) {
+
+ val columns = defaultStringProducingExpressions.zipWithIndex.map {
+ case (expr, index) => s"$expr AS c${index + 1}"
+ }.mkString(", ")
+
+ sql(
+ s"""
+ |CREATE view $testTable
+ |DEFAULT COLLATION $collation
+ |AS SELECT $columns
+ |""".stripMargin)
+
+ (1 to defaultStringProducingExpressions.length).foreach { index =>
+ assertTableColumnCollation(testTable, s"c$index", collation)
+ }
+ }
+ }
+ }
+
+ test("default string producing expressions in view definition - nested in expr tree") {
+ withView(testTable) {
+ sql(
+ s"""
+ |CREATE view $testTable
+ |DEFAULT COLLATION UNICODE AS SELECT
+ |SUBSTRING(current_database(), 1, 1) AS c1,
+ |SUBSTRING(SUBSTRING(current_database(), 1, 2), 1, 1) AS c2,
+ |SUBSTRING(current_database()::STRING, 1, 1) AS c3,
+ |SUBSTRING(CAST(current_database() AS STRING COLLATE UTF8_BINARY), 1, 1) AS c4
+ |""".stripMargin)
+
+ assertTableColumnCollation(testTable, "c1", "UNICODE")
+ assertTableColumnCollation(testTable, "c2", "UNICODE")
+ assertTableColumnCollation(testTable, "c3", "UNICODE")
+ assertTableColumnCollation(testTable, "c4", "UTF8_BINARY")
+ }
+ }
}
class DefaultCollationTestSuiteV2 extends DefaultCollationTestSuite with DatasourceV2SQLBase {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org