You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/28 02:52:43 UTC
(spark) branch branch-4.0 updated: [SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 9dfa73054f2b [SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation
9dfa73054f2b is described below
commit 9dfa73054f2b65741e46f1f4461234dc04895e8b
Author: ilicmarkodb <ma...@databricks.com>
AuthorDate: Wed May 28 10:50:56 2025 +0800
[SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation
Correctly resolve reference data type for Views with default collation.
`CREATE VIEW v DEFAULT COLLATION UTF8_LCASE AS SELECT 'a' AS c1` will resolve `c1` as `AttributeReference` but with non-collated `dataType`. This is because `ResolveReferences` comes before `ApplyDefaultCollationToStringType` in order of rules. Queries were still working correctly because there is `Cast` to collated `dataType` on top of `c1`.
Bug fix.
No.
New test in `DefaultCollationTestSuite.scala`.
No.
Closes #51023 from ilicmarkodb/fix_resolve_ref.
Authored-by: ilicmarkodb <ma...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit 7f1306a309e79449ceef28e15f344c020d3f91ee)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../apache/spark/sql/catalyst/analysis/Analyzer.scala | 12 +++++++++++-
.../spark/sql/collation/DefaultCollationTestSuite.scala | 17 +++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index f0b7d1512eb7..8ff356dee1f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1460,7 +1460,17 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
}
}
- def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+ def apply(plan: LogicalPlan): LogicalPlan = {
+ val collatedPlan =
+ if (conf.getConf(SQLConf.RUN_COLLATION_TYPE_CASTS_BEFORE_ALIAS_ASSIGNMENT)) {
+ CollationRulesRunner(plan)
+ } else {
+ plan
+ }
+ doApply(collatedPlan)
+ }
+
+ def doApply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
// Don't wait other rules to resolve the child plans of `InsertIntoStatement` as we need
// to resolve column "DEFAULT" in the child plans so that they must be unresolved.
case i: InsertIntoStatement => resolveColumnDefaultInCommandInputQuery(i)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
index 031af8200d7e..3373ab9a6dba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
@@ -18,6 +18,8 @@
package org.apache.spark.sql.collation
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.plans.logical.Project
import org.apache.spark.sql.catalyst.util.CollationFactory
import org.apache.spark.sql.connector.DatasourceV2SQLBase
import org.apache.spark.sql.test.SharedSparkSession
@@ -267,6 +269,21 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
class DefaultCollationTestSuiteV1 extends DefaultCollationTestSuite {
+ test("Check AttributeReference dataType from View with default collation") {
+ withView(testView) {
+ sql(s"CREATE VIEW $testView DEFAULT COLLATION UTF8_LCASE AS SELECT 'a' AS c1")
+
+ val df = sql(s"SELECT * FROM $testView")
+ val analyzedPlan = df.queryExecution.analyzed
+ analyzedPlan match {
+ case Project(Seq(AttributeReference("c1", dataType, _, _)), _) =>
+ assert(dataType == StringType("UTF8_LCASE"))
+ case _ =>
+ assert(false)
+ }
+ }
+ }
+
test("create/alter view created from a table") {
withTable(testTable) {
sql(s"CREATE TABLE $testTable (c1 STRING, c2 STRING COLLATE UNICODE_CI) USING $dataSource")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org