You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/05/28 02:52:43 UTC

(spark) branch branch-4.0 updated: [SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 9dfa73054f2b [SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation
9dfa73054f2b is described below

commit 9dfa73054f2b65741e46f1f4461234dc04895e8b
Author: ilicmarkodb <ma...@databricks.com>
AuthorDate: Wed May 28 10:50:56 2025 +0800

    [SPARK-52313][SQL] Correctly resolve reference data type for Views with default collation
    
    Correctly resolve reference data type for Views with default collation.
    `CREATE VIEW v DEFAULT COLLATION UTF8_LCASE AS SELECT 'a' AS c1` will resolve `c1` as `AttributeReference` but with non-collated `dataType`. This is because `ResolveReferences` comes before `ApplyDefaultCollationToStringType` in order of rules. Queries were still working correctly because there is `Cast` to collated `dataType` on top of `c1`.
    
    Bug fix.
    
    No.
    
    New test in `DefaultCollationTestSuite.scala`.
    
    No.
    
    Closes #51023 from ilicmarkodb/fix_resolve_ref.
    
    Authored-by: ilicmarkodb <ma...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
    (cherry picked from commit 7f1306a309e79449ceef28e15f344c020d3f91ee)
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../apache/spark/sql/catalyst/analysis/Analyzer.scala   | 12 +++++++++++-
 .../spark/sql/collation/DefaultCollationTestSuite.scala | 17 +++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index f0b7d1512eb7..8ff356dee1f1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -1460,7 +1460,17 @@ class Analyzer(override val catalogManager: CatalogManager) extends RuleExecutor
       }
     }
 
-    def apply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
+    def apply(plan: LogicalPlan): LogicalPlan = {
+      val collatedPlan =
+        if (conf.getConf(SQLConf.RUN_COLLATION_TYPE_CASTS_BEFORE_ALIAS_ASSIGNMENT)) {
+          CollationRulesRunner(plan)
+        } else {
+          plan
+        }
+      doApply(collatedPlan)
+    }
+
+    def doApply(plan: LogicalPlan): LogicalPlan = plan.resolveOperatorsUp {
       // Don't wait other rules to resolve the child plans of `InsertIntoStatement` as we need
       // to resolve column "DEFAULT" in the child plans so that they must be unresolved.
       case i: InsertIntoStatement => resolveColumnDefaultInCommandInputQuery(i)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
index 031af8200d7e..3373ab9a6dba 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/collation/DefaultCollationTestSuite.scala
@@ -18,6 +18,8 @@
 package org.apache.spark.sql.collation
 
 import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, Row}
+import org.apache.spark.sql.catalyst.expressions.AttributeReference
+import org.apache.spark.sql.catalyst.plans.logical.Project
 import org.apache.spark.sql.catalyst.util.CollationFactory
 import org.apache.spark.sql.connector.DatasourceV2SQLBase
 import org.apache.spark.sql.test.SharedSparkSession
@@ -267,6 +269,21 @@ abstract class DefaultCollationTestSuite extends QueryTest with SharedSparkSessi
 
 class DefaultCollationTestSuiteV1 extends DefaultCollationTestSuite {
 
+  test("Check AttributeReference dataType from View with default collation") {
+    withView(testView) {
+      sql(s"CREATE VIEW $testView DEFAULT COLLATION UTF8_LCASE AS SELECT 'a' AS c1")
+
+      val df = sql(s"SELECT * FROM $testView")
+      val analyzedPlan = df.queryExecution.analyzed
+      analyzedPlan match {
+        case Project(Seq(AttributeReference("c1", dataType, _, _)), _) =>
+          assert(dataType == StringType("UTF8_LCASE"))
+        case _ =>
+          assert(false)
+      }
+    }
+  }
+
   test("create/alter view created from a table") {
     withTable(testTable) {
       sql(s"CREATE TABLE $testTable (c1 STRING, c2 STRING COLLATE UNICODE_CI) USING $dataSource")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org