[spark] branch branch-3.4 updated: [SPARK-44313][SQL] Fix generated column expression validation when there is a char/varchar column in the schema

[ya...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yao pushed a commit to branch branch-3.4
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.4 by this push:
     new 85e252e8503 [SPARK-44313][SQL] Fix generated column expression validation when there is a char/varchar column in the schema
85e252e8503 is described below

commit 85e252e8503534009f4fb5ea005d44c9eda31447
Author: Allison Portis <al...@databricks.com>
AuthorDate: Thu Jul 6 10:25:57 2023 +0800

    [SPARK-44313][SQL] Fix generated column expression validation when there is a char/varchar column in the schema
    
    ### What changes were proposed in this pull request?
    
    #38823 added support for defining generated columns in create table statements. This included generation expression validation. This validation currently erroneously fails when there are char or varchar columns anywhere in the table schema since the checkAnalysis fails here https://github.com/apache/spark/blob/master/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GeneratedColumn.scala#L123.
    
    This PR replaces any char/varchar columns in the schema with a string before analysis.
    
    ### Why are the changes needed?
    
    This should not fail.
    ```
    CREATE TABLE default.example (
        name VARCHAR(64),
        tstamp TIMESTAMP,
        tstamp_date DATE GENERATED ALWAYS AS (CAST(tstamp as DATE))
    )
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Adds a unit test.
    
    Closes #41868 from allisonport-db/validateGeneratedColumns-charvarchar.
    
    Authored-by: Allison Portis <al...@databricks.com>
    Signed-off-by: Kent Yao <ya...@apache.org>
    (cherry picked from commit f0e18284d0c8938cb35f2ad3668aff0e4ef4891c)
    Signed-off-by: Kent Yao <ya...@apache.org>
---
 .../spark/sql/catalyst/util/GeneratedColumn.scala      |  3 ++-
 .../spark/sql/connector/DataSourceV2SQLSuite.scala     | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GeneratedColumn.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GeneratedColumn.scala
index 9a1ce5b0295..5dd278e3fea 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GeneratedColumn.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/GeneratedColumn.scala
@@ -116,7 +116,8 @@ object GeneratedColumn {
     val allowedBaseColumns = schema
       .filterNot(_.name == fieldName) // Can't reference itself
       .filterNot(isGeneratedColumn) // Can't reference other generated columns
-    val relation = new LocalRelation(StructType(allowedBaseColumns).toAttributes)
+    val relation = new LocalRelation(CharVarcharUtils.replaceCharVarcharWithStringInSchema(
+      StructType(allowedBaseColumns)).toAttributes)
     val plan = try {
       val analyzer: Analyzer = GeneratedColumnAnalyzer
       val analyzed = analyzer.execute(Project(Seq(Alias(parsed, fieldName)()), relation))
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 13ffa6d9bfd..c7dc924c117 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -1694,6 +1694,24 @@ class DataSourceV2SQLSuiteV1Filter
     )
   }
 
+  test("SPARK-44313: generation expression validation passes when there is a char/varchar column") {
+    val tblName = "my_tab"
+    // InMemoryTableCatalog.capabilities() = {SUPPORTS_CREATE_TABLE_WITH_GENERATED_COLUMNS}
+    for (charVarCharCol <- Seq("name VARCHAR(64)", "name CHAR(64)")) {
+      withTable(s"testcat.$tblName") {
+        sql(
+          s"""
+             |CREATE TABLE testcat.$tblName(
+             |  $charVarCharCol,
+             |  tstamp TIMESTAMP,
+             |  tstamp_date DATE GENERATED ALWAYS AS (CAST(tstamp AS DATE))
+             |) USING foo
+             |""".stripMargin)
+        assert(catalog("testcat").asTableCatalog.tableExists(Identifier.of(Array(), tblName)))
+      }
+    }
+  }
+
   test("ShowCurrentNamespace: basic tests") {
     def testShowCurrentNamespace(expectedCatalogName: String, expectedNamespace: String): Unit = {
       val schema = new StructType()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org