(spark) branch master updated: [SPARK-54028][SQL] Use empty schema when altering a view which is not Hive compatible

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 3d292dc7b1c5 [SPARK-54028][SQL] Use empty schema when altering a view which is not Hive compatible
3d292dc7b1c5 is described below

commit 3d292dc7b1c5b5ff977c178a88f8ee73deaee88f
Author: Chiran Ravani <ch...@gmail.com>
AuthorDate: Sat Nov 1 03:06:32 2025 +0800

    [SPARK-54028][SQL] Use empty schema when altering a view which is not Hive compatible
    
    ### What changes were proposed in this pull request?
    Spark attempts to save views in a Hive-compatible format and only sets the schema to empty if the save operation fails.
    
    However, due to certain Hive compatibility issues, the save operation may succeed while subsequent read operations fail. This issue arises after the change introduced in [SPARK-46934](https://issues.apache.org/jira/browse/SPARK-46934), which removed the verifyColumnDataType check during the ALTER TABLE operation.
    
    ### Why are the changes needed?
    to not save malformed views that no one can read.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, the malformed view will be saved in non hive compatible way so that Spark can read it.
    
    ### How was this patch tested?
    Updated Test case
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No
    
    Closes #52730 from cravani/SPARK-54028.
    
    Authored-by: Chiran Ravani <ch...@gmail.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/hive/HiveExternalCatalog.scala       | 12 ++++++-
 .../spark/sql/hive/HiveMetastoreCatalogSuite.scala | 37 ++++++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
index 6389e130b6a2..21bdd28dfd88 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveExternalCatalog.scala
@@ -598,7 +598,17 @@ private[spark] class HiveExternalCatalog(conf: SparkConf, hadoopConf: Configurat
 
     if (tableDefinition.tableType == VIEW) {
       val newTableProps = tableDefinition.properties ++ tableMetaToTableProps(tableDefinition).toMap
-      val newTable = tableDefinition.copy(properties = newTableProps)
+      val schemaWithNoCollation = removeCollation(tableDefinition.schema)
+      val hiveCompatibleSchema =
+        // Spark-created views do not have to be Hive compatible. If the data type is not
+        // Hive compatible, we can set schema to empty so that Spark can still read this
+        // view as the schema is also encoded in the table properties.
+        if (schemaWithNoCollation.exists(f => !isHiveCompatibleDataType(f.dataType))) {
+          EMPTY_DATA_SCHEMA
+        } else {
+          schemaWithNoCollation
+        }
+      val newTable = tableDefinition.copy(schema = hiveCompatibleSchema, properties = newTableProps)
       try {
         client.alterTable(newTable)
       } catch {
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index fad374827418..a7d43ebbef07 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -438,6 +438,43 @@ class DataSourceWithHiveMetastoreCatalogSuite
     }
   }
 
+  test("SPARK-54028: Table and View with complex nested schema and ALTER operations") {
+    withTable("t") {
+      val schema =
+          "struct_field STRUCT<" +
+          "`colon:field_name`:STRING" +
+          ">"
+      sql("CREATE TABLE t (" + schema + ")")
+
+      // Verify initial table schema
+      assert(spark.table("t").schema === CatalystSqlParser.parseTableSchema(schema))
+
+      withView("v") {
+        sql("CREATE VIEW v AS SELECT `struct_field` FROM t")
+
+        // Verify view schema matches the original schema
+        val expectedViewSchema = CatalystSqlParser.parseTableSchema(schema)
+        assert(spark.table("v").schema === expectedViewSchema)
+
+        // Add new column to table
+        sql("ALTER TABLE t ADD COLUMN (field_1 INT)")
+
+        // Update schema string to include new column
+        val updatedSchema = schema + ",field_1 INT"
+
+        // Verify table schema after ALTER
+        assert(spark.table("t").schema === CatalystSqlParser.parseTableSchema(updatedSchema))
+
+        // Alter view to include new column
+        sql("ALTER VIEW v AS " +
+          "SELECT `struct_field`,`field_1` FROM t")
+
+        // Verify view schema after ALTER
+        assert(spark.table("v").schema === CatalystSqlParser.parseTableSchema(updatedSchema))
+      }
+    }
+  }
+
   test("SPARK-46934: Handle special characters in struct types with CTAS") {
     withTable("t") {
       val schema = "`a.b` struct<`a.b.b`:array<string>, `a b c`:map<int, string>>"


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org