You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2026/05/20 06:21:17 UTC

(hudi) 04/45: fix: filter spark.sql.* properties in SparkCatalogMetaStoreClient.toCatalogTable (#18654)

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch release-1.2.0
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit bf5f4e3eb8b53e2c30a03ddad6edfb805d35f03f
Author: Prashant Wason <pw...@uber.com>
AuthorDate: Wed Apr 29 19:40:56 2026 -0700

    fix: filter spark.sql.* properties in SparkCatalogMetaStoreClient.toCatalogTable (#18654)
    
    Spark's HiveExternalCatalog.alterTable / createTable rejects table
    properties whose keys start with "spark.sql." with:
    
      AnalysisException: Cannot persist <table> into Hive metastore as table
      property keys may not start with 'spark.sql.': [spark.sql.create.version,
      spark.sql.sources.provider, spark.sql.sources.schema.partCol.0,
      spark.sql.sources.schema.numParts, spark.sql.sources.schema.numPartCols,
      spark.sql.sources.schema.part.0]
    
    These keys are reserved for Spark's internal use (provider, schema parts,
    create version) and Spark itself writes them when persisting a CatalogTable.
    On the way back through getTable they appear in the parameters map, and
    toCatalogTable currently passes them straight through. The next alter_table
    call then trips the validation and the entire HoodieHiveSyncClient flow
    fails - no actual sync happens.
    
    Strip "spark.sql.*" keys in toCatalogTable before constructing the
    CatalogTable. Spark re-derives and writes them from the CatalogTable, so
    dropping them on the way in is safe.
    
    Co-authored-by: Claude Opus 4.7 <no...@anthropic.com>
---
 .../apache/spark/sql/hive/SparkCatalogMetaStoreClient.scala    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hive/SparkCatalogMetaStoreClient.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hive/SparkCatalogMetaStoreClient.scala
index d238341bed46..bf73a5bb064f 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hive/SparkCatalogMetaStoreClient.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hive/SparkCatalogMetaStoreClient.scala
@@ -302,6 +302,14 @@ class SparkCatalogMetaStoreClient(syncConfig: HiveSyncConfig)
     val dataFields = cols.map(fs => StructField(fs.getName, CatalystSqlParser.parseDataType(fs.getType), nullable = true, Metadata.empty))
     val partitionFields = partCols.map(fs => StructField(fs.getName, CatalystSqlParser.parseDataType(fs.getType), nullable = true, Metadata.empty))
 
+    // Strip "spark.sql.*" properties before handing off to Spark's external catalog.
+    // HiveExternalCatalog.alterTable / createTable rejects such keys ("Cannot persist ...
+    // table property keys may not start with 'spark.sql.'") because they are reserved for
+    // Spark's internal use (provider, schema parts, create version). Spark re-derives and
+    // writes these from the CatalogTable itself, so dropping them on the way in is safe.
+    val tableProperties = Option(table.getParameters).map(_.asScala.toMap).getOrElse(Map.empty)
+      .filterNot { case (k, _) => k.startsWith("spark.sql.") }
+
     CatalogTable(
       identifier = TableIdentifier(tbl, Some(db)),
       tableType = if ("EXTERNAL_TABLE".equalsIgnoreCase(table.getTableType)) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED,
@@ -315,7 +323,7 @@ class SparkCatalogMetaStoreClient(syncConfig: HiveSyncConfig)
       schema = StructType(dataFields ++ partitionFields),
       provider = Some("hudi"),
       partitionColumnNames = partCols.map(_.getName),
-      properties = Option(table.getParameters).map(_.asScala.toMap).getOrElse(Map.empty))
+      properties = tableProperties)
   }
 
   private def fromCatalogTable(table: CatalogTable): Table = {