[hudi] branch master updated: [HUDI-6411] Make spark SQL options insensitive (#9020)

[xu...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new e82c5675554 [HUDI-6411] Make spark SQL options insensitive (#9020)
e82c5675554 is described below

commit e82c5675554f4e94ab7a766558b8ed738d6a0f00
Author: Aditya Goenka <63...@users.noreply.github.com>
AuthorDate: Tue Jun 27 21:11:30 2023 +0530

    [HUDI-6411] Make spark SQL options insensitive (#9020)
---
 .../sql/catalyst/catalog/HoodieCatalogTable.scala  |  2 +-
 .../apache/spark/sql/hudi/HoodieOptionConfig.scala | 14 +++++++++++-
 .../apache/spark/sql/hudi/TestCreateTable.scala    | 26 ++++++++++++++++++++++
 3 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
index a329a943969..b1ee6924e11 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
@@ -63,7 +63,7 @@ class HoodieCatalogTable(val spark: SparkSession, var table: CatalogTable) exten
   /**
    * properties defined in catalog.
    */
-  val catalogProperties: Map[String, String] = table.storage.properties ++ table.properties
+  val catalogProperties: Map[String, String] = HoodieOptionConfig.makeOptionsCaseInsensitive(table.storage.properties ++ table.properties)
 
   /**
    * hoodie table's location.
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
index a541bbb8e52..d715a108d62 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/hudi/HoodieOptionConfig.scala
@@ -192,7 +192,6 @@ object HoodieOptionConfig {
   // validate primaryKey, preCombineField and type options
   def validateTable(spark: SparkSession, schema: StructType, sqlOptions: Map[String, String]): Unit = {
     val resolver = spark.sessionState.conf.resolver
-
     // validate primary key
     val primaryKeys = sqlOptions.get(SQL_KEY_TABLE_PRIMARY_KEY.sqlKeyName)
       .map(_.split(",").filter(_.length > 0))
@@ -222,6 +221,19 @@ object HoodieOptionConfig {
   def buildConf[T](): HoodieSQLOptionBuilder[T] = {
     new HoodieSQLOptionBuilder[T]
   }
+
+  def makeOptionsCaseInsensitive(sqlOptions: Map[String, String]): Map[String, String] = {
+    // Make Keys Case Insensitive
+    val standardOptions = Seq(SQL_KEY_TABLE_PRIMARY_KEY, SQL_KEY_PRECOMBINE_FIELD,
+    SQL_KEY_TABLE_TYPE, SQL_PAYLOAD_CLASS, SQL_RECORD_MERGER_STRATEGY).map(key => key.sqlKeyName)
+
+    sqlOptions.map(option => {
+      standardOptions.find(x => x.toLowerCase().contains(option._1.toLowerCase())) match {
+        case Some(standardKey) => (standardKey, option._2)
+        case None => (option._1, option._2)
+      }
+    })
+  }
 }
 
 case class HoodieSQLOption[T](
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
index 11d21d43417..7cfc143be05 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestCreateTable.scala
@@ -1106,4 +1106,30 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
     val hoodieSchema = HoodieSqlCommonUtils.getTableSqlSchema(hoodieCatalogTable.metaClient, true)
     assertResult(hoodieSchema.get)(table.schema)
   }
+
+  test("Test Create Hoodie Table With Options in different case") {
+    val tableName = generateTableName
+    spark.sql(
+      s"""
+         | create table $tableName (
+         |  id int,
+         |  name string,
+         |  price double,
+         |  ts long,
+         |  dt string
+         | ) using hudi
+         | partitioned by (dt)
+         | options (
+         |   hoodie.database.name = "databaseName",
+         |   hoodie.table.name = "tableName",
+         |   PRIMARYKEY = 'id',
+         |   precombineField = 'ts',
+         |   hoodie.datasource.write.operation = 'upsert'
+         | )
+       """.stripMargin)
+    val table = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
+    assertResult(table.properties("type"))("cow")
+    assertResult(table.properties("primaryKey"))("id")
+    assertResult(table.properties("preCombineField"))("ts")
+  }
 }