[spark] branch master updated: [SPARK-44866][SQL] Add `SnowflakeDialect` to handle BOOLEAN type correctly

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new c8fa821a873 [SPARK-44866][SQL] Add `SnowflakeDialect` to handle BOOLEAN type correctly
c8fa821a873 is described below

commit c8fa821a873b9abc500e17b9c5bbb95d77364c5f
Author: Hayssam Saleh <Ha...@starlake.ai>
AuthorDate: Fri Sep 8 15:14:17 2023 -0700

    [SPARK-44866][SQL] Add `SnowflakeDialect` to handle BOOLEAN type correctly
    
    ### What changes were proposed in this pull request?
    
    In Snowflake a BOOLEAN data type exist but not the BIT data type.
    This PR adds `SnowflakeDialect` to override the default JdbcDialect and redefine the default mapping behaviour for the _boolean_ type.  Currently, it's mapped to `BIT(1)` type.
    
    https://github.com/apache/spark/blob/a663c0bf0c5b104170c0612f37a0b0cdf75cd45b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala#L149
    
    ### Why are the changes needed?
    
    The BIT type does not exist in Snowflake. This cause the Spark Job to fail on table creation.
    
    ### Does this PR introduce _any_ user-facing change?
    No
    
    ### How was this patch tested?
    Unit test and directly on Snowflake
    
    Closes #42545 from hayssams/master.
    
    Authored-by: Hayssam Saleh <Ha...@starlake.ai>
    Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
 .../org/apache/spark/sql/jdbc/JdbcDialects.scala   |  1 +
 .../apache/spark/sql/jdbc/SnowflakeDialect.scala   | 36 ++++++++++++++++++++++
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala      |  5 +++
 3 files changed, 42 insertions(+)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
index 2f5e813dcb6..d775f5fce2f 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala
@@ -702,6 +702,7 @@ object JdbcDialects {
   registerDialect(OracleDialect)
   registerDialect(TeradataDialect)
   registerDialect(H2Dialect)
+  registerDialect(SnowflakeDialect)
 
   /**
    * Fetch the JdbcDialect class corresponding to a given database url.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/SnowflakeDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/SnowflakeDialect.scala
new file mode 100644
index 00000000000..d8a8fe6ba4a
--- /dev/null
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/SnowflakeDialect.scala
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.util.Locale
+
+import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils
+import org.apache.spark.sql.types.{BooleanType, DataType}
+
+private case object SnowflakeDialect extends JdbcDialect {
+  override def canHandle(url: String): Boolean =
+    url.toLowerCase(Locale.ROOT).startsWith("jdbc:snowflake")
+
+  override def getJDBCType(dt: DataType): Option[JdbcType] = dt match {
+    case BooleanType =>
+      // By default, BOOLEAN is mapped to BIT(1).
+      // but Snowflake does not have a BIT type. It uses BOOLEAN instead.
+      Some(JdbcType("BOOLEAN", java.sql.Types.BOOLEAN))
+    case _ => JdbcUtils.getCommonJDBCType(dt)
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 93b6652d516..6f60e0f7394 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -2057,4 +2057,9 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
     val df = sql("SELECT * FROM composite_name WHERE `last name` = 'smith'")
     assert(df.collect.toSet === Set(Row("smith", 1)))
   }
+
+  test("SPARK-44866: SnowflakeDialect BOOLEAN type mapping") {
+    val snowflakeDialect = JdbcDialects.get("jdbc:snowflake://account.snowflakecomputing.com")
+    assert(snowflakeDialect.getJDBCType(BooleanType).map(_.databaseTypeDefinition).get == "BOOLEAN")
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org