You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/11/12 06:07:14 UTC
(spark) branch master updated: [SPARK-54207][CONNECT] Supports Date type data in SparkConnectResultSet
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a8ca6f4ba099 [SPARK-54207][CONNECT] Supports Date type data in SparkConnectResultSet
a8ca6f4ba099 is described below
commit a8ca6f4ba09994d280b1ae224f180557edd22e26
Author: vinodkc <vi...@gmail.com>
AuthorDate: Tue Nov 11 22:07:02 2025 -0800
[SPARK-54207][CONNECT] Supports Date type data in SparkConnectResultSet
### What changes were proposed in this pull request?
Add DATE type support to the Spark Connect JDBC client
### Why are the changes needed?
DATE is a fundamental SQL type required for JDBC compliance and interoperability.
### Does this PR introduce _any_ user-facing change?
Yes, it's part of a new feature under Spark connect JDBC support.
### How was this patch tested?
Added new UTs in `SparkConnectJdbcDataTypeSuite`
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #53010 from vinodkc/br_SPARK-54207.
Authored-by: vinodkc <vi...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../client/jdbc/SparkConnectResultSet.scala | 25 +++--
.../connect/client/jdbc/util/JdbcTypeUtils.scala | 9 +-
.../jdbc/SparkConnectJdbcDataTypeSuite.scala | 109 +++++++++++++++++++++
3 files changed, 135 insertions(+), 8 deletions(-)
diff --git a/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectResultSet.scala b/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectResultSet.scala
index ff02cd73dcc6..5544dee6df5f 100644
--- a/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectResultSet.scala
+++ b/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectResultSet.scala
@@ -140,8 +140,9 @@ class SparkConnectResultSet(
override def getBytes(columnIndex: Int): Array[Byte] =
throw new SQLFeatureNotSupportedException
- override def getDate(columnIndex: Int): Date =
- throw new SQLFeatureNotSupportedException
+ override def getDate(columnIndex: Int): Date = {
+ getColumnValue(columnIndex, null: Date) { idx => currentRow.getDate(idx) }
+ }
override def getTime(columnIndex: Int): Time =
throw new SQLFeatureNotSupportedException
@@ -189,7 +190,7 @@ class SparkConnectResultSet(
throw new SQLFeatureNotSupportedException
override def getDate(columnLabel: String): Date =
- throw new SQLFeatureNotSupportedException
+ getDate(findColumn(columnLabel))
override def getTime(columnLabel: String): Time =
throw new SQLFeatureNotSupportedException
@@ -496,11 +497,23 @@ class SparkConnectResultSet(
override def getArray(columnLabel: String): JdbcArray =
throw new SQLFeatureNotSupportedException
- override def getDate(columnIndex: Int, cal: Calendar): Date =
- throw new SQLFeatureNotSupportedException
+ override def getDate(columnIndex: Int, cal: Calendar): Date = {
+ val date = getDate(columnIndex)
+ if (date == null || cal == null) {
+ return date
+ }
+
+ val targetCalendar = cal.clone().asInstanceOf[Calendar]
+ targetCalendar.setTime(date)
+ targetCalendar.set(Calendar.HOUR_OF_DAY, 0)
+ targetCalendar.set(Calendar.MINUTE, 0)
+ targetCalendar.set(Calendar.SECOND, 0)
+ targetCalendar.set(Calendar.MILLISECOND, 0)
+ new Date(targetCalendar.getTimeInMillis)
+ }
override def getDate(columnLabel: String, cal: Calendar): Date =
- throw new SQLFeatureNotSupportedException
+ getDate(findColumn(columnLabel), cal)
override def getTime(columnIndex: Int, cal: Calendar): Time =
throw new SQLFeatureNotSupportedException
diff --git a/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/util/JdbcTypeUtils.scala b/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/util/JdbcTypeUtils.scala
index c2b27128caa7..4af67f1582d3 100644
--- a/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/util/JdbcTypeUtils.scala
+++ b/sql/connect/client/jdbc/src/main/scala/org/apache/spark/sql/connect/client/jdbc/util/JdbcTypeUtils.scala
@@ -36,6 +36,7 @@ private[jdbc] object JdbcTypeUtils {
case DoubleType => Types.DOUBLE
case StringType => Types.VARCHAR
case _: DecimalType => Types.DECIMAL
+ case DateType => Types.DATE
case other =>
throw new SQLFeatureNotSupportedException(s"DataType $other is not supported yet.")
}
@@ -51,6 +52,7 @@ private[jdbc] object JdbcTypeUtils {
case DoubleType => classOf[JDouble].getName
case StringType => classOf[String].getName
case _: DecimalType => classOf[JBigDecimal].getName
+ case DateType => classOf[Date].getName
case other =>
throw new SQLFeatureNotSupportedException(s"DataType $other is not supported yet.")
}
@@ -58,7 +60,7 @@ private[jdbc] object JdbcTypeUtils {
def isSigned(field: StructField): Boolean = field.dataType match {
case ByteType | ShortType | IntegerType | LongType | FloatType | DoubleType |
_: DecimalType => true
- case NullType | BooleanType | StringType => false
+ case NullType | BooleanType | StringType | DateType => false
case other =>
throw new SQLFeatureNotSupportedException(s"DataType $other is not supported yet.")
}
@@ -74,6 +76,7 @@ private[jdbc] object JdbcTypeUtils {
case DoubleType => 15
case StringType => 255
case DecimalType.Fixed(p, _) => p
+ case DateType => 10
case other =>
throw new SQLFeatureNotSupportedException(s"DataType $other is not supported yet.")
}
@@ -81,7 +84,8 @@ private[jdbc] object JdbcTypeUtils {
def getScale(field: StructField): Int = field.dataType match {
case FloatType => 7
case DoubleType => 15
- case NullType | BooleanType | ByteType | ShortType | IntegerType | LongType | StringType => 0
+ case NullType | BooleanType | ByteType | ShortType | IntegerType | LongType | StringType |
+ DateType => 0
case DecimalType.Fixed(_, s) => s
case other =>
throw new SQLFeatureNotSupportedException(s"DataType $other is not supported yet.")
@@ -96,6 +100,7 @@ private[jdbc] object JdbcTypeUtils {
case DoubleType => 24
case StringType =>
getPrecision(field)
+ case DateType => 10 // length of `YYYY-MM-DD`
// precision + negative sign + leading zero + decimal point, like DECIMAL(5,5) = -0.12345
case DecimalType.Fixed(p, s) if p == s => p + 3
// precision + negative sign, like DECIMAL(5,0) = -12345
diff --git a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectJdbcDataTypeSuite.scala b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectJdbcDataTypeSuite.scala
index 217142287b13..1820f30732da 100644
--- a/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectJdbcDataTypeSuite.scala
+++ b/sql/connect/client/jdbc/src/test/scala/org/apache/spark/sql/connect/client/jdbc/SparkConnectJdbcDataTypeSuite.scala
@@ -301,4 +301,113 @@ class SparkConnectJdbcDataTypeSuite extends ConnectFunSuite with RemoteSparkSess
assert(exception.getMessage() === "JDBC Statement is closed.")
}
}
+
+ test("get date type") {
+ withExecuteQuery("SELECT date '2023-11-15'") { rs =>
+ assert(rs.next())
+ assert(rs.getDate(1) === java.sql.Date.valueOf("2023-11-15"))
+ assert(!rs.wasNull)
+ assert(!rs.next())
+
+ val metaData = rs.getMetaData
+ assert(metaData.getColumnCount === 1)
+ assert(metaData.getColumnName(1) === "DATE '2023-11-15'")
+ assert(metaData.getColumnLabel(1) === "DATE '2023-11-15'")
+ assert(metaData.getColumnType(1) === Types.DATE)
+ assert(metaData.getColumnTypeName(1) === "DATE")
+ assert(metaData.getColumnClassName(1) === "java.sql.Date")
+ assert(metaData.isSigned(1) === false)
+ assert(metaData.getPrecision(1) === 10)
+ assert(metaData.getScale(1) === 0)
+ assert(metaData.getColumnDisplaySize(1) === 10)
+ }
+ }
+
+ test("get date type with null") {
+ withExecuteQuery("SELECT cast(null as date)") { rs =>
+ assert(rs.next())
+ assert(rs.getDate(1) === null)
+ assert(rs.wasNull)
+ assert(!rs.next())
+
+ val metaData = rs.getMetaData
+ assert(metaData.getColumnCount === 1)
+ assert(metaData.getColumnName(1) === "CAST(NULL AS DATE)")
+ assert(metaData.getColumnLabel(1) === "CAST(NULL AS DATE)")
+ assert(metaData.getColumnType(1) === Types.DATE)
+ assert(metaData.getColumnTypeName(1) === "DATE")
+ assert(metaData.getColumnClassName(1) === "java.sql.Date")
+ assert(metaData.isSigned(1) === false)
+ assert(metaData.getPrecision(1) === 10)
+ assert(metaData.getScale(1) === 0)
+ assert(metaData.getColumnDisplaySize(1) === 10)
+ }
+ }
+
+ test("get date type by column label") {
+ withExecuteQuery("SELECT date '2025-11-15' as test_date") { rs =>
+ assert(rs.next())
+ assert(rs.getDate("test_date") === java.sql.Date.valueOf("2025-11-15"))
+ assert(!rs.wasNull)
+ assert(!rs.next())
+ }
+ }
+
+ test("get date type with calendar by column index") {
+ withExecuteQuery("SELECT date '2025-11-15'") { rs =>
+ assert(rs.next())
+
+ val calUTC = java.util.Calendar.getInstance(java.util.TimeZone.getTimeZone("UTC"))
+ val dateUTC = rs.getDate(1, calUTC)
+ assert(dateUTC !== null)
+ assert(!rs.wasNull)
+
+ val calPST = java.util.Calendar.getInstance(
+ java.util.TimeZone.getTimeZone("America/Los_Angeles"))
+ val datePST = rs.getDate(1, calPST)
+ assert(datePST !== null)
+ assert(!rs.wasNull)
+ assert(!rs.next())
+ }
+ }
+
+ test("get date type with calendar by column label") {
+ withExecuteQuery("SELECT date '2025-11-15' as test_date") { rs =>
+ assert(rs.next())
+
+ val cal = java.util.Calendar.getInstance(java.util.TimeZone.getTimeZone("UTC"))
+ val date = rs.getDate("test_date", cal)
+ assert(date !== null)
+ assert(!rs.wasNull)
+ assert(!rs.next())
+
+ val metaData = rs.getMetaData
+ assert(metaData.getColumnCount === 1)
+ assert(metaData.getColumnName(1) === "test_date")
+ assert(metaData.getColumnLabel(1) === "test_date")
+ }
+ }
+
+ test("get date type with calendar for null value") {
+ withExecuteQuery("SELECT cast(null as date)") { rs =>
+ assert(rs.next())
+
+ val cal = java.util.Calendar.getInstance(java.util.TimeZone.getTimeZone("UTC"))
+ val date = rs.getDate(1, cal)
+ assert(date === null)
+ assert(rs.wasNull)
+ assert(!rs.next())
+ }
+ }
+
+ test("get date type with null calendar") {
+ withExecuteQuery("SELECT date '2025-11-15'") { rs =>
+ assert(rs.next())
+
+ val date = rs.getDate(1, null)
+ assert(date === java.sql.Date.valueOf("2025-11-15"))
+ assert(!rs.wasNull)
+ assert(!rs.next())
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org