You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/09/14 13:38:52 UTC
(spark) branch branch-4.x updated: [SPARK-59469][SQL] Report structured errors for invalid JDBC partition bounds
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 7b1b6a416245 [SPARK-59469][SQL] Report structured errors for invalid JDBC partition bounds
7b1b6a416245 is described below
commit 7b1b6a416245ec792f747e6cdd63259017a12d37
Author: Uros Stankovic <ur...@databricks.com>
AuthorDate: Mon Sep 14 13:38:14 2026 +0000
[SPARK-59469][SQL] Report structured errors for invalid JDBC partition bounds
### What changes were proposed in this pull request?
- Report invalid JDBC date and timestamp partition bounds with `SparkIllegalArgumentException`, condition `INVALID_JDBC_PARTITION_BOUND`, and SQLSTATE `42616`.
- Include the invalid option (`lowerBound` or `upperBound`), supplied value, and expected data type in the error message.
- Use `42616` ("Invalid options specified"), following Avro and Protobuf option-validation errors.
- Draft pending an Apache Spark JIRA.
### Why are the changes needed?
- Invalid temporal bounds currently throw a plain `IllegalArgumentException` without an error condition or SQLSTATE.
- The existing message, such as `Cannot parse the bound value {bound} as timestamp`, does not identify which JDBC option to correct.
### Does this PR introduce _any_ user-facing change?
- Yes. Clearer error messages for invalid JDBC date and timestamp bounds when partitioned scan is requested. Validation logic is unchanged.
### How was this patch tested?
- Added `JDBCRelationSuite` coverage for invalid lower and upper bounds across `DATE`, `TIMESTAMP`, and `TIMESTAMP_NTZ`, including placeholders, empty strings, and invalid calendar dates. Positive cases check the generated partition predicates.
- Updated the existing `JDBCSuite` timezone-bound regression test to check the structured error.
```text
build/sbt "sql/testOnly org.apache.spark.sql.execution.datasources.jdbc.JDBCRelationSuite" \
"core/testOnly org.apache.spark.SparkThrowableSuite" \
"sql/testOnly org.apache.spark.sql.jdbc.JDBCSuite -- -z columnPartition"
```
- Both affected test expectations now use `42616`; these suites have not been rerun for the SQLSTATE-only update.
```text
build/sbt "catalyst/scalastyle" "sql/scalastyle" "sql/Test/scalastyle"
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex CLI 0.153.4
Closes #58762 from urosstan-db/urosstan-db/jdbc-partition-bound-errors.
Authored-by: Uros Stankovic <ur...@databricks.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
(cherry picked from commit 54862c407e432d2387cab42dcc99bc2bb8a6271f)
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../src/main/resources/error/error-conditions.json | 6 ++
.../spark/sql/errors/QueryCompilationErrors.scala | 14 +++-
.../execution/datasources/jdbc/JDBCRelation.scala | 12 ++--
.../datasources/jdbc/JDBCRelationSuite.scala | 74 ++++++++++++++++++++++
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 12 +++-
5 files changed, 109 insertions(+), 9 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 798ecbace0bc..e3056e927a80 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -4547,6 +4547,12 @@
],
"sqlState" : "46121"
},
+ "INVALID_JDBC_PARTITION_BOUND" : {
+ "message" : [
+ "Cannot parse the value <value> for JDBC data source option <option> as <dataType>."
+ ],
+ "sqlState" : "42616"
+ },
"INVALID_JOIN_TYPE_FOR_JOINWITH" : {
"message" : [
"Invalid join type in joinWith: <joinType>."
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
index 04e5320808e5..faaeae9d91a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala
@@ -21,7 +21,7 @@ import java.util.Locale
import org.apache.hadoop.fs.Path
-import org.apache.spark.{SPARK_DOC_ROOT, SparkException, SparkThrowable, SparkUnsupportedOperationException}
+import org.apache.spark.{SPARK_DOC_ROOT, SparkException, SparkIllegalArgumentException, SparkThrowable, SparkUnsupportedOperationException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.catalyst.{ExtendedAnalysisException, FunctionIdentifier, InternalRow, QualifiedTableName, TableIdentifier}
import org.apache.spark.sql.catalyst.analysis.{CannotReplaceMissingTableException, FunctionAlreadyExistsException, NamedRelation, NamespaceAlreadyExistsException, NoSuchFunctionException, NoSuchNamespaceException, NoSuchPartitionException, NoSuchTableException, Star, TableAlreadyExistsException, UnresolvedRegex}
@@ -3697,6 +3697,18 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase with Compilat
"dataType" -> column.dataType.catalogString))
}
+ def invalidJdbcPartitionBoundError(
+ optionName: String,
+ value: String,
+ dataType: DataType): SparkIllegalArgumentException = {
+ new SparkIllegalArgumentException(
+ errorClass = "INVALID_JDBC_PARTITION_BOUND",
+ messageParameters = Map(
+ "option" -> toDSOption(optionName),
+ "value" -> toSQLConfVal(value),
+ "dataType" -> toSQLType(dataType)))
+ }
+
def tableOrViewAlreadyExistsError(name: String): Throwable = {
new AnalysisException(
errorClass = "_LEGACY_ERROR_TEMP_1288",
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index 1c529279744e..91e45d0a99e8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -93,8 +93,10 @@ private[sql] object JDBCRelation extends Logging {
val (column, columnType) = verifyAndGetNormalizedPartitionColumn(
schema, partitionColumn.get, resolver, jdbcOptions)
- val lowerBoundValue = toInternalBoundValue(lowerBound.get, columnType, timeZoneId)
- val upperBoundValue = toInternalBoundValue(upperBound.get, columnType, timeZoneId)
+ val lowerBoundValue = toInternalBoundValue(
+ lowerBound.get, columnType, timeZoneId, JDBC_LOWER_BOUND)
+ val upperBoundValue = toInternalBoundValue(
+ upperBound.get, columnType, timeZoneId, JDBC_UPPER_BOUND)
JDBCPartitioningInfo(
column, columnType, lowerBoundValue, upperBoundValue, numPartitions.get)
}
@@ -198,11 +200,11 @@ private[sql] object JDBCRelation extends Logging {
private def toInternalBoundValue(
value: String,
columnType: DataType,
- timeZoneId: String): Long = {
+ timeZoneId: String,
+ optionName: String): Long = {
def parse[T](f: UTF8String => Option[T]): T = {
f(UTF8String.fromString(value)).getOrElse {
- throw new IllegalArgumentException(
- s"Cannot parse the bound value $value as ${columnType.catalogString}")
+ throw QueryCompilationErrors.invalidJdbcPartitionBoundError(optionName, value, columnType)
}
}
columnType match {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelationSuite.scala
new file mode 100644
index 000000000000..d6c41f063644
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelationSuite.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.jdbc
+
+import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
+import org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution
+import org.apache.spark.sql.types.{DateType, StructType, TimestampNTZType, TimestampType}
+
+class JDBCRelationSuite extends SparkFunSuite {
+
+ private val options = Map(
+ "url" -> "jdbc:h2:mem:jdbcRelation",
+ "driver" -> "org.h2.Driver",
+ "dbtable" -> "test",
+ "partitionColumn" -> "t",
+ "lowerBound" -> "2020-01-01",
+ "upperBound" -> "2020-01-03",
+ "numPartitions" -> "2")
+
+ for {
+ dataType <- Seq(DateType, TimestampType, TimestampNTZType)
+ bound <- Seq("lowerBound", "upperBound")
+ value <- Seq("{bound}", "", "2020-02-30")
+ } {
+ test(s"invalid $bound for ${dataType.sql}: '$value'") {
+ val jdbcOptions = new JDBCOptions(options.updated(bound, value))
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ JDBCRelation.columnPartition(
+ new StructType().add("t", dataType),
+ caseSensitiveResolution,
+ timeZoneId = "UTC",
+ jdbcOptions = jdbcOptions)
+ },
+ condition = "INVALID_JDBC_PARTITION_BOUND",
+ sqlState = Some("42616"),
+ parameters = Map(
+ "option" -> s""""$bound"""",
+ "value" -> s""""$value"""",
+ "dataType" -> s""""${dataType.sql}""""))
+ }
+ }
+
+ for (dataType <- Seq(DateType, TimestampType, TimestampNTZType)) {
+ test(s"valid ${dataType.sql} bounds generate partition predicates") {
+ val partitions = JDBCRelation.columnPartition(
+ new StructType().add("t", dataType),
+ caseSensitiveResolution,
+ timeZoneId = "UTC",
+ jdbcOptions = new JDBCOptions(options))
+ val midpoint = if (dataType == DateType) "2020-01-02" else "2020-01-02 00:00:00"
+ assertResult(Seq(
+ s""""t" < '$midpoint' or "t" is null""",
+ s""""t" >= '$midpoint'""")) {
+ partitions.map(_.asInstanceOf[JDBCPartition].whereClause).toSeq
+ }
+ }
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index c79fe8dee0a8..77e2ce82d34f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -526,7 +526,7 @@ class JDBCSuite extends SharedSparkSession {
))
// allowTimeZone = false: a bound carrying a zone offset is rejected rather than silently
// shifted, so NTZ bounds stay zoneless.
- val e = intercept[IllegalArgumentException] {
+ val e = intercept[SparkIllegalArgumentException] {
JDBCRelation.columnPartition(
schema,
analysis.caseInsensitiveResolution,
@@ -537,8 +537,14 @@ class JDBCSuite extends SharedSparkSession {
"numPartitions" -> "2",
"partitionColumn" -> "PartitionColumn")))
}
- assert(e.getMessage.contains("Cannot parse the bound value"))
- assert(e.getMessage.contains("2018-07-06 10:00:00+05:00"))
+ checkError(
+ exception = e,
+ condition = "INVALID_JDBC_PARTITION_BOUND",
+ sqlState = Some("42616"),
+ parameters = Map(
+ "option" -> "\"lowerBound\"",
+ "value" -> "\"2018-07-06 10:00:00+05:00\"",
+ "dataType" -> "\"TIMESTAMP_NTZ\""))
}
test("overflow of partition bound difference does not give negative stride") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org