You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/09/24 13:49:23 UTC
(spark) branch master updated: [SPARK-53677][SQL] Improve debuggability for JDBC data source when query contains syntax error
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 997e5383c680 [SPARK-53677][SQL] Improve debuggability for JDBC data source when query contains syntax error
997e5383c680 is described below
commit 997e5383c680a06daa79ac4d02f28ed5d93db90b
Author: Uros Stankovic <ur...@databricks.com>
AuthorDate: Wed Sep 24 21:49:07 2025 +0800
[SPARK-53677][SQL] Improve debuggability for JDBC data source when query contains syntax error
### What changes were proposed in this pull request?
- Improve the error message for syntax errors in the JDBC data source, since the customer complains that it is hard to fix a query currently, as they don't have hints from the remote engine parser.
### Why are the changes needed?
- Improve UX
### Does this PR introduce _any_ user-facing change?
Yes, slight difference in error message
### How was this patch tested?
Modified existing test
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #52421 from urosstan-db/SPARK-53677-improve-jdbc-data-source-external-engine-syntax-error-debugability.
Authored-by: Uros Stankovic <ur...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../utils/src/main/resources/error/error-conditions.json | 2 +-
.../spark/sql/jdbc/SharedJDBCIntegrationSuite.scala | 15 ++++++++++++---
.../spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 10 ++++++++--
3 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json
index 60bce107186f..d497586b64c3 100644
--- a/common/utils/src/main/resources/error/error-conditions.json
+++ b/common/utils/src/main/resources/error/error-conditions.json
@@ -4039,7 +4039,7 @@
},
"JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR" : {
"message" : [
- "JDBC external engine syntax error. The error was caused by the query <jdbcQuery>."
+ "JDBC external engine syntax error. The error was caused by the query <jdbcQuery>. <externalEngineError>."
],
"subClass" : {
"DURING_OUTPUT_SCHEMA_RESOLUTION" : {
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
index e42d04ead087..05338e79998c 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/SharedJDBCIntegrationSuite.scala
@@ -55,13 +55,22 @@ abstract class SharedJDBCIntegrationSuite extends DockerJDBCIntegrationSuite {
}
test("SPARK-52184: Wrap external engine syntax error") {
- val e = intercept[SparkException] {
-
+ val ex = intercept[SparkException] {
spark.read.format("jdbc")
.option("url", jdbcUrl)
.option("query", "THIS IS NOT VALID SQL").load()
}
- assert(e.getCondition.startsWith("JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR"))
+
+ // Exception should be detected in analysis phase first when we resolve a schema from
+ // through JDBC by sending SELECT * FROM (<subquery>) [LIMIT 1][WHERE 1=0] query.
+ checkErrorMatchPVals(
+ ex,
+ condition = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_OUTPUT_SCHEMA_RESOLUTION",
+ parameters = Map(
+ "jdbcQuery" -> "SELECT \\* FROM \\(.*",
+ "externalEngineError" -> "[\\s\\S]*"
+ )
+ )
}
test("SPARK-53386: Parameter `query` should work when ending with semicolons") {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 33cb5342fd3d..47f5f180789e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -67,7 +67,10 @@ object JDBCRDD extends Logging {
case e: SQLException if dialect.isSyntaxErrorBestEffort(e) =>
throw new SparkException(
errorClass = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_OUTPUT_SCHEMA_RESOLUTION",
- messageParameters = Map("jdbcQuery" -> fullQuery),
+ messageParameters = Map(
+ "jdbcQuery" -> fullQuery,
+ "externalEngineError" -> e.getMessage.replaceAll("\\.+$", "")
+ ),
cause = e)
}
}
@@ -335,7 +338,10 @@ class JDBCRDD(
case e: SQLException if dialect.isSyntaxErrorBestEffort(e) =>
throw new SparkException(
errorClass = "JDBC_EXTERNAL_ENGINE_SYNTAX_ERROR.DURING_QUERY_EXECUTION",
- messageParameters = Map("jdbcQuery" -> sqlText),
+ messageParameters = Map(
+ "jdbcQuery" -> sqlText,
+ "externalEngineError" -> e.getMessage.replaceAll("\\.+$", "")
+ ),
cause = e)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org