You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2026/07/14 23:09:28 UTC

(spark) branch master updated: [SPARK-52825][SQL] Recognize well-known JDBC URL prefixes in MySQL/PostgresDialect

This is an automated email from the ASF dual-hosted git repository.

sarutak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new c8e756d6d6ee [SPARK-52825][SQL] Recognize well-known JDBC URL prefixes in MySQL/PostgresDialect
c8e756d6d6ee is described below

commit c8e756d6d6ee08a9d3f9a725276ed73ef6bbda50
Author: Aaron <aa...@improving.com>
AuthorDate: Wed Jul 15 08:09:11 2026 +0900

    [SPARK-52825][SQL] Recognize well-known JDBC URL prefixes in MySQL/PostgresDialect
    
    ### What changes were proposed in this pull request?
    Update `MySQLDialect` and `PostgresDialect` to use regex-based `canHandle` matching
    that recognizes well-known JDBC URL prefixes in addition to the bare
    `jdbc:mysql` / `jdbc:postgresql` prefixes.
    
    For MySQL, covered URL patterns include:
    - `jdbc:mysql:` (standard)
    - `jdbc:mysql+srv:` (SRV-based discovery)
    - `jdbc:mysql:replication:` / `jdbc:mysql:loadbalance:` (Connector/J multi-host)
    - `jdbc:mysqlx:` (X DevAPI)
    - `jdbc:aws-wrapper:mysql:` (aws-advanced-jdbc-wrapper)
    - `jdbc:p6spy:mysql:` (P6Spy)
    
    For PostgreSQL, covered patterns include:
    - `jdbc:postgresql:` (standard)
    - `jdbc:aws-wrapper:postgresql:` (aws-advanced-jdbc-wrapper)
    - `jdbc:p6spy:postgresql:` (P6Spy)
    
    ### Why are the changes needed?
    Some JDBC tools (such as the AWS advanced JDBC wrapper, P6Spy, and others)
    use URL prefixes like `jdbc:<tool>:<db>://` so the JVM selects the tool as the
    JDBC driver. The underlying connection still speaks standard MySQL or PostgreSQL -
    there is no dialect difference. However, because `canHandle` previously only matched
    the bare `jdbc:mysql` / `jdbc:postgresql` prefixes, Spark fell through to `NoopDialect`
    for any of these tool URLs, causing issues like unrecognized SQL type errors
    ([example](https://github.com/aws/aws-advanced-jdbc-wrapper/issues/1370)).
    
    This fix replaces the `startsWith` check with a regex that generically matches any
    intermediate prefix between `jdbc:` and the database name. This avoids
    hardcoding vendor-specific strings while covering all known wrapper patterns with a
    single rule.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes. Users connecting to MySQL or PostgreSQL through drivers with alternative prefixes (AWS JDBC
    wrapper, P6Spy, etc.) will now get the correct dialect automatically, fixing SQL type
    errors and other dialect-mismatch bugs.
    
    ### How was this patch tested?
    Added assertions to the existing "Default jdbc dialect registration" test verifying that
    alternative URL patterns (e.g. `jdbc:aws-wrapper:mysql://`, `jdbc:aws-wrapper:postgresql://)) resolve to `MySQLDialect` and
    `PostgresDialect` respectively.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Yes - co-authored with generative AI tooling for identifying relevant code locations and
    generating the fix, which I reviewed and submitted.
    
    Closes #53902 from aaron-congo/support-aws-protocols.
    
    Authored-by: Aaron <aa...@improving.com>
    Signed-off-by: Kousuke Saruta <sa...@apache.org>
---
 sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala    | 2 +-
 sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala | 2 +-
 sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala       | 2 ++
 3 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
index b301c0c0bd5b..d5a1aa8bd712 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/MySQLDialect.scala
@@ -37,7 +37,7 @@ import org.apache.spark.sql.types._
 private case class MySQLDialect() extends JdbcDialect with SQLConfHelper with NoLegacyJDBCError {
 
   override def canHandle(url : String): Boolean =
-    url.toLowerCase(Locale.ROOT).startsWith("jdbc:mysql")
+    url.toLowerCase(Locale.ROOT).matches("jdbc:(.*:)?mysql.*")
 
   private val distinctUnsupportedAggregateFunctions =
     Set("VAR_POP", "VAR_SAMP", "STDDEV_POP", "STDDEV_SAMP")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
index 9fc69594932b..068c62b1736b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/PostgresDialect.scala
@@ -41,7 +41,7 @@ private case class PostgresDialect()
   extends JdbcDialect with SQLConfHelper with NoLegacyJDBCError {
 
   override def canHandle(url: String): Boolean =
-    url.toLowerCase(Locale.ROOT).startsWith("jdbc:postgresql")
+    url.toLowerCase(Locale.ROOT).matches("jdbc:(.*:)?postgresql.*")
 
   // See https://www.postgresql.org/docs/8.4/functions-aggregate.html
   private val supportedAggregateFunctions = Set("MAX", "MIN", "SUM", "COUNT", "AVG",
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index ebf1ce36dd56..58797e68b8ab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -912,7 +912,9 @@ class JDBCSuite extends SharedSparkSession {
 
   test("Default jdbc dialect registration") {
     assert(JdbcDialects.get("jdbc:mysql://127.0.0.1/db") === MySQLDialect())
+    assert(JdbcDialects.get("jdbc:aws-wrapper:mysql://127.0.0.1/db") === MySQLDialect())
     assert(JdbcDialects.get("jdbc:postgresql://127.0.0.1/db") === PostgresDialect())
+    assert(JdbcDialects.get("jdbc:aws-wrapper:postgresql://127.0.0.1/db") === PostgresDialect())
     assert(JdbcDialects.get("jdbc:db2://127.0.0.1/db") === DB2Dialect())
     assert(JdbcDialects.get("jdbc:sqlserver://127.0.0.1/db") === MsSqlServerDialect())
     assert(JdbcDialects.get("jdbc:derby:db") === DerbyDialect())


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org