(spark) branch master updated: [SPARK-51264][SQL][JDBC] Close driver connection before writing

[ur...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 535175ba9cf1 [SPARK-51264][SQL][JDBC] Close driver connection before writing
535175ba9cf1 is described below

commit 535175ba9cf1621ccc314ab0cf25b28c6ad43549
Author: Xinyao Zhang <43...@users.noreply.github.com>
AuthorDate: Mon Sep 28 12:21:15 2026 +0000

    [SPARK-51264][SQL][JDBC] Close driver connection before writing
    
    ### What changes were proposed in this pull request?
    
    Close the driver-side JDBC connection after Spark has finished checking, truncating, dropping, or creating the target table, and before it starts the distributed write.
    
    The write path opens its own JDBC connections for partitions, so keeping the setup connection open for the entire write is unnecessary.
    
    ### Why are the changes needed?
    
    The setup connection can remain idle for minutes or hours while a large write runs. Closing it before the distributed write avoids unnecessary resource consumption and prevents an idle connection from being dropped by a database, firewall, or proxy.
    
    ### Does this PR introduce any user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    `./build/sbt 'sql/testOnly org.apache.spark.sql.jdbc.JDBCWriteSuite'`
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes. Generated-by: Codex (GPT-5)
    
    Closes #58588 from zhangxinyao88/codex/SPARK-51264-close-jdbc-driver-connection.
    
    Authored-by: Xinyao Zhang <43...@users.noreply.github.com>
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
 .../datasources/jdbc/JdbcRelationProvider.scala          | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index 6b9d0520f2eb..244064762f28 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -58,7 +58,7 @@ class JdbcRelationProvider extends CreatableRelationProvider
     val isCaseSensitive = sqlContext.sparkSession.sessionState.conf.caseSensitiveAnalysis
     val dialect = JdbcDialects.get(options.url)
     val conn = dialect.createConnectionFactory(options)(-1)
-    try {
+    val (shouldSave, tableSchema) = try {
       val tableExists = JdbcUtils.tableExists(conn, options)
       if (tableExists) {
         mode match {
@@ -66,18 +66,16 @@ class JdbcRelationProvider extends CreatableRelationProvider
             if (options.isTruncate && isCascadingTruncateTable(options.url) == Some(false)) {
               // In this case, we should truncate table and then load.
               truncateTable(conn, options)
-              val tableSchema = JdbcUtils.getSchemaOption(conn, options)
-              saveTable(df, tableSchema, isCaseSensitive, options)
+              (true, JdbcUtils.getSchemaOption(conn, options))
             } else {
               // Otherwise, do not truncate the table, instead drop and recreate it
               dropTable(conn, options.table, options)
               createTable(conn, options.table, df.schema, isCaseSensitive, options)
-              saveTable(df, Some(df.schema), isCaseSensitive, options)
+              (true, Some(df.schema))
             }
 
           case SaveMode.Append =>
-            val tableSchema = JdbcUtils.getSchemaOption(conn, options)
-            saveTable(df, tableSchema, isCaseSensitive, options)
+            (true, JdbcUtils.getSchemaOption(conn, options))
 
           case SaveMode.ErrorIfExists =>
             throw QueryCompilationErrors.tableOrViewAlreadyExistsError(options.table)
@@ -86,14 +84,18 @@ class JdbcRelationProvider extends CreatableRelationProvider
             // With `SaveMode.Ignore` mode, if table already exists, the save operation is expected
             // to not save the contents of the DataFrame and to not change the existing data.
             // Therefore, it is okay to do nothing here and then just return the relation below.
+            (false, None)
         }
       } else {
         createTable(conn, options.table, df.schema, isCaseSensitive, options)
-        saveTable(df, Some(df.schema), isCaseSensitive, options)
+        (true, Some(df.schema))
       }
     } finally {
       conn.close()
     }
+    if (shouldSave) {
+      saveTable(df, tableSchema, isCaseSensitive, options)
+    }
 
     createRelation(sqlContext, parameters)
   }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org