You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/07 02:16:48 UTC
spark git commit: [SPARK-8463][SQL] Use DriverRegistry to load jdbc
driver at writing path
Repository: spark
Updated Branches:
refs/heads/master 09a06418d -> d4d6d31db
[SPARK-8463][SQL] Use DriverRegistry to load jdbc driver at writing path
JIRA: https://issues.apache.org/jira/browse/SPARK-8463
Currently, at the reading path, `DriverRegistry` is used to load needed jdbc driver at executors. However, at the writing path, we also need `DriverRegistry` to load jdbc driver.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #6900 from viirya/jdbc_write_driver and squashes the following commits:
16cd04b [Liang-Chi Hsieh] Use DriverRegistry to load jdbc driver at writing path.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d4d6d31d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d4d6d31d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d4d6d31d
Branch: refs/heads/master
Commit: d4d6d31db5cc5c69ac369f754b7489f444c9ba2f
Parents: 09a0641
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Mon Jul 6 17:16:44 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Mon Jul 6 17:16:44 2015 -0700
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d4d6d31d/sql/core/src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala
index dd8aaf6..f7ea852 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/jdbc.scala
@@ -58,13 +58,12 @@ package object jdbc {
* are used.
*/
def savePartition(
- url: String,
+ getConnection: () => Connection,
table: String,
iterator: Iterator[Row],
rddSchema: StructType,
- nullTypes: Array[Int],
- properties: Properties): Iterator[Byte] = {
- val conn = DriverManager.getConnection(url, properties)
+ nullTypes: Array[Int]): Iterator[Byte] = {
+ val conn = getConnection()
var committed = false
try {
conn.setAutoCommit(false) // Everything in the same db transaction.
@@ -185,8 +184,10 @@ package object jdbc {
}
val rddSchema = df.schema
+ val driver: String = DriverRegistry.getDriverClassName(url)
+ val getConnection: () => Connection = JDBCRDD.getConnector(driver, url, properties)
df.foreachPartition { iterator =>
- JDBCWriteDetails.savePartition(url, table, iterator, rddSchema, nullTypes, properties)
+ JDBCWriteDetails.savePartition(getConnection, table, iterator, rddSchema, nullTypes)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org