You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/10/18 20:48:54 UTC
spark git commit: [SPARK-17955][SQL] Make DataFrameReader.jdbc call
DataFrameReader.format("jdbc").load
Repository: spark
Updated Branches:
refs/heads/master 4518642ab -> b3130c7b6
[SPARK-17955][SQL] Make DataFrameReader.jdbc call DataFrameReader.format("jdbc").load
## What changes were proposed in this pull request?
This PR proposes to make `DataFrameReader.jdbc` call `DataFrameReader.format("jdbc").load` consistently with other APIs in `DataFrameReader`/`DataFrameWriter` and avoid calling `sparkSession.baseRelationToDataFrame(..)` here and there.
The changes were mostly copied from `DataFrameWriter.jdbc()` which was recently updated.
```diff
- val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
- val options = new JDBCOptions(url, table, params)
- val relation = JDBCRelation(parts, options)(sparkSession)
- sparkSession.baseRelationToDataFrame(relation)
+ this.extraOptions = this.extraOptions ++ connectionProperties.asScala
+ // explicit url and dbtable should override all
+ this.extraOptions += ("url" -> url, "dbtable" -> table)
+ format("jdbc").load()
```
## How was this patch tested?
Existing tests should cover this.
Author: hyukjinkwon <gu...@gmail.com>
Closes #15499 from HyukjinKwon/SPARK-17955.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b3130c7b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b3130c7b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b3130c7b
Branch: refs/heads/master
Commit: b3130c7b6a1ab4975023f08c3ab02ee8d2c7e995
Parents: 4518642
Author: hyukjinkwon <gu...@gmail.com>
Authored: Tue Oct 18 13:49:02 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Oct 18 13:49:02 2016 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/DataFrameReader.scala | 8 ++++----
.../main/scala/org/apache/spark/sql/DataFrameWriter.scala | 4 ++--
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b3130c7b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index ac33585..b7b2203 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -232,10 +232,10 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends Logging {
parts: Array[Partition],
connectionProperties: Properties): DataFrame = {
// connectionProperties should override settings in extraOptions.
- val params = extraOptions.toMap ++ connectionProperties.asScala.toMap
- val options = new JDBCOptions(url, table, params)
- val relation = JDBCRelation(parts, options)(sparkSession)
- sparkSession.baseRelationToDataFrame(relation)
+ this.extraOptions = this.extraOptions ++ connectionProperties.asScala
+ // explicit url and dbtable should override all
+ this.extraOptions += ("url" -> url, "dbtable" -> table)
+ format("jdbc").load()
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/b3130c7b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 35ef050..5be3277 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -426,8 +426,8 @@ final class DataFrameWriter[T] private[sql](ds: Dataset[T]) {
def jdbc(url: String, table: String, connectionProperties: Properties): Unit = {
assertNotPartitioned("jdbc")
assertNotBucketed("jdbc")
- // connectionProperties should override settings in extraOptions
- this.extraOptions = this.extraOptions ++ (connectionProperties.asScala)
+ // connectionProperties should override settings in extraOptions.
+ this.extraOptions = this.extraOptions ++ connectionProperties.asScala
// explicit url and dbtable should override all
this.extraOptions += ("url" -> url, "dbtable" -> table)
format("jdbc").save()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org