You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by be...@apache.org on 2025/02/27 07:36:13 UTC
(spark) branch branch-4.0 updated: [SPARK-51302][CONNECT] Spark Connect supports JDBC should use the DataFrameReader API
This is an automated email from the ASF dual-hosted git repository.
beliefer pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 77c726568f11 [SPARK-51302][CONNECT] Spark Connect supports JDBC should use the DataFrameReader API
77c726568f11 is described below
commit 77c726568f117061455c166848a15f7f8442edc0
Author: beliefer <be...@163.com>
AuthorDate: Thu Feb 27 15:25:10 2025 +0800
[SPARK-51302][CONNECT] Spark Connect supports JDBC should use the DataFrameReader API
### What changes were proposed in this pull request?
This PR proposes to unify the calling to the `DataFrameReader` API in Spark Connect where supports the jdbc API.
### Why are the changes needed?
The origin code is good at a little advance of performance, but it is bad if we change the logic of jdbc API.
I think we should unify the code path here.
### Does this PR introduce _any_ user-facing change?
'No'.
### How was this patch tested?
GA.
### Was this patch authored or co-authored using generative AI tooling?
'No'.
Closes #50059 from beliefer/SPARK-51302.
Authored-by: beliefer <be...@163.com>
Signed-off-by: beliefer <be...@163.com>
(cherry picked from commit 70d532a796e28d2d614dbf9795a02f2b74e30901)
Signed-off-by: beliefer <be...@163.com>
---
.../sql/connect/planner/SparkConnectPlanner.scala | 29 ++++++++++------------
1 file changed, 13 insertions(+), 16 deletions(-)
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 8aaa3a7151a2..1cb300d5aaf1 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql.connect.planner
+import java.util.Properties
import java.util.UUID
import scala.collection.mutable
@@ -31,7 +32,7 @@ import io.grpc.{Context, Status, StatusRuntimeException}
import io.grpc.stub.StreamObserver
import org.apache.commons.lang3.exception.ExceptionUtils
-import org.apache.spark.{Partition, SparkEnv, TaskContext}
+import org.apache.spark.{SparkEnv, TaskContext}
import org.apache.spark.annotation.{DeveloperApi, Since}
import org.apache.spark.api.python.{PythonEvalType, SimplePythonFunction}
import org.apache.spark.connect.proto
@@ -72,8 +73,7 @@ import org.apache.spark.sql.execution.QueryExecution
import org.apache.spark.sql.execution.aggregate.{ScalaAggregator, TypedAggregateExpression}
import org.apache.spark.sql.execution.arrow.ArrowConverters
import org.apache.spark.sql.execution.command.{CreateViewCommand, ExternalCommandExecutor}
-import org.apache.spark.sql.execution.datasources.LogicalRelation
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCPartition, JDBCRelation}
+import org.apache.spark.sql.execution.datasources.jdbc.JDBCOptions
import org.apache.spark.sql.execution.datasources.v2.python.UserDefinedPythonDataSource
import org.apache.spark.sql.execution.python.{UserDefinedPythonFunction, UserDefinedPythonTableFunction}
import org.apache.spark.sql.execution.python.streaming.PythonForeachWriter
@@ -1383,28 +1383,25 @@ class SparkConnectPlanner(
isStreaming = rel.getIsStreaming)
case proto.Read.ReadTypeCase.DATA_SOURCE if !rel.getIsStreaming =>
- val localMap = CaseInsensitiveMap[String](rel.getDataSource.getOptionsMap.asScala.toMap)
val reader = session.read
- if (rel.getDataSource.hasFormat) {
- reader.format(rel.getDataSource.getFormat)
- }
- localMap.foreach { case (key, value) => reader.option(key, value) }
if (rel.getDataSource.getFormat == "jdbc" && rel.getDataSource.getPredicatesCount > 0) {
- if (!localMap.contains(JDBCOptions.JDBC_URL) ||
- !localMap.contains(JDBCOptions.JDBC_TABLE_NAME)) {
+ if (!rel.getDataSource.getOptionsMap.containsKey(JDBCOptions.JDBC_URL) ||
+ !rel.getDataSource.getOptionsMap.containsKey(JDBCOptions.JDBC_TABLE_NAME)) {
throw InvalidPlanInput(s"Invalid jdbc params, please specify jdbc url and table.")
}
val url = rel.getDataSource.getOptionsMap.get(JDBCOptions.JDBC_URL)
val table = rel.getDataSource.getOptionsMap.get(JDBCOptions.JDBC_TABLE_NAME)
- val options = new JDBCOptions(url, table, localMap)
val predicates = rel.getDataSource.getPredicatesList.asScala.toArray
- val parts: Array[Partition] = predicates.zipWithIndex.map { case (part, i) =>
- JDBCPartition(part, i): Partition
- }
- val relation = JDBCRelation(parts, options)(session)
- LogicalRelation(relation)
+ val properties = new Properties()
+ properties.putAll(rel.getDataSource.getOptionsMap)
+ reader.jdbc(url, table, predicates, properties).queryExecution.analyzed
} else if (rel.getDataSource.getPredicatesCount == 0) {
+ val localMap = CaseInsensitiveMap[String](rel.getDataSource.getOptionsMap.asScala.toMap)
+ if (rel.getDataSource.hasFormat) {
+ reader.format(rel.getDataSource.getFormat)
+ }
+ localMap.foreach { case (key, value) => reader.option(key, value) }
if (rel.getDataSource.hasSchema && rel.getDataSource.getSchema.nonEmpty) {
reader.schema(parseSchema(rel.getDataSource.getSchema))
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org