You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/08/01 09:43:29 UTC
(spark) branch branch-4.0 updated: [SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new c951611e8403 [SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior
c951611e8403 is described below
commit c951611e8403953f102a756df5aab9b3951ab08d
Author: Robert Dillitz <r....@gmail.com>
AuthorDate: Fri Aug 1 18:42:15 2025 +0900
[SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior
See title.
Scala Spark Connect does not adhere to the [documented](https://spark.apache.org/docs/3.5.6/sql-data-sources-load-save-functions.html) behavior.
As documented in [Generic Load/Save Functions - Spark 3.5.6 Documentation](https://spark.apache.org/docs/3.5.6/sql-data-sources-load-save-functions.html), and similar to Spark Classic and the Python Spark Connect, Scala Spark Connect's `DataFrameReader` will now also default to the format set via the `spark.sql.sources.default` SQL configuration.
**Before**: `spark.read.load("..."`) throws
```
java.lang.IllegalArgumentException: The source format must be specified.
```
**Now**: `spark.read.load("...")` uses the format specified via `spark.sql.sources.default`
Test case added to ClientE2ETestSuite.
No.
Closes #51761 from dillitz/fix-default-format-master.
Lead-authored-by: Robert Dillitz <r....@gmail.com>
Co-authored-by: Robert Dillitz <ro...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 0dd0e022b411fcd513d0ed13a66a5b27176111e8)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../org/apache/spark/sql/connect/ClientE2ETestSuite.scala | 12 ++++++++++++
.../scala/org/apache/spark/sql/connect/DataFrameReader.scala | 9 +--------
2 files changed, 13 insertions(+), 8 deletions(-)
diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
index 566584f7e7a8..3b6dd090caf6 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
@@ -1669,6 +1669,18 @@ class ClientE2ETestSuite
}
checkAnswer(df, (0 until 6).map(i => Row(i)))
}
+
+ test("SPARK-53054: DataFrameReader defaults to spark.sql.sources.default") {
+ withTempPath { file =>
+ val path = file.getAbsoluteFile.toURI.toString
+ spark.range(100).write.parquet(file.toPath.toAbsolutePath.toString)
+
+ spark.conf.set("spark.sql.sources.default", "parquet")
+
+ val df = spark.read.load(path)
+ assert(df.count() == 100)
+ }
+ }
}
private[sql] case class ClassData(a: String, b: Int)
diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
index 0af603e0f6cc..67a4d983f56b 100644
--- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
+++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
@@ -79,8 +79,7 @@ class DataFrameReader private[sql] (sparkSession: SparkSession) extends sql.Data
def load(paths: String*): DataFrame = {
sparkSession.newDataFrame { builder =>
val dataSourceBuilder = builder.getReadBuilder.getDataSourceBuilder
- assertSourceFormatSpecified()
- dataSourceBuilder.setFormat(source)
+ Option(source).foreach(dataSourceBuilder.setFormat)
userSpecifiedSchema.foreach(schema => dataSourceBuilder.setSchema(schema.toDDL))
extraOptions.foreach { case (k, v) =>
dataSourceBuilder.putOptions(k, v)
@@ -211,12 +210,6 @@ class DataFrameReader private[sql] (sparkSession: SparkSession) extends sql.Data
@scala.annotation.varargs
override def textFile(paths: String*): Dataset[String] = super.textFile(paths: _*)
- private def assertSourceFormatSpecified(): Unit = {
- if (source == null) {
- throw new IllegalArgumentException("The source format must be specified.")
- }
- }
-
private def parse(ds: Dataset[String], format: ParseFormat): DataFrame = {
sparkSession.newDataFrame { builder =>
val parseBuilder = builder.getParseBuilder
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org