You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/08/01 09:43:29 UTC

(spark) branch branch-4.0 updated: [SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new c951611e8403 [SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior
c951611e8403 is described below

commit c951611e8403953f102a756df5aab9b3951ab08d
Author: Robert Dillitz <r....@gmail.com>
AuthorDate: Fri Aug 1 18:42:15 2025 +0900

    [SPARK-53054][CONNECT] Fix the connect.DataFrameReader default format behavior
    
    See title.
    
    Scala Spark Connect does not adhere to the [documented](https://spark.apache.org/docs/3.5.6/sql-data-sources-load-save-functions.html) behavior.
    
    As documented in [Generic Load/Save Functions - Spark 3.5.6 Documentation](https://spark.apache.org/docs/3.5.6/sql-data-sources-load-save-functions.html), and similar to Spark Classic and the Python Spark Connect, Scala Spark Connect's `DataFrameReader` will now also default to the format set via the `spark.sql.sources.default` SQL configuration.
    
    **Before**: `spark.read.load("..."`) throws
    
    ```
    java.lang.IllegalArgumentException: The source format must be specified.
    ```
    **Now**: `spark.read.load("...")` uses the format specified via `spark.sql.sources.default`
    
    Test case added to ClientE2ETestSuite.
    
    No.
    
    Closes #51761 from dillitz/fix-default-format-master.
    
    Lead-authored-by: Robert Dillitz <r....@gmail.com>
    Co-authored-by: Robert Dillitz <ro...@databricks.com>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
    (cherry picked from commit 0dd0e022b411fcd513d0ed13a66a5b27176111e8)
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 .../org/apache/spark/sql/connect/ClientE2ETestSuite.scala    | 12 ++++++++++++
 .../scala/org/apache/spark/sql/connect/DataFrameReader.scala |  9 +--------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
index 566584f7e7a8..3b6dd090caf6 100644
--- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
+++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/connect/ClientE2ETestSuite.scala
@@ -1669,6 +1669,18 @@ class ClientE2ETestSuite
     }
     checkAnswer(df, (0 until 6).map(i => Row(i)))
   }
+
+  test("SPARK-53054: DataFrameReader defaults to spark.sql.sources.default") {
+    withTempPath { file =>
+      val path = file.getAbsoluteFile.toURI.toString
+      spark.range(100).write.parquet(file.toPath.toAbsolutePath.toString)
+
+      spark.conf.set("spark.sql.sources.default", "parquet")
+
+      val df = spark.read.load(path)
+      assert(df.count() == 100)
+    }
+  }
 }
 
 private[sql] case class ClassData(a: String, b: Int)
diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
index 0af603e0f6cc..67a4d983f56b 100644
--- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
+++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/DataFrameReader.scala
@@ -79,8 +79,7 @@ class DataFrameReader private[sql] (sparkSession: SparkSession) extends sql.Data
   def load(paths: String*): DataFrame = {
     sparkSession.newDataFrame { builder =>
       val dataSourceBuilder = builder.getReadBuilder.getDataSourceBuilder
-      assertSourceFormatSpecified()
-      dataSourceBuilder.setFormat(source)
+      Option(source).foreach(dataSourceBuilder.setFormat)
       userSpecifiedSchema.foreach(schema => dataSourceBuilder.setSchema(schema.toDDL))
       extraOptions.foreach { case (k, v) =>
         dataSourceBuilder.putOptions(k, v)
@@ -211,12 +210,6 @@ class DataFrameReader private[sql] (sparkSession: SparkSession) extends sql.Data
   @scala.annotation.varargs
   override def textFile(paths: String*): Dataset[String] = super.textFile(paths: _*)
 
-  private def assertSourceFormatSpecified(): Unit = {
-    if (source == null) {
-      throw new IllegalArgumentException("The source format must be specified.")
-    }
-  }
-
   private def parse(ds: Dataset[String], format: ParseFormat): DataFrame = {
     sparkSession.newDataFrame { builder =>
       val parseBuilder = builder.getParseBuilder


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org