You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2025/02/26 10:00:52 UTC

(spark) branch master updated: [SPARK-51312][SQL] Fix createDataFrame from RDD[Row]

This is an automated email from the ASF dual-hosted git repository.

maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 381d5bc65e4d [SPARK-51312][SQL] Fix createDataFrame from RDD[Row]
381d5bc65e4d is described below

commit 381d5bc65e4d1f49cb89b8736f9d1c8437345fbb
Author: Mihailo Milosevic <mi...@databricks.com>
AuthorDate: Wed Feb 26 11:00:34 2025 +0100

    [SPARK-51312][SQL] Fix createDataFrame from RDD[Row]
    
    ### What changes were proposed in this pull request?
    Making createDataFrame use lenient Encoder for serialization.
    
    ### Why are the changes needed?
    Passing Rows is something defined by users, so if we do not allow this, we would fail queries like this.
    ```
    val schema = new org.apache.spark.sql.types.StructType().add("a", "date")
    val rdd = spark.sparkContext.parallelize(
    org.apache.spark.sql.Row(java.time.LocalDate.of(2020, 5, 13)) :: Nil)
    spark.createDataFrame(rdd, schema).collect()
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    Yes.
    The change is that we fix failing queries when we try to read Date/LocalDate independent of java8API config value.
    
    ### How was this patch tested?
    Added appropriate tests.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    No.
    
    Closes #50079 from mihailom-db/fixCreateDataFrame.
    
    Authored-by: Mihailo Milosevic <mi...@databricks.com>
    Signed-off-by: Max Gekk <ma...@gmail.com>
---
 .../scala/org/apache/spark/sql/classic/SparkSession.scala |  2 +-
 .../test/scala/org/apache/spark/sql/DatasetSuite.scala    | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
index 14ae46c25a90..034a3359da94 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
@@ -305,7 +305,7 @@ class SparkSession private(
     val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
     // TODO: use MutableProjection when rowRDD is another DataFrame and the applied
     // schema differs from the existing schema on any field data type.
-    val encoder = ExpressionEncoder(replaced)
+    val encoder = ExpressionEncoder(replaced, lenient = true)
     val toRow = encoder.createSerializer()
     val catalystRows = rowRDD.map(toRow)
     internalCreateDataFrame(catalystRows.setName(rowRDD.name), schema)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 0e6b1840a6ff..7844acc8bdcf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -2779,6 +2779,21 @@ class DatasetSuite extends QueryTest
     }
   }
 
+  test("SPARK-51312: createDataFrame should work with both Date and LocalDate") {
+    for (confVal <- Seq("true", "false")) {
+      withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> confVal) {
+        val schema = new org.apache.spark.sql.types.StructType().add("a", "date").add("b", "date")
+        val rdd = spark.sparkContext.parallelize(
+          Seq(Row(java.time.LocalDate.of(2020, 5, 13), java.sql.Date.valueOf("2020-05-13")))
+        )
+        checkAnswer(
+          spark.createDataFrame(rdd, schema),
+          sql("select date'2020-05-13' as a, date'2020-05-13' as b")
+        )
+      }
+    }
+  }
+
   test("SPARK-46791: Dataset with set field") {
     val ds = Seq(WithSet(0, HashSet("foo", "bar")), WithSet(1, HashSet("bar", "zoo"))).toDS()
     checkDataset(ds.map(t => t),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org