You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2025/02/26 10:00:52 UTC
(spark) branch master updated: [SPARK-51312][SQL] Fix createDataFrame from RDD[Row]
This is an automated email from the ASF dual-hosted git repository.
maxgekk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 381d5bc65e4d [SPARK-51312][SQL] Fix createDataFrame from RDD[Row]
381d5bc65e4d is described below
commit 381d5bc65e4d1f49cb89b8736f9d1c8437345fbb
Author: Mihailo Milosevic <mi...@databricks.com>
AuthorDate: Wed Feb 26 11:00:34 2025 +0100
[SPARK-51312][SQL] Fix createDataFrame from RDD[Row]
### What changes were proposed in this pull request?
Making createDataFrame use lenient Encoder for serialization.
### Why are the changes needed?
Passing Rows is something defined by users, so if we do not allow this, we would fail queries like this.
```
val schema = new org.apache.spark.sql.types.StructType().add("a", "date")
val rdd = spark.sparkContext.parallelize(
org.apache.spark.sql.Row(java.time.LocalDate.of(2020, 5, 13)) :: Nil)
spark.createDataFrame(rdd, schema).collect()
```
### Does this PR introduce _any_ user-facing change?
Yes.
The change is that we fix failing queries when we try to read Date/LocalDate independent of java8API config value.
### How was this patch tested?
Added appropriate tests.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50079 from mihailom-db/fixCreateDataFrame.
Authored-by: Mihailo Milosevic <mi...@databricks.com>
Signed-off-by: Max Gekk <ma...@gmail.com>
---
.../scala/org/apache/spark/sql/classic/SparkSession.scala | 2 +-
.../test/scala/org/apache/spark/sql/DatasetSuite.scala | 15 +++++++++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
index 14ae46c25a90..034a3359da94 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/SparkSession.scala
@@ -305,7 +305,7 @@ class SparkSession private(
val replaced = CharVarcharUtils.failIfHasCharVarchar(schema).asInstanceOf[StructType]
// TODO: use MutableProjection when rowRDD is another DataFrame and the applied
// schema differs from the existing schema on any field data type.
- val encoder = ExpressionEncoder(replaced)
+ val encoder = ExpressionEncoder(replaced, lenient = true)
val toRow = encoder.createSerializer()
val catalystRows = rowRDD.map(toRow)
internalCreateDataFrame(catalystRows.setName(rowRDD.name), schema)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 0e6b1840a6ff..7844acc8bdcf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -2779,6 +2779,21 @@ class DatasetSuite extends QueryTest
}
}
+ test("SPARK-51312: createDataFrame should work with both Date and LocalDate") {
+ for (confVal <- Seq("true", "false")) {
+ withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> confVal) {
+ val schema = new org.apache.spark.sql.types.StructType().add("a", "date").add("b", "date")
+ val rdd = spark.sparkContext.parallelize(
+ Seq(Row(java.time.LocalDate.of(2020, 5, 13), java.sql.Date.valueOf("2020-05-13")))
+ )
+ checkAnswer(
+ spark.createDataFrame(rdd, schema),
+ sql("select date'2020-05-13' as a, date'2020-05-13' as b")
+ )
+ }
+ }
+ }
+
test("SPARK-46791: Dataset with set field") {
val ds = Seq(WithSet(0, HashSet("foo", "bar")), WithSet(1, HashSet("bar", "zoo"))).toDS()
checkDataset(ds.map(t => t),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org