You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2024/02/13 00:38:20 UTC
(spark) branch master updated: [SPARK-47026][SQL][TESTS] Enable JSON sources in default value nested type tests
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5da73d32e36a [SPARK-47026][SQL][TESTS] Enable JSON sources in default value nested type tests
5da73d32e36a is described below
commit 5da73d32e36aed80ae4a3eabe1599cf630d0a753
Author: Mark Jarvin <ma...@databricks.com>
AuthorDate: Tue Feb 13 09:38:08 2024 +0900
[SPARK-47026][SQL][TESTS] Enable JSON sources in default value nested type tests
### What changes were proposed in this pull request?
Enable JSON sources in default value nested type tests.
### Why are the changes needed?
This exposes a bug in default value handling; filed https://issues.apache.org/jira/browse/SPARK-47029 to track and added a comment to the code.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Test-only change; tested with existing tests
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #45086 from markj-db/SPARK-47026.
Authored-by: Mark Jarvin <ma...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../org/apache/spark/sql/sources/InsertSuite.scala | 48 +++++++++++++++-------
1 file changed, 33 insertions(+), 15 deletions(-)
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
index 0eb268cff951..ccf032f27281 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
@@ -434,7 +434,7 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
)
sql(s"CREATE TABLE target2(a INT, b STRING) USING JSON")
- val e = sql(
+ sql(
"""
|WITH tbl AS (SELECT * FROM jt)
|FROM tbl
@@ -2026,6 +2026,11 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
Config(
"parquet",
useDataFrames = true),
+ Config(
+ "json"),
+ Config(
+ "json",
+ useDataFrames = true),
Config(
"orc"),
Config(
@@ -2083,6 +2088,11 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
Config(
"parquet",
useDataFrames = true),
+ Config(
+ "json"),
+ Config(
+ "json",
+ useDataFrames = true),
Config(
"orc"),
Config(
@@ -2141,6 +2151,14 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
Config(
"parquet",
useDataFrames = true),
+ // SPARK-47029: ALTER COLUMN DROP DEFAULT fails to work correctly with JSON data sources.
+ /*
+ Config(
+ "json"),
+ */
+ Config(
+ "json",
+ useDataFrames = true),
Config(
"orc"),
Config(
@@ -2153,8 +2171,8 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
} else {
sql("insert into t select false")
}
- sql("alter table t add column s map<boolean, string> default map(true, 'abc')")
- checkAnswer(spark.table("t"), Row(false, Map(true -> "abc")))
+ sql("alter table t add column s map<string, boolean> default map('abc', true)")
+ checkAnswer(spark.table("t"), Row(false, Map("abc" -> true)))
}
withTable("t") {
sql(
@@ -2165,12 +2183,12 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
x array<
struct<a int, b int>>,
y array<
- map<boolean, string>>>
+ map<string, boolean>>>
default struct(
array(
struct(1, 2)),
array(
- map(false, 'def', true, 'jkl'))))
+ map('def', false, 'jkl', true))))
using ${config.dataSource}""")
sql("insert into t select 1, default")
sql("alter table t alter column s drop default")
@@ -2186,30 +2204,30 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
array(
struct(3, 4)),
array(
- map(false, 'mno', true, 'pqr')))""")
+ map('mno', false, 'pqr', true)))""")
sql("insert into t select 3, default")
sql(
"""
alter table t
add column t array<
- map<boolean, string>>
+ map<string, boolean>>
default array(
- map(true, 'xyz'))""")
+ map('xyz', true))""")
sql("insert into t(i, s) select 4, default")
checkAnswer(spark.table("t"),
Seq(
Row(1,
- Row(Seq(Row(1, 2)), Seq(Map(false -> "def", true -> "jkl"))),
- Seq(Map(true -> "xyz"))),
+ Row(Seq(Row(1, 2)), Seq(Map("def" -> false, "jkl" -> true))),
+ Seq(Map("xyz" -> true))),
Row(2,
null,
- Seq(Map(true -> "xyz"))),
+ Seq(Map("xyz" -> true))),
Row(3,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz"))),
+ Row(Seq(Row(3, 4)), Seq(Map("mno" -> false, "pqr" -> true))),
+ Seq(Map("xyz" -> true))),
Row(4,
- Row(Seq(Row(3, 4)), Seq(Map(false -> "mno", true -> "pqr"))),
- Seq(Map(true -> "xyz")))))
+ Row(Seq(Row(3, 4)), Seq(Map("mno" -> false, "pqr" -> true))),
+ Seq(Map("xyz" -> true)))))
}
}
// Negative tests: provided map element types must match their corresponding DEFAULT
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org