You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2021/08/06 11:58:38 UTC
[spark] branch branch-3.2 updated: Revert "[SPARK-36429][SQL]
JacksonParser should throw exception when data type unsupported"
This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 586eb5d Revert "[SPARK-36429][SQL] JacksonParser should throw exception when data type unsupported"
586eb5d is described below
commit 586eb5d4c6b01b008cb0ace076f94f49580201de
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Fri Aug 6 20:56:24 2021 +0900
Revert "[SPARK-36429][SQL] JacksonParser should throw exception when data type unsupported"
### What changes were proposed in this pull request?
This PR reverts the change in SPARK-36429 (#33654).
See [conversation](https://github.com/apache/spark/pull/33654#issuecomment-894160037).
### Why are the changes needed?
To recover CIs.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
N/A
Closes #33670 from sarutak/revert-SPARK-36429.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
(cherry picked from commit e17612d0bfa1b1dc719f6f2c202e2a4ea7870ff1)
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
---
.../scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala | 8 ++++++--
.../sql-tests/results/timestampNTZ/timestamp-ansi.sql.out | 5 ++---
.../resources/sql-tests/results/timestampNTZ/timestamp.sql.out | 5 ++---
3 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index 2761c52..04a0f1a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -330,8 +330,12 @@ class JacksonParser(
case udt: UserDefinedType[_] =>
makeConverter(udt.sqlType)
- // We don't actually hit this exception though, we keep it for understandability
- case _ => throw QueryExecutionErrors.unsupportedTypeError(dataType)
+ case _ =>
+ (parser: JsonParser) =>
+ // Here, we pass empty `PartialFunction` so that this case can be
+ // handled as a failed conversion. It will throw an exception as
+ // long as the value is not null.
+ parseJsonToken[AnyRef](parser, dataType)(PartialFunction.empty[JsonToken, AnyRef])
}
/**
diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
index fae7721..fe83675 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp-ansi.sql.out
@@ -661,10 +661,9 @@ You may get a different result due to the upgrading of Spark 3.0: Fail to recogn
-- !query
select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
-- !query schema
-struct<>
+struct<from_json({"t":"26/October/2015"}):struct<t:timestamp_ntz>>
-- !query output
-java.lang.Exception
-Unsupported type: timestamp_ntz
+{"t":null}
-- !query
diff --git a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out
index c6de535..b8a6800 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestampNTZ/timestamp.sql.out
@@ -642,10 +642,9 @@ You may get a different result due to the upgrading of Spark 3.0: Fail to recogn
-- !query
select from_json('{"t":"26/October/2015"}', 't Timestamp', map('timestampFormat', 'dd/MMMMM/yyyy'))
-- !query schema
-struct<>
+struct<from_json({"t":"26/October/2015"}):struct<t:timestamp_ntz>>
-- !query output
-java.lang.Exception
-Unsupported type: timestamp_ntz
+{"t":null}
-- !query
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org