You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/04/02 14:26:54 UTC
spark git commit: [SPARK-20143][SQL] DataType.fromJson should throw
an exception with better message
Repository: spark
Updated Branches:
refs/heads/master 2287f3d0b -> d40cbb861
[SPARK-20143][SQL] DataType.fromJson should throw an exception with better message
## What changes were proposed in this pull request?
Currently, `DataType.fromJson` throws `scala.MatchError` or `java.util.NoSuchElementException` in some cases when the JSON input is invalid as below:
```scala
DataType.fromJson(""""abcd"""")
```
```
java.util.NoSuchElementException: key not found: abcd
at ...
```
```scala
DataType.fromJson("""{"abcd":"a"}""")
```
```
scala.MatchError: JObject(List((abcd,JString(a)))) (of class org.json4s.JsonAST$JObject)
at ...
```
```scala
DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""")
```
```
scala.MatchError: JObject(List((a,JInt(123)))) (of class org.json4s.JsonAST$JObject)
at ...
```
After this PR,
```scala
DataType.fromJson(""""abcd"""")
```
```
java.lang.IllegalArgumentException: Failed to convert the JSON string 'abcd' to a data type.
at ...
```
```scala
DataType.fromJson("""{"abcd":"a"}""")
```
```
java.lang.IllegalArgumentException: Failed to convert the JSON string '{"abcd":"a"}' to a data type.
at ...
```
```scala
DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""")
at ...
```
```
java.lang.IllegalArgumentException: Failed to convert the JSON string '{"a":123}' to a field.
```
## How was this patch tested?
Unit test added in `DataTypeSuite`.
Author: hyukjinkwon <gu...@gmail.com>
Closes #17468 from HyukjinKwon/fromjson_exception.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d40cbb86
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d40cbb86
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d40cbb86
Branch: refs/heads/master
Commit: d40cbb861898de881621d5053a468af570d72127
Parents: 2287f3d
Author: hyukjinkwon <gu...@gmail.com>
Authored: Sun Apr 2 07:26:49 2017 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Sun Apr 2 07:26:49 2017 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/types/DataType.scala | 12 ++++++++-
.../apache/spark/sql/types/DataTypeSuite.scala | 28 ++++++++++++++++++++
2 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d40cbb86/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
index 2642d93..2687125 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/DataType.scala
@@ -115,7 +115,10 @@ object DataType {
name match {
case "decimal" => DecimalType.USER_DEFAULT
case FIXED_DECIMAL(precision, scale) => DecimalType(precision.toInt, scale.toInt)
- case other => nonDecimalNameToType(other)
+ case other => nonDecimalNameToType.getOrElse(
+ other,
+ throw new IllegalArgumentException(
+ s"Failed to convert the JSON string '$name' to a data type."))
}
}
@@ -164,6 +167,10 @@ object DataType {
("sqlType", v: JValue),
("type", JString("udt"))) =>
new PythonUserDefinedType(parseDataType(v), pyClass, serialized)
+
+ case other =>
+ throw new IllegalArgumentException(
+ s"Failed to convert the JSON string '${compact(render(other))}' to a data type.")
}
private def parseStructField(json: JValue): StructField = json match {
@@ -179,6 +186,9 @@ object DataType {
("nullable", JBool(nullable)),
("type", dataType: JValue)) =>
StructField(name, parseDataType(dataType), nullable)
+ case other =>
+ throw new IllegalArgumentException(
+ s"Failed to convert the JSON string '${compact(render(other))}' to a field.")
}
protected[types] def buildFormattedString(
http://git-wip-us.apache.org/repos/asf/spark/blob/d40cbb86/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
index 05cb999..f078ef0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.types
+import com.fasterxml.jackson.core.JsonParseException
+
import org.apache.spark.{SparkException, SparkFunSuite}
import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
@@ -246,6 +248,32 @@ class DataTypeSuite extends SparkFunSuite {
checkDataTypeFromJson(structType)
checkDataTypeFromDDL(structType)
+ test("fromJson throws an exception when given type string is invalid") {
+ var message = intercept[IllegalArgumentException] {
+ DataType.fromJson(""""abcd"""")
+ }.getMessage
+ assert(message.contains(
+ "Failed to convert the JSON string 'abcd' to a data type."))
+
+ message = intercept[IllegalArgumentException] {
+ DataType.fromJson("""{"abcd":"a"}""")
+ }.getMessage
+ assert(message.contains(
+ """Failed to convert the JSON string '{"abcd":"a"}' to a data type"""))
+
+ message = intercept[IllegalArgumentException] {
+ DataType.fromJson("""{"fields": [{"a":123}], "type": "struct"}""")
+ }.getMessage
+ assert(message.contains(
+ """Failed to convert the JSON string '{"a":123}' to a field."""))
+
+ // Malformed JSON string
+ message = intercept[JsonParseException] {
+ DataType.fromJson("abcd")
+ }.getMessage
+ assert(message.contains("Unrecognized token 'abcd'"))
+ }
+
def checkDefaultSize(dataType: DataType, expectedDefaultSize: Int): Unit = {
test(s"Check the default size of $dataType") {
assert(dataType.defaultSize === expectedDefaultSize)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org