You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/12/02 14:52:05 UTC
spark git commit: [SPARK-26198][SQL] Fix Metadata serialize null
values throw NPE
Repository: spark
Updated Branches:
refs/heads/master 9cda9a892 -> 676bbb244
[SPARK-26198][SQL] Fix Metadata serialize null values throw NPE
## What changes were proposed in this pull request?
How to reproduce this issue:
```scala
scala> val meta = new org.apache.spark.sql.types.MetadataBuilder().putNull("key").build().json
java.lang.NullPointerException
at org.apache.spark.sql.types.Metadata$.org$apache$spark$sql$types$Metadata$$toJsonValue(Metadata.scala:196)
at org.apache.spark.sql.types.Metadata$$anonfun$1.apply(Metadata.scala:180)
```
This pr fix `NullPointerException` when `Metadata` serialize `null` values.
## How was this patch tested?
unit tests
Closes #23164 from wangyum/SPARK-26198.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Sean Owen <se...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/676bbb24
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/676bbb24
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/676bbb24
Branch: refs/heads/master
Commit: 676bbb2446af1f281b8f76a5428b7ba75b7588b3
Parents: 9cda9a8
Author: Yuming Wang <yu...@ebay.com>
Authored: Sun Dec 2 08:52:01 2018 -0600
Committer: Sean Owen <se...@databricks.com>
Committed: Sun Dec 2 08:52:01 2018 -0600
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/types/Metadata.scala | 2 ++
.../test/scala/org/apache/spark/sql/types/MetadataSuite.scala | 5 +++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/676bbb24/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
index 4979ace..b6a859b 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/Metadata.scala
@@ -190,6 +190,8 @@ object Metadata {
JBool(x)
case x: String =>
JString(x)
+ case null =>
+ JNull
case x: Metadata =>
toJsonValue(x.map)
case other =>
http://git-wip-us.apache.org/repos/asf/spark/blob/676bbb24/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
index 210e657..b4aeac5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/types/MetadataSuite.scala
@@ -26,6 +26,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getString("key") === "value")
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getString("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -36,6 +37,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getLong("key") === 12)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getLong("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -46,6 +48,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getDouble("key") === 12)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getDouble("no_such_key"))
intercept[ClassCastException](meta.getBoolean("key"))
}
@@ -56,6 +59,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.## !== 0)
assert(meta.getBoolean("key") === true)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getBoolean("no_such_key"))
intercept[ClassCastException](meta.getString("key"))
}
@@ -69,6 +73,7 @@ class MetadataSuite extends SparkFunSuite {
assert(meta.getLong("key") === 0)
assert(meta.getBoolean("key") === false)
assert(meta.contains("key"))
+ assert(meta === Metadata.fromJson(meta.json))
intercept[NoSuchElementException](meta.getLong("no_such_key"))
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org