You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/11/05 07:07:56 UTC
spark git commit: [SPARK-18260] Make from_json null safe
Repository: spark
Updated Branches:
refs/heads/master 8a9ca1924 -> 6e2701815
[SPARK-18260] Make from_json null safe
## What changes were proposed in this pull request?
`from_json` is currently not safe against `null` rows. This PR adds a fix and a regression test for it.
## How was this patch tested?
Regression test
Author: Burak Yavuz <br...@gmail.com>
Closes #15771 from brkyvz/json_fix.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6e270181
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6e270181
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6e270181
Branch: refs/heads/master
Commit: 6e2701815761d5870111cb56300e30d3059b39ed
Parents: 8a9ca19
Author: Burak Yavuz <br...@gmail.com>
Authored: Sat Nov 5 00:07:51 2016 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Sat Nov 5 00:07:51 2016 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/jsonExpressions.scala | 4 +++-
.../sql/catalyst/expressions/JsonExpressionsSuite.scala | 8 ++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6e270181/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index e034735..89fe7c4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -498,7 +498,9 @@ case class JsonToStruct(schema: StructType, options: Map[String, String], child:
override def children: Seq[Expression] = child :: Nil
override def eval(input: InternalRow): Any = {
- try parser.parse(child.eval(input).toString).head catch {
+ val json = child.eval(input)
+ if (json == null) return null
+ try parser.parse(json.toString).head catch {
case _: SparkSQLJsonProcessingException => null
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6e270181/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index f9db649..3bfa0bf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -344,6 +344,14 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
)
}
+ test("from_json null input column") {
+ val schema = StructType(StructField("a", IntegerType) :: Nil)
+ checkEvaluation(
+ JsonToStruct(schema, Map.empty, Literal(null)),
+ null
+ )
+ }
+
test("to_json") {
val schema = StructType(StructField("a", IntegerType) :: Nil)
val struct = Literal.create(create_row(1), schema)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org