You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/12/11 02:22:46 UTC
spark git commit: [SPARK-12258][SQL] passing null into ScalaUDF
Repository: spark
Updated Branches:
refs/heads/master 24d3357d6 -> b1b4ee7f3
[SPARK-12258][SQL] passing null into ScalaUDF
Check nullability and passing them into ScalaUDF.
Closes #10249
Author: Davies Liu <da...@databricks.com>
Closes #10259 from davies/udf_null.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b1b4ee7f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b1b4ee7f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b1b4ee7f
Branch: refs/heads/master
Commit: b1b4ee7f3541d92c8bc2b0b4fdadf46cfdb09504
Parents: 24d3357
Author: Davies Liu <da...@databricks.com>
Authored: Thu Dec 10 17:22:18 2015 -0800
Committer: Yin Huai <yh...@databricks.com>
Committed: Thu Dec 10 17:22:18 2015 -0800
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/expressions/ScalaUDF.scala | 7 +++++--
.../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 9 +++++----
2 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b1b4ee7f/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
index 03b8922..5deb2f8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ScalaUDF.scala
@@ -1029,8 +1029,11 @@ case class ScalaUDF(
// such as IntegerType, its javaType is `int` and the returned type of user-defined
// function is Object. Trying to convert an Object to `int` will cause casting exception.
val evalCode = evals.map(_.code).mkString
- val funcArguments = converterTerms.zip(evals).map {
- case (converter, eval) => s"$converter.apply(${eval.value})"
+ val funcArguments = converterTerms.zipWithIndex.map {
+ case (converter, i) =>
+ val eval = evals(i)
+ val dt = children(i).dataType
+ s"$converter.apply(${eval.isNull} ? null : (${ctx.boxedType(dt)}) ${eval.value})"
}.mkString(",")
val callFunc = s"${ctx.boxedType(ctx.javaType(dataType))} $resultTerm = " +
s"(${ctx.boxedType(ctx.javaType(dataType))})${catalystConverterTerm}" +
http://git-wip-us.apache.org/repos/asf/spark/blob/b1b4ee7f/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 605a654..8887dc6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1138,14 +1138,15 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
}
test("SPARK-11725: correctly handle null inputs for ScalaUDF") {
- val df = Seq(
+ val df = sparkContext.parallelize(Seq(
new java.lang.Integer(22) -> "John",
- null.asInstanceOf[java.lang.Integer] -> "Lucy").toDF("age", "name")
+ null.asInstanceOf[java.lang.Integer] -> "Lucy")).toDF("age", "name")
+ // passing null into the UDF that could handle it
val boxedUDF = udf[java.lang.Integer, java.lang.Integer] {
- (i: java.lang.Integer) => if (i == null) null else i * 2
+ (i: java.lang.Integer) => if (i == null) -10 else i * 2
}
- checkAnswer(df.select(boxedUDF($"age")), Row(44) :: Row(null) :: Nil)
+ checkAnswer(df.select(boxedUDF($"age")), Row(44) :: Row(-10) :: Nil)
val primitiveUDF = udf((i: Int) => i * 2)
checkAnswer(df.select(primitiveUDF($"age")), Row(44) :: Row(null) :: Nil)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org