You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/11/23 19:14:03 UTC
spark git commit: [SPARK-11894][SQL] fix isNull for
GetInternalRowField
Repository: spark
Updated Branches:
refs/heads/master 94ce65dfc -> 1a5baaa65
[SPARK-11894][SQL] fix isNull for GetInternalRowField
We should use `InternalRow.isNullAt` to check if the field is null before calling `InternalRow.getXXX`
Thanks gatorsmile who discovered this bug.
Author: Wenchen Fan <we...@databricks.com>
Closes #9904 from cloud-fan/null.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1a5baaa6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1a5baaa6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1a5baaa6
Branch: refs/heads/master
Commit: 1a5baaa6517872b9a4fd6cd41c4b2cf1e390f6d1
Parents: 94ce65d
Author: Wenchen Fan <we...@databricks.com>
Authored: Mon Nov 23 10:13:59 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Nov 23 10:13:59 2015 -0800
----------------------------------------------------------------------
.../sql/catalyst/expressions/objects.scala | 23 ++++++++------------
.../org/apache/spark/sql/DatasetSuite.scala | 15 ++++++++++++-
2 files changed, 23 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1a5baaa6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
index 82317d3..4a1f419 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
@@ -236,11 +236,6 @@ case class NewInstance(
}
if (propagateNull) {
- val objNullCheck = if (ctx.defaultValue(dataType) == "null") {
- s"${ev.isNull} = ${ev.value} == null;"
- } else {
- ""
- }
val argsNonNull = s"!(${argGen.map(_.isNull).mkString(" || ")})"
s"""
@@ -531,15 +526,15 @@ case class GetInternalRowField(child: Expression, ordinal: Int, dataType: DataTy
throw new UnsupportedOperationException("Only code-generated evaluation is supported")
override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
- val row = child.gen(ctx)
- s"""
- ${row.code}
- final boolean ${ev.isNull} = ${row.isNull};
- ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};
- if (!${ev.isNull}) {
- ${ev.value} = ${ctx.getValue(row.value, dataType, ordinal.toString)};
- }
- """
+ nullSafeCodeGen(ctx, ev, eval => {
+ s"""
+ if ($eval.isNullAt($ordinal)) {
+ ${ev.isNull} = true;
+ } else {
+ ${ev.value} = ${ctx.getValue(eval, dataType, ordinal.toString)};
+ }
+ """
+ })
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/1a5baaa6/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 9da0255..cc8e432 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -386,7 +386,7 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
Seq((JavaData(1), 1L), (JavaData(2), 1L)))
}
- ignore("Java encoder self join") {
+ test("Java encoder self join") {
implicit val kryoEncoder = Encoders.javaSerialization[JavaData]
val ds = Seq(JavaData(1), JavaData(2)).toDS()
assert(ds.joinWith(ds, lit(true)).collect().toSet ==
@@ -396,6 +396,19 @@ class DatasetSuite extends QueryTest with SharedSQLContext {
(JavaData(2), JavaData(1)),
(JavaData(2), JavaData(2))))
}
+
+ test("SPARK-11894: Incorrect results are returned when using null") {
+ val nullInt = null.asInstanceOf[java.lang.Integer]
+ val ds1 = Seq((nullInt, "1"), (new java.lang.Integer(22), "2")).toDS()
+ val ds2 = Seq((nullInt, "1"), (new java.lang.Integer(22), "2")).toDS()
+
+ checkAnswer(
+ ds1.joinWith(ds2, lit(true)),
+ ((nullInt, "1"), (nullInt, "1")),
+ ((new java.lang.Integer(22), "2"), (nullInt, "1")),
+ ((nullInt, "1"), (new java.lang.Integer(22), "2")),
+ ((new java.lang.Integer(22), "2"), (new java.lang.Integer(22), "2")))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org