You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/08/24 17:36:40 UTC
spark git commit: [SPARK-21826][SQL][2.1][2.0] outer broadcast hash
join should not throw NPE
Repository: spark
Updated Branches:
refs/heads/branch-2.1 3d3be4dca -> 576975356
[SPARK-21826][SQL][2.1][2.0] outer broadcast hash join should not throw NPE
backport https://github.com/apache/spark/pull/19036 to branch 2.1 and 2.0
Author: Wenchen Fan <we...@databricks.com>
Closes #19040 from cloud-fan/bug.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/57697535
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/57697535
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/57697535
Branch: refs/heads/branch-2.1
Commit: 576975356357ead203e452d0d794794349ba4578
Parents: 3d3be4d
Author: Wenchen Fan <we...@databricks.com>
Authored: Thu Aug 24 10:36:37 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Thu Aug 24 10:36:37 2017 -0700
----------------------------------------------------------------------
.../execution/joins/BroadcastHashJoinExec.scala | 2 +-
.../scala/org/apache/spark/sql/JoinSuite.scala | 20 ++++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/57697535/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
index 0bc261d..69715ab 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala
@@ -257,8 +257,8 @@ case class BroadcastHashJoinExec(
s"""
|boolean $conditionPassed = true;
|${eval.trim}
- |${ev.code}
|if ($matched != null) {
+ | ${ev.code}
| $conditionPassed = !${ev.isNull} && ${ev.value};
|}
""".stripMargin
http://git-wip-us.apache.org/repos/asf/spark/blob/57697535/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
index 913b2ae..3416532 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/JoinSuite.scala
@@ -17,6 +17,7 @@
package org.apache.spark.sql
+import scala.collection.JavaConverters._
import scala.language.existentials
import org.apache.spark.sql.catalyst.analysis.UnresolvedRelation
@@ -24,6 +25,7 @@ import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.execution.joins._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.sql.types.StructType
class JoinSuite extends QueryTest with SharedSQLContext {
@@ -604,4 +606,22 @@ class JoinSuite extends QueryTest with SharedSQLContext {
cartesianQueries.foreach(checkCartesianDetection)
}
+
+ test("outer broadcast hash join should not throw NPE") {
+ withTempView("v1", "v2") {
+ withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "true") {
+ Seq(2 -> 2).toDF("x", "y").createTempView("v1")
+
+ spark.createDataFrame(
+ Seq(Row(1, "a")).asJava,
+ new StructType().add("i", "int", nullable = false).add("j", "string", nullable = false)
+ ).createTempView("v2")
+
+ checkAnswer(
+ sql("select x, y, i, j from v1 left join v2 on x = i and y < length(j)"),
+ Row(2, 2, null, null)
+ )
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org