You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2016/10/25 19:08:20 UTC
spark git commit: [SPARK-18070][SQL] binary operator should not
consider nullability when comparing input types
Repository: spark
Updated Branches:
refs/heads/master c5fe3dd4f -> a21791e31
[SPARK-18070][SQL] binary operator should not consider nullability when comparing input types
## What changes were proposed in this pull request?
Binary operator requires its inputs to be of same type, but it should not consider nullability, e.g. `EqualTo` should be able to compare an element-nullable array and an element-non-nullable array.
## How was this patch tested?
a regression test in `DataFrameSuite`
Author: Wenchen Fan <we...@databricks.com>
Closes #15606 from cloud-fan/type-bug.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a21791e3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a21791e3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a21791e3
Branch: refs/heads/master
Commit: a21791e3164f4e6546fbe0a90017a4394a05deb1
Parents: c5fe3dd
Author: Wenchen Fan <we...@databricks.com>
Authored: Tue Oct 25 12:08:17 2016 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Oct 25 12:08:17 2016 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/expressions/Expression.scala | 2 +-
.../test/scala/org/apache/spark/sql/DataFrameSuite.scala | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a21791e3/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
index fa1a2ad..9edc1ce 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Expression.scala
@@ -511,7 +511,7 @@ abstract class BinaryOperator extends BinaryExpression with ExpectsInputTypes {
override def checkInputDataTypes(): TypeCheckResult = {
// First check whether left and right have the same type, then check if the type is acceptable.
- if (left.dataType != right.dataType) {
+ if (!left.dataType.sameType(right.dataType)) {
TypeCheckResult.TypeCheckFailure(s"differing types in '$sql' " +
s"(${left.dataType.simpleString} and ${right.dataType.simpleString}).")
} else if (!inputType.acceptsType(left.dataType)) {
http://git-wip-us.apache.org/repos/asf/spark/blob/a21791e3/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 3fb7eee..33b3b78 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1649,4 +1649,13 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
dates.except(widenTypedRows).collect()
dates.intersect(widenTypedRows).collect()
}
+
+ test("SPARK-18070 binary operator should not consider nullability when comparing input types") {
+ val rows = Seq(Row(Seq(1), Seq(1)))
+ val schema = new StructType()
+ .add("array1", ArrayType(IntegerType))
+ .add("array2", ArrayType(IntegerType, containsNull = false))
+ val df = spark.createDataFrame(spark.sparkContext.makeRDD(rows), schema)
+ assert(df.filter($"array1" === $"array2").count() == 1)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org