You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/07/07 03:34:37 UTC
spark git commit: [SPARK-24749][SQL] Use sameType to compare Array's
element type in ArrayContains
Repository: spark
Updated Branches:
refs/heads/master 4de0425df -> fc43690d3
[SPARK-24749][SQL] Use sameType to compare Array's element type in ArrayContains
## What changes were proposed in this pull request?
We should use `DataType.sameType` to compare element type in `ArrayContains`, otherwise nullability affects comparison result.
## How was this patch tested?
Added test.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #21724 from viirya/SPARK-24749.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc43690d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc43690d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc43690d
Branch: refs/heads/master
Commit: fc43690d36e7a17e45826a69ab86935fb0ee2be4
Parents: 4de0425
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Sat Jul 7 11:34:30 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Sat Jul 7 11:34:30 2018 +0800
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/collectionOperations.scala | 2 +-
.../sql/catalyst/expressions/CollectionExpressionsSuite.scala | 7 +++++++
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fc43690d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 8b278f0..fcac3a5 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1085,7 +1085,7 @@ case class ArrayContains(left: Expression, right: Expression)
if (right.dataType == NullType) {
TypeCheckResult.TypeCheckFailure("Null typed values cannot be used as arguments")
} else if (!left.dataType.isInstanceOf[ArrayType]
- || left.dataType.asInstanceOf[ArrayType].elementType != right.dataType) {
+ || !left.dataType.asInstanceOf[ArrayType].elementType.sameType(right.dataType)) {
TypeCheckResult.TypeCheckFailure(
"Arguments must be an array followed by a value of same type as the array members")
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/fc43690d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index d7744eb..496ee1d 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -213,6 +213,8 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
val a1 = Literal.create(Seq[String](null, ""), ArrayType(StringType))
val a2 = Literal.create(Seq(null), ArrayType(LongType))
val a3 = Literal.create(null, ArrayType(StringType))
+ val a4 = Literal.create(Seq(create_row(1)), ArrayType(StructType(Seq(
+ StructField("a", IntegerType, true)))))
checkEvaluation(ArrayContains(a0, Literal(1)), true)
checkEvaluation(ArrayContains(a0, Literal(0)), false)
@@ -228,6 +230,11 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(ArrayContains(a3, Literal("")), null)
checkEvaluation(ArrayContains(a3, Literal.create(null, StringType)), null)
+ checkEvaluation(ArrayContains(a4, Literal.create(create_row(1), StructType(Seq(
+ StructField("a", IntegerType, false))))), true)
+ checkEvaluation(ArrayContains(a4, Literal.create(create_row(0), StructType(Seq(
+ StructField("a", IntegerType, false))))), false)
+
// binary
val b0 = Literal.create(Seq[Array[Byte]](Array[Byte](5, 6), Array[Byte](1, 2)),
ArrayType(BinaryType))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org