You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/08/27 06:55:33 UTC
[spark] branch branch-3.3 updated (784cb0f68af -> 2de364a4134)
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a change to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
from 784cb0f68af [SPARK-40218][SQL] GROUPING SETS should preserve the grouping columns
new 167f3ff4d75 [SPARK-40152][SQL][TESTS] Move tests from SplitPart to elementAt
new 2de364a4134 [SPARK-40152][SQL][TESTS][FOLLOW-UP] Disable ANSI for out of bound test at ElementAt
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../expressions/CollectionExpressionsSuite.scala | 46 +++++++++++-----------
1 file changed, 23 insertions(+), 23 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 01/02: [SPARK-40152][SQL][TESTS] Move tests from SplitPart to elementAt
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 167f3ff4d752c6f51b71a38378deb47c97f745f0
Author: Yuming Wang <yu...@ebay.com>
AuthorDate: Wed Aug 24 13:33:26 2022 +0900
[SPARK-40152][SQL][TESTS] Move tests from SplitPart to elementAt
Move tests from SplitPart to elementAt in CollectionExpressionsSuite.
Simplify test.
No.
N/A.
Closes #37637 from wangyum/SPARK-40152-3.
Authored-by: Yuming Wang <yu...@ebay.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 06997d6eb73f271aede5b159d86d1db80a73b89f)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../expressions/CollectionExpressionsSuite.scala | 38 ++++++++++------------
1 file changed, 18 insertions(+), 20 deletions(-)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 8fb04cd1ac7..27187a15f43 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -1535,6 +1535,24 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
}
checkEvaluation(ElementAt(mb0, Literal(Array[Byte](2, 1), BinaryType)), "2")
checkEvaluation(ElementAt(mb0, Literal(Array[Byte](3, 4))), null)
+
+ // test defaultValueOutOfBound
+ val delimiter = Literal.create(".", StringType)
+ val str = StringSplitSQL(Literal.create("11.12.13", StringType), delimiter)
+ val outOfBoundValue = Some(Literal.create("", StringType))
+
+ checkEvaluation(ElementAt(str, Literal(3), outOfBoundValue), UTF8String.fromString("13"))
+ checkEvaluation(ElementAt(str, Literal(1), outOfBoundValue), UTF8String.fromString("11"))
+ checkEvaluation(ElementAt(str, Literal(10), outOfBoundValue), UTF8String.fromString(""))
+ checkEvaluation(ElementAt(str, Literal(-10), outOfBoundValue), UTF8String.fromString(""))
+
+ checkEvaluation(ElementAt(StringSplitSQL(Literal.create(null, StringType), delimiter),
+ Literal(1), outOfBoundValue), null)
+ checkEvaluation(ElementAt(StringSplitSQL(Literal.create("11.12.13", StringType),
+ Literal.create(null, StringType)), Literal(1), outOfBoundValue), null)
+
+ checkExceptionInExpression[Exception](
+ ElementAt(str, Literal(0), outOfBoundValue), "The index 0 is invalid")
}
test("correctly handles ElementAt nullability for arrays") {
@@ -2532,24 +2550,4 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Date.valueOf("2017-02-12")))
}
}
-
- test("SplitPart") {
- val delimiter = Literal.create(".", StringType)
- val str = StringSplitSQL(Literal.create("11.12.13", StringType), delimiter)
- val outOfBoundValue = Some(Literal.create("", StringType))
-
- checkEvaluation(ElementAt(str, Literal(3), outOfBoundValue), UTF8String.fromString("13"))
- checkEvaluation(ElementAt(str, Literal(1), outOfBoundValue), UTF8String.fromString("11"))
- checkEvaluation(ElementAt(str, Literal(10), outOfBoundValue), UTF8String.fromString(""))
- checkEvaluation(ElementAt(str, Literal(-10), outOfBoundValue), UTF8String.fromString(""))
-
- checkEvaluation(ElementAt(StringSplitSQL(Literal.create(null, StringType), delimiter),
- Literal(1), outOfBoundValue), null)
- checkEvaluation(ElementAt(StringSplitSQL(Literal.create("11.12.13", StringType),
- Literal.create(null, StringType)), Literal(1), outOfBoundValue), null)
-
- intercept[Exception] {
- checkEvaluation(ElementAt(str, Literal(0), outOfBoundValue), null)
- }.getMessage.contains("The index 0 is invalid")
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
[spark] 02/02: [SPARK-40152][SQL][TESTS][FOLLOW-UP] Disable ANSI for out of bound test at ElementAt
Posted by gu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
commit 2de364a4134670b46f606acabd59f204bcbd7dc2
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Sat Aug 27 14:58:56 2022 +0900
[SPARK-40152][SQL][TESTS][FOLLOW-UP] Disable ANSI for out of bound test at ElementAt
This PR proposes to fix the test to pass with ANSI mode on. Currently `elementAt` test fails when ANSI mode is on:
```
[info] - elementAt *** FAILED *** (309 milliseconds)
[info] Exception evaluating element_at(stringsplitsql(11.12.13, .), 10, Some(), true) (ExpressionEvalHelper.scala:205)
[info] org.scalatest.exceptions.TestFailedException:
[info] at org.scalatest.Assertions.newAssertionFailedException(Assertions.scala:472)
[info] at org.scalatest.Assertions.newAssertionFailedException$(Assertions.scala:471)
[info] at org.scalatest.funsuite.AnyFunSuite.newAssertionFailedException(AnyFunSuite.scala:1563)
[info] at org.scalatest.Assertions.fail(Assertions.scala:949)
[info] at org.scalatest.Assertions.fail$(Assertions.scala:945)
[info] at org.scalatest.funsuite.AnyFunSuite.fail(AnyFunSuite.scala:1563)
[info] at org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper.checkEvaluationWithoutCodegen(ExpressionEvalHelper.scala:205)
[info] at org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper.checkEvaluationWithoutCodegen$(ExpressionEvalHelper.scala:199)
[info] at org.apache.spark.sql.catalyst.expressions.CollectionExpressionsSuite.checkEvaluationWithoutCodegen(CollectionExpressionsSuite.scala:39)
[info] at org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper.checkEvaluation(ExpressionEvalHelper.scala:87)
[info] at org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper.checkEvaluation$(ExpressionEvalHelper.scala:82)
[info] at org.apache.spark.sql.catalyst.expressions.CollectionExpressionsSuite.checkEvaluation(CollectionExpressionsSuite.scala:39)
[info] at org.apache.spark.sql.catalyst.expressions.CollectionExpressionsSuite.$anonfun$new$333(CollectionExpressionsSuite.scala:1546)
```
https://github.com/apache/spark/runs/8046961366?check_suite_focus=true
To recover the build with ANSI mode.
No, test-only.
Unittest fixed.
Closes #37684 from HyukjinKwon/SPARK-40152.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 4b0c3bab1ab082565a051990bf45774f15962deb)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../expressions/CollectionExpressionsSuite.scala | 40 ++++++++++++----------
1 file changed, 21 insertions(+), 19 deletions(-)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 27187a15f43..e121dab4c33 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -1483,7 +1483,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(ElementAt(a0, Literal(0)), null)
}.getMessage.contains("SQL array indices start at 1")
intercept[Exception] { checkEvaluation(ElementAt(a0, Literal(1.1)), null) }
- withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) {
checkEvaluation(ElementAt(a0, Literal(4)), null)
checkEvaluation(ElementAt(a0, Literal(-4)), null)
}
@@ -1512,7 +1512,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
assert(ElementAt(m0, Literal(1.0)).checkInputDataTypes().isFailure)
- withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) {
checkEvaluation(ElementAt(m0, Literal("d")), null)
checkEvaluation(ElementAt(m1, Literal("a")), null)
}
@@ -1529,7 +1529,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
MapType(BinaryType, StringType))
val mb1 = Literal.create(Map[Array[Byte], String](), MapType(BinaryType, StringType))
- withSQLConf(SQLConf.ANSI_ENABLED.key -> "false") {
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) {
checkEvaluation(ElementAt(mb0, Literal(Array[Byte](1, 2, 3))), null)
checkEvaluation(ElementAt(mb1, Literal(Array[Byte](1, 2))), null)
}
@@ -1537,22 +1537,24 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
checkEvaluation(ElementAt(mb0, Literal(Array[Byte](3, 4))), null)
// test defaultValueOutOfBound
- val delimiter = Literal.create(".", StringType)
- val str = StringSplitSQL(Literal.create("11.12.13", StringType), delimiter)
- val outOfBoundValue = Some(Literal.create("", StringType))
-
- checkEvaluation(ElementAt(str, Literal(3), outOfBoundValue), UTF8String.fromString("13"))
- checkEvaluation(ElementAt(str, Literal(1), outOfBoundValue), UTF8String.fromString("11"))
- checkEvaluation(ElementAt(str, Literal(10), outOfBoundValue), UTF8String.fromString(""))
- checkEvaluation(ElementAt(str, Literal(-10), outOfBoundValue), UTF8String.fromString(""))
-
- checkEvaluation(ElementAt(StringSplitSQL(Literal.create(null, StringType), delimiter),
- Literal(1), outOfBoundValue), null)
- checkEvaluation(ElementAt(StringSplitSQL(Literal.create("11.12.13", StringType),
- Literal.create(null, StringType)), Literal(1), outOfBoundValue), null)
-
- checkExceptionInExpression[Exception](
- ElementAt(str, Literal(0), outOfBoundValue), "The index 0 is invalid")
+ withSQLConf(SQLConf.ANSI_ENABLED.key -> false.toString) {
+ val delimiter = Literal.create(".", StringType)
+ val str = StringSplitSQL(Literal.create("11.12.13", StringType), delimiter)
+ val outOfBoundValue = Some(Literal.create("", StringType))
+
+ checkEvaluation(ElementAt(str, Literal(3), outOfBoundValue), UTF8String.fromString("13"))
+ checkEvaluation(ElementAt(str, Literal(1), outOfBoundValue), UTF8String.fromString("11"))
+ checkEvaluation(ElementAt(str, Literal(10), outOfBoundValue), UTF8String.fromString(""))
+ checkEvaluation(ElementAt(str, Literal(-10), outOfBoundValue), UTF8String.fromString(""))
+
+ checkEvaluation(ElementAt(StringSplitSQL(Literal.create(null, StringType), delimiter),
+ Literal(1), outOfBoundValue), null)
+ checkEvaluation(ElementAt(StringSplitSQL(Literal.create("11.12.13", StringType),
+ Literal.create(null, StringType)), Literal(1), outOfBoundValue), null)
+
+ checkExceptionInExpression[Exception](
+ ElementAt(str, Literal(0), outOfBoundValue), "The index 0 is invalid")
+ }
}
test("correctly handles ElementAt nullability for arrays") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org