You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sa...@apache.org on 2021/09/03 14:26:46 UTC
[spark] branch branch-3.2 updated: [SPARK-36639][SQL] Fix an issue
that sequence builtin function causes ArrayIndexOutOfBoundsException if the
arguments are under the condition of start == stop && step < 0
This is an automated email from the ASF dual-hosted git repository.
sarutak pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new a3901ed [SPARK-36639][SQL] Fix an issue that sequence builtin function causes ArrayIndexOutOfBoundsException if the arguments are under the condition of start == stop && step < 0
a3901ed is described below
commit a3901ed3848d21fd36bb5aa265ef8e8d74d8e324
Author: Kousuke Saruta <sa...@oss.nttdata.com>
AuthorDate: Fri Sep 3 23:25:18 2021 +0900
[SPARK-36639][SQL] Fix an issue that sequence builtin function causes ArrayIndexOutOfBoundsException if the arguments are under the condition of start == stop && step < 0
### What changes were proposed in this pull request?
This PR fixes an issue that `sequence` builtin function causes `ArrayIndexOutOfBoundsException` if the arguments are under the condition of `start == stop && step < 0`.
This is an example.
```
SELECT sequence(timestamp'2021-08-31', timestamp'2021-08-31', -INTERVAL 1 month);
21/09/02 04:14:42 ERROR SparkSQLDriver: Failed in [SELECT sequence(timestamp'2021-08-31', timestamp'2021-08-31', -INTERVAL 1 month)]
java.lang.ArrayIndexOutOfBoundsException: 1
```
Actually, this example succeeded before SPARK-31980 (#28819) was merged.
### Why are the changes needed?
Bug fix.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New tests.
Closes #33895 from sarutak/fix-sequence-issue.
Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
(cherry picked from commit cf3bc65e69dcb0f8ba3dee89642d082265edab31)
Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
---
.../catalyst/expressions/collectionOperations.scala | 4 ++--
.../expressions/CollectionExpressionsSuite.scala | 18 ++++++++++++++++++
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index 6cbab86..ce17231 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -2903,7 +2903,7 @@ object Sequence {
val maxEstimatedArrayLength =
getSequenceLength(startMicros, stopMicros, input3, intervalStepInMicros)
- val stepSign = if (stopMicros >= startMicros) +1 else -1
+ val stepSign = if (intervalStepInMicros > 0) +1 else -1
val exclusiveItem = stopMicros + stepSign
val arr = new Array[T](maxEstimatedArrayLength)
var t = startMicros
@@ -2989,7 +2989,7 @@ object Sequence {
|
| $sequenceLengthCode
|
- | final int $stepSign = $stopMicros >= $startMicros ? +1 : -1;
+ | final int $stepSign = $intervalInMicros > 0 ? +1 : -1;
| final long $exclusiveItem = $stopMicros + $stepSign;
|
| $arr = new $elemType[$arrLength];
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index caa5e96..e8f5f07 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -2232,6 +2232,24 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Seq(Date.valueOf("2018-01-01")))
}
+ test("SPARK-36639: Start and end equal in month range with a negative step") {
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval -1 day"))),
+ Seq(Date.valueOf("2018-01-01")))
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval -1 month"))),
+ Seq(Date.valueOf("2018-01-01")))
+ checkEvaluation(new Sequence(
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(Date.valueOf("2018-01-01")),
+ Literal(stringToInterval("interval -1 year"))),
+ Seq(Date.valueOf("2018-01-01")))
+ }
+
test("SPARK-33386: element_at ArrayIndexOutOfBoundsException") {
Seq(true, false).foreach { ansiEnabled =>
withSQLConf(SQLConf.ANSI_ENABLED.key -> ansiEnabled.toString) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org