You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/07/31 14:00:29 UTC
(spark) branch master updated: [SPARK-58431][SQL] Preserve ANSI array index errors during optimization
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a027e552eb9b [SPARK-58431][SQL] Preserve ANSI array index errors during optimization
a027e552eb9b is described below
commit a027e552eb9b28b6cd1b48c4d4177d3722332ae1
Author: Marcus Lin <ml...@gmail.com>
AuthorDate: Fri Jul 31 22:00:04 2026 +0800
[SPARK-58431][SQL] Preserve ANSI array index errors during optimization
### What changes were proposed in this pull request?
This PR updates `SimplifyExtractValueOps` so it does not fold out-of-bounds `GetArrayItem(CreateArray(...), literalIndex)` expressions to `NULL` when ANSI error behavior is enabled.
Valid literal indexes are still simplified to the selected element, and non-ANSI out-of-bounds access still folds to `NULL` as before.
This PR also adds regression coverage for both optimizer behavior and the end-to-end SQL error path.
### Why are the changes needed?
Under ANSI mode, out-of-bounds array access should raise `INVALID_ARRAY_INDEX`. The optimizer previously simplified `array(...)[out_of_bounds]` to a null literal even when the expression had `failOnError = true`, which skipped the runtime error and changed query semantics.
### Does this PR introduce _any_ user-facing change?
Yes. This fixes a bug where ANSI out-of-bounds array access could incorrectly return `NULL` after optimization instead of raising `INVALID_ARRAY_INDEX`.
For example, `select array(id, 2, 3)[5] from range(1)` now preserves the expected ANSI error behavior.
### How was this patch tested?
Added regression tests and ran:
`build/sbt 'sql/testOnly org.apache.spark.sql.errors.QueryExecutionAnsiErrorsSuite -- -z "INVALID_ARRAY_INDEX: get element from array"'`
`build/sbt 'catalyst/testOnly org.apache.spark.sql.catalyst.optimizer.ComplexTypesSuite -- -z "SPARK-58431"'`
Also ran `git diff --check`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (OpenAI GPT-5) was used for code assistance and review support; the patch was guided by the contributor.
Closes #57640 from marcuslin123/SPARK-58431-ansi-array-index.
Authored-by: Marcus Lin <ml...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/catalyst/optimizer/ComplexTypes.scala | 3 +++
.../sql/catalyst/optimizer/complexTypesSuite.scala | 18 ++++++++++++++++++
.../sql/errors/QueryExecutionAnsiErrorsSuite.scala | 8 ++++++++
3 files changed, 29 insertions(+)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
index 5c1967c094ff..34c7aa442d4e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/ComplexTypes.scala
@@ -59,6 +59,9 @@ object SimplifyExtractValueOps extends Rule[LogicalPlan] {
if (idx >= 0 && idx < elems.size) {
// valid index
elems(idx)
+ } else if (ga.failOnError) {
+ // Keep ANSI runtime behavior, which raises on out-of-bounds access.
+ ga
} else {
// out of bounds, mimic the runtime behavior and return null
Literal(null, ga.dataType)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index 71acbdfdd2fc..9d4cf9dff953 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -409,6 +409,24 @@ class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
checkRule(mapRel, mapExpected)
}
+ test("SPARK-58431: don't simplify ANSI out-of-bounds array access to null") {
+ val ansiQuery = relation
+ .select(
+ GetArrayItem(
+ CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+ 5,
+ failOnError = true) as "a1")
+ val nonAnsiQuery = relation
+ .select(
+ GetArrayItem(
+ CreateArray(Seq($"nullable_id", $"nullable_id" + 1L)),
+ 5,
+ failOnError = false) as "a1")
+
+ checkRule(ansiQuery, ansiQuery)
+ checkRule(nonAnsiQuery, relation.select(Literal.create(null, LongType) as "a1"))
+ }
+
test("SPARK-23500: Ensure that aggregation expressions are not simplified") {
// Make sure that aggregation exprs are correctly ignored. Maps can't be used in
// grouping exprs so aren't tested here.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
index d5185289e021..1ce04c8064ab 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryExecutionAnsiErrorsSuite.scala
@@ -195,6 +195,14 @@ class QueryExecutionAnsiErrorsSuite extends SharedSparkSession {
context = ExpectedContext(
fragment = "apply",
callSitePattern = getCurrentClassCallSitePattern))
+
+ checkError(
+ exception = intercept[SparkArrayIndexOutOfBoundsException] {
+ sql("select array(id, 2, 3)[5] from range(1)").collect()
+ },
+ condition = "INVALID_ARRAY_INDEX",
+ parameters = Map("indexValue" -> "5", "arraySize" -> "3"),
+ context = ExpectedContext(fragment = "array(id, 2, 3)[5]", start = 7, stop = 24))
}
test("INVALID_ARRAY_INDEX_IN_ELEMENT_AT: element_at from array") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org