You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/09/02 11:22:07 UTC
(spark) branch branch-4.x updated: [SPARK-58443][SQL] Fix wrong results/exception in instr and substring_index for an Integer.MIN_VALUE position or count
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 439a757720be [SPARK-58443][SQL] Fix wrong results/exception in instr and substring_index for an Integer.MIN_VALUE position or count
439a757720be is described below
commit 439a757720becc961efa7453da5818a5b02ae404
Author: Eric Yang <ji...@gmail.com>
AuthorDate: Wed Sep 2 11:21:35 2026 +0000
[SPARK-58443][SQL] Fix wrong results/exception in instr and substring_index for an Integer.MIN_VALUE position or count
### What changes were proposed in this pull request?
Four backward-search paths negate a user-supplied `int` and use the result as a number of characters or delimiters to skip. `-Integer.MIN_VALUE` wraps back to `Integer.MIN_VALUE`, so the value stays negative: the search loop never runs and the out-of-range check that should have fired is bypassed.
Each site now performs the negation in `long` (`-(long) start` / `-(long) count`), so `Integer.MIN_VALUE` stays positive after negation and flows through the same range checks and search loops as every other negative value. No value is special-cased, and the loops still terminate as soon as the string runs out of characters or delimiters.
| Site | Function | Collations | Behavior before |
|---|---|---|---|
| `UTF8String.indexOf(pattern, start, occurrence)` | `instr` | UTF8_BINARY, UTF8_LCASE (all-ASCII fast path) | behaves as if `start = -1` instead of returning 0 |
| `UTF8String.subStringIndex` | `substring_index` | UTF8_BINARY | `NegativeArraySizeException` |
| `CollationAwareUTF8String.lowercaseSubStringIndex` | `substring_index` | UTF8_LCASE | empty string |
| `CollationAwareUTF8String.subStringIndex` | `substring_index` | ICU | `AssertionError` under `-ea`; correct otherwise |
The last row is latent: assertions are enabled in Spark's test JVMs but not in deployments, so it is hardening rather than a user-visible defect.
### Why are the changes needed?
```sql
SELECT instr('abcabc', 'abc', -2147483648, 1); -- returns 4, expected 0
SELECT substring_index('a.b.c', '.', -2147483648); -- NegativeArraySizeException, expected a.b.c
SELECT substring_index(collate('a.b.c', 'UTF8_LCASE'), '.', -2147483648); -- returns '', expected a.b.c
```
### Does this PR introduce _any_ user-facing change?
Yes - bug fix
### How was this patch tested?
New test cases added
### Was this patch authored or co-authored using generative AI tooling?
Yes
Closes #57648 from jiwen624/SPARK-58443.
Authored-by: Eric Yang <ji...@gmail.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
(cherry picked from commit a9e9132bac5093119362d6dbfd0d3b80af2eebfc)
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../spark/sql/catalyst/util/CollationAwareUTF8String.java | 12 +++++++-----
.../java/org/apache/spark/unsafe/types/UTF8String.java | 13 ++++++++-----
.../apache/spark/unsafe/types/CollationSupportSuite.java | 12 ++++++++++++
.../org/apache/spark/unsafe/types/UTF8StringSuite.java | 6 ++++++
.../sql/catalyst/expressions/StringExpressionsSuite.scala | 14 ++++++++++++++
5 files changed, 47 insertions(+), 10 deletions(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
index 0da7af3ca640..5e239212dfc6 100644
--- a/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/sql/catalyst/util/CollationAwareUTF8String.java
@@ -979,7 +979,7 @@ public class CollationAwareUTF8String {
return index;
}
- private static int findIndexReverse(final StringSearch stringSearch, int count) {
+ private static int findIndexReverse(final StringSearch stringSearch, long count) {
assert(count >= 0);
int index = 0;
while (count > 0) {
@@ -1026,7 +1026,8 @@ public class CollationAwareUTF8String {
}
} else {
// If the count is negative, we search for the count-th delimiter from the right.
- int searchIndex = findIndexReverse(stringSearch, -count);
+ // The count is widened to a long because negating it overflows for Integer.MIN_VALUE.
+ int searchIndex = findIndexReverse(stringSearch, -(long) count);
if (searchIndex == MATCH_NOT_FOUND) {
return string;
} else if (searchIndex == str.length()) {
@@ -1057,10 +1058,11 @@ public class CollationAwareUTF8String {
} else {
// Search right to left (note: the end code point is exclusive).
int matchLength = string.numChars() + 1;
- count = -count;
- while (count > 0) {
+ // `remaining` is a long because negating `count` overflows for Integer.MIN_VALUE.
+ long remaining = -(long) count;
+ while (remaining > 0) {
matchLength = lowercaseRFind(string, lowercaseDelimiter, matchLength - 1);
- if (matchLength > MATCH_NOT_FOUND) --count; // Found a delimiter.
+ if (matchLength > MATCH_NOT_FOUND) --remaining; // Found a delimiter.
else return string; // Cannot find enough delimiters in the string.
}
return string.substring(matchLength, string.numChars());
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 03c42785edb2..a9d393381697 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -1289,7 +1289,9 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
}
int charCount = 0;
- int charsToSkip, byteIdx;
+ int byteIdx;
+ // `charsToSkip` is a long because negating `start` overflows for Integer.MIN_VALUE.
+ long charsToSkip;
if (start > 0) {
byteIdx = 0; // position in byte
charsToSkip = start - 1; // skip character count
@@ -1300,7 +1302,7 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
} else {
// For negative start, skip |start| characters from the end to position
// byteIdx at the starting byte of the first character to compare.
- charsToSkip = -start;
+ charsToSkip = -(long) start;
byteIdx = numBytes;
while (byteIdx > 0 && charCount < charsToSkip) {
byteIdx = prevCharStart(byteIdx);
@@ -1452,11 +1454,12 @@ public final class UTF8String implements Comparable<UTF8String>, Externalizable,
} else {
int idx = numBytes - delim.numBytes + 1;
- count = -count;
- while (count > 0) {
+ // `remaining` is a long because negating `count` overflows for Integer.MIN_VALUE.
+ long remaining = -(long) count;
+ while (remaining > 0) {
idx = rfind(delim, idx - 1);
if (idx >= 0) {
- count --;
+ remaining --;
} else {
// can not find enough delim
return this;
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
index 9e7cf3893185..7f3c1a925e02 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CollationSupportSuite.java
@@ -2737,6 +2737,10 @@ public class CollationSupportSuite {
assertSubstringIndex("a🙃b🙃c", "d", -1, UTF8_LCASE, "a🙃b🙃c");
assertSubstringIndex("a🙃b🙃c", "d", -1, UNICODE, "a🙃b🙃c");
assertSubstringIndex("a🙃b🙃c", "d", -1, UNICODE_CI, "a🙃b🙃c");
+ assertSubstringIndex("a.b.c", ".", Integer.MIN_VALUE, UTF8_BINARY, "a.b.c");
+ assertSubstringIndex("a.b.c", ".", Integer.MIN_VALUE, UTF8_LCASE, "a.b.c");
+ assertSubstringIndex("a.b.c", ".", Integer.MIN_VALUE, UNICODE, "a.b.c");
+ assertSubstringIndex("a.b.c", ".", Integer.MIN_VALUE, UNICODE_CI, "a.b.c");
}
/**
@@ -4192,6 +4196,14 @@ public class CollationSupportSuite {
// Boundary: start out of range (forward/backward)
assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", 10, 1, UTF8_LCASE, 0);
assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", -10, 1, UNICODE_CI, 0);
+ assertStringInstrWithOccurrence("abcabc", "abc", Integer.MIN_VALUE, 1, UTF8_BINARY, 0);
+ assertStringInstrWithOccurrence("abcabc", "abc", Integer.MIN_VALUE, 1, UTF8_LCASE, 0);
+ assertStringInstrWithOccurrence("abcabc", "abc", Integer.MIN_VALUE, 1, UNICODE, 0);
+ assertStringInstrWithOccurrence("abcabc", "abc", Integer.MIN_VALUE, 1, UNICODE_CI, 0);
+ assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", Integer.MIN_VALUE, 1, UTF8_BINARY, 0);
+ assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", Integer.MIN_VALUE, 1, UTF8_LCASE, 0);
+ assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", Integer.MIN_VALUE, 1, UNICODE, 0);
+ assertStringInstrWithOccurrence("İoi\u0307oİo", "İo", Integer.MIN_VALUE, 1, UNICODE_CI, 0);
String sigmaStr = "σΣςσΣς"; // 1:σ, 2:Σ, 3:ς, 4:σ, 5:Σ, 6:ς
// UTF8_BINARY: all sigma forms are distinct, only exact byte matches succeed
assertStringInstrWithOccurrence("σΣςσΣς", "Σ", 1, 2, UTF8_BINARY, 5);
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
index 420e49d0a26e..9d67b7b611b1 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java
@@ -332,6 +332,8 @@ public class UTF8StringSuite {
assertEquals(-1, fromString("hello").indexOf(fromString("o"), 6, 1));
assertEquals(0, fromString("hello").indexOf(fromString("h"), -5, 1));
assertEquals(-1, fromString("hello").indexOf(fromString("h"), -6, 1));
+ assertEquals(-1, fromString("abcabc").indexOf(fromString("abc"), Integer.MIN_VALUE, 1));
+ assertEquals(-1, fromString("abcabc").indexOf(fromString("abc"), Integer.MIN_VALUE + 1, 1));
// Target larger than string
assertEquals(-1, fromString("ab").indexOf(fromString("abc"), 1, 1));
@@ -381,6 +383,10 @@ public class UTF8StringSuite {
// overlapped delim
assertEquals(fromString("||"), fromString("||||||").subStringIndex(fromString("|||"), 3));
assertEquals(fromString("|||"), fromString("||||||").subStringIndex(fromString("|||"), -4));
+ assertEquals(fromString("www.apache.org"),
+ fromString("www.apache.org").subStringIndex(fromString("."), Integer.MIN_VALUE));
+ assertEquals(fromString("www.apache.org"),
+ fromString("www.apache.org").subStringIndex(fromString("."), Integer.MIN_VALUE + 1));
}
@Test
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 0cb944192cb8..d6153c29f7b0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -346,6 +346,14 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
SubstringIndex(Literal("www.apache.org"), Literal("."), Literal(-2)), "apache.org")
checkEvaluation(
SubstringIndex(Literal("www.apache.org"), Literal("."), Literal(-1)), "org")
+ // SPARK-58443: negating a count of Integer.MIN_VALUE overflows back to Integer.MIN_VALUE;
+ // such a count is always out of range, so the whole string is returned.
+ checkEvaluation(
+ SubstringIndex(Literal("www.apache.org"), Literal("."), Literal(Int.MinValue)),
+ "www.apache.org")
+ checkEvaluation(
+ SubstringIndex(Literal("www.apache.org"), Literal("."), Literal(Int.MinValue + 1)),
+ "www.apache.org")
checkEvaluation(
SubstringIndex(Literal(""), Literal("."), Literal(-2)), "")
checkEvaluation(
@@ -1115,6 +1123,12 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
Literal("abcabc"), Literal("ab"), Literal(-2), Literal(3)), 0)
checkEvaluation(StringInstrWithOccurrence(
Literal("abcabc"), Literal("ab"), Literal(-3), Literal(3)), 0)
+ // SPARK-58443: negating a start of Integer.MIN_VALUE overflows back to Integer.MIN_VALUE;
+ // such a start is always out of range, so no match is reported.
+ checkEvaluation(StringInstrWithOccurrence(
+ Literal("abcabc"), Literal("abc"), Literal(Int.MinValue), Literal(1)), 0)
+ checkEvaluation(StringInstrWithOccurrence(
+ Literal("abcabc"), Literal("abc"), Literal(Int.MinValue + 1), Literal(1)), 0)
checkEvaluation(StringInstrWithOccurrence(
Literal("abc"), Literal("b"), Literal(0), Literal(1)), 0)
checkEvaluation(StringInstrWithOccurrence(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org