You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/11/23 06:57:09 UTC
(spark) branch branch-4.1 updated: [SPARK-50072][SQL] Handle ArithmeticException in interval parsing with large values
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 13d62dc60f28 [SPARK-50072][SQL] Handle ArithmeticException in interval parsing with large values
13d62dc60f28 is described below
commit 13d62dc60f28c084875ecb87ad59a9d4dd40fbb3
Author: AbinayaJayaprakasam <ja...@gmail.com>
AuthorDate: Sat Nov 22 22:56:45 2025 -0800
[SPARK-50072][SQL] Handle ArithmeticException in interval parsing with large values
SPARK-50072][SQL] Handle ArithmeticException in interval parsing with large values
### What changes were proposed in this pull request?
This PR will fixe SPARK-50072 where parsing intervals with large day values throws an raw java.lang.ArithmeticException instead of a user-friendly SparkArithmeticException.
### Why are the changes needed?
When executing queries like 'SELECT interval 106751991 day 4 hour 0 minute 54.776 second', Spark throws a raw ArithmeticException with 'long overflow' message instead of a SparkArithmeticException with error class INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION. This will improve user experience in terms of understanding the error.
### Does this PR introduce any user-facing change?
Yes. Users will now see a proper SparkArithmeticException with error class INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION instead of a raw ArithmeticException.
### How was this patch tested?
- I Added unit test 'interval overflow with large day values' in IntervalUtilsSuite
- Updated existing 'interval duration' test to expect SparkArithmeticException
- Added SQL integration test in interval.sql -i have manually tested with spark-shell
### Was this patch authored or co-authored using generative AI tooling?
No.
### What changes were proposed in this pull request?
### Why are the changes needed?
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
### Was this patch authored or co-authored using generative AI tooling?
Closes #53172 from AbinayaJayaprakasam/SPARK-50072-fix.
Authored-by: AbinayaJayaprakasam <ja...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 3861ba498f6d78f9603949dc145395e162f51cdc)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../spark/sql/catalyst/util/IntervalUtils.scala | 21 ++++++++++-------
.../sql/catalyst/util/IntervalUtilsSuite.scala | 27 ++++++++++++++++++----
.../sql-tests/analyzer-results/interval.sql.out | 24 +++++++++++++++----
.../analyzer-results/nonansi/interval.sql.out | 24 +++++++++++++++----
.../test/resources/sql-tests/inputs/interval.sql | 5 ++++
.../resources/sql-tests/results/interval.sql.out | 26 +++++++++++++++++----
.../sql-tests/results/nonansi/interval.sql.out | 26 +++++++++++++++++----
7 files changed, 125 insertions(+), 28 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
index 8793c0407a9b..ef10a308cff9 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala
@@ -594,14 +594,19 @@ object IntervalUtils extends SparkIntervalUtils {
interval: CalendarInterval,
targetUnit: TimeUnit,
daysPerMonth: Int = 31): Long = {
- val monthsDuration = Math.multiplyExact(
- daysPerMonth * MICROS_PER_DAY,
- interval.months)
- val daysDuration = Math.multiplyExact(
- MICROS_PER_DAY,
- interval.days)
- val result = Math.addExact(interval.microseconds, Math.addExact(daysDuration, monthsDuration))
- targetUnit.convert(result, TimeUnit.MICROSECONDS)
+ try {
+ val monthsDuration = Math.multiplyExact(
+ daysPerMonth * MICROS_PER_DAY,
+ interval.months)
+ val daysDuration = Math.multiplyExact(
+ MICROS_PER_DAY,
+ interval.days)
+ val result = Math.addExact(interval.microseconds, Math.addExact(daysDuration, monthsDuration))
+ targetUnit.convert(result, TimeUnit.MICROSECONDS)
+ } catch {
+ case _: ArithmeticException =>
+ throw QueryExecutionErrors.withoutSuggestionIntervalArithmeticOverflowError(context = null)
+ }
}
/**
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
index cbaebfa12238..a87d599711cf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.util
import java.time.{Duration, Period}
import java.util.concurrent.TimeUnit
-import org.apache.spark.{SparkFunSuite, SparkIllegalArgumentException}
+import org.apache.spark.{SparkArithmeticException, SparkFunSuite, SparkIllegalArgumentException}
import org.apache.spark.sql.catalyst.plans.SQLHelper
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.DateTimeUtils.millisToMicros
@@ -364,10 +364,29 @@ class IntervalUtilsSuite extends SparkFunSuite with SQLHelper {
assert(duration("1 microsecond", TimeUnit.MICROSECONDS, 30) === 1)
assert(duration("1 month -30 days", TimeUnit.DAYS, 31) === 1)
- val e = intercept[ArithmeticException] {
- duration(s"${Integer.MAX_VALUE} month", TimeUnit.SECONDS, 31)
+ checkError(
+ exception = intercept[SparkArithmeticException] {
+ duration(s"${Integer.MAX_VALUE} month", TimeUnit.SECONDS, 31)
+ },
+ condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ parameters = Map.empty
+ )
+ }
+
+ test("interval overflow with large day values") {
+ // Test case for SPARK-50072: handling ArithmeticException during interval parsing
+ // The value 106751991 days causes overflow when converted to microseconds
+ def duration(s: String, unit: TimeUnit): Long = {
+ IntervalUtils.getDuration(stringToInterval(UTF8String.fromString(s)), unit)
}
- assert(e.getMessage.contains("overflow"))
+
+ checkError(
+ exception = intercept[SparkArithmeticException] {
+ duration("106751991 days 4 hours 0 minutes 54.776 seconds", TimeUnit.MICROSECONDS)
+ },
+ condition = "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ parameters = Map.empty
+ )
}
test("negative interval") {
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/interval.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/interval.sql.out
index c023e3b56f11..88121e4dc3c2 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/interval.sql.out
@@ -61,15 +61,21 @@ Project [(INTERVAL '2147483647' MONTH / 0.5) AS (INTERVAL '2147483647' MONTH / 0
-- !query
select interval 2147483647 day * 2
-- !query analysis
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
select interval 2147483647 day / 0.5
-- !query analysis
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -3212,3 +3218,13 @@ SELECT width_bucket(INTERVAL '-59' MINUTE, INTERVAL -'1 01' DAY TO HOUR, INTERVA
-- !query analysis
Project [width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, cast(10 as bigint)) AS width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, 10)#xL]
+- OneRowRelation
+
+
+-- !query
+SELECT interval 106751991 day 4 hour 0 minute 54.776 second
+-- !query analysis
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/interval.sql.out
index c0196bbe118e..259cb2bff5ef 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/nonansi/interval.sql.out
@@ -61,15 +61,21 @@ Project [(INTERVAL '2147483647' MONTH / 0.5) AS (INTERVAL '2147483647' MONTH / 0
-- !query
select interval 2147483647 day * 2
-- !query analysis
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
select interval 2147483647 day / 0.5
-- !query analysis
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -3212,3 +3218,13 @@ SELECT width_bucket(INTERVAL '-59' MINUTE, INTERVAL -'1 01' DAY TO HOUR, INTERVA
-- !query analysis
Project [width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, cast(10 as bigint)) AS width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, 10)#xL]
+- OneRowRelation
+
+
+-- !query
+SELECT interval 106751991 day 4 hour 0 minute 54.776 second
+-- !query analysis
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
index e4da28c2e758..e8e10d089dec 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql
@@ -385,3 +385,8 @@ SELECT width_bucket(INTERVAL '0' YEAR, INTERVAL '0' YEAR, INTERVAL '10' YEAR, 10
SELECT width_bucket(INTERVAL '-1' YEAR, INTERVAL -'1-2' YEAR TO MONTH, INTERVAL '1-2' YEAR TO MONTH, 10);
SELECT width_bucket(INTERVAL '0' DAY, INTERVAL '0' DAY, INTERVAL '10' DAY, 10);
SELECT width_bucket(INTERVAL '-59' MINUTE, INTERVAL -'1 01' DAY TO HOUR, INTERVAL '1 2:3:4.001' DAY TO SECOND, 10);
+
+-- interval overflow with large day values (SPARK-50072)
+-- This should throw INTERVAL_ARITHMETIC_OVERFLOW error
+SELECT interval 106751991 day 4 hour 0 minute 54.776 second;
+
diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
index c32ecc19448a..79fce5906df8 100644
--- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out
@@ -75,8 +75,11 @@ select interval 2147483647 day * 2
-- !query schema
struct<>
-- !query output
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -84,8 +87,11 @@ select interval 2147483647 day / 0.5
-- !query schema
struct<>
-- !query output
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -3897,3 +3903,15 @@ SELECT width_bucket(INTERVAL '-59' MINUTE, INTERVAL -'1 01' DAY TO HOUR, INTERVA
struct<width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, 10):bigint>
-- !query output
5
+
+
+-- !query
+SELECT interval 106751991 day 4 hour 0 minute 54.776 second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
diff --git a/sql/core/src/test/resources/sql-tests/results/nonansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/nonansi/interval.sql.out
index a8a0423bdb3e..e40053d1bc79 100644
--- a/sql/core/src/test/resources/sql-tests/results/nonansi/interval.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/nonansi/interval.sql.out
@@ -75,8 +75,11 @@ select interval 2147483647 day * 2
-- !query schema
struct<>
-- !query output
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -84,8 +87,11 @@ select interval 2147483647 day / 0.5
-- !query schema
struct<>
-- !query output
-java.lang.ArithmeticException
-long overflow
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
-- !query
@@ -3710,3 +3716,15 @@ SELECT width_bucket(INTERVAL '-59' MINUTE, INTERVAL -'1 01' DAY TO HOUR, INTERVA
struct<width_bucket(INTERVAL '-59' MINUTE, INTERVAL '-1 01' DAY TO HOUR, INTERVAL '1 02:03:04.001' DAY TO SECOND, 10):bigint>
-- !query output
5
+
+
+-- !query
+SELECT interval 106751991 day 4 hour 0 minute 54.776 second
+-- !query schema
+struct<>
+-- !query output
+org.apache.spark.SparkArithmeticException
+{
+ "errorClass" : "INTERVAL_ARITHMETIC_OVERFLOW.WITHOUT_SUGGESTION",
+ "sqlState" : "22015"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org