You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/10/12 08:55:19 UTC

(spark) branch branch-4.x updated: [SPARK-59978][SQL] Handle BIN BY range and boundary overflows

This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new a05ac609ff35 [SPARK-59978][SQL] Handle BIN BY range and boundary overflows
a05ac609ff35 is described below

commit a05ac609ff35033418b7f0165011ab4e98d176e9
Author: Nikolina Vraneš <ni...@databricks.com>
AuthorDate: Mon Oct 12 08:54:57 2026 +0000

    [SPARK-59978][SQL] Handle BIN BY range and boundary overflows
    
    ### What changes were proposed in this pull request?
    
    This PR uses checked arithmetic for BIN BY range duration and bucket boundary calculations. It
    translates arithmetic failures from these timestamp calculations to `DATETIME_OVERFLOW` and adds
    regression tests for each overflow path.
    
    ### Why are the changes needed?
    
    The next bucket index could wrap from `Long.MaxValue` to `Long.MinValue` and produce a bin end
    before its start. Other extreme ranges could expose a generic arithmetic overflow while locating
    or advancing bucket boundaries. BIN BY now fails consistently with a datetime-specific error.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Extreme BIN BY ranges now raise `DATETIME_OVERFLOW` instead of returning an invalid boundary
    or exposing a generic arithmetic overflow.
    
    ### How was this patch tested?
    
    ```
    build/sbt 'sql/testOnly *BinBySuite'
    build/sbt 'sql/scalastyle' 'sql/Test/scalastyle'
    ```
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-5)
    
    Closes #59239 from vranes/fix-bin-by-boundary-index-overflow.
    
    Authored-by: Nikolina Vraneš <ni...@databricks.com>
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
    (cherry picked from commit 354da2a3207f6fb12e2924dcee64a90cb235f113)
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
 .../spark/sql/errors/QueryExecutionErrors.scala    |  8 +++
 .../org/apache/spark/sql/execution/BinByExec.scala | 36 +++++++---
 .../scala/org/apache/spark/sql/BinBySuite.scala    | 80 +++++++++++++++++++++-
 3 files changed, 114 insertions(+), 10 deletions(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
index e066ed191c38..3f9423f8ce90 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryExecutionErrors.scala
@@ -3616,4 +3616,12 @@ private[sql] object QueryExecutionErrors extends QueryErrorsBase with ExecutionE
       messageParameters = Map("rangeStart" -> rangeStart, "rangeEnd" -> rangeEnd)
     )
   }
+
+  def binByOverflowError(): SparkArithmeticException = {
+    new SparkArithmeticException(
+      errorClass = "DATETIME_OVERFLOW",
+      messageParameters = Map("operation" -> "compute BIN BY bucket boundaries"),
+      context = Array.empty,
+      summary = "")
+  }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/BinByExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/BinByExec.scala
index cc35971aa898..dd61082f1d5a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/BinByExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/BinByExec.scala
@@ -24,7 +24,7 @@ import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeMap, AttributeSet, BindReferences, BoundReference, Cast, Expression, GenericInternalRow, JoinedRow, Literal, Multiply, NamedExpression, UnsafeProjection}
 import org.apache.spark.sql.catalyst.plans.logical.BinBy
-import org.apache.spark.sql.catalyst.util.{DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateTimeUtils, MathUtils, TimestampFormatter}
 import org.apache.spark.sql.errors.QueryExecutionErrors
 import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.types.DoubleType
@@ -109,6 +109,15 @@ case class BinByExec(
       val joined = new JoinedRow
       val fmt = TimestampFormatter.getFractionFormatter(zone)
 
+      def withBinByOverflow[T](f: => T): T = {
+        try {
+          f
+        } catch {
+          case _: ArithmeticException =>
+            throw QueryExecutionErrors.binByOverflowError()
+        }
+      }
+
       rows.flatMap { row =>
         joined.withLeft(row)
         val rowIter: Iterator[InternalRow] =
@@ -120,31 +129,40 @@ case class BinByExec(
             if (rs > re) {
               throw QueryExecutionErrors.binByInvalidRangeError(fmt.format(rs), fmt.format(re))
             } else if (rs == re) {
-              val (k, binStart) = DateTimeUtils.timeBucketFromTimestampDTInterval(width, rs, origin,
-                zone)
-              val binEnd = DateTimeUtils.timeBucketFromIndexDTInterval(width, k + 1, origin, zone)
+              val (k, binStart) = withBinByOverflow {
+                DateTimeUtils.timeBucketFromTimestampDTInterval(width, rs, origin, zone)
+              }
+              val binEnd = withBinByOverflow {
+                val nextK = MathUtils.addExact(k, 1L)
+                DateTimeUtils.timeBucketFromIndexDTInterval(width, nextK, origin, zone)
+              }
               appended.update(0, binStart)
               appended.update(1, binEnd)
               appended.update(2, 1.0d)
               Iterator.single(proj(joined.withRight(appended)))
             } else {
-              val total = Math.subtractExact(re, rs)
+              val total = withBinByOverflow(MathUtils.subtractExact(re, rs))
               new Iterator[InternalRow] {
                 // Advance the bucket index and read each boundary off the grid, rather than
                 // walking one bin end to the next start.
-                private var (k, curStart) =
+                private var (k, curStart) = withBinByOverflow {
                   DateTimeUtils.timeBucketFromTimestampDTInterval(width, rs, origin, zone)
+                }
 
                 override def hasNext: Boolean = curStart < re
 
                 override def next(): InternalRow = {
-                  val curEnd =
-                    DateTimeUtils.timeBucketFromIndexDTInterval(width, k + 1, origin, zone)
+                  val (nextK, curEnd) = withBinByOverflow {
+                    val index = MathUtils.addExact(k, 1L)
+                    val end =
+                      DateTimeUtils.timeBucketFromIndexDTInterval(width, index, origin, zone)
+                    (index, end)
+                  }
                   val overlap = math.min(re, curEnd) - math.max(rs, curStart)
                   appended.update(0, curStart)
                   appended.update(1, curEnd)
                   appended.update(2, overlap.toDouble / total.toDouble)
-                  k += 1
+                  k = nextK
                   curStart = curEnd
                   proj(joined.withRight(appended))
                 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/BinBySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/BinBySuite.scala
index fe5af74d4d2a..e51f9955adc6 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/BinBySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/BinBySuite.scala
@@ -20,12 +20,14 @@ package org.apache.spark.sql
 import java.sql.Timestamp
 import java.time.{LocalDateTime, ZoneId, ZoneOffset}
 
-import org.apache.spark.SparkThrowable
+import org.apache.spark.{SparkArithmeticException, SparkThrowable}
+import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import org.apache.spark.sql.execution.BinByExec
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 
 class BinBySuite extends QueryTest with SharedSparkSession {
+  import testImplicits._
 
   private def tsAt(s: String, zone: ZoneId = ZoneOffset.UTC): Timestamp =
     Timestamp.from(LocalDateTime.parse(s.replace(' ', 'T')).atZone(zone).toInstant)
@@ -35,6 +37,32 @@ class BinBySuite extends QueryTest with SharedSparkSession {
   private def ratio(overlapMicros: Long, totalMicros: Long): Double =
     overlapMicros.toDouble / totalMicros.toDouble
 
+  private def checkBinByDatetimeOverflow(
+      rangeStart: LocalDateTime,
+      rangeEnd: LocalDateTime,
+      binWidthSql: String,
+      originSql: String): Unit = {
+    withSQLConf(SQLConf.BIN_BY_ENABLED.key -> "true") {
+      withTempView("bin_by_overflow") {
+        Seq((rangeStart, rangeEnd, 1.0d))
+          .toDF("ts_start", "ts_end", "value")
+          .createOrReplaceTempView("bin_by_overflow")
+
+        checkError(
+          exception = intercept[SparkArithmeticException] {
+            spark.sql(
+              s"""SELECT * FROM bin_by_overflow
+                 |BIN BY (
+                 |  RANGE ts_start TO ts_end BIN WIDTH $binWidthSql
+                 |  ALIGN TO $originSql
+                 |  DISTRIBUTE UNIFORM (value))""".stripMargin).collect()
+          },
+          condition = "DATETIME_OVERFLOW",
+          parameters = Map("operation" -> "compute BIN BY bucket boundaries"))
+      }
+    }
+  }
+
   private def binByExecArgs(sqlText: String): String = {
     val plan = spark.sql(sqlText).queryExecution.executedPlan
     val binByExec = plan.collectFirst {
@@ -428,6 +456,56 @@ class BinBySuite extends QueryTest with SharedSparkSession {
     }
   }
 
+  test("BIN BY raises DATETIME_OVERFLOW when the next bin index is not representable") {
+    val maxTimestamp = DateTimeUtils.microsToLocalDateTime(Long.MaxValue)
+    checkBinByDatetimeOverflow(
+      rangeStart = maxTimestamp,
+      rangeEnd = maxTimestamp,
+      binWidthSql = "INTERVAL '0.000001' SECOND",
+      originSql = "TIMESTAMP_NTZ '1970-01-01 00:00:00'")
+  }
+
+  test("BIN BY raises DATETIME_OVERFLOW when the range duration is not representable") {
+    checkBinByDatetimeOverflow(
+      rangeStart = DateTimeUtils.microsToLocalDateTime(Long.MinValue),
+      rangeEnd = DateTimeUtils.microsToLocalDateTime(Long.MaxValue),
+      binWidthSql = "INTERVAL '1' SECOND",
+      originSql = "TIMESTAMP_NTZ '1970-01-01 00:00:00'")
+  }
+
+  test("BIN BY raises DATETIME_OVERFLOW when locating a zero-length range bin overflows") {
+    val maxTimestamp = DateTimeUtils.microsToLocalDateTime(Long.MaxValue)
+    checkBinByDatetimeOverflow(
+      rangeStart = maxTimestamp,
+      rangeEnd = maxTimestamp,
+      binWidthSql = "INTERVAL '1' DAY",
+      originSql = "TIMESTAMP_NTZ '1969-12-31 23:59:59.999999'")
+  }
+
+  test("BIN BY raises DATETIME_OVERFLOW when locating a non-empty range bin overflows") {
+    checkBinByDatetimeOverflow(
+      rangeStart = DateTimeUtils.microsToLocalDateTime(Long.MaxValue - 1L),
+      rangeEnd = DateTimeUtils.microsToLocalDateTime(Long.MaxValue),
+      binWidthSql = "INTERVAL '1' DAY",
+      originSql = "TIMESTAMP_NTZ '1969-12-31 23:59:59.999998'")
+  }
+
+  test("BIN BY raises DATETIME_OVERFLOW when advancing a bin index overflows") {
+    checkBinByDatetimeOverflow(
+      rangeStart = DateTimeUtils.microsToLocalDateTime(Long.MaxValue - 1L),
+      rangeEnd = DateTimeUtils.microsToLocalDateTime(Long.MaxValue),
+      binWidthSql = "INTERVAL '0.000001' SECOND",
+      originSql = "TIMESTAMP_NTZ '1969-12-31 23:59:59.999999'")
+  }
+
+  test("BIN BY raises DATETIME_OVERFLOW when a multi-day bin boundary overflows") {
+    checkBinByDatetimeOverflow(
+      rangeStart = ntz("2020-01-01 00:00:00"),
+      rangeEnd = ntz("2020-01-02 00:00:00"),
+      binWidthSql = "INTERVAL '106751990' DAY",
+      originSql = "TIMESTAMP_NTZ '2000-01-01 00:00:00'")
+  }
+
   test("BIN BY raises BIN_BY_INVALID_RANGE for an inverted range") {
     withSQLConf(
         SQLConf.BIN_BY_ENABLED.key -> "true",


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org