You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/09/06 20:01:02 UTC
(spark) branch master updated: [SPARK-59282][SQL][TEST] Reuse the shared nanosecond truncation helper in timestamp nanos tests
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f9b447d3e6d6 [SPARK-59282][SQL][TEST] Reuse the shared nanosecond truncation helper in timestamp nanos tests
f9b447d3e6d6 is described below
commit f9b447d3e6d6127ff223a3897762a6587c2cde88
Author: Stevo Mitric <st...@gmail.com>
AuthorDate: Sun Sep 6 20:00:49 2026 +0000
[SPARK-59282][SQL][TEST] Reuse the shared nanosecond truncation helper in timestamp nanos tests
### What changes were proposed in this pull request?
Replace the locally-duplicated sub-microsecond truncation helpers `floorNanosToPrecision` (in `CastSuiteBase`) and `flooredNano` (in `TimestampNanosRenderingSuiteBase`) with the shared production helper `SparkDateTimeUtils.truncateNanosWithinMicroToPrecision`, and delete the comments that described the duplicated rule. Pure test-only refactor: no production code and no assertion values change.
### Why are the changes needed?
Both suites re-implemented, inline, the same sub-microsecond truncation rule that the code under test uses, so the expected values drifted away from a single source of truth. Both local copies were also non-exhaustive (`MatchError` on a precision outside [7, 9]) whereas the shared helper raises a clear internal error.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing suites pass unchanged: `CastWithAnsiOnSuite` / `CastWithAnsiOffSuite` (270 tests) and `TimestampNanosRenderingAnsiOnSuite` / `TimestampNanosRenderingAnsiOffSuite` (10 tests).
### Was this patch authored or co-authored using generative AI tooling?
Co-authored-by: Claude Opus 5
Closes #58548 from stevomitric/stevomitric/nanos-truncation-dedup.
Authored-by: Stevo Mitric <st...@gmail.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../sql/catalyst/expressions/CastSuiteBase.scala | 38 +++++++++-------------
.../sql/TimestampNanosRenderingSuiteBase.scala | 20 +++++++-----
2 files changed, 26 insertions(+), 32 deletions(-)
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
index 8f37f17c1069..643e28ce9fa0 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala
@@ -1484,14 +1484,6 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
}
}
- // Floors a sub-microsecond nanos component (0..999) to the given timestamp precision in [7, 9],
- // mirroring the production truncation rule used by the cross-precision cast.
- private def floorNanosToPrecision(nanosWithinMicro: Int, precision: Int): Int = precision match {
- case 7 => (nanosWithinMicro / 100) * 100
- case 8 => (nanosWithinMicro / 10) * 10
- case 9 => nanosWithinMicro
- }
-
test("SPARK-57490: cast between timestamp_ntz nanos types of different precision") {
// Same physical (epochMicros, nanosWithinMicro) pair; only the sub-microsecond part is
// re-floored to the target precision. epochMicros never changes and no time zone is involved.
@@ -1506,8 +1498,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
// The source value is already valid at p1 (its sub-micro digits are floored to p1); casting
// to p2 re-floors to p2, so the net effect is flooring to min(p1, p2). Widening (p2 >= p1)
// is lossless, narrowing (p2 < p1) drops the extra sub-microsecond digits.
- val srcNanos = floorNanosToPrecision(789, p1)
- val expectedNanos = floorNanosToPrecision(srcNanos, p2)
+ val srcNanos = nanoOfSecTruncator(p1)(789)
+ val expectedNanos = nanoOfSecTruncator(p2)(srcNanos)
checkEvaluation(
cast(Literal.create(nanosVal(micros, srcNanos), TimestampNTZNanosType(p1)),
TimestampNTZNanosType(p2)),
@@ -1534,8 +1526,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
p1 <- TimestampLTZNanosType.MIN_PRECISION to TimestampLTZNanosType.MAX_PRECISION
p2 <- TimestampLTZNanosType.MIN_PRECISION to TimestampLTZNanosType.MAX_PRECISION
} {
- val srcNanos = floorNanosToPrecision(789, p1)
- val expectedNanos = floorNanosToPrecision(srcNanos, p2)
+ val srcNanos = nanoOfSecTruncator(p1)(789)
+ val expectedNanos = nanoOfSecTruncator(p2)(srcNanos)
checkEvaluation(
cast(Literal.create(nanosVal(micros, srcNanos), TimestampLTZNanosType(p1)),
TimestampLTZNanosType(p2), UTC_OPT),
@@ -1569,8 +1561,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
} {
// The source already floors to p; the cross-family cast then floors to q, i.e. to
// min(p, q). Only the wall-clock epoch-micros part shifts with the zone.
- val srcNanos = floorNanosToPrecision(789, p)
- val expectedNanos = floorNanosToPrecision(srcNanos, q)
+ val srcNanos = nanoOfSecTruncator(p)(789)
+ val expectedNanos = nanoOfSecTruncator(q)(srcNanos)
checkEvaluation(
cast(Literal.create(nanosVal(srcMicros, srcNanos), TimestampLTZNanosType(p)),
TimestampNTZNanosType(q), zid),
@@ -1604,8 +1596,8 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
p <- TimestampLTZNanosType.MIN_PRECISION to TimestampLTZNanosType.MAX_PRECISION
q <- TimestampNTZNanosType.MIN_PRECISION to TimestampNTZNanosType.MAX_PRECISION
} {
- val srcNanos = floorNanosToPrecision(789, q)
- val expectedNanos = floorNanosToPrecision(srcNanos, p)
+ val srcNanos = nanoOfSecTruncator(q)(789)
+ val expectedNanos = nanoOfSecTruncator(p)(srcNanos)
checkEvaluation(
cast(Literal.create(nanosVal(srcMicros, srcNanos), TimestampNTZNanosType(q)),
TimestampLTZNanosType(p), zid),
@@ -1630,7 +1622,7 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
foreachNanosPrecision { p =>
val src = nanosVal(
instantToNanosVal(timestampLTZ(2020, 7, 1, 6, 15, 30, 123456789)).epochMicros,
- floorNanosToPrecision(789, p))
+ nanoOfSecTruncator(p)(789))
checkEvaluation(
cast(
cast(Literal.create(src, TimestampLTZNanosType(p)), TimestampNTZNanosType(p), zid),
@@ -1668,17 +1660,17 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
q <- TimestampNTZNanosType.MIN_PRECISION to TimestampNTZNanosType.MAX_PRECISION
} {
// NTZ(q) -> LTZ(p): the epoch-micros part resolves DST exactly like the micro cast.
- val ntzToLtzSrc = floorNanosToPrecision(789, q)
+ val ntzToLtzSrc = nanoOfSecTruncator(q)(789)
checkEvaluation(
cast(Literal.create(nanosVal(ntzMicros, ntzToLtzSrc), TimestampNTZNanosType(q)),
TimestampLTZNanosType(p), la),
- nanosVal(microLtz, floorNanosToPrecision(ntzToLtzSrc, p)))
+ nanosVal(microLtz, nanoOfSecTruncator(p)(ntzToLtzSrc)))
// LTZ(p) -> NTZ(q): rendering the resolved instant matches the micro cast too.
- val ltzToNtzSrc = floorNanosToPrecision(789, p)
+ val ltzToNtzSrc = nanoOfSecTruncator(p)(789)
checkEvaluation(
cast(Literal.create(nanosVal(microLtz, ltzToNtzSrc), TimestampLTZNanosType(p)),
TimestampNTZNanosType(q), la),
- nanosVal(microNtz, floorNanosToPrecision(ltzToNtzSrc, q)))
+ nanosVal(microNtz, nanoOfSecTruncator(q)(ltzToNtzSrc)))
}
}
}
@@ -1709,12 +1701,12 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
// TIMESTAMP_NTZ(p) -> TIMESTAMP (LTZ(6)): drops the sub-microsecond digits before the zone
// reinterpretation. A non-zero nanosWithinMicro on the source proves the truncation.
checkEvaluation(
- cast(Literal.create(nanosVal(ntzMicros, floorNanosToPrecision(789, p)),
+ cast(Literal.create(nanosVal(ntzMicros, nanoOfSecTruncator(p)(789)),
TimestampNTZNanosType(p)), TimestampType, zid),
convertTz(ntzMicros, zone, UTC))
// TIMESTAMP_LTZ(p) -> TIMESTAMP_NTZ (NTZ(6)): drops the sub-microsecond digits likewise.
checkEvaluation(
- cast(Literal.create(nanosVal(ltzMicros, floorNanosToPrecision(789, p)),
+ cast(Literal.create(nanosVal(ltzMicros, nanoOfSecTruncator(p)(789)),
TimestampLTZNanosType(p)), TimestampNTZType, zid),
convertTz(ltzMicros, UTC, zone))
// Null input in all four directions.
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosRenderingSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosRenderingSuiteBase.scala
index f3525de4af0e..e36f1111325a 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosRenderingSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosRenderingSuiteBase.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql
import java.time.{Instant, LocalDateTime}
import org.apache.spark.SparkConf
+import org.apache.spark.sql.catalyst.util.TimestampNanosTestUtils
// castToImpl: `spark.createDataFrame` is typed as the public sql.DataFrame, but `showString` is a
// classic-only method; this implicit narrows the receiver to classic.Dataset for that call.
import org.apache.spark.sql.classic.ClassicConversions.castToImpl
@@ -135,12 +136,6 @@ abstract class TimestampNanosRenderingSuiteBase extends QueryTest with SharedSpa
// remainder floored to the type precision. Two values sharing epochMicros are distinguishable.
// ==========================================================================================
// .000000123 floors to 100ns at p=7, 120ns at p=8, 123ns at p=9; .000000999 -> 900 / 990 / 999.
- private def flooredNano(base: Int, p: Int): Int = p match {
- case 7 => base / 100 * 100
- case 8 => base / 10 * 10
- case 9 => base
- }
-
test("collect() over nanosecond TIMESTAMP_NTZ preserves the precision-floored remainder") {
Seq(7, 8, 9).foreach { p =>
val df = ntzDF(Seq(
@@ -148,9 +143,13 @@ abstract class TimestampNanosRenderingSuiteBase extends QueryTest with SharedSpa
"2020-01-01T00:00:00.000000999",
null), p)
val got = df.collect().map(r => Option(r.getAs[LocalDateTime]("c")).map(_.getNano)).toSet
- assert(got === Set(Some(flooredNano(123, p)), Some(flooredNano(999, p)), None))
+ assert(got === Set(
+ Some(TimestampNanosTestUtils.nanoOfSecTruncator(p)(123)),
+ Some(TimestampNanosTestUtils.nanoOfSecTruncator(p)(999)),
+ None))
// The two non-null values share epochMicros yet stay distinct at every supported precision.
- assert(flooredNano(123, p) != flooredNano(999, p))
+ assert(TimestampNanosTestUtils.nanoOfSecTruncator(p)(123) !=
+ TimestampNanosTestUtils.nanoOfSecTruncator(p)(999))
}
}
@@ -161,7 +160,10 @@ abstract class TimestampNanosRenderingSuiteBase extends QueryTest with SharedSpa
"2020-01-01T00:00:00.000000999Z",
null), p)
val got = df.collect().map(r => Option(r.getAs[Instant]("c")).map(_.getNano)).toSet
- assert(got === Set(Some(flooredNano(123, p)), Some(flooredNano(999, p)), None))
+ assert(got === Set(
+ Some(TimestampNanosTestUtils.nanoOfSecTruncator(p)(123)),
+ Some(TimestampNanosTestUtils.nanoOfSecTruncator(p)(999)),
+ None))
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org