You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ur...@apache.org on 2026/08/18 09:55:03 UTC
(spark) branch master updated: [SPARK-58741][SQL][TEST] Add nanosecond-timestamp coverage for collect_list
This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 07495f4dc0fc [SPARK-58741][SQL][TEST] Add nanosecond-timestamp coverage for collect_list
07495f4dc0fc is described below
commit 07495f4dc0fc912bcddcb891bafe26306c0b469f
Author: Stevo Mitric <st...@gmail.com>
AuthorDate: Tue Aug 18 09:54:50 2026 +0000
[SPARK-58741][SQL][TEST] Add nanosecond-timestamp coverage for collect_list
### What changes were proposed in this pull request?
Add an end-to-end test to `TimestampNanosFunctionsSuiteBase` (run in both ANSI modes) asserting that collect_list preserves the input precision and family (NTZ/LTZ) and the sub-microsecond remainder, drops NULLs, and agrees between the SQL and Scala Column API paths. Also add a deterministic golden SQL case (`sort_array(collect_list(...))`) to `timestamp-ntz-nanos.sql` / `timestamp-ltz-nanos.sql`.
### Why are the changes needed?
Extend test coverage for timestamp nanosecond precision datatype.
### Does this PR introduce _any_ user-facing change?
No, test only change
### How was this patch tested?
Test-only change.
### Was this patch authored or co-authored using generative AI tooling?
Co-Authored-by: Claude Code 4.8
Closes #57964 from stevomitric/stevomitric/nanos-collect-list-tests.
Authored-by: Stevo Mitric <st...@gmail.com>
Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
.../analyzer-results/timestamp-ltz-nanos.sql.out | 12 ++++++
.../analyzer-results/timestamp-ntz-nanos.sql.out | 12 ++++++
.../sql-tests/inputs/timestamp-ltz-nanos.sql | 11 +++++
.../sql-tests/inputs/timestamp-ntz-nanos.sql | 10 +++++
.../sql-tests/results/timestamp-ltz-nanos.sql.out | 12 ++++++
.../sql-tests/results/timestamp-ntz-nanos.sql.out | 12 ++++++
.../sql/TimestampNanosFunctionsSuiteBase.scala | 47 ++++++++++++++++++++++
7 files changed, 116 insertions(+)
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
index 553fe363f815..6c028a1c3127 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ltz-nanos.sql.out
@@ -777,6 +777,18 @@ Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS sort_array(collect_
+- LocalRelation [c#x]
+-- !query
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (CAST(NULL AS timestamp_ltz(9))) AS t(c)
+-- !query analysis
+Aggregate [sort_array(collect_list(c#x, 0, 0, true), true) AS sort_array(collect_list(c), true)#x]
++- SubqueryAlias t
+ +- LocalRelation [c#x]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789')
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
index 5eb9a2878ce3..c64be7281b47 100644
--- a/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/analyzer-results/timestamp-ntz-nanos.sql.out
@@ -705,6 +705,18 @@ Aggregate [sort_array(collect_set(c#x, 0, 0, true), true) AS sort_array(collect_
+- LocalRelation [c#x]
+-- !query
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (CAST(NULL AS timestamp_ntz(9))) AS t(c)
+-- !query analysis
+Aggregate [sort_array(collect_list(c#x, 0, 0, true), true) AS sort_array(collect_list(c), true)#x]
++- SubqueryAlias t
+ +- LocalRelation [c#x]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789')
-- !query analysis
diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
index 236facfcf88c..0f18303867ba 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ltz-nanos.sql
@@ -250,6 +250,17 @@ SELECT sort_array(collect_set(c)) FROM VALUES
(TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
(TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC') AS t(c);
+-- SPARK-56822: collect_list over nanosecond-precision TIMESTAMP_LTZ. The buffer holds the full
+-- nanos value, so the sub-microsecond remainder survives and the result element type stays
+-- TIMESTAMP_LTZ(9); values render in the session time zone (America/Los_Angeles). collect_list
+-- order is non-deterministic, so the output is stabilized with sort_array; duplicates are kept and
+-- NULLs are dropped.
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (CAST(NULL AS timestamp_ltz(9))) AS t(c);
+
-- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is
-- whole-second BIGINT; the sub-second digits are dropped. A literal without an explicit zone is
-- read in the session time zone (America/Los_Angeles, UTC-08:00); an explicit-zone literal fixes
diff --git a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
index 43c83810c750..478356098d8c 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/timestamp-ntz-nanos.sql
@@ -221,6 +221,16 @@ SELECT sort_array(collect_set(c)) FROM VALUES
(TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
(TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001') AS t(c);
+-- SPARK-56822: collect_list over nanosecond-precision TIMESTAMP_NTZ. The buffer holds the full
+-- nanos value, so the sub-microsecond remainder survives and the result element type stays
+-- TIMESTAMP_NTZ(9). collect_list order is non-deterministic, so the output is stabilized with
+-- sort_array; duplicates are kept and NULLs are dropped.
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (CAST(NULL AS timestamp_ntz(9))) AS t(c);
+
-- SPARK-57528: unix_timestamp / to_unix_timestamp over nanosecond-precision values. The result is
-- whole-second BIGINT; the sub-second digits are dropped and NTZ applies no zone shift, so the
-- wall-clock value is read as the epoch instant.
diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
index 5da97df45d9b..2d1ae31273c1 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ltz-nanos.sql.out
@@ -870,6 +870,18 @@ struct<sort_array(collect_set(c), true):array<timestamp_ltz(9)>>
[2019-12-31 16:00:00.000000001,2019-12-31 16:00:00.000000999]
+-- !query
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000999 UTC'),
+ (TIMESTAMP_LTZ '2020-01-01 00:00:00.000000001 UTC'),
+ (CAST(NULL AS timestamp_ltz(9))) AS t(c)
+-- !query schema
+struct<sort_array(collect_list(c), true):array<timestamp_ltz(9)>>
+-- !query output
+[2019-12-31 16:00:00.000000001,2019-12-31 16:00:00.000000001,2019-12-31 16:00:00.000000999]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_LTZ '2020-01-01 13:24:35.123456789')
-- !query schema
diff --git a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
index 26ce0e0fd8b2..709a64321fec 100644
--- a/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/timestamp-ntz-nanos.sql.out
@@ -788,6 +788,18 @@ struct<sort_array(collect_set(c), true):array<timestamp_ntz(9)>>
[2020-01-01 00:00:00.000000001,2020-01-01 00:00:00.000000999]
+-- !query
+SELECT sort_array(collect_list(c)) FROM VALUES
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000999'),
+ (TIMESTAMP_NTZ '2020-01-01 00:00:00.000000001'),
+ (CAST(NULL AS timestamp_ntz(9))) AS t(c)
+-- !query schema
+struct<sort_array(collect_list(c), true):array<timestamp_ntz(9)>>
+-- !query output
+[2020-01-01 00:00:00.000000001,2020-01-01 00:00:00.000000001,2020-01-01 00:00:00.000000999]
+
+
-- !query
SELECT unix_timestamp(TIMESTAMP_NTZ '2020-01-01 13:24:35.123456789')
-- !query schema
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
index f6f3d105d462..4a73b2c448df 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TimestampNanosFunctionsSuiteBase.scala
@@ -774,6 +774,53 @@ abstract class TimestampNanosFunctionsSuiteBase extends SharedSparkSession {
}
}
+ // collect_list over nanosecond-precision timestamps (SPARK-56822). `CollectList` is
+ // type-agnostic: its `ArrayBuffer` buffer holds the physical `TimestampNanosVal` and the
+ // result element type is exactly `child.dataType`, so the input precision, family (NTZ/LTZ)
+ // and sub-microsecond remainder survive with no truncation to micros. The collection order
+ // after aggregation is not deterministic, so the contents are compared as a set.
+
+ test("SPARK-56822: collect_list over nanosecond-precision timestamps preserves type and nanos " +
+ "remainder") {
+ Seq(7, 8, 9).foreach { p =>
+ val schema = new StructType()
+ .add("ntz", TimestampNTZNanosType(p))
+ .add("ltz", TimestampLTZNanosType(p))
+ // The sub-microsecond parts are multiples of 100ns, so they are exact at every p in [7, 9]
+ // (no flooring) yet non-zero -- a value truncated to micros would be visibly wrong. The NULL
+ // row is dropped (collect_list ignores nulls by default).
+ val data = Seq(
+ Row(LocalDateTime.parse("2020-01-01T12:34:56.000000100"),
+ Instant.parse("2020-01-01T12:34:56.000000100Z")),
+ Row(LocalDateTime.parse("2020-01-02T00:00:00.000000900"),
+ Instant.parse("2020-01-02T00:00:00.000000900Z")),
+ Row(null, null))
+ val df = spark.createDataFrame(spark.sparkContext.parallelize(data), schema)
+
+ val sqlRes = df.selectExpr("collect_list(ntz)", "collect_list(ltz)")
+ // The result keeps the family (NTZ/LTZ) and precision; nulls are dropped so
+ // containsNull=false.
+ assert(sqlRes.schema.map(_.dataType) === Seq(
+ ArrayType(TimestampNTZNanosType(p), containsNull = false),
+ ArrayType(TimestampLTZNanosType(p), containsNull = false)))
+
+ val expectedNtz = Set(
+ LocalDateTime.parse("2020-01-01T12:34:56.000000100"),
+ LocalDateTime.parse("2020-01-02T00:00:00.000000900"))
+ val expectedLtz = Set(
+ Instant.parse("2020-01-01T12:34:56.000000100Z"),
+ Instant.parse("2020-01-02T00:00:00.000000900Z"))
+ // The Scala Column API path agrees with the SQL path; both drop the null and keep
+ // both values.
+ val sqlRow = sqlRes.collect().head
+ val colRow = df.select(collect_list(col("ntz")), collect_list(col("ltz"))).collect().head
+ assert(sqlRow.getSeq[LocalDateTime](0).toSet === expectedNtz)
+ assert(sqlRow.getSeq[Instant](1).toSet === expectedLtz)
+ assert(colRow.getSeq[LocalDateTime](0).toSet === expectedNtz)
+ assert(colRow.getSeq[Instant](1).toSet === expectedLtz)
+ }
+ }
+
test("SPARK-57816: date_format / to_char / to_varchar over nanosecond-precision timestamps") {
// The 9-`S` pattern is a fixed-width fraction field, so it always emits 9 digits; truncating to
// precision `p` zeros the low digits (floor); it does not drop them. The session zone is
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org