(spark) branch master updated: [SPARK-58740][SQL][TEST] Add to_xml test coverage for nanosecond-precision timestamp types

[ur...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 94cbfbaf818a [SPARK-58740][SQL][TEST] Add to_xml test coverage for nanosecond-precision timestamp types
94cbfbaf818a is described below

commit 94cbfbaf818aa071eb989620aee4cf20a24133ad
Author: Stevo Mitric <st...@gmail.com>
AuthorDate: Thu Aug 13 05:50:45 2026 +0000

    [SPARK-58740][SQL][TEST] Add to_xml test coverage for nanosecond-precision timestamp types
    
    ### What changes were proposed in this pull request?
    
    This PR adds test coverage for the `to_xml` write path with the nanosecond-precision timestamp types (`TIMESTAMP_NTZ(p)` / `TIMESTAMP_LTZ(p)`, `p in [7, 9]`). The write path exists in `StaxXmlGenerator` but had no dedicated tests, while `from_xml` (read) and both JSON directions were already covered.
    
    Two tests are added to `XmlFunctionsSuite`:
    - `to_xml with nanos timestamp types` — asserts the exact XML output for both NTZ and LTZ nanos types at every valid precision.
    - `roundtrip in to_xml and from_xml - nanos timestamps` — verifies the write -> read round-trip is loss-free at every valid precision.
    
    ### Why are the changes needed?
    
    The `to_xml` nanosecond write path was implemented but never exercised by a test, leaving a coverage gap for the XML datasource nanosecond support (SPARK-57458).
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. Test-only change.
    
    ### How was this patch tested?
    
    `build/sbt "sql/testOnly org.apache.spark.sql.XmlFunctionsSuite"`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    Co-authored-by: Claude Code 4.8
    
    Closes #57963 from stevomitric/stevomitric/spark-57458-to-xml-nanos-tests.
    
    Lead-authored-by: Stevo Mitric <st...@gmail.com>
    Co-authored-by: Uros Bojanic <22...@users.noreply.github.com>
    Signed-off-by: Uros Bojanic <22...@users.noreply.github.com>
---
 .../org/apache/spark/sql/XmlFunctionsSuite.scala   | 63 ++++++++++++++++++++++
 1 file changed, 63 insertions(+)

diff --git a/sql/core/src/test/scala/org/apache/spark/sql/XmlFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/XmlFunctionsSuite.scala
index 73cdc7df139a..bfe46e681261 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/XmlFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/XmlFunctionsSuite.scala
@@ -361,6 +361,69 @@ class XmlFunctionsSuite extends SharedSparkSession {
     checkAnswer(dfTwo, readBackTwo)
   }
 
+  test("SPARK-57458: to_xml with nanos timestamp types") {
+    withSQLConf(
+        SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+      foreachNanosPrecision { p =>
+        // The pattern must carry `p` fractional digits to emit the full declared precision; the
+        // stored value is truncated to precision `p`, so the rendered fraction is the first
+        // `p` digits of 123456789.
+        val fracPat = "S" * p
+        val frac = "123456789".take(p)
+        val ldt = LocalDateTime.of(2020, 1, 1, 0, 0, 0, 123456789)
+        Seq(
+          (TimestampNTZNanosType(p): DataType, "timestampNTZFormat",
+            s"yyyy-MM-dd'T'HH:mm:ss.$fracPat", ldt: Any,
+            s"2020-01-01T00:00:00.$frac"),
+          (TimestampLTZNanosType(p): DataType, "timestampFormat",
+            s"yyyy-MM-dd'T'HH:mm:ss.${fracPat}XXX", ldt.toInstant(ZoneOffset.UTC): Any,
+            s"2020-01-01T00:00:00.${frac}Z")).foreach {
+          case (nanosType, optKey, fmt, value, expectedTs) =>
+            val schema = new StructType().add("ts", nanosType)
+            val df = spark.createDataFrame(
+              spark.sparkContext.parallelize(Seq(Row(value))), schema)
+            val expectedXml =
+              s"""|<ROW>
+                  |    <ts>$expectedTs</ts>
+                  |</ROW>""".stripMargin
+            checkAnswer(
+              df.select(to_xml(struct($"ts"), Map(optKey -> fmt).asJava)),
+              Row(expectedXml))
+        }
+      }
+    }
+  }
+
+  test("SPARK-57458: roundtrip in to_xml and from_xml - nanos timestamps") {
+    withSQLConf(
+        SQLConf.TIMESTAMP_NANOS_TYPES_ENABLED.key -> "true",
+        SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
+      foreachNanosPrecision { p =>
+        val fracPat = "S" * p
+        val ldt = LocalDateTime.of(2020, 1, 1, 0, 0, 0, 123456789)
+        Seq(
+          (TimestampNTZNanosType(p): DataType, "timestampNTZFormat",
+            s"yyyy-MM-dd'T'HH:mm:ss.$fracPat", ldt: Any),
+          (TimestampLTZNanosType(p): DataType, "timestampFormat",
+            s"yyyy-MM-dd'T'HH:mm:ss.${fracPat}XXX", ldt.toInstant(ZoneOffset.UTC): Any)).foreach {
+          case (nanosType, optKey, fmt, value) =>
+            val schema = new StructType().add("ts", nanosType)
+            val df = spark.createDataFrame(
+              spark.sparkContext.parallelize(Seq(Row(value))), schema)
+            val options = Map(optKey -> fmt).asJava
+            // The input column already carries precision `p`, so the to_xml -> from_xml
+            // round-trip with a `p`-digit pattern is loss-free.
+            val readBack = df
+              .select(to_xml(struct($"ts"), options).as("xml"))
+              .select(from_xml($"xml", schema, options).as("data"))
+              .select($"data.ts".as("ts"))
+            checkAnswer(readBack, df.select($"ts"))
+        }
+      }
+    }
+  }
+
   test("Support to_xml in SQL") {
     val schemaOne = StructType(StructField("a", IntegerType, nullable = false) :: Nil)
     val dataOne = Seq(Row(1))


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org