(spark-connect-swift) branch main updated: [SPARK-59203] Support nanosecond timestamp literals in `lit` and SQL parameter binding

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 9ae3226  [SPARK-59203] Support nanosecond timestamp literals in `lit` and SQL parameter binding
9ae3226 is described below

commit 9ae3226a81160fa6da530390041e23c2d0041ce3
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 3 08:27:50 2026 -0700

    [SPARK-59203] Support nanosecond timestamp literals in `lit` and SQL parameter binding
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support nanosecond timestamp literals in `lit` and SQL parameter binding.
    
    - Add `lit(_ value: TimestampNanos)` which builds `Expression.Literal.TimestampLTZNanos` with `epochMicros`, `nanosWithinMicro`, and `precision = 9`.
    - Conform `TimestampNanos` to `SparkLiteral` so it can be used directly as an operand of `Column` operators, e.g. `col("t") == ts`.
    - Handle `TimestampNanos` in `ExpressionLiteral.init(_:)` so it can be bound as a positional (`?`) or named (`:name`) parameter of `spark.sql`.
    
    Like `Date`, which maps to `TIMESTAMP` (`TIMESTAMP_LTZ`), `TimestampNanos` is an absolute point in time and maps to `TIMESTAMP_LTZ(9)`.
    
    ```swift
    try await spark.conf.set("spark.sql.timestampNanosTypes.enabled", "true")
    let ts = TimestampNanos(epochNanos: 1_767_225_600_123_456_789)
    spark.range(1).select(lit(ts))                     // timestamp_ltz(9)
    try await spark.sql("SELECT ?", ts)
    try await spark.sql("SELECT :t", args: ["t": ts])
    ```
    
    ### Why are the changes needed?
    
    SPARK-59198 covered only the read path (`DataFrame.collect`). This completes the write path for expressions so that Swift users can send nanosecond timestamp values to the server via `lit`, `Column` operators, and parameterized SQL without losing sub-microsecond digits.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, `TimestampNanos` is newly accepted by `lit`, `Column` operators, and `spark.sql` parameters. There is no behavior change for existing code.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5.1
    
    Closes #512 from dongjoon-hyun/SPARK-59203.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 Sources/SparkConnect/Extension.swift            |  6 ++++++
 Sources/SparkConnect/Functions.swift            | 15 +++++++++++++++
 Tests/SparkConnectTests/DataFrameTests.swift    | 18 ++++++++++++++++++
 Tests/SparkConnectTests/FunctionsTests.swift    |  8 ++++++++
 Tests/SparkConnectTests/SparkSessionTests.swift | 16 ++++++++++++++++
 5 files changed, 63 insertions(+)

diff --git a/Sources/SparkConnect/Extension.swift b/Sources/SparkConnect/Extension.swift
index c33f07d..dc0575f 100644
--- a/Sources/SparkConnect/Extension.swift
+++ b/Sources/SparkConnect/Extension.swift
@@ -200,6 +200,12 @@ extension ExpressionLiteral {
     case let value as LocalTime:
       self.time.nano = value.nanoOfDay
       self.time.precision = 6
+    case let value as TimestampNanos:
+      // Like `Date`, `TimestampNanos` is an absolute point in time, so it maps to
+      // Spark's `TIMESTAMP_LTZ(9)`.
+      self.timestampLtzNanos.epochMicros = value.epochMicros
+      self.timestampLtzNanos.nanosWithinMicro = Int32(value.nanosWithinMicro)
+      self.timestampLtzNanos.precision = 9
     default:
       if case Optional<Any>.none = value as Any {
         var dataType = ProtoDataType()
diff --git a/Sources/SparkConnect/Functions.swift b/Sources/SparkConnect/Functions.swift
index d9c5631..dac4b9c 100644
--- a/Sources/SparkConnect/Functions.swift
+++ b/Sources/SparkConnect/Functions.swift
@@ -122,6 +122,17 @@ public func lit(_ value: LocalTime) -> Column {
   return litColumn(literal)
 }
 
+/// Creates a ``Column`` of `TIMESTAMP_LTZ(9)` literal value.
+/// - Parameter value: A literal value.
+/// - Returns: A ``Column``.
+public func lit(_ value: TimestampNanos) -> Column {
+  var literal = ExpressionLiteral()
+  literal.timestampLtzNanos.epochMicros = value.epochMicros
+  literal.timestampLtzNanos.nanosWithinMicro = Int32(value.nanosWithinMicro)
+  literal.timestampLtzNanos.precision = 9
+  return litColumn(literal)
+}
+
 /// A type that can be used as a literal operand of ``Column`` operators.
 ///
 /// This allows Swift literals to be mixed with ``Column`` expressions directly
@@ -175,6 +186,10 @@ extension LocalTime: SparkLiteral {
   public var toLiteralColumn: Column { lit(self) }
 }
 
+extension TimestampNanos: SparkLiteral {
+  public var toLiteralColumn: Column { lit(self) }
+}
+
 /// Returns a sort expression based on the ascending order of the given column name.
 /// - Parameter name: A column name.
 /// - Returns: A ``Column`` with ascending sort order.
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift
index 52a39a8..c9511b0 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -233,6 +233,24 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func selectTimestampNanosLiteral() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    if await isSparkVersionAtLeast(spark.version, "4.3") {
+      try await spark.conf.set("spark.sql.timestampNanosTypes.enabled", "true")
+      for ts in [
+        TimestampNanos(epochNanos: 1_767_225_600_123_456_789),
+        TimestampNanos(epochNanos: 0),
+        TimestampNanos(epochNanos: -1),
+      ] {
+        let df = try await spark.range(1).select(lit(ts))
+        #expect(try await df.dtypes[0].1 == "timestamp_ltz(9)")
+        #expect(try await df.collect() == [Row(ts)])
+      }
+    }
+    await spark.stop()
+  }
+
   @Test
   func selectTimeLiteral() async throws {
     let spark = try await SparkSession.builder.getOrCreate()
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift b/Tests/SparkConnectTests/FunctionsTests.swift
index 92df869..7087437 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -60,6 +60,10 @@ struct FunctionsTests {
     let time = try #require(LocalTime(hour: 12, minute: 34, second: 56))
     #expect(lit(time).expr.literal.time.nano == 45_296_000_000_000)
     #expect(lit(time).expr.literal.time.precision == 6)
+    let ts = TimestampNanos(epochNanos: 1_767_225_600_123_456_789)
+    #expect(lit(ts).expr.literal.timestampLtzNanos.epochMicros == 1_767_225_600_123_456)
+    #expect(lit(ts).expr.literal.timestampLtzNanos.nanosWithinMicro == 789)
+    #expect(lit(ts).expr.literal.timestampLtzNanos.precision == 9)
   }
 
   @Test
@@ -260,6 +264,10 @@ struct FunctionsTests {
     #expect(
       (col("a") == time).expr.unresolvedFunction.arguments[1].literal.time.nano
         == 45_296_000_000_000)
+    let ts = TimestampNanos(epochNanos: 1_767_225_600_123_456_789)
+    #expect(
+      (col("a") == ts).expr.unresolvedFunction.arguments[1].literal.timestampLtzNanos.epochMicros
+        == 1_767_225_600_123_456)
   }
 
   @Test
diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift
index 0d92e9b..2d17096 100644
--- a/Tests/SparkConnectTests/SparkSessionTests.swift
+++ b/Tests/SparkConnectTests/SparkSessionTests.swift
@@ -305,6 +305,22 @@ struct SparkSessionTests {
     await spark.stop()
   }
 
+  @Test
+  func sqlWithTimestampNanosParameter() async throws {
+    await SparkSession.builder.clear()
+    let spark = try await SparkSession.builder.getOrCreate()
+    if await isSparkVersionAtLeast(spark.version, "4.3") {
+      try await spark.conf.set("spark.sql.timestampNanosTypes.enabled", "true")
+      let ts = TimestampNanos(epochNanos: 1_767_225_600_123_456_789)
+      #expect(try await spark.sql("SELECT typeof(?)", ts).collect() == [Row("timestamp_ltz(9)")])
+      #expect(try await spark.sql("SELECT ?", ts).collect() == [Row(ts)])
+      #expect(try await spark.sql("SELECT :t", args: ["t": ts]).collect() == [Row(ts)])
+      let before1970 = TimestampNanos(epochNanos: -1)
+      #expect(try await spark.sql("SELECT :t", args: ["t": before1970]).collect() == [Row(before1970)])
+    }
+    await spark.stop()
+  }
+
   @Test
   func sqlWithUnsupportedParameterType() async throws {
     await SparkSession.builder.clear()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org