(spark-connect-swift) branch main updated: [SPARK-59473] Support top-level `Decimal` decoding in `DataFrame.collect(as:)`

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 0d10cfc  [SPARK-59473] Support top-level `Decimal` decoding in `DataFrame.collect(as:)`
0d10cfc is described below

commit 0d10cfc948e727c62056d4a2937887b81724322d
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Sep 13 12:46:16 2026 -0700

    [SPARK-59473] Support top-level `Decimal` decoding in `DataFrame.collect(as:)`
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support top-level `Decimal` decoding in `DataFrame.collect(as:)`.
    
    ### Why are the changes needed?
    
    Foundation's `Decimal.init(from:)` requests a keyed container of its internal fields. So, the following throws `ArrowError.invalid("Column for key \"exponent\" not found")`, while Scala `as[BigDecimal]` returns `1.50`.
    
    ```swift
    try await spark.sql("SELECT CAST(1.5 AS DECIMAL(10,2)) AS v").collect(as: Decimal.self)
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `collect(as: Decimal.self)` and `collect(as: Decimal?.self)` now return values instead of throwing errors.
    
    ### How was this patch tested?
    
    Pass the CIs with the updated `CreateDataFrameTests.collectAsDecimal`.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5
    
    Closes #560 from dongjoon-hyun/SPARK-59473.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 Sources/SparkConnect/ArrowDecoder.swift            | 15 ++++++++++++---
 Tests/SparkConnectTests/CreateDataFrameTests.swift | 15 +++++++++++++++
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/Sources/SparkConnect/ArrowDecoder.swift b/Sources/SparkConnect/ArrowDecoder.swift
index ddc2f6f..4f77fc9 100644
--- a/Sources/SparkConnect/ArrowDecoder.swift
+++ b/Sources/SparkConnect/ArrowDecoder.swift
@@ -70,9 +70,9 @@ public class ArrowDecoder: Decoder {
     for index in 0..<rb.length {
       self.rbIndex = index
       self.singleRBCol = 0
-      let key = try T.init(from: self)
+      let key = try self.decodeSingleValue(T.self)
       self.singleRBCol = 1
-      let value = try U.init(from: self)
+      let value = try self.decodeSingleValue(U.self)
       output[key] = value
     }
 
@@ -84,12 +84,21 @@ public class ArrowDecoder: Decoder {
     var output = [T]()
     for index in 0..<rb.length {
       self.rbIndex = index
-      output.append(try type.init(from: self))
+      output.append(try self.decodeSingleValue(type))
     }
 
     return output
   }
 
+  /// Foundation's `Decimal.init(from:)` asks for a keyed container of its internal fields,
+  /// so decode `Decimal` from the current column directly.
+  func decodeSingleValue<T: Decodable>(_ type: T.Type) throws -> T {
+    if type == Decimal.self {
+      return try self.doDecode(self.singleRBCol)
+    }
+    return try type.init(from: self)
+  }
+
   public func container<Key>(
     keyedBy type: Key.Type
   ) -> KeyedDecodingContainer<Key> where Key: CodingKey {
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index 591f6ae..f19527b 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -504,10 +504,25 @@ struct CreateDataFrameTests {
       #expect(
         try await spark.sql("SELECT \(expr) AS v").collect(as: DecimalS.self)
           == [DecimalS(v: expected)])
+      #expect(try await spark.sql("SELECT \(expr) AS v").collect(as: Decimal.self) == [expected])
+      #expect(try await spark.sql("SELECT \(expr) AS v").collect(as: Decimal?.self) == [expected])
     }
     #expect(
       try await spark.sql("SELECT CAST(NULL AS DECIMAL(10,2)) AS v").collect(as: DecimalS.self)
         == [DecimalS(v: nil)])
+    #expect(
+      try await spark.sql("SELECT CAST(NULL AS DECIMAL(10,2)) AS v").collect(as: Decimal?.self)
+        == [nil])
+    #expect(
+      try await spark.sql(
+        "SELECT CAST(-1.5 AS DECIMAL(10,2)) AS a, CAST(12345678901234567890.5 AS DECIMAL(38,2)) AS b"
+      ).collect(as: [Decimal].self)
+        == [[Decimal(string: "-1.50")!, Decimal(string: "12345678901234567890.50")!]])
+
+    // `Decimal` is printed as `NSDecimal` on Darwin, so only the error type is checked.
+    await #expect(throws: ArrowError.self) {
+      try await spark.sql("SELECT CAST(NULL AS DECIMAL(10,2)) AS v").collect(as: Decimal.self)
+    }
     await spark.stop()
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org