(spark-connect-swift) branch main updated: [SPARK-58344] Support Arrow `Null` type to fix `collect()` crash on void columns

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 99f6d39  [SPARK-58344] Support Arrow `Null` type to fix `collect()` crash on void columns
99f6d39 is described below

commit 99f6d3978de9018b56223bc2d42c46afb9cc1342
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Fri Jul 24 19:55:37 2026 -0700

    [SPARK-58344] Support Arrow `Null` type to fix `collect()` crash on void columns
    
    ### What changes were proposed in this pull request?
    
    This PR adds Arrow `Null` type support to the vendored Arrow reader so that collecting a result with a NULL-typed (`void`) column returns `nil` values instead of crashing.
    
    - `ArrowType.swift`: add `ArrowType.ArrowNull` and handle `.null` in `getStride()` (previously `fatalError`).
    - `ArrowReaderHelper.swift`: map the flatbuffer `.null` field type in `findArrowType` and add `makeNullHolder` which builds an `ArrowData` with no buffers.
    - `ArrowReader.swift`: add a `.null` branch in `loadField` (`loadNullData`) that consumes only the field node, since Arrow `Null` arrays carry no validity or data buffers.
    - `ArrowArray.swift`: add `NullArray` (always returns `nil`) and a `.null` case in `ArrowArrayHolderImpl.loadArray` for null columns nested inside structs.
    - `ArrowData.swift`: `isNull` returns `true` for the null type instead of accessing the non-existent validity buffer.
    
    ### Why are the changes needed?
    
    Any query whose result schema contains a `NullType` column crashes the client process with `SIGTRAP`.
    
    ```swift
    let spark = try await SparkSession.builder.getOrCreate()
    _ = try await spark.sql("SELECT null").collect()
    // Fatal error: Unexpectedly found nil while unwrapping an Optional value
    ```
    
    The reader did not recognize the Arrow `Null` type, so `loadSchema` failed and `DataFrame.execute()` proceeded with a `nil` `messageSchema`, crashing on a force-unwrap in `ArrowReader.fromMessage`. This is easy to hit, e.g. `SELECT :x` with a null named parameter.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, this is a bug fix. Previously, `collect()` on a result containing a `void` column crashed the process; now it returns `nil` for those values (e.g. `SELECT null` returns `[Row(nil)]`).
    
    ### How was this patch tested?
    
    Pass the CIs with a newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Fable 5
    
    Closes #445 from dongjoon-hyun/SPARK-58344.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 Sources/SparkConnect/ArrowArray.swift        |  9 +++++++++
 Sources/SparkConnect/ArrowData.swift         |  3 +++
 Sources/SparkConnect/ArrowReader.swift       | 16 ++++++++++++++++
 Sources/SparkConnect/ArrowReaderHelper.swift | 17 +++++++++++++++++
 Sources/SparkConnect/ArrowType.swift         |  3 +++
 Tests/SparkConnectTests/DataFrameTests.swift | 10 ++++++++++
 6 files changed, 58 insertions(+)

diff --git a/Sources/SparkConnect/ArrowArray.swift b/Sources/SparkConnect/ArrowArray.swift
index 8154f90..dff8efd 100644
--- a/Sources/SparkConnect/ArrowArray.swift
+++ b/Sources/SparkConnect/ArrowArray.swift
@@ -128,6 +128,8 @@ public class ArrowArrayHolderImpl: ArrowArrayHolder {
       return try ArrowArrayHolderImpl(NestedArray(with))
     case .list:
       return try ArrowArrayHolderImpl(NestedArray(with))
+    case .null:
+      return try ArrowArrayHolderImpl(NullArray(with))
     default:
       throw ArrowError.invalid("Array not found for type: \(arrowType)")
     }
@@ -174,6 +176,13 @@ public class ArrowArray<T>: AsString, AnyArray {
   }
 }
 
+/// @nodoc
+public class NullArray: ArrowArray<Any> {
+  public override subscript(_ index: UInt) -> Any? {
+    return nil
+  }
+}
+
 /// @nodoc
 public class FixedArray<T>: ArrowArray<T> {
   public override subscript(_ index: UInt) -> T? {
diff --git a/Sources/SparkConnect/ArrowData.swift b/Sources/SparkConnect/ArrowData.swift
index 0c747ee..97974d2 100644
--- a/Sources/SparkConnect/ArrowData.swift
+++ b/Sources/SparkConnect/ArrowData.swift
@@ -70,6 +70,9 @@ public class ArrowData {
   }
 
   public func isNull(_ at: UInt) -> Bool {
+    if self.type.id == .null {
+      return true
+    }
     let nullBuffer = buffers[0]
     return nullBuffer.length > 0 && !BitUtility.isSet(at, buffer: nullBuffer)
   }
diff --git a/Sources/SparkConnect/ArrowReader.swift b/Sources/SparkConnect/ArrowReader.swift
index 76c2470..c51569e 100644
--- a/Sources/SparkConnect/ArrowReader.swift
+++ b/Sources/SparkConnect/ArrowReader.swift
@@ -168,6 +168,20 @@ public class ArrowReader {  // swiftlint:disable:this type_body_length
     }
   }
 
+  private func loadNullData(
+    _ loadInfo: DataLoadInfo,
+    field: org_apache_arrow_flatbuf_Field
+  )
+    -> Result<ArrowArrayHolder, ArrowError>
+  {
+    guard let node = loadInfo.batchData.nextNode() else {
+      return .failure(.invalid("Node not found"))
+    }
+
+    // Arrow Null arrays carry no validity or data buffers.
+    return makeNullHolder(UInt(node.length))
+  }
+
   private func loadPrimitiveData(
     _ loadInfo: DataLoadInfo,
     field: org_apache_arrow_flatbuf_Field
@@ -244,6 +258,8 @@ public class ArrowReader {  // swiftlint:disable:this type_body_length
     -> Result<ArrowArrayHolder, ArrowError>
   {
     switch field.typeType {
+    case .null:
+      return loadNullData(loadInfo, field: field)
     case .struct_:
       return loadStructData(loadInfo, field: field)
     case .list:
diff --git a/Sources/SparkConnect/ArrowReaderHelper.swift b/Sources/SparkConnect/ArrowReaderHelper.swift
index 5642346..eeefdfc 100644
--- a/Sources/SparkConnect/ArrowReaderHelper.swift
+++ b/Sources/SparkConnect/ArrowReaderHelper.swift
@@ -150,6 +150,21 @@ private func makeBoolHolder(
   }
 }
 
+func makeNullHolder(
+  _ length: UInt
+) -> Result<ArrowArrayHolder, ArrowError> {
+  do {
+    let arrowType = ArrowType(ArrowType.ArrowNull)
+    let arrowData = try ArrowData(
+      arrowType, buffers: [], children: [], nullCount: length, length: length)
+    return .success(ArrowArrayHolderImpl(try NullArray(arrowData)))
+  } catch let error as ArrowError {
+    return .failure(error)
+  } catch {
+    return .failure(.unknownError("\(error)"))
+  }
+}
+
 private func makeFixedHolder<T>(
   _: T.Type, field: ArrowField, buffers: [ArrowBuffer],
   nullCount: UInt
@@ -277,6 +292,8 @@ func findArrowType(  // swiftlint:disable:this cyclomatic_complexity function_bo
 ) -> ArrowType {
   let type = field.typeType
   switch type {
+  case .null:
+    return ArrowType(ArrowType.ArrowNull)
   case .int:
     let intType = field.type(type: org_apache_arrow_flatbuf_Int.self)!
     let bitWidth = intType.bitWidth
diff --git a/Sources/SparkConnect/ArrowType.swift b/Sources/SparkConnect/ArrowType.swift
index 7d47a90..2ca02ec 100644
--- a/Sources/SparkConnect/ArrowType.swift
+++ b/Sources/SparkConnect/ArrowType.swift
@@ -240,6 +240,7 @@ public class ArrowType {
   public static let ArrowTimestamp = Info.timeInfo(ArrowTypeId.timestamp)
   public static let ArrowStruct = Info.complexInfo(ArrowTypeId.strct)
   public static let ArrowList = Info.complexInfo(ArrowTypeId.list)
+  public static let ArrowNull = Info.primitiveInfo(ArrowTypeId.null)
 
   public init(_ info: ArrowType.Info) {
     self.info = info
@@ -375,6 +376,8 @@ public class ArrowType {
       return MemoryLayout<Int8>.stride
     case .strct, .list:
       return 0
+    case .null:
+      return 0
     default:
       fatalError("Stride requested for unknown type: \(self)")
     }
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift
index e495b13..e052095 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -190,6 +190,16 @@ struct DataFrameTests {
     await spark.stop()
   }
 
+  @Test
+  func collectNull() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    #expect(try await spark.sql("SELECT null").collect() == [Row(nil)])
+    #expect(try await spark.sql("SELECT null, 1").collect() == [Row(nil, 1)])
+    #expect(try await spark.sql("SELECT null FROM RANGE(2)").collect() == [Row(nil), Row(nil)])
+    #expect(try await spark.sql("SELECT struct(null)").collect().count == 1)
+    await spark.stop()
+  }
+
   @Test
   func selectNone() async throws {
     let spark = try await SparkSession.builder.getOrCreate()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org