You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/13 20:56:06 UTC
(spark-connect-swift) branch main updated: [SPARK-59474] Support `DECIMAL` type in `createDataFrame` with `[[Sendable?]]` rows
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new bd359a4 [SPARK-59474] Support `DECIMAL` type in `createDataFrame` with `[[Sendable?]]` rows
bd359a4 is described below
commit bd359a485d419dd8ece6694ded18dedc17a5e13b
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Sep 13 13:55:59 2026 -0700
[SPARK-59474] Support `DECIMAL` type in `createDataFrame` with `[[Sendable?]]` rows
### What changes were proposed in this pull request?
This PR aims to support the `DECIMAL` type in `createDataFrame` with `[[Sendable?]]` rows and a DDL schema string.
- Add a `.decimal` case to `ConvertToArrow.toArrowColumn`, which uses the precision and scale of the schema.
- Accept `Decimal` and integer values. Other values throw `SparkConnectError.InvalidType`.
- Like `createDataFrame` with `Encodable` types, values are rounded `HALF_UP` to the scale, and values which do not fit in the precision throw an error.
### Why are the changes needed?
Currently, `createDataFrame` throws `SparkConnectError.InvalidType` for `DECIMAL` columns.
```swift
try await spark.createDataFrame([[Decimal(1)]], "id DECIMAL(10, 2)")
```
### Does this PR introduce _any_ user-facing change?
No behavior change because previous it wasn't supported.
`createDataFrame` now works with `DECIMAL(p, s)` columns. The results match Apache Spark 4.2.0 `createDataFrame(rows, schema)` in ANSI mode, which is the default.
### How was this patch tested?
Pass the CIs with the newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5
Closes #561 from dongjoon-hyun/SPARK-59474.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
Sources/SparkConnect/ConvertToArrow.swift | 4 ++
Tests/SparkConnectTests/CreateDataFrameTests.swift | 66 +++++++++++++++++++++-
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git a/Sources/SparkConnect/ConvertToArrow.swift b/Sources/SparkConnect/ConvertToArrow.swift
index 823e4e2..79b909b 100644
--- a/Sources/SparkConnect/ConvertToArrow.swift
+++ b/Sources/SparkConnect/ConvertToArrow.swift
@@ -105,6 +105,10 @@ enum ConvertToArrow {
case .double:
return try fill(ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Double>,
column) { ($0 as? Double) ?? ($0 as? Float).map(Double.init) }
+ case .decimal(let decimal):
+ return try fill(
+ ArrowArrayBuilders.loadDecimal128ArrayBuilder(decimal.precision, decimal.scale), column
+ ) { ($0 as? Decimal) ?? toInt64($0).map { Decimal($0) } }
case .string:
return try fill(ArrowArrayBuilders.loadStringArrayBuilder(), column) { $0 as? String }
case .binary:
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index f19527b..15f73c5 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -185,11 +185,75 @@ struct CreateDataFrameTests {
await spark.stop()
}
+ @Test
+ func createDataFrameWithDecimal() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let a = Decimal(string: "-1.5")!
+ let b = Decimal(string: "12345678901234567890.50")!
+ let c = Decimal(string: "99999999999999999999999999999999999999")!
+ let d = Decimal(string: "-0.000000000000000001")!
+ let df = try await spark.createDataFrame(
+ [[a, b, c, d], [Decimal(0), Decimal(0), -c, Decimal(0)], [nil, nil, nil, nil]],
+ "a DECIMAL(10, 2), b DECIMAL(38, 2), c DECIMAL(38, 0), d DECIMAL(38, 18)")
+ #expect(
+ try await df.dtypes.map { $0.1 }
+ == ["decimal(10,2)", "decimal(38,2)", "decimal(38,0)", "decimal(38,18)"])
+ #expect(
+ try await df.selectExpr(
+ "CAST(a AS STRING)", "CAST(b AS STRING)", "CAST(c AS STRING)", "CAST(d AS STRING)"
+ ).collect() == [
+ Row(
+ "-1.50", "12345678901234567890.50", "99999999999999999999999999999999999999",
+ "-0.000000000000000001"),
+ Row("0.00", "0.00", "-99999999999999999999999999999999999999", "0.000000000000000000"),
+ Row(nil, nil, nil, nil),
+ ])
+ #expect(
+ try await df.collect() == [
+ Row(a, b, c, d), Row(Decimal(0), Decimal(0), -c, Decimal(0)), Row(nil, nil, nil, nil),
+ ])
+ await spark.stop()
+ }
+
+ @Test
+ func createDataFrameWithDecimalRoundingAndOverflow() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ // Like Spark, values are rounded `HALF_UP` to the scale. Integers are also accepted.
+ let df = try await spark.createDataFrame(
+ [
+ [Decimal(string: "1.005")!], [Decimal(string: "-1.005")!], [Decimal(string: "1.004")!],
+ [1], [Int8(-2)],
+ ], "v DECIMAL(10, 2)")
+ #expect(
+ try await df.selectExpr("CAST(v AS STRING)").collect()
+ == [Row("1.01"), Row("-1.01"), Row("1.00"), Row("1.00"), Row("-2.00")])
+
+ // Like Spark in ANSI mode, values which do not fit in the precision throw an error.
+ for value in ["123456789.5", "99999999.995"] {
+ await #expect(throws: ArrowError.self) {
+ try await spark.createDataFrame([[Decimal(string: value)!]], "v DECIMAL(10, 2)")
+ }
+ }
+ await #expect(throws: ArrowError.self) {
+ try await spark.createDataFrame(
+ [[Decimal(string: "100000000000000000000000000000000000000")!]], "v DECIMAL(38, 0)")
+ }
+
+ // Non-convertible values throw an error.
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([["1.5"]], "v DECIMAL(10, 2)")
+ }
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.createDataFrame([[1.5]], "v DECIMAL(10, 2)")
+ }
+ await spark.stop()
+ }
+
@Test
func unsupportedType() async throws {
let spark = try await SparkSession.builder.getOrCreate()
await #expect(throws: SparkConnectError.InvalidType) {
- try await spark.createDataFrame([[Decimal(1)]], "id DECIMAL(10, 2)")
+ try await spark.createDataFrame([[[1]]], "id ARRAY<INT>")
}
await spark.stop()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org