You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/08/07 20:27:08 UTC
(spark-connect-swift) branch main updated: [SPARK-58651] Support `parseDDL` in `SparkSession`
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new aaccf26 [SPARK-58651] Support `parseDDL` in `SparkSession`
aaccf26 is described below
commit aaccf2680e4a275e38c70f601cde9c0ede52abca
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Fri Aug 7 13:27:01 2026 -0700
[SPARK-58651] Support `parseDDL` in `SparkSession`
### What changes were proposed in this pull request?
This PR adds `SparkSession.parseDDL` which parses a DDL-formatted schema string into the public `StructType` model using the server-side parser.
```swift
let schema = try await spark.parseDDL("id BIGINT NOT NULL, name STRING")
// StructType(fields: [
// StructField(name: "id", dataType: .long, nullable: false),
// StructField(name: "name", dataType: .string),
// ])
```
Internally, it reuses the existing `AnalyzePlan` `ddlParse` round-trip (already used by `createDataFrame` and `DataFrameReader.schema`) and converts the returned protobuf `DataType` via the `StructType` initializer added in SPARK-58647. A non-struct top-level type (e.g. `array<int>`) throws `SparkConnectError.InvalidType`, and an invalid DDL string throws `SparkConnectError.ParseSyntaxError` through the existing `ErrorInfo`-based error conversion.
Note that this follows the PySpark Connect behavior where `DataType.fromDDL` delegates to the server-side parser (`SparkSession._parse_ddl` via the `ddl_parse` analyze method), instead of implementing a client-local SQL type parser.
### Why are the changes needed?
For feature parity with Scala (`StructType.fromDDL`) and PySpark (`DataType.fromDDL`), so users can obtain a traversable `StructType` from a DDL string. This is the missing inverse of `StructType.toDDL`.
### Does this PR introduce _any_ user-facing change?
No. This adds a new API without changing existing behavior.
### How was this patch tested?
Pass the CIs with the newly added test cases .
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #489 from dongjoon-hyun/SPARK-58651.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
Sources/SparkConnect/SparkSession.swift | 19 +++++++
Tests/SparkConnectTests/SparkSessionTests.swift | 71 +++++++++++++++++++++++++
2 files changed, 90 insertions(+)
diff --git a/Sources/SparkConnect/SparkSession.swift b/Sources/SparkConnect/SparkSession.swift
index 27e1f38..7dd1057 100644
--- a/Sources/SparkConnect/SparkSession.swift
+++ b/Sources/SparkConnect/SparkSession.swift
@@ -186,6 +186,25 @@ public actor SparkSession {
return try await createDataFrame(data, schema.toDDL)
}
+ /// Parse a DDL-formatted schema string into a ``StructType`` using the server-side parser.
+ ///
+ /// ```swift
+ /// let schema = try await spark.parseDDL("id INT NOT NULL, name STRING")
+ /// ```
+ ///
+ /// - Parameter ddl: A DDL-formatted schema string, e.g. `id INT, name STRING` or
+ /// `struct<id:int,name:string>`.
+ /// - Returns: A ``StructType`` instance.
+ /// - Throws: `SparkConnectError.ParseSyntaxError` if the DDL string is invalid, or
+ /// `SparkConnectError.InvalidType` if the parsed type is not a struct type.
+ public func parseDDL(_ ddl: String) async throws -> StructType {
+ let dataType = try await client.ddlParse(ddl)
+ guard case .struct(let structType) = dataType.kind else {
+ throw SparkConnectError.InvalidType
+ }
+ return try StructType(structType)
+ }
+
/// Create a ``DataFrame`` with a single `Int64` column name `id`, containing elements in a
/// range from 0 to `end` (exclusive) with step value 1.
///
diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift
index ac68fc2..511c998 100644
--- a/Tests/SparkConnectTests/SparkSessionTests.swift
+++ b/Tests/SparkConnectTests/SparkSessionTests.swift
@@ -174,6 +174,77 @@ struct SparkSessionTests {
await spark.stop()
}
+ @Test
+ func parseDDL() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ let expected = StructType(fields: [
+ StructField(name: "id", dataType: .long, nullable: false),
+ StructField(name: "name", dataType: .string),
+ ])
+ #expect(try await spark.parseDDL("id BIGINT NOT NULL, name STRING") == expected)
+ #expect(
+ try await spark.parseDDL("struct<id:bigint,name:string>")
+ == StructType(fields: [
+ StructField(name: "id", dataType: .long),
+ StructField(name: "name", dataType: .string),
+ ]))
+ #expect(try await spark.parseDDL(expected.toDDL) == expected)
+ await spark.stop()
+ }
+
+ @Test
+ func parseDDLWithNestedTypes() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ #expect(
+ try await spark.parseDDL("a ARRAY<INT>, m MAP<STRING, DOUBLE>, s STRUCT<x: INT NOT NULL>")
+ == StructType(fields: [
+ StructField(name: "a", dataType: .array(elementType: .integer, containsNull: true)),
+ StructField(
+ name: "m", dataType: .map(keyType: .string, valueType: .double, valueContainsNull: true)
+ ),
+ StructField(
+ name: "s",
+ dataType: .struct(
+ StructType(fields: [StructField(name: "x", dataType: .integer, nullable: false)]))),
+ ]))
+ await spark.stop()
+ }
+
+ @Test
+ func parseDDLWithParameterizedTypes() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ #expect(
+ try await spark.parseDDL("d DECIMAL(10,2), c CHAR(5), v VARCHAR(10)")
+ == StructType(fields: [
+ StructField(name: "d", dataType: .decimal(precision: 10, scale: 2)),
+ StructField(name: "c", dataType: .char(length: 5)),
+ StructField(name: "v", dataType: .varchar(length: 10)),
+ ]))
+ if await spark.version >= "4.2" {
+ try await spark.conf.set("spark.sql.timeType.enabled", "true")
+ #expect(
+ try await spark.parseDDL("t TIME(3)")
+ == StructType(fields: [StructField(name: "t", dataType: .time(precision: 3))]))
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func parseDDLWithInvalidDDL() async throws {
+ await SparkSession.builder.clear()
+ let spark = try await SparkSession.builder.getOrCreate()
+ await #expect(throws: SparkConnectError.ParseSyntaxError) {
+ try await spark.parseDDL("a INT,")
+ }
+ await #expect(throws: SparkConnectError.InvalidType) {
+ try await spark.parseDDL("array<int>")
+ }
+ await spark.stop()
+ }
+
@Test
func sql() async throws {
await SparkSession.builder.clear()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org