You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/03 05:31:29 UTC
(spark-connect-swift) branch main updated: [SPARK-59194] Use numeric Spark version comparison in tests
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 5840405 [SPARK-59194] Use numeric Spark version comparison in tests
5840405 is described below
commit 5840405dc19cabba3a547228d47f834d56d79b4d
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Wed Sep 2 22:31:22 2026 -0700
[SPARK-59194] Use numeric Spark version comparison in tests
### What changes were proposed in this pull request?
This PR replaces the lexicographic `String` comparisons in the test version gates with a numeric one.
- Adds `isSparkVersionAtLeast(_:_:)` in a new `Tests/SparkConnectTests/SparkVersionUtils.swift`, covered by `SparkVersionUtilsTests`.
- Migrates all 60 version gates across 12 test files to it.
- Turns the prefix checks left over from SPARK-59177 into lower bounds: `starts(with: "4.")` becomes a `4.0` bound, and `starts(with: "4.1")` in `ConstraintTests` a `4.1` bound.
### Why are the changes needed?
`String` comparison is lexicographic, so the gates break from Apache Spark 4.10 on:
```swift
"4.10" >= "4.2" // false
"4.10".starts(with: "4.1") // true, for a 4.1-only gate
```
Every `>= "4.2"` gate would silently skip its body on a 4.10 server, and the 4.1 gates in `ConstraintTests` would incorrectly run. The lower-bound style from SPARK-59177 is right, but it only holds if the comparison is numeric. The remaining prefix checks match the 4.x line only, so they would also skip on Spark 5; each guards a feature available since Apache Spark 4.0.
`SparkSessionTests.version()` keeps its `starts(with: "4.")`, which asserts the server is in the supported 4.x range rather than gating a test.
### Does this PR introduce _any_ user-facing change?
No. This is a test-only change.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5
Closes #508 from dongjoon-hyun/SPARK-59194.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
Tests/SparkConnectTests/CatalogTests.swift | 26 ++++-----
Tests/SparkConnectTests/ConstraintTests.swift | 8 +--
Tests/SparkConnectTests/CreateDataFrameTests.swift | 2 +-
.../SparkConnectTests/DataFrameInternalTests.swift | 2 +-
Tests/SparkConnectTests/DataFrameReaderTests.swift | 8 +--
Tests/SparkConnectTests/DataFrameTests.swift | 20 +++----
Tests/SparkConnectTests/DataFrameWriterTests.swift | 2 +-
Tests/SparkConnectTests/DataTypeTests.swift | 2 +-
.../SparkConnectTests/DateTimeFunctionsTests.swift | 6 +-
Tests/SparkConnectTests/MergeIntoWriterTests.swift | 6 +-
Tests/SparkConnectTests/SQLTests.swift | 2 +-
.../SparkConnectClientTests.swift | 12 ++--
Tests/SparkConnectTests/SparkSessionTests.swift | 24 ++++----
Tests/SparkConnectTests/SparkVersionUtils.swift | 48 ++++++++++++++++
.../SparkConnectTests/SparkVersionUtilsTests.swift | 67 ++++++++++++++++++++++
15 files changed, 175 insertions(+), 60 deletions(-)
diff --git a/Tests/SparkConnectTests/CatalogTests.swift b/Tests/SparkConnectTests/CatalogTests.swift
index 2f18cb1..e05a9f4 100644
--- a/Tests/SparkConnectTests/CatalogTests.swift
+++ b/Tests/SparkConnectTests/CatalogTests.swift
@@ -40,7 +40,7 @@ struct CatalogTests {
func setCurrentCatalog() async throws {
let spark = try await SparkSession.builder.getOrCreate()
try await spark.catalog.setCurrentCatalog("spark_catalog")
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
try await #require(throws: SparkConnectError.CatalogNotFound) {
try await spark.catalog.setCurrentCatalog("not_exist_catalog")
}
@@ -112,7 +112,7 @@ struct CatalogTests {
@Test
func createDatabase() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let dbName = "DB_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withDatabase(spark, dbName)({
#expect(try await spark.catalog.databaseExists(dbName) == false)
@@ -131,7 +131,7 @@ struct CatalogTests {
@Test
func dropDatabase() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let dbName = "DB_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withDatabase(spark, dbName)({
try await spark.catalog.createDatabase(dbName)
@@ -193,7 +193,7 @@ struct CatalogTests {
@Test
func listViews() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
#expect(try await spark.catalog.listViews().count == 0)
let viewName = ("VIEW_" + UUID().uuidString.replacingOccurrences(of: "-", with: ""))
@@ -241,7 +241,7 @@ struct CatalogTests {
@Test
func getTableProperties() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = ("TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: ""))
.lowercased()
try await SQLHelper.withTable(spark, tableName)({
@@ -304,7 +304,7 @@ struct CatalogTests {
try await SQLHelper.withTable(spark, tableName)({
try await spark.range(2).write.orc(path)
let expected =
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
[Row("id", nil, "bigint", true, false, false, false)]
} else {
[Row("id", nil, "bigint", true, false, false)]
@@ -319,7 +319,7 @@ struct CatalogTests {
try await SQLHelper.withTempView(spark, viewName)({
try await spark.range(1).createTempView(viewName)
let expected =
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
[Row("id", nil, "bigint", false, false, false, false)]
} else {
[Row("id", nil, "bigint", false, false, false)]
@@ -333,7 +333,7 @@ struct CatalogTests {
@Test
func listPartitions() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.sql(
@@ -510,7 +510,7 @@ struct CatalogTests {
@Test
func dropTable() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.range(1).write.saveAsTable(tableName)
@@ -530,7 +530,7 @@ struct CatalogTests {
@Test
func dropView() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let viewName = "VIEW_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, viewName)({
try await spark.sql("CREATE VIEW \(viewName) AS SELECT 1").count()
@@ -605,7 +605,7 @@ struct CatalogTests {
@Test
func getCreateTableString() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.range(1).write.saveAsTable(tableName)
@@ -624,7 +624,7 @@ struct CatalogTests {
@Test
func truncateTable() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.range(10).write.saveAsTable(tableName)
@@ -660,7 +660,7 @@ struct CatalogTests {
@Test
func analyzeTable() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.range(10).write.saveAsTable(tableName)
diff --git a/Tests/SparkConnectTests/ConstraintTests.swift b/Tests/SparkConnectTests/ConstraintTests.swift
index 3d7b7d3..802f688 100644
--- a/Tests/SparkConnectTests/ConstraintTests.swift
+++ b/Tests/SparkConnectTests/ConstraintTests.swift
@@ -34,7 +34,7 @@ struct ConstraintTests {
@Test
func primary_key() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.1") {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.sql("CREATE TABLE \(tableName)(a INT, PRIMARY KEY(a)) USING ORC").count()
@@ -47,7 +47,7 @@ struct ConstraintTests {
@Test
func foreign_key() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.1") {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let tableName1 = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
let tableName2 = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName1, tableName2)({
@@ -63,7 +63,7 @@ struct ConstraintTests {
@Test
func unique() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.1") {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.sql("CREATE TABLE \(tableName)(a INT UNIQUE) USING ORC").count()
@@ -76,7 +76,7 @@ struct ConstraintTests {
@Test
func check() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.1") {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
try await spark.sql(
diff --git a/Tests/SparkConnectTests/CreateDataFrameTests.swift b/Tests/SparkConnectTests/CreateDataFrameTests.swift
index eb4881b..542860c 100644
--- a/Tests/SparkConnectTests/CreateDataFrameTests.swift
+++ b/Tests/SparkConnectTests/CreateDataFrameTests.swift
@@ -82,7 +82,7 @@ struct CreateDataFrameTests {
@Test
func timeType() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.3" {
+ if await isSparkVersionAtLeast(spark.version, "4.3") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let time = try #require(LocalTime(hour: 12, minute: 34, second: 56, nanosecond: 123_456_000))
let df = try await spark.createDataFrame([[time], [nil]], "t TIME")
diff --git a/Tests/SparkConnectTests/DataFrameInternalTests.swift b/Tests/SparkConnectTests/DataFrameInternalTests.swift
index feed245..c71414f 100644
--- a/Tests/SparkConnectTests/DataFrameInternalTests.swift
+++ b/Tests/SparkConnectTests/DataFrameInternalTests.swift
@@ -86,7 +86,7 @@ struct DataFrameInternalTests {
@Test
func removeCachedRemoteRelation() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
let df = try await spark.range(10).localCheckpoint()
#expect(try await df.count() == 10)
let cachedRemoteRelation = await df.plan.root.cachedRemoteRelation
diff --git a/Tests/SparkConnectTests/DataFrameReaderTests.swift b/Tests/SparkConnectTests/DataFrameReaderTests.swift
index 71210c6..5843758 100644
--- a/Tests/SparkConnectTests/DataFrameReaderTests.swift
+++ b/Tests/SparkConnectTests/DataFrameReaderTests.swift
@@ -42,7 +42,7 @@ struct DataFrameReaderTests {
@Test
func csvDataset() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.2.0") {
let csvDF = try await spark.sql(
"SELECT * FROM VALUES "
+ "('Alice,25'), "
@@ -66,7 +66,7 @@ struct DataFrameReaderTests {
@Test
func jsonDataset() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.2.0") {
let jsonDF = try await spark.sql(
"SELECT * FROM VALUES "
+ "('{\"name\":\"Alice\",\"age\":25}'), "
@@ -80,7 +80,7 @@ struct DataFrameReaderTests {
@Test
func xml() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
let path = "../examples/src/main/resources/people.xml"
#expect(try await spark.read.option("rowTag", "person").format("xml").load(path).count() == 3)
#expect(try await spark.read.option("rowTag", "person").xml(path).count() == 3)
@@ -92,7 +92,7 @@ struct DataFrameReaderTests {
@Test
func xmlDataset() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.2.0") {
let xmlDF = try await spark.sql(
"SELECT * FROM VALUES "
+ "('<person><name>Alice</name><age>25</age></person>'), "
diff --git a/Tests/SparkConnectTests/DataFrameTests.swift b/Tests/SparkConnectTests/DataFrameTests.swift
index 5df8109..80dc6a0 100644
--- a/Tests/SparkConnectTests/DataFrameTests.swift
+++ b/Tests/SparkConnectTests/DataFrameTests.swift
@@ -137,7 +137,7 @@ struct DataFrameTests {
@Test
func dtypesGeospatial() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let expected = [
("st_geomfromwkb(X'0101000000000000000000F03F0000000000000040')", "geometry(0)"),
(
@@ -156,7 +156,7 @@ struct DataFrameTests {
@Test
func dtypesTime() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let expected = [
("TIME'12:34:56'", "time(6)"),
@@ -173,7 +173,7 @@ struct DataFrameTests {
@Test
func collectTime() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let expected = [
("TIME'00:00:00'", LocalTime(hour: 0, minute: 0)),
@@ -199,7 +199,7 @@ struct DataFrameTests {
@Test
func selectTimeLiteral() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let time = try #require(LocalTime(hour: 12, minute: 34, second: 56, nanosecond: 123_456_000))
let df = try await spark.range(1).select(lit(time))
@@ -541,7 +541,7 @@ struct DataFrameTests {
let version = await spark.version
// Since Spark 4.1, `AnalyzePlan` no longer executes commands eagerly (SPARK-51818),
// so command plans are not local anymore.
- let expected = version < "4.1"
+ let expected = !isSparkVersionAtLeast(version, "4.1")
#expect(try await spark.sql("SHOW DATABASES").isLocal() == expected)
#expect(try await spark.sql("SHOW TABLES").isLocal() == expected)
#expect(try await spark.range(1).isLocal() == false)
@@ -754,7 +754,7 @@ struct DataFrameTests {
@Test
func checkpoint() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
// By default, reliable checkpoint location is required.
try await #require(throws: Error.self) {
try await spark.range(10).checkpoint()
@@ -769,7 +769,7 @@ struct DataFrameTests {
@Test
func localCheckpoint() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
#expect(try await spark.range(10).localCheckpoint().count() == 10)
}
await spark.stop()
@@ -878,7 +878,7 @@ struct DataFrameTests {
@Test
func lateralJoin() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
let df1 = try await spark.sql("SELECT * FROM VALUES ('a', 1), ('b', 2) AS T(a, b)")
let df2 = try await spark.sql("SELECT * FROM VALUES ('c', 2), ('d', 3) AS S(c, b)")
let expectedCross = [
@@ -903,7 +903,7 @@ struct DataFrameTests {
func nearestByJoin() async throws {
let spark = try await SparkSession.builder.getOrCreate()
let version = await spark.version
- if version >= "4.2" {
+ if isSparkVersionAtLeast(version, "4.2") {
let users = try await spark.sql(
"SELECT * FROM VALUES (1, 10.0), (2, 20.0), (3, 30.0) AS T(user_id, score)")
let products = try await spark.sql(
@@ -1355,7 +1355,7 @@ struct DataFrameTests {
@Test
func transpose() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
#expect(try await spark.range(1).transpose().columns == ["key", "0"])
#expect(try await spark.range(1).transpose().count() == 0)
diff --git a/Tests/SparkConnectTests/DataFrameWriterTests.swift b/Tests/SparkConnectTests/DataFrameWriterTests.swift
index 94f6f43..1580ab3 100644
--- a/Tests/SparkConnectTests/DataFrameWriterTests.swift
+++ b/Tests/SparkConnectTests/DataFrameWriterTests.swift
@@ -51,7 +51,7 @@ struct DataFrameWriterTests {
func xml() async throws {
let tmpDir = "/tmp/" + UUID().uuidString
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
try await spark.range(2025).write.option("rowTag", "person").xml(tmpDir)
#expect(try await spark.read.option("rowTag", "person").xml(tmpDir).count() == 2025)
}
diff --git a/Tests/SparkConnectTests/DataTypeTests.swift b/Tests/SparkConnectTests/DataTypeTests.swift
index a732e69..cd7175b 100644
--- a/Tests/SparkConnectTests/DataTypeTests.swift
+++ b/Tests/SparkConnectTests/DataTypeTests.swift
@@ -162,7 +162,7 @@ struct DataTypeTests {
@Test
func schemaTimeType() async throws {
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let df = try await spark.sql(
"SELECT TIME'12:34:56' t6, CAST(TIME'12:34:56' AS TIME(0)) t0")
diff --git a/Tests/SparkConnectTests/DateTimeFunctionsTests.swift b/Tests/SparkConnectTests/DateTimeFunctionsTests.swift
index 922fd20..1ae14fb 100644
--- a/Tests/SparkConnectTests/DateTimeFunctionsTests.swift
+++ b/Tests/SparkConnectTests/DateTimeFunctionsTests.swift
@@ -275,7 +275,7 @@ struct DateTimeFunctionsTests {
).collect()
#expect(dates == [Row("2025-05-01", "2025-05-01", "2025-05-01")])
- if await spark.version >= "4.1" {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let tryDates = try await df.select(
try_to_date(col("s")).cast("string"), try_to_date(col("s"), "yyyy-MM-dd").cast("string"),
try_to_date(lit("abc"))
@@ -330,7 +330,7 @@ struct DateTimeFunctionsTests {
func selectTimeFunctions() async throws {
let spark = try await SparkSession.builder.getOrCreate()
try await spark.conf.set("spark.sql.timeType.enabled", "true")
- if await spark.version >= "4.1" {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let df = try await spark.sql("SELECT TIME'12:34:56' AS tm")
let times = try await df.select(
make_time(lit(12), lit(34), lit(56)).cast("string"), to_time(lit("12:34:56")).cast("string"),
@@ -345,7 +345,7 @@ struct DateTimeFunctionsTests {
#expect(rows.count == 1)
#expect(rows[0].length == 2)
}
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let df = try await spark.sql("SELECT TIMESTAMP'2025-05-01 12:34:56' AS t")
let rows = try await df.select(
time_from_seconds(lit(3661)).cast("string"), time_from_millis(lit(1000)).cast("string"),
diff --git a/Tests/SparkConnectTests/MergeIntoWriterTests.swift b/Tests/SparkConnectTests/MergeIntoWriterTests.swift
index 2d5c8e9..8bd319e 100644
--- a/Tests/SparkConnectTests/MergeIntoWriterTests.swift
+++ b/Tests/SparkConnectTests/MergeIntoWriterTests.swift
@@ -35,7 +35,7 @@ struct MergeIntoWriterTests {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
let mergeInto = try await spark.range(1).mergeInto(tableName, "true")
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
try await #require(throws: SparkConnectError.TableOrViewNotFound) {
try await mergeInto.whenMatched().delete().merge()
}
@@ -60,7 +60,7 @@ struct MergeIntoWriterTests {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
let mergeInto = try await spark.range(1).mergeInto(tableName, "true")
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
try await #require(throws: SparkConnectError.TableOrViewNotFound) {
try await mergeInto.whenNotMatched().insertAll().merge()
}
@@ -85,7 +85,7 @@ struct MergeIntoWriterTests {
let tableName = "TABLE_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await SQLHelper.withTable(spark, tableName)({
let mergeInto = try await spark.range(1).mergeInto(tableName, "true")
- if await spark.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
try await #require(throws: SparkConnectError.TableOrViewNotFound) {
try await mergeInto.whenNotMatchedBySource().delete().merge()
}
diff --git a/Tests/SparkConnectTests/SQLTests.swift b/Tests/SparkConnectTests/SQLTests.swift
index bd83c38..f3a4bb4 100644
--- a/Tests/SparkConnectTests/SQLTests.swift
+++ b/Tests/SparkConnectTests/SQLTests.swift
@@ -104,7 +104,7 @@ struct SQLTests {
for name in try! fm.contentsOfDirectory(atPath: path).sorted() {
guard name.hasSuffix(".sql") else { continue }
print(name)
- if await spark.version < "4.2" && queriesForSpark42AndLater.contains(name) {
+ if await !isSparkVersionAtLeast(spark.version, "4.2") && queriesForSpark42AndLater.contains(name) {
print("Skip query \(name) which requires Spark 4.2 or later")
continue
}
diff --git a/Tests/SparkConnectTests/SparkConnectClientTests.swift b/Tests/SparkConnectTests/SparkConnectClientTests.swift
index d9bd367..f5a6597 100644
--- a/Tests/SparkConnectTests/SparkConnectClientTests.swift
+++ b/Tests/SparkConnectTests/SparkConnectClientTests.swift
@@ -151,7 +151,7 @@ struct SparkConnectClientTests {
func jsonToDdl() async throws {
let client = try SparkConnectClient(remote: TEST_REMOTE)
let response = try await client.connect(UUID().uuidString)
- if response.sparkVersion.version.starts(with: "4.") {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.0") {
let json =
#"{"type":"struct","fields":[{"name":"id","type":"long","nullable":false,"metadata":{}}]}"#
#expect(try await client.jsonToDdl(json) == "id BIGINT NOT NULL")
@@ -163,7 +163,7 @@ struct SparkConnectClientTests {
func createDataflowGraph() async throws {
let client = try SparkConnectClient(remote: TEST_REMOTE)
let response = try await client.connect(UUID().uuidString)
- if response.sparkVersion.version >= "4.1" {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.1") {
let dataflowGraphID = try await client.createDataflowGraph()
#expect(UUID(uuidString: dataflowGraphID) != nil)
}
@@ -179,7 +179,7 @@ struct SparkConnectClientTests {
try await client.startRun("not-a-uuid-format")
}
- if response.sparkVersion.version >= "4.1" {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.1") {
let dataflowGraphID = try await client.createDataflowGraph()
#expect(UUID(uuidString: dataflowGraphID) != nil)
#expect(try await client.startRun(dataflowGraphID))
@@ -196,7 +196,7 @@ struct SparkConnectClientTests {
try await client.defineOutput("not-a-uuid-format", "ds1", "table")
}
- if response.sparkVersion.version >= "4.1" {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.1") {
let dataflowGraphID = try await client.createDataflowGraph()
#expect(UUID(uuidString: dataflowGraphID) != nil)
try await #require(throws: SparkConnectError.OutputTypeUnspecified) {
@@ -231,7 +231,7 @@ struct SparkConnectClientTests {
try await client.defineFlow("not-a-uuid-format", "f1", "ds1", Relation())
}
- if response.sparkVersion.version >= "4.1" {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.1") {
let dataflowGraphID = try await client.createDataflowGraph()
#expect(UUID(uuidString: dataflowGraphID) != nil)
let relation = await client.getLocalRelation().root
@@ -249,7 +249,7 @@ struct SparkConnectClientTests {
try await client.defineSqlGraphElements("not-a-uuid-format", "path", "sql")
}
- if response.sparkVersion.version >= "4.1" {
+ if isSparkVersionAtLeast(response.sparkVersion.version, "4.1") {
let dataflowGraphID = try await client.createDataflowGraph()
let sqlText = "CREATE MATERIALIZED VIEW mv1 AS SELECT 1"
#expect(UUID(uuidString: dataflowGraphID) != nil)
diff --git a/Tests/SparkConnectTests/SparkSessionTests.swift b/Tests/SparkConnectTests/SparkSessionTests.swift
index 511c998..c2b2255 100644
--- a/Tests/SparkConnectTests/SparkSessionTests.swift
+++ b/Tests/SparkConnectTests/SparkSessionTests.swift
@@ -61,7 +61,7 @@ struct SparkSessionTests {
func cloneSession() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.1" {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let viewName = "VIEW_" + UUID().uuidString.replacingOccurrences(of: "-", with: "")
try await spark.conf.set("spark.test.cloneSession", "original")
try await spark.range(3).createTempView(viewName)
@@ -87,7 +87,7 @@ struct SparkSessionTests {
func cloneSessionWithSessionID() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.1" {
+ if await isSparkVersionAtLeast(spark.version, "4.1") {
let newSessionID = UUID().uuidString.lowercased()
let cloned = try await spark.cloneSession(newSessionID)
#expect(cloned.sessionID == newSessionID)
@@ -113,7 +113,7 @@ struct SparkSessionTests {
func closedSessionID() async throws {
await SparkSession.builder.clear()
let spark1 = try await SparkSession.builder.getOrCreate()
- if await spark1.version >= "4.0.0" {
+ if await isSparkVersionAtLeast(spark1.version, "4.0.0") {
let sessionID = spark1.sessionID
await spark1.stop()
let remote = ProcessInfo.processInfo.environment["SPARK_REMOTE"] ?? "sc://localhost"
@@ -223,7 +223,7 @@ struct SparkSessionTests {
StructField(name: "c", dataType: .char(length: 5)),
StructField(name: "v", dataType: .varchar(length: 10)),
]))
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
#expect(
try await spark.parseDDL("t TIME(3)")
@@ -250,7 +250,7 @@ struct SparkSessionTests {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
let expected = [Row(true, 1, "a")]
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
#expect(try await spark.sql("SELECT ?, ?, ?", true, 1, "a").collect() == expected)
#expect(
try await spark.sql("SELECT :x, :y, :z", args: ["x": true, "y": 1, "z": "a"]).collect()
@@ -263,7 +263,7 @@ struct SparkSessionTests {
func sqlWithMoreParameterTypes() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
// Use at most 4 positional parameters per query because Spark Connect
// servers (up to 4.2.0) bind 5 or more positional parameters in the wrong order.
#expect(
@@ -286,7 +286,7 @@ struct SparkSessionTests {
func sqlWithTimeParameter() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
try await spark.conf.set("spark.sql.timeType.enabled", "true")
let time = try #require(LocalTime(hour: 12, minute: 34, second: 56, nanosecond: 123_456_000))
#expect(try await spark.sql("SELECT typeof(?)", time).collect() == [Row("time(6)")])
@@ -328,7 +328,7 @@ struct SparkSessionTests {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
#expect(fm.createFile(atPath: path, contents: "abc".data(using: .utf8)))
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
try await spark.addArtifact(path)
try await spark.addArtifact(url)
}
@@ -345,7 +345,7 @@ struct SparkSessionTests {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
#expect(fm.createFile(atPath: path, contents: "abc".data(using: .utf8)))
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
try await spark.addArtifacts(url, url)
}
try fm.removeItem(atPath: path)
@@ -356,7 +356,7 @@ struct SparkSessionTests {
func executeCommand() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version.starts(with: "4.") {
+ if await isSparkVersionAtLeast(spark.version, "4.0") {
await #expect(throws: SparkConnectError.DataSourceNotFound) {
try await spark.executeCommand("runner", "command", [:]).show()
}
@@ -442,7 +442,7 @@ struct SparkSessionTests {
func getOperationStatuses() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
#expect(try await spark.getOperationStatuses() == [])
}
await spark.stop()
@@ -452,7 +452,7 @@ struct SparkSessionTests {
func getOperationStatusesByIds() async throws {
await SparkSession.builder.clear()
let spark = try await SparkSession.builder.getOrCreate()
- if await spark.version >= "4.2" {
+ if await isSparkVersionAtLeast(spark.version, "4.2") {
let statuses = try await spark.getOperationStatuses(["id"])
#expect(statuses.count == 1)
#expect(statuses[0].operationID == "id")
diff --git a/Tests/SparkConnectTests/SparkVersionUtils.swift b/Tests/SparkConnectTests/SparkVersionUtils.swift
new file mode 100644
index 0000000..b7bee27
--- /dev/null
+++ b/Tests/SparkConnectTests/SparkVersionUtils.swift
@@ -0,0 +1,48 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+/// Returns whether `version` is greater than or equal to `target`.
+///
+/// The versions are compared component by component numerically, so this is correct where
+/// the lexicographical `String` comparison is not, e.g. `"4.10" >= "4.2"` is `false` while
+/// `isSparkVersionAtLeast("4.10", "4.2")` is `true`.
+///
+/// A missing component is treated as zero (`"4.2"` and `"4.2.0"` are equivalent) and a
+/// non-numeric suffix is ignored (`"4.3.0-SNAPSHOT"` is treated as `"4.3.0"`).
+func isSparkVersionAtLeast(_ version: String, _ target: String) -> Bool {
+ let lhs = numericComponents(version)
+ let rhs = numericComponents(target)
+ for i in 0..<max(lhs.count, rhs.count) {
+ let l = i < lhs.count ? lhs[i] : 0
+ let r = i < rhs.count ? rhs[i] : 0
+ if l != r {
+ return l > r
+ }
+ }
+ return true
+}
+
+/// Returns the leading numeric components of a version string, e.g. `[4, 3, 0]` for
+/// `"4.3.0-SNAPSHOT"`.
+private func numericComponents(_ version: String) -> [Int] {
+ version
+ .prefix(while: { $0.isNumber || $0 == "." })
+ .split(separator: ".")
+ .map { Int($0) ?? 0 }
+}
diff --git a/Tests/SparkConnectTests/SparkVersionUtilsTests.swift b/Tests/SparkConnectTests/SparkVersionUtilsTests.swift
new file mode 100644
index 0000000..c6f13c1
--- /dev/null
+++ b/Tests/SparkConnectTests/SparkVersionUtilsTests.swift
@@ -0,0 +1,67 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import Testing
+
+/// A test suite for `isSparkVersionAtLeast`. This does not require a Spark Connect server.
+struct SparkVersionUtilsTests {
+ @Test
+ func equalVersions() {
+ #expect(isSparkVersionAtLeast("4.2.0", "4.2.0"))
+ #expect(isSparkVersionAtLeast("4.2", "4.2.0"))
+ #expect(isSparkVersionAtLeast("4.2.0", "4.2"))
+ #expect(isSparkVersionAtLeast("4", "4.0.0"))
+ }
+
+ @Test
+ func compareVersions() {
+ #expect(isSparkVersionAtLeast("4.2.0", "4.1.0"))
+ #expect(isSparkVersionAtLeast("4.2.1", "4.2.0"))
+ #expect(isSparkVersionAtLeast("5.0.0", "4.0.0"))
+ #expect(!isSparkVersionAtLeast("4.1.0", "4.2.0"))
+ #expect(!isSparkVersionAtLeast("4.2.0", "4.2.1"))
+ #expect(!isSparkVersionAtLeast("3.5.0", "4.0.0"))
+ }
+
+ @Test
+ func compareVersionsNumerically() {
+ #expect(isSparkVersionAtLeast("4.10", "4.2"))
+ #expect(isSparkVersionAtLeast("4.10.0", "4.2.0"))
+ #expect(isSparkVersionAtLeast("4.2.10", "4.2.9"))
+ #expect(isSparkVersionAtLeast("10.0.0", "9.0.0"))
+ #expect(!isSparkVersionAtLeast("4.2", "4.10"))
+ #expect(!isSparkVersionAtLeast("4.9", "4.10"))
+ }
+
+ @Test
+ func ignoreNonNumericSuffix() {
+ #expect(isSparkVersionAtLeast("4.3.0-SNAPSHOT", "4.3.0"))
+ #expect(isSparkVersionAtLeast("4.3.0-SNAPSHOT", "4.2.0"))
+ #expect(isSparkVersionAtLeast("4.10.0-SNAPSHOT", "4.2.0"))
+ #expect(isSparkVersionAtLeast("4.1.0-preview1", "4.1.0"))
+ #expect(!isSparkVersionAtLeast("4.1.0-SNAPSHOT", "4.2.0"))
+ }
+
+ @Test
+ func emptyVersion() {
+ #expect(!isSparkVersionAtLeast("", "4.0.0"))
+ #expect(isSparkVersionAtLeast("", "0"))
+ #expect(isSparkVersionAtLeast("4.0.0", ""))
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org