You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/03 17:52:56 UTC
(spark-connect-swift) branch main updated: [SPARK-59211] Support conditional functions
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 66e8a6b [SPARK-59211] Support conditional functions
66e8a6b is described below
commit 66e8a6bacc07dd1bb2a8f5cd67c659a5b247d796
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 3 10:52:49 2026 -0700
[SPARK-59211] Support conditional functions
### What changes were proposed in this pull request?
This PR aims to support the following 8 conditional SQL functions.
- `coalesce`
- `ifnull`
- `nanvl`
- `nullif`
- `nullifzero`
- `nvl`
- `nvl2`
- `zeroifnull`
In addition, this PR moves the existing `when` functions from `Functions.swift` into the new `ConditionalFunctions.swift` file together with their test cases.
### Why are the changes needed?
To improve the feature parity with the other Apache Spark Connect clients like PySpark and Scala.
Like `HashFunctions.swift`, these functions are grouped into a dedicated file instead of `Functions.swift` in order to follow the existing per-category layout of this repository. The resulting file matches the `Conditional Functions` group of the PySpark API reference, which consists of `coalesce`, `ifnull`, `nanvl`, `nullif`, `nullifzero`, `nvl`, `nvl2`, `when`, and `zeroifnull`.
### Does this PR introduce _any_ user-facing change?
No, the new functions are added to the unreleased branch, and moving `when` to another file does not change the public API.
### How was this patch tested?
Pass the CIs with newly added test cases.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5
Closes #518 from dongjoon-hyun/SPARK-59211.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
Sources/SparkConnect/ConditionalFunctions.swift | 112 +++++++++++++++++
Sources/SparkConnect/Functions.swift | 26 ----
.../ConditionalFunctionsTests.swift | 139 +++++++++++++++++++++
Tests/SparkConnectTests/FunctionsTests.swift | 47 -------
4 files changed, 251 insertions(+), 73 deletions(-)
diff --git a/Sources/SparkConnect/ConditionalFunctions.swift b/Sources/SparkConnect/ConditionalFunctions.swift
new file mode 100644
index 0000000..4a3e1e8
--- /dev/null
+++ b/Sources/SparkConnect/ConditionalFunctions.swift
@@ -0,0 +1,112 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+// MARK: - Conditional functions
+
+/// Returns the first column that is not null, or null if all inputs are null.
+/// - Parameter cols: ``Column``s to coalesce.
+/// - Returns: A ``Column``.
+public func coalesce(_ cols: Column...) -> Column {
+ return fn("coalesce", cols)
+}
+
+/// Returns `col2` if `col1` is null, or `col1` otherwise.
+/// - Parameters:
+/// - col1: A ``Column`` to test for null.
+/// - col2: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func ifnull(_ col1: Column, _ col2: Column) -> Column {
+ return fn("ifnull", col1, col2)
+}
+
+/// Returns `col1` if it is not NaN, or `col2` if `col1` is NaN.
+/// Both inputs should be floating point columns (`DoubleType` or `FloatType`).
+/// - Parameters:
+/// - col1: A ``Column`` to test for NaN.
+/// - col2: A ``Column`` to return when `col1` is NaN.
+/// - Returns: A ``Column``.
+public func nanvl(_ col1: Column, _ col2: Column) -> Column {
+ return fn("nanvl", col1, col2)
+}
+
+/// Returns null if `col1` equals `col2`, or `col1` otherwise.
+/// - Parameters:
+/// - col1: A ``Column`` to return when the two columns are not equal.
+/// - col2: A ``Column`` to compare with.
+/// - Returns: A ``Column``.
+public func nullif(_ col1: Column, _ col2: Column) -> Column {
+ return fn("nullif", col1, col2)
+}
+
+/// Returns null if `col` is equal to zero, or `col` otherwise.
+/// - Parameter col: A ``Column`` to test for zero.
+/// - Returns: A ``Column``.
+public func nullifzero(_ col: Column) -> Column {
+ return fn("nullifzero", col)
+}
+
+/// Returns `col2` if `col1` is null, or `col1` otherwise.
+/// - Parameters:
+/// - col1: A ``Column`` to test for null.
+/// - col2: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func nvl(_ col1: Column, _ col2: Column) -> Column {
+ return fn("nvl", col1, col2)
+}
+
+/// Returns `col2` if `col1` is not null, or `col3` otherwise.
+/// - Parameters:
+/// - col1: A ``Column`` to test for null.
+/// - col2: A ``Column`` to return when `col1` is not null.
+/// - col3: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func nvl2(_ col1: Column, _ col2: Column, _ col3: Column) -> Column {
+ return fn("nvl2", col1, col2, col3)
+}
+
+/// Evaluates a list of conditions and returns one of multiple possible result expressions.
+/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
+///
+/// ```swift
+/// df.select(when(col("age") > 21, "adult").otherwise("minor").alias("group"))
+/// ```
+/// - Parameters:
+/// - condition: A condition ``Column``.
+/// - value: A value ``Column`` to return when the condition is true.
+/// - Returns: A ``Column``.
+public func when(_ condition: Column, _ value: Column) -> Column {
+ return fn("when", condition, value)
+}
+
+/// Evaluates a list of conditions and returns one of multiple possible result expressions.
+/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
+/// - Parameters:
+/// - condition: A condition ``Column``.
+/// - value: A literal value to return when the condition is true.
+/// - Returns: A ``Column``.
+public func when(_ condition: Column, _ value: some SparkLiteral) -> Column {
+ return when(condition, value.toLiteralColumn)
+}
+
+/// Returns zero if `col` is null, or `col` otherwise.
+/// - Parameter col: A ``Column`` to test for null.
+/// - Returns: A ``Column``.
+public func zeroifnull(_ col: Column) -> Column {
+ return fn("zeroifnull", col)
+}
diff --git a/Sources/SparkConnect/Functions.swift b/Sources/SparkConnect/Functions.swift
index dac4b9c..3904b6f 100644
--- a/Sources/SparkConnect/Functions.swift
+++ b/Sources/SparkConnect/Functions.swift
@@ -258,32 +258,6 @@ public func max(_ col: Column) -> Column {
return fn("max", col)
}
-// MARK: - Condition functions
-
-/// Evaluates a list of conditions and returns one of multiple possible result expressions.
-/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
-///
-/// ```swift
-/// df.select(when(col("age") > 21, "adult").otherwise("minor").alias("group"))
-/// ```
-/// - Parameters:
-/// - condition: A condition ``Column``.
-/// - value: A value ``Column`` to return when the condition is true.
-/// - Returns: A ``Column``.
-public func when(_ condition: Column, _ value: Column) -> Column {
- return fn("when", condition, value)
-}
-
-/// Evaluates a list of conditions and returns one of multiple possible result expressions.
-/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
-/// - Parameters:
-/// - condition: A condition ``Column``.
-/// - value: A literal value to return when the condition is true.
-/// - Returns: A ``Column``.
-public func when(_ condition: Column, _ value: some SparkLiteral) -> Column {
- return when(condition, value.toLiteralColumn)
-}
-
func fn(_ name: String, _ args: Column...) -> Column {
var function = Spark_Connect_Expression.UnresolvedFunction()
function.functionName = name
diff --git a/Tests/SparkConnectTests/ConditionalFunctionsTests.swift b/Tests/SparkConnectTests/ConditionalFunctionsTests.swift
new file mode 100644
index 0000000..7361b24
--- /dev/null
+++ b/Tests/SparkConnectTests/ConditionalFunctionsTests.swift
@@ -0,0 +1,139 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import Testing
+
+@testable import SparkConnect
+
+/// A test suite for `ConditionalFunctions`
+@Suite(.serialized)
+struct ConditionalFunctionsTests {
+
+ @Test
+ func conditionalFunctions() throws {
+ for (column, name) in [
+ (nullifzero(col("a")), "nullifzero"),
+ (zeroifnull(col("a")), "zeroifnull"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+ }
+
+ for (column, name) in [
+ (coalesce(col("a"), col("b")), "coalesce"),
+ (ifnull(col("a"), col("b")), "ifnull"),
+ (nanvl(col("a"), col("b")), "nanvl"),
+ (nullif(col("a"), col("b")), "nullif"),
+ (nvl(col("a"), col("b")), "nvl"),
+ ] {
+ let expr = column.expr
+ #expect(expr.unresolvedFunction.functionName == name)
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+ #expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+ #expect(expr.unresolvedFunction.arguments[1].unresolvedAttribute.unparsedIdentifier == "b")
+ }
+
+ let coalesced = coalesce(col("a"), col("b"), col("c")).expr
+ #expect(coalesced.unresolvedFunction.functionName == "coalesce")
+ #expect(coalesced.unresolvedFunction.arguments.count == 3)
+
+ let nvl2Expr = nvl2(col("a"), col("b"), col("c")).expr
+ #expect(nvl2Expr.unresolvedFunction.functionName == "nvl2")
+ #expect(nvl2Expr.unresolvedFunction.arguments.count == 3)
+ }
+
+ @Test
+ func whenFunction() throws {
+ let expr = when(col("a") > 1, lit(10)).expr
+ #expect(expr.unresolvedFunction.functionName == "when")
+ #expect(expr.unresolvedFunction.arguments.count == 2)
+
+ let chained = try when(col("a") > 2, 10).when(col("a") > 1, 20).otherwise(0).expr
+ #expect(chained.unresolvedFunction.functionName == "when")
+ #expect(chained.unresolvedFunction.arguments.count == 5)
+ #expect(chained.unresolvedFunction.arguments[1].literal.long == 10)
+ #expect(chained.unresolvedFunction.arguments[3].literal.long == 20)
+ #expect(chained.unresolvedFunction.arguments[4].literal.long == 0)
+ }
+
+ @Test
+ func whenMisuse() throws {
+ // `when`/`otherwise` can only be applied on a column generated by `when`.
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try col("a").when(col("b") > 1, 1)
+ }
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try col("a").otherwise(1)
+ }
+ // `when`/`otherwise` cannot be applied once `otherwise` is applied.
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try when(col("a") > 1, 1).otherwise(2).when(col("a") > 0, 3)
+ }
+ #expect(throws: SparkConnectError.InvalidArgument) {
+ try when(col("a") > 1, 1).otherwise(2).otherwise(3)
+ }
+ }
+
+ @Test
+ func selectConditionalFunctions() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES (1, 10), (CAST(NULL AS INT), 20) T(a, b)")
+ #expect(try await df.select(coalesce(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+ #expect(try await df.select(ifnull(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+ #expect(try await df.select(nvl(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+ #expect(
+ try await df.select(nvl2(col("a"), lit("yes"), lit("no"))).collect()
+ == [Row("yes"), Row("no")])
+ #expect(try await df.select(nullif(col("b"), lit(10))).collect() == [Row(nil), Row(20)])
+
+ let doubles = try await spark.sql("SELECT CAST('NaN' AS DOUBLE) AS a, 1.0D AS b")
+ #expect(try await doubles.select(nanvl(col("a"), col("b"))).collect() == [Row(1.0)])
+ #expect(try await doubles.select(nanvl(col("b"), col("a"))).collect() == [Row(1.0)])
+ await spark.stop()
+ }
+
+ @Test
+ func selectNullifzeroAndZeroifnull() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ if await isSparkVersionAtLeast(spark.version, "4.0.0") {
+ let df = try await spark.sql(
+ "SELECT * FROM VALUES (0, CAST(NULL AS INT)), (1, 2) T(a, b)")
+ #expect(try await df.select(nullifzero(col("a"))).collect() == [Row(nil), Row(1)])
+ #expect(try await df.select(zeroifnull(col("b"))).collect() == [Row(0), Row(2)])
+ }
+ await spark.stop()
+ }
+
+ @Test
+ func selectWhenOtherwise() async throws {
+ let spark = try await SparkSession.builder.getOrCreate()
+ let df = try await spark.range(4)
+ let rows = try await df.select(
+ when(col("id") < 1, "zero").when(col("id") < 3, "few").otherwise("many").alias("group")
+ ).collect()
+ #expect(rows == [Row("zero"), Row("few"), Row("few"), Row("many")])
+
+ // Without `otherwise`, null is returned for unmatched conditions.
+ let nullable = try await df.select(when(col("id") == 0, "zero").alias("group")).collect()
+ #expect(nullable == [Row("zero"), Row(nil), Row(nil), Row(nil)])
+ await spark.stop()
+ }
+}
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift b/Tests/SparkConnectTests/FunctionsTests.swift
index 7087437..86b49db 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -119,38 +119,6 @@ struct FunctionsTests {
}
}
- @Test
- func whenFunction() throws {
- let expr = when(col("a") > 1, lit(10)).expr
- #expect(expr.unresolvedFunction.functionName == "when")
- #expect(expr.unresolvedFunction.arguments.count == 2)
-
- let chained = try when(col("a") > 2, 10).when(col("a") > 1, 20).otherwise(0).expr
- #expect(chained.unresolvedFunction.functionName == "when")
- #expect(chained.unresolvedFunction.arguments.count == 5)
- #expect(chained.unresolvedFunction.arguments[1].literal.long == 10)
- #expect(chained.unresolvedFunction.arguments[3].literal.long == 20)
- #expect(chained.unresolvedFunction.arguments[4].literal.long == 0)
- }
-
- @Test
- func whenMisuse() throws {
- // `when`/`otherwise` can only be applied on a column generated by `when`.
- #expect(throws: SparkConnectError.InvalidArgument) {
- try col("a").when(col("b") > 1, 1)
- }
- #expect(throws: SparkConnectError.InvalidArgument) {
- try col("a").otherwise(1)
- }
- // `when`/`otherwise` cannot be applied once `otherwise` is applied.
- #expect(throws: SparkConnectError.InvalidArgument) {
- try when(col("a") > 1, 1).otherwise(2).when(col("a") > 0, 3)
- }
- #expect(throws: SparkConnectError.InvalidArgument) {
- try when(col("a") > 1, 1).otherwise(2).otherwise(3)
- }
- }
-
@Test
func comparisonOperators() throws {
for (column, name) in [
@@ -475,19 +443,4 @@ struct FunctionsTests {
#expect(rows == [Row(1, 10, 7)])
await spark.stop()
}
-
- @Test
- func selectWhenOtherwise() async throws {
- let spark = try await SparkSession.builder.getOrCreate()
- let df = try await spark.range(4)
- let rows = try await df.select(
- when(col("id") < 1, "zero").when(col("id") < 3, "few").otherwise("many").alias("group")
- ).collect()
- #expect(rows == [Row("zero"), Row("few"), Row("few"), Row("many")])
-
- // Without `otherwise`, null is returned for unmatched conditions.
- let nullable = try await df.select(when(col("id") == 0, "zero").alias("group")).collect()
- #expect(nullable == [Row("zero"), Row(nil), Row(nil), Row(nil)])
- await spark.stop()
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org