You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/03 17:52:56 UTC

(spark-connect-swift) branch main updated: [SPARK-59211] Support conditional functions

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git


The following commit(s) were added to refs/heads/main by this push:
     new 66e8a6b  [SPARK-59211] Support conditional functions
66e8a6b is described below

commit 66e8a6bacc07dd1bb2a8f5cd67c659a5b247d796
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 3 10:52:49 2026 -0700

    [SPARK-59211] Support conditional functions
    
    ### What changes were proposed in this pull request?
    
    This PR aims to support the following 8 conditional SQL functions.
    
    - `coalesce`
    - `ifnull`
    - `nanvl`
    - `nullif`
    - `nullifzero`
    - `nvl`
    - `nvl2`
    - `zeroifnull`
    
    In addition, this PR moves the existing `when` functions from `Functions.swift` into the new `ConditionalFunctions.swift` file together with their test cases.
    
    ### Why are the changes needed?
    
    To improve the feature parity with the other Apache Spark Connect clients like PySpark and Scala.
    
    Like `HashFunctions.swift`, these functions are grouped into a dedicated file instead of `Functions.swift` in order to follow the existing per-category layout of this repository. The resulting file matches the `Conditional Functions` group of the PySpark API reference, which consists of `coalesce`, `ifnull`, `nanvl`, `nullif`, `nullifzero`, `nvl`, `nvl2`, `when`, and `zeroifnull`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No, the new functions are added to the unreleased branch, and moving `when` to another file does not change the public API.
    
    ### How was this patch tested?
    
    Pass the CIs with newly added test cases.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5
    
    Closes #518 from dongjoon-hyun/SPARK-59211.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 Sources/SparkConnect/ConditionalFunctions.swift    | 112 +++++++++++++++++
 Sources/SparkConnect/Functions.swift               |  26 ----
 .../ConditionalFunctionsTests.swift                | 139 +++++++++++++++++++++
 Tests/SparkConnectTests/FunctionsTests.swift       |  47 -------
 4 files changed, 251 insertions(+), 73 deletions(-)

diff --git a/Sources/SparkConnect/ConditionalFunctions.swift b/Sources/SparkConnect/ConditionalFunctions.swift
new file mode 100644
index 0000000..4a3e1e8
--- /dev/null
+++ b/Sources/SparkConnect/ConditionalFunctions.swift
@@ -0,0 +1,112 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+// MARK: - Conditional functions
+
+/// Returns the first column that is not null, or null if all inputs are null.
+/// - Parameter cols: ``Column``s to coalesce.
+/// - Returns: A ``Column``.
+public func coalesce(_ cols: Column...) -> Column {
+  return fn("coalesce", cols)
+}
+
+/// Returns `col2` if `col1` is null, or `col1` otherwise.
+/// - Parameters:
+///   - col1: A ``Column`` to test for null.
+///   - col2: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func ifnull(_ col1: Column, _ col2: Column) -> Column {
+  return fn("ifnull", col1, col2)
+}
+
+/// Returns `col1` if it is not NaN, or `col2` if `col1` is NaN.
+/// Both inputs should be floating point columns (`DoubleType` or `FloatType`).
+/// - Parameters:
+///   - col1: A ``Column`` to test for NaN.
+///   - col2: A ``Column`` to return when `col1` is NaN.
+/// - Returns: A ``Column``.
+public func nanvl(_ col1: Column, _ col2: Column) -> Column {
+  return fn("nanvl", col1, col2)
+}
+
+/// Returns null if `col1` equals `col2`, or `col1` otherwise.
+/// - Parameters:
+///   - col1: A ``Column`` to return when the two columns are not equal.
+///   - col2: A ``Column`` to compare with.
+/// - Returns: A ``Column``.
+public func nullif(_ col1: Column, _ col2: Column) -> Column {
+  return fn("nullif", col1, col2)
+}
+
+/// Returns null if `col` is equal to zero, or `col` otherwise.
+/// - Parameter col: A ``Column`` to test for zero.
+/// - Returns: A ``Column``.
+public func nullifzero(_ col: Column) -> Column {
+  return fn("nullifzero", col)
+}
+
+/// Returns `col2` if `col1` is null, or `col1` otherwise.
+/// - Parameters:
+///   - col1: A ``Column`` to test for null.
+///   - col2: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func nvl(_ col1: Column, _ col2: Column) -> Column {
+  return fn("nvl", col1, col2)
+}
+
+/// Returns `col2` if `col1` is not null, or `col3` otherwise.
+/// - Parameters:
+///   - col1: A ``Column`` to test for null.
+///   - col2: A ``Column`` to return when `col1` is not null.
+///   - col3: A ``Column`` to return when `col1` is null.
+/// - Returns: A ``Column``.
+public func nvl2(_ col1: Column, _ col2: Column, _ col3: Column) -> Column {
+  return fn("nvl2", col1, col2, col3)
+}
+
+/// Evaluates a list of conditions and returns one of multiple possible result expressions.
+/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
+///
+/// ```swift
+/// df.select(when(col("age") > 21, "adult").otherwise("minor").alias("group"))
+/// ```
+/// - Parameters:
+///   - condition: A condition ``Column``.
+///   - value: A value ``Column`` to return when the condition is true.
+/// - Returns: A ``Column``.
+public func when(_ condition: Column, _ value: Column) -> Column {
+  return fn("when", condition, value)
+}
+
+/// Evaluates a list of conditions and returns one of multiple possible result expressions.
+/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
+/// - Parameters:
+///   - condition: A condition ``Column``.
+///   - value: A literal value to return when the condition is true.
+/// - Returns: A ``Column``.
+public func when(_ condition: Column, _ value: some SparkLiteral) -> Column {
+  return when(condition, value.toLiteralColumn)
+}
+
+/// Returns zero if `col` is null, or `col` otherwise.
+/// - Parameter col: A ``Column`` to test for null.
+/// - Returns: A ``Column``.
+public func zeroifnull(_ col: Column) -> Column {
+  return fn("zeroifnull", col)
+}
diff --git a/Sources/SparkConnect/Functions.swift b/Sources/SparkConnect/Functions.swift
index dac4b9c..3904b6f 100644
--- a/Sources/SparkConnect/Functions.swift
+++ b/Sources/SparkConnect/Functions.swift
@@ -258,32 +258,6 @@ public func max(_ col: Column) -> Column {
   return fn("max", col)
 }
 
-// MARK: - Condition functions
-
-/// Evaluates a list of conditions and returns one of multiple possible result expressions.
-/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
-///
-/// ```swift
-/// df.select(when(col("age") > 21, "adult").otherwise("minor").alias("group"))
-/// ```
-/// - Parameters:
-///   - condition: A condition ``Column``.
-///   - value: A value ``Column`` to return when the condition is true.
-/// - Returns: A ``Column``.
-public func when(_ condition: Column, _ value: Column) -> Column {
-  return fn("when", condition, value)
-}
-
-/// Evaluates a list of conditions and returns one of multiple possible result expressions.
-/// If `otherwise` is not defined at the end, null is returned for unmatched conditions.
-/// - Parameters:
-///   - condition: A condition ``Column``.
-///   - value: A literal value to return when the condition is true.
-/// - Returns: A ``Column``.
-public func when(_ condition: Column, _ value: some SparkLiteral) -> Column {
-  return when(condition, value.toLiteralColumn)
-}
-
 func fn(_ name: String, _ args: Column...) -> Column {
   var function = Spark_Connect_Expression.UnresolvedFunction()
   function.functionName = name
diff --git a/Tests/SparkConnectTests/ConditionalFunctionsTests.swift b/Tests/SparkConnectTests/ConditionalFunctionsTests.swift
new file mode 100644
index 0000000..7361b24
--- /dev/null
+++ b/Tests/SparkConnectTests/ConditionalFunctionsTests.swift
@@ -0,0 +1,139 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//  http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+
+import Testing
+
+@testable import SparkConnect
+
+/// A test suite for `ConditionalFunctions`
+@Suite(.serialized)
+struct ConditionalFunctionsTests {
+
+  @Test
+  func conditionalFunctions() throws {
+    for (column, name) in [
+      (nullifzero(col("a")), "nullifzero"),
+      (zeroifnull(col("a")), "zeroifnull"),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+    }
+
+    for (column, name) in [
+      (coalesce(col("a"), col("b")), "coalesce"),
+      (ifnull(col("a"), col("b")), "ifnull"),
+      (nanvl(col("a"), col("b")), "nanvl"),
+      (nullif(col("a"), col("b")), "nullif"),
+      (nvl(col("a"), col("b")), "nvl"),
+    ] {
+      let expr = column.expr
+      #expect(expr.unresolvedFunction.functionName == name)
+      #expect(expr.unresolvedFunction.arguments.count == 2)
+      #expect(expr.unresolvedFunction.arguments[0].unresolvedAttribute.unparsedIdentifier == "a")
+      #expect(expr.unresolvedFunction.arguments[1].unresolvedAttribute.unparsedIdentifier == "b")
+    }
+
+    let coalesced = coalesce(col("a"), col("b"), col("c")).expr
+    #expect(coalesced.unresolvedFunction.functionName == "coalesce")
+    #expect(coalesced.unresolvedFunction.arguments.count == 3)
+
+    let nvl2Expr = nvl2(col("a"), col("b"), col("c")).expr
+    #expect(nvl2Expr.unresolvedFunction.functionName == "nvl2")
+    #expect(nvl2Expr.unresolvedFunction.arguments.count == 3)
+  }
+
+  @Test
+  func whenFunction() throws {
+    let expr = when(col("a") > 1, lit(10)).expr
+    #expect(expr.unresolvedFunction.functionName == "when")
+    #expect(expr.unresolvedFunction.arguments.count == 2)
+
+    let chained = try when(col("a") > 2, 10).when(col("a") > 1, 20).otherwise(0).expr
+    #expect(chained.unresolvedFunction.functionName == "when")
+    #expect(chained.unresolvedFunction.arguments.count == 5)
+    #expect(chained.unresolvedFunction.arguments[1].literal.long == 10)
+    #expect(chained.unresolvedFunction.arguments[3].literal.long == 20)
+    #expect(chained.unresolvedFunction.arguments[4].literal.long == 0)
+  }
+
+  @Test
+  func whenMisuse() throws {
+    // `when`/`otherwise` can only be applied on a column generated by `when`.
+    #expect(throws: SparkConnectError.InvalidArgument) {
+      try col("a").when(col("b") > 1, 1)
+    }
+    #expect(throws: SparkConnectError.InvalidArgument) {
+      try col("a").otherwise(1)
+    }
+    // `when`/`otherwise` cannot be applied once `otherwise` is applied.
+    #expect(throws: SparkConnectError.InvalidArgument) {
+      try when(col("a") > 1, 1).otherwise(2).when(col("a") > 0, 3)
+    }
+    #expect(throws: SparkConnectError.InvalidArgument) {
+      try when(col("a") > 1, 1).otherwise(2).otherwise(3)
+    }
+  }
+
+  @Test
+  func selectConditionalFunctions() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.sql(
+      "SELECT * FROM VALUES (1, 10), (CAST(NULL AS INT), 20) T(a, b)")
+    #expect(try await df.select(coalesce(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+    #expect(try await df.select(ifnull(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+    #expect(try await df.select(nvl(col("a"), col("b"))).collect() == [Row(1), Row(20)])
+    #expect(
+      try await df.select(nvl2(col("a"), lit("yes"), lit("no"))).collect()
+        == [Row("yes"), Row("no")])
+    #expect(try await df.select(nullif(col("b"), lit(10))).collect() == [Row(nil), Row(20)])
+
+    let doubles = try await spark.sql("SELECT CAST('NaN' AS DOUBLE) AS a, 1.0D AS b")
+    #expect(try await doubles.select(nanvl(col("a"), col("b"))).collect() == [Row(1.0)])
+    #expect(try await doubles.select(nanvl(col("b"), col("a"))).collect() == [Row(1.0)])
+    await spark.stop()
+  }
+
+  @Test
+  func selectNullifzeroAndZeroifnull() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    if await isSparkVersionAtLeast(spark.version, "4.0.0") {
+      let df = try await spark.sql(
+        "SELECT * FROM VALUES (0, CAST(NULL AS INT)), (1, 2) T(a, b)")
+      #expect(try await df.select(nullifzero(col("a"))).collect() == [Row(nil), Row(1)])
+      #expect(try await df.select(zeroifnull(col("b"))).collect() == [Row(0), Row(2)])
+    }
+    await spark.stop()
+  }
+
+  @Test
+  func selectWhenOtherwise() async throws {
+    let spark = try await SparkSession.builder.getOrCreate()
+    let df = try await spark.range(4)
+    let rows = try await df.select(
+      when(col("id") < 1, "zero").when(col("id") < 3, "few").otherwise("many").alias("group")
+    ).collect()
+    #expect(rows == [Row("zero"), Row("few"), Row("few"), Row("many")])
+
+    // Without `otherwise`, null is returned for unmatched conditions.
+    let nullable = try await df.select(when(col("id") == 0, "zero").alias("group")).collect()
+    #expect(nullable == [Row("zero"), Row(nil), Row(nil), Row(nil)])
+    await spark.stop()
+  }
+}
diff --git a/Tests/SparkConnectTests/FunctionsTests.swift b/Tests/SparkConnectTests/FunctionsTests.swift
index 7087437..86b49db 100644
--- a/Tests/SparkConnectTests/FunctionsTests.swift
+++ b/Tests/SparkConnectTests/FunctionsTests.swift
@@ -119,38 +119,6 @@ struct FunctionsTests {
     }
   }
 
-  @Test
-  func whenFunction() throws {
-    let expr = when(col("a") > 1, lit(10)).expr
-    #expect(expr.unresolvedFunction.functionName == "when")
-    #expect(expr.unresolvedFunction.arguments.count == 2)
-
-    let chained = try when(col("a") > 2, 10).when(col("a") > 1, 20).otherwise(0).expr
-    #expect(chained.unresolvedFunction.functionName == "when")
-    #expect(chained.unresolvedFunction.arguments.count == 5)
-    #expect(chained.unresolvedFunction.arguments[1].literal.long == 10)
-    #expect(chained.unresolvedFunction.arguments[3].literal.long == 20)
-    #expect(chained.unresolvedFunction.arguments[4].literal.long == 0)
-  }
-
-  @Test
-  func whenMisuse() throws {
-    // `when`/`otherwise` can only be applied on a column generated by `when`.
-    #expect(throws: SparkConnectError.InvalidArgument) {
-      try col("a").when(col("b") > 1, 1)
-    }
-    #expect(throws: SparkConnectError.InvalidArgument) {
-      try col("a").otherwise(1)
-    }
-    // `when`/`otherwise` cannot be applied once `otherwise` is applied.
-    #expect(throws: SparkConnectError.InvalidArgument) {
-      try when(col("a") > 1, 1).otherwise(2).when(col("a") > 0, 3)
-    }
-    #expect(throws: SparkConnectError.InvalidArgument) {
-      try when(col("a") > 1, 1).otherwise(2).otherwise(3)
-    }
-  }
-
   @Test
   func comparisonOperators() throws {
     for (column, name) in [
@@ -475,19 +443,4 @@ struct FunctionsTests {
     #expect(rows == [Row(1, 10, 7)])
     await spark.stop()
   }
-
-  @Test
-  func selectWhenOtherwise() async throws {
-    let spark = try await SparkSession.builder.getOrCreate()
-    let df = try await spark.range(4)
-    let rows = try await df.select(
-      when(col("id") < 1, "zero").when(col("id") < 3, "few").otherwise("many").alias("group")
-    ).collect()
-    #expect(rows == [Row("zero"), Row("few"), Row("few"), Row("many")])
-
-    // Without `otherwise`, null is returned for unmatched conditions.
-    let nullable = try await df.select(when(col("id") == 0, "zero").alias("group")).collect()
-    #expect(nullable == [Row("zero"), Row(nil), Row(nil), Row(nil)])
-    await spark.stop()
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org