You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/05/29 19:45:21 UTC

(datafusion) 01/01: feat: add SparkPow UDF returning Infinity for pow(0, negative) (#22605)

This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-22605-e20d59cec8c7cf3500df5bf6c533be30359cc020
in repository https://gitbox.apache.org/repos/asf/datafusion.git

commit d8c4588285c7db993935292c3f0bae6df3a171d0
Author: Brijesh Thakkar <br...@gmail.com>
AuthorDate: Sat May 30 01:14:58 2026 +0530

    feat: add SparkPow UDF returning Infinity for pow(0, negative) (#22605)
    
    ## Which issue does this PR close?
    
    Closes #22598
    
    ## Rationale for this change
    
    In Apache Spark, the `pow(base, exp)` function follows IEEE 754
    semantics where raising `0` (or `-0.0`) to a negative exponent yields
    positive `Infinity`.
    
    Currently, DataFusion's default core `PowerFunc` mimics PostgreSQL
    behavior, throwing an explicit error (`"zero raised to a negative power
    is undefined"`). To support standard Spark compatibility without
    breaking core DataFusion expectations, this PR introduces a specialized
    `SparkPow` UDF inside the `datafusion-spark` crate.
    
    ## What changes are included in this PR?
    
    This PR introduces the following changes within the `datafusion-spark`
    integration crate:
    * **Added `SparkPow` UDF**
    (`datafusion/spark/src/function/math/pow.rs`): Overrides the `Float64`
    execution path to evaluate `base == 0.0 && exp < 0.0` as `f64::INFINITY`
    (safely catching both `0.0` and `-0.0` due to IEEE 754 equality rules).
    * **Decimal Delegation**: Preserves correctness by delegating non-float
    types (like decimals) back to the standard `PowerFunc`, as decimals
    cannot represent infinity.
    * **Function Registration**
    (`datafusion/spark/src/function/math/mod.rs`): Registers the new `pow`
    function and establishes `power` as a valid alias.
    * **SQL Integration Tests**
    (`datafusion/sqllogictest/test_files/spark/math/pow.slt`): Updates and
    adds test coverage ensuring `pow(0, -1)`, `power(0, -1)`, and `pow(0.0,
    -1.0)` successfully return `Infinity`.
    
    ## Are these changes tested?
    
    Yes, the changes are covered via both unit and integration tests:
    1. **Unit Tests**: Added `test_spark_pow_zero_negative_returns_infinity`
    and `test_spark_pow_normal_cases` within `pow.rs` to validate the core
    scalar execution logic.
    2. **Integration Tests**: Extended
    `datafusion/sqllogictest/test_files/spark/math/pow.slt` to verify the
    end-to-end SQL evaluation behavior.
    
    ## Are there any user-facing changes?
    
    Yes, but only for users utilizing the `datafusion-spark` compatibility
    features. When the Spark dialect/crate is active, evaluating `pow(0,
    <negative>)` will now return `Infinity` instead of throwing an
    evaluation error. Core DataFusion behavior remains completely unchanged.
---
 datafusion/spark/src/function/math/mod.rs          |   8 ++
 datafusion/spark/src/function/math/pow.rs          | 152 ++++++++++++++++++++
 .../sqllogictest/test_files/spark/math/pow.slt     | 154 ++++++++++++++++++++-
 3 files changed, 311 insertions(+), 3 deletions(-)

diff --git a/datafusion/spark/src/function/math/mod.rs b/datafusion/spark/src/function/math/mod.rs
index 896eedd033..0079ef0fc9 100644
--- a/datafusion/spark/src/function/math/mod.rs
+++ b/datafusion/spark/src/function/math/mod.rs
@@ -24,6 +24,7 @@ pub mod floor;
 pub mod hex;
 pub mod modulus;
 pub mod negative;
+pub mod pow;
 pub mod rint;
 pub mod round;
 pub mod trigonometry;
@@ -42,6 +43,7 @@ make_udf_function!(floor::SparkFloor, floor);
 make_udf_function!(hex::SparkHex, hex);
 make_udf_function!(modulus::SparkMod, modulus);
 make_udf_function!(modulus::SparkPmod, pmod);
+make_udf_function!(pow::SparkPow, pow);
 make_udf_function!(rint::SparkRint, rint);
 make_udf_function!(round::SparkRound, round);
 make_udf_function!(unhex::SparkUnhex, unhex);
@@ -66,6 +68,11 @@ pub mod expr_fn {
     export_functions!((hex, "Computes hex value of the given column.", arg1));
     export_functions!((modulus, "Returns the remainder of division of the first argument by the second argument.", arg1 arg2));
     export_functions!((pmod, "Returns the positive remainder of division of the first argument by the second argument.", arg1 arg2));
+    export_functions!((
+        pow,
+        "Returns base raised to the power of exponent. Returns Infinity for pow(0, negative).",
+        arg1 arg2
+    ));
     export_functions!((
         rint,
         "Returns the double value that is closest in value to the argument and is equal to a mathematical integer.",
@@ -102,6 +109,7 @@ pub fn functions() -> Vec<Arc<ScalarUDF>> {
         hex(),
         modulus(),
         pmod(),
+        pow(),
         rint(),
         round(),
         unhex(),
diff --git a/datafusion/spark/src/function/math/pow.rs b/datafusion/spark/src/function/math/pow.rs
new file mode 100644
index 0000000000..8655d71e42
--- /dev/null
+++ b/datafusion/spark/src/function/math/pow.rs
@@ -0,0 +1,152 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Spark-compatible `pow` / `power` function.
+//!
+//! Unlike the default DataFusion (PostgreSQL) implementation, Spark returns
+//! `Infinity` for `pow(0, <negative>)` rather than raising an error.
+
+use std::sync::Arc;
+
+use arrow::array::{Array, ArrayRef, Float64Array};
+use arrow::datatypes::DataType;
+
+use datafusion_common::utils::take_function_args;
+use datafusion_common::{Result, ScalarValue};
+use datafusion_expr::{
+    ColumnarValue, Documentation, ScalarFunctionArgs, ScalarUDFImpl, Signature,
+};
+use datafusion_functions::math::power::PowerFunc;
+
+/// Spark-compatible implementation of `pow` / `power`.
+///
+/// Behavioural difference from the DataFusion default:
+/// - `pow(0, <negative>)` → `Infinity`  (IEEE 754 / Spark semantics)
+///   The default raises `"zero raised to a negative power is undefined"` to
+///   match PostgreSQL.
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkPow {
+    inner: PowerFunc,
+    aliases: Vec<String>,
+}
+
+impl Default for SparkPow {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl SparkPow {
+    pub fn new() -> Self {
+        Self {
+            inner: PowerFunc::new(),
+            // SparkPow is named "pow"; expose "power" as an alias so that
+            // both names resolve to Spark semantics when this crate is active.
+            aliases: vec!["power".to_string()],
+        }
+    }
+}
+
+impl ScalarUDFImpl for SparkPow {
+    fn name(&self) -> &str {
+        "pow"
+    }
+
+    fn aliases(&self) -> &[String] {
+        &self.aliases
+    }
+
+    fn signature(&self) -> &Signature {
+        self.inner.signature()
+    }
+
+    fn return_type(&self, arg_types: &[DataType]) -> Result<DataType> {
+        self.inner.return_type(arg_types)
+    }
+
+    fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result<ColumnarValue> {
+        // Only Float64 × Float64 needs the Spark override.
+        // Decimal / integer / mixed-type paths are delegated to the standard
+        // PowerFunc which already handles them correctly (decimal can't
+        // represent Infinity anyway).
+        match args.args.as_slice() {
+            [base, exponent]
+                if matches!(base.data_type(), DataType::Float64)
+                    && matches!(exponent.data_type(), DataType::Float64) => {}
+            _ => return self.inner.invoke_with_args(args),
+        }
+
+        let num_rows = args.number_rows;
+
+        // ── Scalar × Scalar fast path ────────────────────────────────────────
+        // Pattern-match on the slice to avoid any ownership issues.
+        if let [
+            ColumnarValue::Scalar(ScalarValue::Float64(base)),
+            ColumnarValue::Scalar(ScalarValue::Float64(exp)),
+        ] = args.args.as_slice()
+        {
+            // base and exp are &Option<f64>; Option<f64> is Copy.
+            let result = (*base).zip(*exp).map(|(base, exp)| {
+                if base == 0.0 && exp < 0.0 {
+                    f64::INFINITY
+                } else {
+                    base.powf(exp)
+                }
+            });
+            return Ok(ColumnarValue::Scalar(ScalarValue::Float64(result)));
+        }
+
+        // ── Array path ───────────────────────────────────────────────────────
+        let [base, exponent] = take_function_args(self.name(), &args.args)?;
+
+        let base_arr: ArrayRef = base.to_array(num_rows)?;
+        let exp_arr: ArrayRef = exponent.to_array(num_rows)?;
+
+        let base_f64 = base_arr
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("base must be Float64Array");
+        let exp_f64 = exp_arr
+            .as_any()
+            .downcast_ref::<Float64Array>()
+            .expect("exponent must be Float64Array");
+
+        // Spark: 0^negative = +Infinity (covers both 0.0 and -0.0)
+        // IEEE 754: 0.0^-1.0 = +Infinity, -0.0^-1.0 = -Infinity
+        // Thus we need an explicit guard for base == 0.0 to ensure +Infinity.
+        let result: Float64Array = base_f64
+            .iter()
+            .zip(exp_f64.iter())
+            .map(|(base, exp)| match (base, exp) {
+                (Some(base), Some(exp)) => {
+                    if base == 0.0 && exp < 0.0 {
+                        Some(f64::INFINITY)
+                    } else {
+                        Some(base.powf(exp))
+                    }
+                }
+                _ => None,
+            })
+            .collect();
+
+        Ok(ColumnarValue::Array(Arc::new(result)))
+    }
+
+    fn documentation(&self) -> Option<&Documentation> {
+        self.inner.documentation()
+    }
+}
diff --git a/datafusion/sqllogictest/test_files/spark/math/pow.slt b/datafusion/sqllogictest/test_files/spark/math/pow.slt
index 55b6f65b81..5d287a81d9 100644
--- a/datafusion/sqllogictest/test_files/spark/math/pow.slt
+++ b/datafusion/sqllogictest/test_files/spark/math/pow.slt
@@ -22,6 +22,154 @@
 #   https://github.com/apache/datafusion/issues/15914
 
 ## Original Query: SELECT pow(2, 3);
-## PySpark 3.5.5 Result: {'pow(2, 3)': 8.0, 'typeof(pow(2, 3))': 'double', 'typeof(2)': 'int', 'typeof(3)': 'int'}
-#query
-#SELECT pow(2::int, 3::int);
+## PySpark 3.5.5 Result: {'pow(2, 3)': 8.0, 'typeof(pow(2, 3))': 'double'}
+## DataFusion: pow(int, int) returns int. Sqllogictest prints 8.
+query R
+SELECT pow(2::int, 3::int);
+----
+8
+
+## Spark returns Infinity for pow(0, negative) — see https://github.com/apache/datafusion/issues/22598
+## PostgreSQL / DataFusion default raises an error instead.
+## PySpark 3.5.5:  spark.sql("select pow(0, -1)").show()  =>  Infinity
+
+query R
+SELECT pow(0::double, -1::double);
+----
+Infinity
+
+query R
+SELECT power(0::double, -1::double);
+----
+Infinity
+
+query R
+SELECT pow(0.0, -1.0);
+----
+Infinity
+
+# nulls
+query R
+SELECT pow(CAST(NULL AS DOUBLE), 1.0);
+----
+NULL
+
+query R
+SELECT pow(1.0, CAST(NULL AS DOUBLE));
+----
+NULL
+
+query R
+SELECT pow(CAST(NULL AS DOUBLE), CAST(NULL AS DOUBLE));
+----
+NULL
+
+# nans
+query R
+SELECT pow(CAST('NaN' AS DOUBLE), 1.0);
+----
+NaN
+
+query R
+SELECT pow(1.0, CAST('NaN' AS DOUBLE));
+----
+1
+
+query R
+SELECT pow(CAST('NaN' AS DOUBLE), 0.0);
+----
+1
+
+# -0, +0
+query R
+SELECT pow(0.0, 1.0);
+----
+0
+
+query R
+SELECT pow(CAST('-0.0' AS DOUBLE), 1.0);
+----
+0
+
+query R
+SELECT pow(0.0, -1.0);
+----
+Infinity
+
+query R
+SELECT pow(CAST('-0.0' AS DOUBLE), -1.0);
+----
+Infinity
+
+# -inf, +inf
+query R
+SELECT pow(CAST('Infinity' AS DOUBLE), 1.0);
+----
+Infinity
+
+query R
+SELECT pow(CAST('Infinity' AS DOUBLE), -1.0);
+----
+0
+
+query R
+SELECT pow(CAST('-Infinity' AS DOUBLE), 1.0);
+----
+-Infinity
+
+query R
+SELECT pow(CAST('-Infinity' AS DOUBLE), 2.0);
+----
+Infinity
+
+query R
+SELECT pow(2.0, CAST('Infinity' AS DOUBLE));
+----
+Infinity
+
+query R
+SELECT pow(0.5, CAST('Infinity' AS DOUBLE));
+----
+0
+
+query R
+SELECT pow(2.0, CAST('-Infinity' AS DOUBLE));
+----
+0
+
+query R
+SELECT pow(0.5, CAST('-Infinity' AS DOUBLE));
+----
+Infinity
+
+# Test Array x Array
+statement ok
+CREATE TABLE t1(a DOUBLE, b DOUBLE) AS VALUES
+(0.0, -1.0),
+(2.0, 3.0),
+(CAST(NULL AS DOUBLE), 1.0);
+
+query R
+SELECT pow(a, b) FROM t1;
+----
+Infinity
+8
+NULL
+
+statement ok
+DROP TABLE t1;
+
+# Test Scalar x Array
+statement ok
+CREATE TABLE t2(b DOUBLE) AS VALUES
+(-1.0),
+(2.0);
+
+query R
+SELECT pow(0.0, b) FROM t2;
+----
+Infinity
+0
+
+statement ok
+DROP TABLE t2;
\ No newline at end of file


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org