You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/09/04 05:20:46 UTC
(spark) branch branch-4.x updated: [SPARK-59232][PS] Support boolean operands in NumPy ufuncs
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 01bd9293a2f3 [SPARK-59232][PS] Support boolean operands in NumPy ufuncs
01bd9293a2f3 is described below
commit 01bd9293a2f367fdfe8258c4534677e8c7651516
Author: Spenser Sun <hs...@gmail.com>
AuthorDate: Fri Sep 4 13:20:22 2026 +0800
[SPARK-59232][PS] Support boolean operands in NumPy ufuncs
### What changes were proposed in this pull request?
`maybe_dispatch_ufunc_to_spark_func` now casts a boolean operand to `tinyint` before building the expression, the way NumPy promotes one to `int8` before it picks a loop. `np.invert` and `np.negative` reuse the existing `logical_not` mapping instead, since NumPy reads a boolean logically for those two.
### Why are the changes needed?
Most ufuncs in the mapping tables raise `AnalysisException` on a boolean column, because Spark's functions have no implicit boolean-to-numeric cast, where pandas returns a value: `np.sqrt(ps.Series([True, False]))` fails while pandas answers `[1.0, 0.0]`. The accepted-type table added in #58485 already lists `BooleanType` for these ufuncs, so the failure contradicts it.
### Does this PR introduce _any_ user-facing change?
Yes. A NumPy ufunc on a boolean column now returns a value instead of raising; the values match pandas, while the result dtype follows Spark's math functions, which return double.
### How was this patch tested?
New `test_np_boolean_operand` and `test_np_boolean_operand_logical` in `NumPyCompatTestsMixin`, both of which fail without the change.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Closes #58509 from Spenserrrr/numpy-ufunc-boolean-cast.
Authored-by: Spenser Sun <hs...@gmail.com>
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
(cherry picked from commit d0bc9867cc77a5cccfd5768fc393dde827a44dad)
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
python/pyspark/pandas/numpy_compat.py | 15 +++++++
python/pyspark/pandas/tests/test_numpy_compat.py | 53 ++++++++++++++++++++++++
2 files changed, 68 insertions(+)
diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py
index 906f5512d637..9b076798c995 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -28,6 +28,7 @@ from pyspark.sql import functions as F
from pyspark.sql.pandas.functions import pandas_udf
from pyspark.sql.types import (
BooleanType,
+ ByteType,
DataType,
DoubleType,
FloatType,
@@ -546,12 +547,26 @@ def maybe_dispatch_ufunc_to_spark_func(
ser_or_index: IndexOpsMixin, ufunc: Callable, method: str, *inputs: Any, **kwargs: Any
) -> Union[SeriesOrIndex, Tuple[SeriesOrIndex, SeriesOrIndex]]:
from pyspark.pandas.base import column_op
+ from pyspark.pandas.data_type_ops.base import transform_boolean_operand_to_numeric
op_name = ufunc.__name__
# Check before building the expression, so the error comes from the ufunc call itself.
if method == "__call__" and kwargs.get("out") is None:
_check_operand_types(op_name, inputs)
+ if op_name in ("invert", "negative") and isinstance(
+ ser_or_index.spark.data_type, BooleanType
+ ):
+ # np.invert on a boolean is a logical not, and pandas reads np.negative the same way;
+ # Spark keeps that meaning in the logical_not entry instead.
+ op_name = "logical_not"
+ elif op_name in _np_spark_accepted_types:
+ # Spark's functions reject a boolean, so cast it to int8, the narrowest integer NumPy
+ # promotes one to. Only the gated ufuncs are cast, since the rest either return an
+ # operand unchanged, as np.fmax does, or handle a boolean already.
+ inputs = tuple(
+ transform_boolean_operand_to_numeric(inp, spark_type=ByteType()) for inp in inputs
+ )
if (
method == "__call__"
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py b/python/pyspark/pandas/tests/test_numpy_compat.py
index 69f7eb30a11a..2205e2c4f257 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -219,6 +219,59 @@ class NumPyCompatTestsMixin:
np.fmax(psdf["string"], psdf["string"]), np.fmax(pdf["string"], pdf["string"])
)
+ def test_np_boolean_operand(self):
+ from pyspark.pandas.numpy_compat import _np_spark_accepted_types
+
+ pdf = pd.DataFrame({"b": [True, False, True], "i": [7, 8, 9]})
+ psdf = ps.from_pandas(pdf)
+ # The reference is pandas on the same values as int64, where its float loops are float64 as
+ # Spark's math always is; on a boolean or an int8 pandas runs coarser float16 loops.
+ as_int64 = pdf.b.astype("int64")
+ # Skipped: np.sign rejects a boolean; np.log and np.log10 answer nan for a zero where pandas
+ # answers -inf; np.reciprocal divides in int8; np.invert and np.negative are logical.
+ skip = {"sign", "log", "log10", "reciprocal", "invert", "negative"}
+
+ for op_name, accepted_per_operand in _np_spark_accepted_types.items():
+ if op_name in skip:
+ continue
+ np_func = getattr(np, op_name)
+ with self.subTest(name=op_name):
+ result = np_func(*[psdf.b] * len(accepted_per_operand))
+ expected = np_func(*[as_int64] * len(accepted_per_operand))
+ if isinstance(expected, tuple):
+ for one_result, one_expected in zip(result, expected):
+ self.assert_eq(one_result, one_expected, almost=True)
+ else:
+ self.assert_eq(result, expected, almost=True)
+
+ # A boolean scalar operand takes the same promotion as a column.
+ self.assert_eq(np.ldexp(psdf.i, True), np.ldexp(pdf.i, True), almost=True)
+
+ # A nullable boolean keeps an extension dtype and propagates <NA>.
+ nullable_pser = pd.Series(pd.array([True, None, False], dtype="boolean"))
+ nullable = ps.from_pandas(nullable_pser)
+ self.assertIsInstance(np.sqrt(nullable).dtype, pd.Float64Dtype)
+ self.assert_eq(np.sqrt(nullable), np.sqrt(nullable_pser.astype("Int64")), almost=True)
+
+ def test_np_boolean_operand_logical(self):
+ # NumPy applies these logically rather than promoting: np.invert on a boolean is
+ # np.logical_not and pandas reads np.negative the same way; promoting would answer -2, -1.
+ pdf = pd.DataFrame(
+ {
+ "b": [True, False, True],
+ "nullable": pd.array([True, None, False], dtype="boolean"),
+ "i": [-2, 0, 3],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+
+ for np_func in (np.invert, np.negative):
+ with self.subTest(name=np_func.__name__):
+ self.assert_eq(np_func(psdf.b), np_func(pdf.b))
+ self.assert_eq(np_func(psdf.nullable), np_func(pdf.nullable))
+ # An integer column keeps the arithmetic meaning.
+ self.assert_eq(np_func(psdf.i), np_func(pdf.i))
+
def test_np_math_functions(self):
for np_func, values in (
(np.arccosh, [-np.inf, -1.0, 0.0, 1.0, 2.0, 64.0, np.inf, np.nan]),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org