You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/09/01 01:06:44 UTC
(spark) branch master updated: [SPARK-58582][PS][FOLLOWUP] Keep integral operands in integer space for NumPy fmod
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6374c2f83df8 [SPARK-58582][PS][FOLLOWUP] Keep integral operands in integer space for NumPy fmod
6374c2f83df8 is described below
commit 6374c2f83df8b83abade73b80008156972ee602a
Author: Spenser Sun <hs...@gmail.com>
AuthorDate: Tue Sep 1 09:06:26 2026 +0800
[SPARK-58582][PS][FOLLOWUP] Keep integral operands in integer space for NumPy fmod
### What changes were proposed in this pull request?
`_fmod_func` cast both operands to double before taking the remainder, including on the branch reached only when neither operand is floating. Integral operands now take the remainder as longs, and only the result is cast back to double.
### Why are the changes needed?
A double carries 53 bits of mantissa, so an integral operand above 2**53 was rounded before the remainder was taken: `np.fmod(9007199254740993, 2)` gave `0.0` where pandas gives `1`. The same cast also returned `-0.0` for `-64 % 2`, a sign of zero an integral remainder cannot have. The `pandas_udf` this mapping replaced called NumPy directly and was exact.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
New rows in `test_np_fmod` for integral operands above 2**53, compared exactly since a relative tolerance accepts an off-by-one at those magnitudes, plus an `np.signbit` assertion for the sign of zero. Both fail without the change.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 5)
Closes #58319 from Spenserrrr/numpy-fmod-int-precision.
Authored-by: Spenser Sun <hs...@gmail.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/pandas/numpy_compat.py | 28 ++++++++++++++++--------
python/pyspark/pandas/tests/test_numpy_compat.py | 21 ++++++++++++++++++
2 files changed, 40 insertions(+), 9 deletions(-)
diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py
index aaa7d843cd75..5775e46b49a9 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -152,18 +152,28 @@ def _copysign_func(c1: Column, c2: Column) -> Column:
def _fmod_func(c1: Column, c2: Column) -> Column:
c1_double = c1.cast("double")
c2_double = c2.cast("double")
+ integral_types = ["tinyint", "smallint", "int", "bigint"]
- return F.when(
- F.typeof(c1).isin("float", "double") | F.typeof(c2).isin("float", "double"),
- F.when(c1.isNull() | F.isnan(c1), c1_double)
- .when(c2.isNull() | F.isnan(c2), c2_double)
- .when(c2_double == 0, F.lit(float("nan")))
- .otherwise(F.try_mod(c1_double, c2_double)),
- ).otherwise(
+ return (
+ # A null or a nan propagates for every operand type.
F.when(c1.isNull() | F.isnan(c1), c1_double)
.when(c2.isNull() | F.isnan(c2), c2_double)
- .when(c2_double == 0, F.lit(0.0))
- .otherwise(F.try_mod(c1_double, c2_double))
+ # Integral operands take the remainder as longs: a double cannot hold 9007199254740993.
+ .when(
+ F.typeof(c1).isin(integral_types) & F.typeof(c2).isin(integral_types),
+ F.when(c2_double == 0, F.lit(0.0)).otherwise(
+ F.try_mod(c1.cast("long"), c2.cast("long")).cast("double")
+ ),
+ )
+ # Floating operands, where a zero divisor is nan rather than the 0 above.
+ .when(
+ F.typeof(c1).isin("float", "double") | F.typeof(c2).isin("float", "double"),
+ F.when(c2_double == 0, F.lit(float("nan"))).otherwise(F.try_mod(c1_double, c2_double)),
+ )
+ # A decimal falls through here, since typeof carries its precision, as in decimal(10,2).
+ # np.fmod raises on Decimal objects, so there is no NumPy behavior to match: a zero
+ # divisor returns 0 and any other divisor takes the remainder in double.
+ .otherwise(F.when(c2_double == 0, F.lit(0.0)).otherwise(F.try_mod(c1_double, c2_double)))
)
diff --git a/python/pyspark/pandas/tests/test_numpy_compat.py b/python/pyspark/pandas/tests/test_numpy_compat.py
index a733155b7199..04dc60181020 100644
--- a/python/pyspark/pandas/tests/test_numpy_compat.py
+++ b/python/pyspark/pandas/tests/test_numpy_compat.py
@@ -313,6 +313,27 @@ class NumPyCompatTestsMixin:
self.assert_eq(np.fmod(psdf.x1, psdf.x2), np.fmod(pdf.x1, pdf.x2), almost=True)
+ # Integral operands above 2**53, where casting an operand to double would drop its low
+ # bits: fmod(9007199254740993, 2) is 1, not 0. Compared exactly, since almost=True would
+ # accept an off-by-one at these magnitudes.
+ pdf = pd.DataFrame(
+ {
+ "x1": [9007199254740993, 9007199254740995, -9007199254740993, 4611686018427387905],
+ "x2": [2, 4, 2, 1000000000],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(np.fmod(psdf.x1, psdf.x2), np.fmod(pdf.x1, pdf.x2).astype("float64"))
+
+ # almost=True treats -0.0 and 0.0 as equal, so check the sign of zero explicitly:
+ # an integral remainder is never negative zero, unlike a double one.
+ pdf = pd.DataFrame({"x1": [-64, -2, 0, 64], "x2": [2, 2, 3, 2]})
+ psdf = ps.from_pandas(pdf)
+
+ result = np.fmod(psdf.x1, psdf.x2)
+ expected = np.fmod(pdf.x1, pdf.x2)
+ self.assert_eq(np.signbit(result.to_pandas()), np.signbit(expected))
+
def test_np_modf(self):
# np.modf(x) returns a tuple (fractional part, integral part).
for pdf in (
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org