You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/07/28 00:22:33 UTC
[spark] branch branch-3.2 updated: [SPARK-36310][PYTHON] Fix
IndexOpsMixin.hasnans to use isnull().any()
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 0e9e737 [SPARK-36310][PYTHON] Fix IndexOpsMixin.hasnans to use isnull().any()
0e9e737 is described below
commit 0e9e737a8403b73ed19fb35e0f1af7b06b2c7660
Author: Takuya UESHIN <ue...@databricks.com>
AuthorDate: Wed Jul 28 09:21:12 2021 +0900
[SPARK-36310][PYTHON] Fix IndexOpsMixin.hasnans to use isnull().any()
### What changes were proposed in this pull request?
Fix `IndexOpsMixin.hasnans` to use `IndexOpsMixin.isnull().any()`.
### Why are the changes needed?
`IndexOpsMixin.hasnans` has a potential issue to cause `a window function inside an aggregate function` error.
Also it returns a wrong value when the `Series`/`Index` is empty.
```py
>>> ps.Series([]).hasnans
None
```
whereas:
```py
>>> pd.Series([]).hasnans
False
```
`IndexOpsMixin.any()` is safe for both cases.
### Does this PR introduce _any_ user-facing change?
`IndexOpsMixin.hasnans` will return `False` when empty.
### How was this patch tested?
Added some tests.
Closes #33547 from ueshin/issues/SPARK-36310/hasnan.
Authored-by: Takuya UESHIN <ue...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit bcc595c112a23d8e3024ace50f0dbc7eab7144b2)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/base.py | 8 +-------
python/pyspark/pandas/tests/indexes/test_base.py | 5 +++++
python/pyspark/pandas/tests/test_series.py | 5 +++++
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
index f547f71..832d7e8 100644
--- a/python/pyspark/pandas/base.py
+++ b/python/pyspark/pandas/base.py
@@ -522,13 +522,7 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
>>> ps.Series([1, 2, 3]).rename("a").to_frame().set_index("a").index.hasnans
False
"""
- sdf = self._internal.spark_frame
- scol = self.spark.column
-
- if isinstance(self.spark.data_type, (DoubleType, FloatType)):
- return sdf.select(F.max(scol.isNull() | F.isnan(scol))).collect()[0][0]
- else:
- return sdf.select(F.max(scol.isNull())).collect()[0][0]
+ return self.isnull().any()
@property
def is_monotonic(self) -> bool:
diff --git a/python/pyspark/pandas/tests/indexes/test_base.py b/python/pyspark/pandas/tests/indexes/test_base.py
index d941322..65831d1 100644
--- a/python/pyspark/pandas/tests/indexes/test_base.py
+++ b/python/pyspark/pandas/tests/indexes/test_base.py
@@ -1766,6 +1766,11 @@ class IndexesTest(PandasOnSparkTestCase, TestUtils):
psser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, psser.hasnans)
+ # empty
+ pidx = pd.Index([])
+ psidx = ps.from_pandas(pidx)
+ self.assert_eq(pidx.hasnans, psidx.hasnans)
+
# Not supported for MultiIndex
psmidx = ps.Index([("a", 1), ("b", 2)])
self.assertRaises(NotImplementedError, lambda: psmidx.hasnans())
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
index 310e5fb..b42d3cd 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -2444,6 +2444,11 @@ class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
psser = ps.from_pandas(pser)
self.assert_eq(pser.hasnans, psser.hasnans)
+ # empty
+ pser = pd.Series([])
+ psser = ps.from_pandas(pser)
+ self.assert_eq(pser.hasnans, psser.hasnans)
+
def test_last_valid_index(self):
pser = pd.Series([250, 1.5, 320, 1, 0.3, None, None, None, None])
psser = ps.from_pandas(pser)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org