You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2021/09/18 00:49:55 UTC
[spark] branch branch-3.2 updated: [SPARK-36762][PYTHON] Fix
Series.isin when Series has NaN values
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 36ce9cc [SPARK-36762][PYTHON] Fix Series.isin when Series has NaN values
36ce9cc is described below
commit 36ce9cce55619160db6104da10b9b53141c10105
Author: dgd-contributor <dg...@viettel.com.vn>
AuthorDate: Fri Sep 17 17:48:15 2021 -0700
[SPARK-36762][PYTHON] Fix Series.isin when Series has NaN values
### What changes were proposed in this pull request?
Fix Series.isin when Series has NaN values
### Why are the changes needed?
Fix Series.isin when Series has NaN values
``` python
>>> pser = pd.Series([None, 5, None, 3, 2, 1, None, 0, 0])
>>> psser = ps.from_pandas(pser)
>>> pser.isin([1, 3, 5, None])
0 False
1 True
2 False
3 True
4 False
5 True
6 False
7 False
8 False
dtype: bool
>>> psser.isin([1, 3, 5, None])
0 None
1 True
2 None
3 True
4 None
5 True
6 None
7 None
8 None
dtype: object
```
### Does this PR introduce _any_ user-facing change?
After this PR
``` python
>>> pser = pd.Series([None, 5, None, 3, 2, 1, None, 0, 0])
>>> psser = ps.from_pandas(pser)
>>> psser.isin([1, 3, 5, None])
0 False
1 True
2 False
3 True
4 False
5 True
6 False
7 False
8 False
dtype: bool
```
### How was this patch tested?
unit tests
Closes #34005 from dgd-contributor/SPARK-36762_fix_series.isin_when_values_have_NaN.
Authored-by: dgd-contributor <dg...@viettel.com.vn>
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
(cherry picked from commit 32b8512912c211f7f12e717e7029e89645da9d3b)
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
---
python/pyspark/pandas/base.py | 10 ++++++++--
python/pyspark/pandas/tests/test_series.py | 17 +++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
index 58f6c19..bdec3ac 100644
--- a/python/pyspark/pandas/base.py
+++ b/python/pyspark/pandas/base.py
@@ -27,7 +27,7 @@ import numpy as np
import pandas as pd # noqa: F401
from pandas.api.types import is_list_like, CategoricalDtype
from pyspark.sql import functions as F, Column, Window
-from pyspark.sql.types import LongType
+from pyspark.sql.types import LongType, BooleanType
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark.pandas._typing import Axis, Dtype, IndexOpsLike, Label, SeriesOrIndex
@@ -867,7 +867,13 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
)
values = values.tolist() if isinstance(values, np.ndarray) else list(values)
- return self._with_new_scol(self.spark.column.isin([SF.lit(v) for v in values]))
+
+ other = [SF.lit(v) for v in values]
+ scol = self.spark.column.isin(other)
+ field = self._internal.data_fields[0].copy(
+ dtype=np.dtype("bool"), spark_type=BooleanType(), nullable=False
+ )
+ return self._with_new_scol(scol=F.coalesce(scol, F.lit(False)), field=field)
def isnull(self: IndexOpsLike) -> IndexOpsLike:
"""
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
index de5ba60..cbfc999 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -394,6 +394,23 @@ class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
with self.assertRaisesRegex(TypeError, msg):
psser.isin(1)
+ # when Series have NaN
+ pser = pd.Series(["lama", "cow", None, "lama", "beetle", "lama", "hippo", None], name="a")
+ psser = ps.from_pandas(pser)
+
+ self.assert_eq(psser.isin(["cow", "lama"]), pser.isin(["cow", "lama"]))
+
+ pser = pd.Series([None, 5, None, 3, 2, 1, None, 0, 0], name="a")
+ psser = ps.from_pandas(pser)
+
+ if LooseVersion(pd.__version__) >= LooseVersion("1.2"):
+ self.assert_eq(psser.isin([1, 5, 0, None]), pser.isin([1, 5, 0, None]))
+ else:
+ expected = pd.Series(
+ [False, True, False, False, False, True, False, True, True], name="a"
+ )
+ self.assert_eq(psser.isin([1, 5, 0, None]), expected)
+
def test_drop_duplicates(self):
pdf = pd.DataFrame({"animal": ["lama", "cow", "lama", "beetle", "lama", "hippo"]})
psdf = ps.from_pandas(pdf)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org