You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2021/09/03 17:41:57 UTC
[spark] branch master updated: [SPARK-36401][PYTHON] Implement
Series.cov
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 9b262e7 [SPARK-36401][PYTHON] Implement Series.cov
9b262e7 is described below
commit 9b262e722d717bebf7d2aa19af0658cf3b8e1e85
Author: dgd-contributor <dg...@viettel.com.vn>
AuthorDate: Fri Sep 3 10:41:27 2021 -0700
[SPARK-36401][PYTHON] Implement Series.cov
### What changes were proposed in this pull request?
Implement Series.cov
### Why are the changes needed?
That is supported in pandas. We should support that as well.
### Does this PR introduce _any_ user-facing change?
Yes. Series.cov can be used.
```python
>>> from pyspark.pandas.config import set_option, reset_option
>>> set_option("compute.ops_on_diff_frames", True)
>>> s1 = ps.Series([0.90010907, 0.13484424, 0.62036035])
>>> s2 = ps.Series([0.12528585, 0.26962463, 0.51111198])
>>> s1.cov(s2)
-0.016857626527158744
>>> reset_option("compute.ops_on_diff_frames")
```
### How was this patch tested?
Unit tests
Closes #33752 from dgd-contributor/SPARK-36401_Implement_Series.cov.
Authored-by: dgd-contributor <dg...@viettel.com.vn>
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
---
.../source/reference/pyspark.pandas/series.rst | 1 +
python/pyspark/pandas/missing/series.py | 1 -
python/pyspark/pandas/series.py | 50 ++++++++++++++++++++++
.../pandas/tests/test_ops_on_diff_frames.py | 37 ++++++++++++++++
python/pyspark/pandas/tests/test_series.py | 45 +++++++++++++++++++
5 files changed, 133 insertions(+), 1 deletion(-)
diff --git a/python/docs/source/reference/pyspark.pandas/series.rst b/python/docs/source/reference/pyspark.pandas/series.rst
index 1e32773..337f686 100644
--- a/python/docs/source/reference/pyspark.pandas/series.rst
+++ b/python/docs/source/reference/pyspark.pandas/series.rst
@@ -138,6 +138,7 @@ Computations / Descriptive Stats
Series.clip
Series.corr
Series.count
+ Series.cov
Series.cummax
Series.cummin
Series.cumsum
diff --git a/python/pyspark/pandas/missing/series.py b/python/pyspark/pandas/missing/series.py
index ef3f38b..f6ea8d3 100644
--- a/python/pyspark/pandas/missing/series.py
+++ b/python/pyspark/pandas/missing/series.py
@@ -40,7 +40,6 @@ class MissingPandasLikeSeries(object):
autocorr = _unsupported_function("autocorr")
combine = _unsupported_function("combine")
convert_dtypes = _unsupported_function("convert_dtypes")
- cov = _unsupported_function("cov")
ewm = _unsupported_function("ewm")
infer_objects = _unsupported_function("infer_objects")
interpolate = _unsupported_function("interpolate")
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index 568754c..7e45566 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -944,6 +944,56 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
return lmask & rmask
+ def cov(self, other: "Series", min_periods: Optional[int] = None) -> float:
+ """
+ Compute covariance with Series, excluding missing values.
+
+ .. versionadded:: 3.3.0
+
+ Parameters
+ ----------
+ other : Series
+ Series with which to compute the covariance.
+ min_periods : int, optional
+ Minimum number of observations needed to have a valid result.
+
+ Returns
+ -------
+ float
+ Covariance between Series and other
+
+ Examples
+ --------
+ >>> from pyspark.pandas.config import set_option, reset_option
+ >>> set_option("compute.ops_on_diff_frames", True)
+ >>> s1 = ps.Series([0.90010907, 0.13484424, 0.62036035])
+ >>> s2 = ps.Series([0.12528585, 0.26962463, 0.51111198])
+ >>> s1.cov(s2)
+ -0.016857626527158744
+ >>> reset_option("compute.ops_on_diff_frames")
+ """
+ if not isinstance(other, Series):
+ raise TypeError("unsupported type: %s" % type(other))
+ if not np.issubdtype(self.dtype, np.number):
+ raise TypeError("unsupported dtype: %s" % self.dtype)
+ if not np.issubdtype(other.dtype, np.number):
+ raise TypeError("unsupported dtype: %s" % other.dtype)
+
+ min_periods = 1 if min_periods is None else min_periods
+
+ if same_anchor(self, other):
+ sdf = self._internal.spark_frame.select(self.spark.column, other.spark.column)
+ else:
+ combined = combine_frames(self.to_frame(), other.to_frame())
+ sdf = combined._internal.spark_frame.select(*combined._internal.data_spark_columns)
+
+ sdf = sdf.dropna()
+
+ if len(sdf.head(min_periods)) < min_periods:
+ return np.nan
+ else:
+ return sdf.select(F.covar_samp(*sdf.columns)).head(1)[0][0]
+
# TODO: arg should support Series
# TODO: NaN and None
def map(self, arg: Union[Dict, Callable]) -> "Series":
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
index f7fa3e8..3e3bb0d 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
@@ -1793,6 +1793,43 @@ class OpsOnDiffFramesEnabledTest(PandasOnSparkTestCase, SQLTestUtils):
pdf["Col2"].rank().loc[pdf["Col1"] == 20], psdf["Col2"].rank().loc[psdf["Col1"] == 20]
)
+ def test_cov(self):
+ pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035], index=[0, 1, 2])
+ pser2 = pd.Series([0.12528585, 0.26962463, 0.51111198], index=[1, 2, 3])
+ self._test_cov(pser1, pser2)
+
+ pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035], index=[0, 1, 2])
+ pser2 = pd.Series([0.12528585, 0.26962463, 0.51111198, 0.32076008], index=[1, 2, 3, 4])
+ self._test_cov(pser1, pser2)
+
+ pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035, 0.32076008], index=[0, 1, 2, 3])
+ pser2 = pd.Series([0.12528585, 0.26962463], index=[1, 2])
+ self._test_cov(pser1, pser2)
+
+ psser1 = ps.from_pandas(pser1)
+ with self.assertRaisesRegex(TypeError, "unsupported type: <class 'list'>"):
+ psser1.cov([0.12528585, 0.26962463, 0.51111198])
+ with self.assertRaisesRegex(
+ TypeError, "unsupported type: <class 'pandas.core.series.Series'>"
+ ):
+ psser1.cov(pser2)
+
+ def _test_cov(self, pser1, pser2):
+ psser1 = ps.from_pandas(pser1)
+ psser2 = ps.from_pandas(pser2)
+
+ pcov = pser1.cov(pser2)
+ pscov = psser1.cov(psser2)
+ self.assert_eq(pcov, pscov, almost=True)
+
+ pcov = pser1.cov(pser2, min_periods=2)
+ pscov = psser1.cov(psser2, min_periods=2)
+ self.assert_eq(pcov, pscov, almost=True)
+
+ pcov = pser1.cov(pser2, min_periods=3)
+ pscov = psser1.cov(psser2, min_periods=3)
+ self.assert_eq(pcov, pscov, almost=True)
+
class OpsOnDiffFramesDisabledTest(PandasOnSparkTestCase, SQLTestUtils):
@classmethod
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
index a861dd0..ffa8431 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -2905,6 +2905,51 @@ class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
self.assert_eq(psser1.combine_first(psser2), pser1.combine_first(pser2))
+ def test_cov(self):
+ pdf = pd.DataFrame(
+ {
+ "s1": ["a", "b", "c"],
+ "s2": [0.12528585, 0.26962463, 0.51111198],
+ },
+ index=[0, 1, 2],
+ )
+ psdf = ps.from_pandas(pdf)
+ with self.assertRaisesRegex(TypeError, "unsupported dtype: object"):
+ psdf["s1"].cov(psdf["s2"])
+
+ pdf = pd.DataFrame(
+ {
+ "s1": [0.90010907, 0.13484424, 0.62036035],
+ "s2": [0.12528585, 0.26962463, 0.51111198],
+ },
+ index=[0, 1, 2],
+ )
+ self._test_cov(pdf)
+
+ pdf = pd.DataFrame(
+ {
+ "s1": [0.90010907, np.nan, 0.13484424, 0.62036035],
+ "s2": [0.12528585, 0.81131178, 0.26962463, 0.51111198],
+ },
+ index=[0, 1, 2, 3],
+ )
+ self._test_cov(pdf)
+
+ def _test_cov(self, pdf):
+ psdf = ps.from_pandas(pdf)
+
+ pcov = pdf["s1"].cov(pdf["s2"])
+ pscov = psdf["s1"].cov(psdf["s2"])
+ self.assert_eq(pcov, pscov, almost=True)
+
+ pcov = pdf["s1"].cov(pdf["s2"], min_periods=3)
+ pscov = psdf["s1"].cov(psdf["s2"], min_periods=3)
+ self.assert_eq(pcov, pscov, almost=True)
+
+ pcov = pdf["s1"].cov(pdf["s2"], min_periods=4)
+ pscov = psdf["s1"].cov(psdf["s2"], min_periods=4)
+ self.assert_eq(pcov, pscov, almost=True)
+
if __name__ == "__main__":
from pyspark.pandas.tests.test_series import * # noqa: F401
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org