You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2021/09/03 17:41:57 UTC

[spark] branch master updated: [SPARK-36401][PYTHON] Implement Series.cov

This is an automated email from the ASF dual-hosted git repository.

ueshin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 9b262e7  [SPARK-36401][PYTHON] Implement Series.cov
9b262e7 is described below

commit 9b262e722d717bebf7d2aa19af0658cf3b8e1e85
Author: dgd-contributor <dg...@viettel.com.vn>
AuthorDate: Fri Sep 3 10:41:27 2021 -0700

    [SPARK-36401][PYTHON] Implement Series.cov
    
    ### What changes were proposed in this pull request?
    
    Implement Series.cov
    
    ### Why are the changes needed?
    
    That is supported in pandas. We should support that as well.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Series.cov can be used.
    
    ```python
    >>> from pyspark.pandas.config import set_option, reset_option
    >>> set_option("compute.ops_on_diff_frames", True)
    >>> s1 = ps.Series([0.90010907, 0.13484424, 0.62036035])
    >>> s2 = ps.Series([0.12528585, 0.26962463, 0.51111198])
    >>> s1.cov(s2)
    -0.016857626527158744
    >>> reset_option("compute.ops_on_diff_frames")
    ```
    
    ### How was this patch tested?
    
    Unit tests
    
    Closes #33752 from dgd-contributor/SPARK-36401_Implement_Series.cov.
    
    Authored-by: dgd-contributor <dg...@viettel.com.vn>
    Signed-off-by: Takuya UESHIN <ue...@databricks.com>
---
 .../source/reference/pyspark.pandas/series.rst     |  1 +
 python/pyspark/pandas/missing/series.py            |  1 -
 python/pyspark/pandas/series.py                    | 50 ++++++++++++++++++++++
 .../pandas/tests/test_ops_on_diff_frames.py        | 37 ++++++++++++++++
 python/pyspark/pandas/tests/test_series.py         | 45 +++++++++++++++++++
 5 files changed, 133 insertions(+), 1 deletion(-)

diff --git a/python/docs/source/reference/pyspark.pandas/series.rst b/python/docs/source/reference/pyspark.pandas/series.rst
index 1e32773..337f686 100644
--- a/python/docs/source/reference/pyspark.pandas/series.rst
+++ b/python/docs/source/reference/pyspark.pandas/series.rst
@@ -138,6 +138,7 @@ Computations / Descriptive Stats
    Series.clip
    Series.corr
    Series.count
+   Series.cov
    Series.cummax
    Series.cummin
    Series.cumsum
diff --git a/python/pyspark/pandas/missing/series.py b/python/pyspark/pandas/missing/series.py
index ef3f38b..f6ea8d3 100644
--- a/python/pyspark/pandas/missing/series.py
+++ b/python/pyspark/pandas/missing/series.py
@@ -40,7 +40,6 @@ class MissingPandasLikeSeries(object):
     autocorr = _unsupported_function("autocorr")
     combine = _unsupported_function("combine")
     convert_dtypes = _unsupported_function("convert_dtypes")
-    cov = _unsupported_function("cov")
     ewm = _unsupported_function("ewm")
     infer_objects = _unsupported_function("infer_objects")
     interpolate = _unsupported_function("interpolate")
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index 568754c..7e45566 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -944,6 +944,56 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
 
         return lmask & rmask
 
+    def cov(self, other: "Series", min_periods: Optional[int] = None) -> float:
+        """
+        Compute covariance with Series, excluding missing values.
+
+        .. versionadded:: 3.3.0
+
+        Parameters
+        ----------
+        other : Series
+            Series with which to compute the covariance.
+        min_periods : int, optional
+            Minimum number of observations needed to have a valid result.
+
+        Returns
+        -------
+        float
+            Covariance between Series and other
+
+        Examples
+        --------
+        >>> from pyspark.pandas.config import set_option, reset_option
+        >>> set_option("compute.ops_on_diff_frames", True)
+        >>> s1 = ps.Series([0.90010907, 0.13484424, 0.62036035])
+        >>> s2 = ps.Series([0.12528585, 0.26962463, 0.51111198])
+        >>> s1.cov(s2)
+        -0.016857626527158744
+        >>> reset_option("compute.ops_on_diff_frames")
+        """
+        if not isinstance(other, Series):
+            raise TypeError("unsupported type: %s" % type(other))
+        if not np.issubdtype(self.dtype, np.number):
+            raise TypeError("unsupported dtype: %s" % self.dtype)
+        if not np.issubdtype(other.dtype, np.number):
+            raise TypeError("unsupported dtype: %s" % other.dtype)
+
+        min_periods = 1 if min_periods is None else min_periods
+
+        if same_anchor(self, other):
+            sdf = self._internal.spark_frame.select(self.spark.column, other.spark.column)
+        else:
+            combined = combine_frames(self.to_frame(), other.to_frame())
+            sdf = combined._internal.spark_frame.select(*combined._internal.data_spark_columns)
+
+        sdf = sdf.dropna()
+
+        if len(sdf.head(min_periods)) < min_periods:
+            return np.nan
+        else:
+            return sdf.select(F.covar_samp(*sdf.columns)).head(1)[0][0]
+
     # TODO: arg should support Series
     # TODO: NaN and None
     def map(self, arg: Union[Dict, Callable]) -> "Series":
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
index f7fa3e8..3e3bb0d 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames.py
@@ -1793,6 +1793,43 @@ class OpsOnDiffFramesEnabledTest(PandasOnSparkTestCase, SQLTestUtils):
             pdf["Col2"].rank().loc[pdf["Col1"] == 20], psdf["Col2"].rank().loc[psdf["Col1"] == 20]
         )
 
+    def test_cov(self):
+        pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035], index=[0, 1, 2])
+        pser2 = pd.Series([0.12528585, 0.26962463, 0.51111198], index=[1, 2, 3])
+        self._test_cov(pser1, pser2)
+
+        pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035], index=[0, 1, 2])
+        pser2 = pd.Series([0.12528585, 0.26962463, 0.51111198, 0.32076008], index=[1, 2, 3, 4])
+        self._test_cov(pser1, pser2)
+
+        pser1 = pd.Series([0.90010907, 0.13484424, 0.62036035, 0.32076008], index=[0, 1, 2, 3])
+        pser2 = pd.Series([0.12528585, 0.26962463], index=[1, 2])
+        self._test_cov(pser1, pser2)
+
+        psser1 = ps.from_pandas(pser1)
+        with self.assertRaisesRegex(TypeError, "unsupported type: <class 'list'>"):
+            psser1.cov([0.12528585, 0.26962463, 0.51111198])
+        with self.assertRaisesRegex(
+            TypeError, "unsupported type: <class 'pandas.core.series.Series'>"
+        ):
+            psser1.cov(pser2)
+
+    def _test_cov(self, pser1, pser2):
+        psser1 = ps.from_pandas(pser1)
+        psser2 = ps.from_pandas(pser2)
+
+        pcov = pser1.cov(pser2)
+        pscov = psser1.cov(psser2)
+        self.assert_eq(pcov, pscov, almost=True)
+
+        pcov = pser1.cov(pser2, min_periods=2)
+        pscov = psser1.cov(psser2, min_periods=2)
+        self.assert_eq(pcov, pscov, almost=True)
+
+        pcov = pser1.cov(pser2, min_periods=3)
+        pscov = psser1.cov(psser2, min_periods=3)
+        self.assert_eq(pcov, pscov, almost=True)
+
 
 class OpsOnDiffFramesDisabledTest(PandasOnSparkTestCase, SQLTestUtils):
     @classmethod
diff --git a/python/pyspark/pandas/tests/test_series.py b/python/pyspark/pandas/tests/test_series.py
index a861dd0..ffa8431 100644
--- a/python/pyspark/pandas/tests/test_series.py
+++ b/python/pyspark/pandas/tests/test_series.py
@@ -2905,6 +2905,51 @@ class SeriesTest(PandasOnSparkTestCase, SQLTestUtils):
 
         self.assert_eq(psser1.combine_first(psser2), pser1.combine_first(pser2))
 
+    def test_cov(self):
+        pdf = pd.DataFrame(
+            {
+                "s1": ["a", "b", "c"],
+                "s2": [0.12528585, 0.26962463, 0.51111198],
+            },
+            index=[0, 1, 2],
+        )
+        psdf = ps.from_pandas(pdf)
+        with self.assertRaisesRegex(TypeError, "unsupported dtype: object"):
+            psdf["s1"].cov(psdf["s2"])
+
+        pdf = pd.DataFrame(
+            {
+                "s1": [0.90010907, 0.13484424, 0.62036035],
+                "s2": [0.12528585, 0.26962463, 0.51111198],
+            },
+            index=[0, 1, 2],
+        )
+        self._test_cov(pdf)
+
+        pdf = pd.DataFrame(
+            {
+                "s1": [0.90010907, np.nan, 0.13484424, 0.62036035],
+                "s2": [0.12528585, 0.81131178, 0.26962463, 0.51111198],
+            },
+            index=[0, 1, 2, 3],
+        )
+        self._test_cov(pdf)
+
+    def _test_cov(self, pdf):
+        psdf = ps.from_pandas(pdf)
+
+        pcov = pdf["s1"].cov(pdf["s2"])
+        pscov = psdf["s1"].cov(psdf["s2"])
+        self.assert_eq(pcov, pscov, almost=True)
+
+        pcov = pdf["s1"].cov(pdf["s2"], min_periods=3)
+        pscov = psdf["s1"].cov(psdf["s2"], min_periods=3)
+        self.assert_eq(pcov, pscov, almost=True)
+
+        pcov = pdf["s1"].cov(pdf["s2"], min_periods=4)
+        pscov = psdf["s1"].cov(psdf["s2"], min_periods=4)
+        self.assert_eq(pcov, pscov, almost=True)
+
 
 if __name__ == "__main__":
     from pyspark.pandas.tests.test_series import *  # noqa: F401

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org