You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/08/21 00:36:32 UTC
[spark] branch master updated: [SPARK-44842][SPARK-43812][PS] Support stat functions for pandas 2.0.0 and enabling tests
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ef6327545e2 [SPARK-44842][SPARK-43812][PS] Support stat functions for pandas 2.0.0 and enabling tests
ef6327545e2 is described below
commit ef6327545e2a36e36e3d73bb196f9fcdb8562e9d
Author: itholic <ha...@databricks.com>
AuthorDate: Mon Aug 21 08:36:18 2023 +0800
[SPARK-44842][SPARK-43812][PS] Support stat functions for pandas 2.0.0 and enabling tests
### What changes were proposed in this pull request?
This PR proposes to match the behavior with pandas 2.0.0 and above for stat functions, such as `sum`, `quantile`, `prod`, etc. See https://github.com/pandas-dev/pandas/pull/41480 and https://github.com/pandas-dev/pandas/issues/47500 for more detail.
### Why are the changes needed?
To match the behavior to latest pandas.
### Does this PR introduce _any_ user-facing change?
Yes, the behaviors for stat funcs are now matched with pandas 2.0.0 and above.
### How was this patch tested?
Enabling & updating the existing UTs.
Closes #42526 from itholic/pandas_stat.
Authored-by: itholic <ha...@databricks.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/pandas/frame.py | 10 ++---
python/pyspark/pandas/generic.py | 5 ---
python/pyspark/pandas/groupby.py | 20 +++++----
python/pyspark/pandas/series.py | 16 +++-----
.../pandas/tests/computation/test_any_all.py | 14 ++++---
.../pandas/tests/computation/test_compute.py | 38 ++++++++---------
python/pyspark/pandas/tests/groupby/test_stat.py | 33 ++++-----------
python/pyspark/pandas/tests/series/test_stat.py | 47 +++++++---------------
python/pyspark/pandas/tests/test_stats.py | 28 ++++++-------
9 files changed, 80 insertions(+), 131 deletions(-)
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 8fbe1b8f926..41ab03a5c0b 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -83,6 +83,7 @@ from pyspark.sql.types import (
DecimalType,
TimestampType,
TimestampNTZType,
+ NullType,
)
from pyspark.sql.window import Window
@@ -797,7 +798,7 @@ class DataFrame(Frame, Generic[T]):
new_column_labels.append(label)
if len(exprs) == 1:
- return Series([])
+ return Series([], dtype="float64")
sdf = self._internal.spark_frame.select(*exprs)
@@ -12128,11 +12129,6 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
0.50 3.0 7.0
0.75 4.0 8.0
"""
- warnings.warn(
- "Default value of `numeric_only` will be changed to `False` "
- "instead of `True` in 4.0.0.",
- FutureWarning,
- )
axis = validate_axis(axis)
if axis != 0:
raise NotImplementedError('axis should be either 0 or "index" currently.')
@@ -12155,7 +12151,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
def quantile(psser: "Series") -> PySparkColumn:
spark_type = psser.spark.data_type
spark_column = psser.spark.column
- if isinstance(spark_type, (BooleanType, NumericType)):
+ if isinstance(spark_type, (BooleanType, NumericType, NullType)):
return F.percentile_approx(spark_column.cast(DoubleType()), qq, accuracy)
else:
raise TypeError(
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
index c346889863b..b540045f88f 100644
--- a/python/pyspark/pandas/generic.py
+++ b/python/pyspark/pandas/generic.py
@@ -1419,11 +1419,6 @@ class Frame(object, metaclass=ABCMeta):
nan
"""
axis = validate_axis(axis)
- warnings.warn(
- "Default value of `numeric_only` will be changed to `False` "
- "instead of `None` in 4.0.0.",
- FutureWarning,
- )
if numeric_only is None and axis == 0:
numeric_only = True
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index df671d71eec..e66d08400b4 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -614,9 +614,10 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
Parameters
----------
- numeric_only : bool, default False
+ numeric_only : bool, default True
Include only float, int, boolean columns. If None, will attempt to use
- everything, then use only numeric data.
+ everything, then use only numeric data. False is not supported.
+ This parameter is mainly for pandas compatibility.
.. versionadded:: 3.4.0
@@ -646,11 +647,6 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
2 4.0 1.500000 1.000000
"""
self._validate_agg_columns(numeric_only=numeric_only, function_name="median")
- warnings.warn(
- "Default value of `numeric_only` will be changed to `False` "
- "instead of `True` in 4.0.0.",
- FutureWarning,
- )
return self._reduce_for_stat_function(
F.mean, accepted_spark_types=(NumericType,), bool_to_numeric=True
@@ -920,7 +916,7 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
)
# TODO: sync the doc.
- def var(self, ddof: int = 1) -> FrameLike:
+ def var(self, ddof: int = 1, numeric_only: Optional[bool] = True) -> FrameLike:
"""
Compute variance of groups, excluding missing values.
@@ -935,6 +931,13 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
.. versionchanged:: 3.4.0
Supported including arbitary integers.
+ numeric_only : bool, default True
+ Include only float, int, boolean columns. If None, will attempt to use
+ everything, then use only numeric data. False is not supported.
+ This parameter is mainly for pandas compatibility.
+
+ .. versionadded:: 4.0.0
+
Examples
--------
>>> df = ps.DataFrame({"A": [1, 2, 1, 2], "B": [True, False, False, True],
@@ -961,6 +964,7 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
var,
accepted_spark_types=(NumericType,),
bool_to_numeric=True,
+ numeric_only=numeric_only,
)
def skew(self) -> FrameLike:
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index d1b1d763179..7fa08c6d9b2 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -67,6 +67,7 @@ from pyspark.sql.types import (
Row,
StructType,
TimestampType,
+ NullType,
)
from pyspark.sql.window import Window
from pyspark.sql.utils import get_column_class, get_window_class
@@ -4024,7 +4025,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
def quantile(psser: Series) -> PySparkColumn:
spark_type = psser.spark.data_type
spark_column = psser.spark.column
- if isinstance(spark_type, (BooleanType, NumericType)):
+ if isinstance(spark_type, (BooleanType, NumericType, NullType)):
return F.percentile_approx(spark_column.cast(DoubleType()), q_float, accuracy)
else:
raise TypeError(
@@ -4059,7 +4060,8 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
ascending : boolean, default True
False for ranks by high (1) to low (N)
numeric_only : bool, optional
- If set to True, rank numeric Series, or return an empty Series for non-numeric Series
+ If set to True, rank numeric Series, or raise TypeError for non-numeric Series.
+ False is not supported. This parameter is mainly for pandas compatibility.
Returns
-------
@@ -4127,18 +4129,10 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
y b
z c
Name: A, dtype: object
-
- >>> s.rank(numeric_only=True)
- Series([], Name: A, dtype: float64)
"""
- warnings.warn(
- "Default value of `numeric_only` will be changed to `False` "
- "instead of `None` in 4.0.0.",
- FutureWarning,
- )
is_numeric = isinstance(self.spark.data_type, (NumericType, BooleanType))
if numeric_only and not is_numeric:
- return ps.Series([], dtype="float64", name=self.name)
+ raise TypeError("Series.rank does not allow numeric_only=True with non-numeric dtype.")
else:
return self._rank(method, ascending).spark.analyzed
diff --git a/python/pyspark/pandas/tests/computation/test_any_all.py b/python/pyspark/pandas/tests/computation/test_any_all.py
index 3574254d1db..64f293c48d6 100644
--- a/python/pyspark/pandas/tests/computation/test_any_all.py
+++ b/python/pyspark/pandas/tests/computation/test_any_all.py
@@ -39,10 +39,6 @@ class FrameAnyAllMixin:
psdf = ps.from_pandas(pdf)
return pdf, psdf
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43812): Enable DataFrameTests.test_all for pandas 2.0.0.",
- )
def test_all(self):
pdf = pd.DataFrame(
{
@@ -105,9 +101,15 @@ class FrameAnyAllMixin:
self.assert_eq(psdf.all(skipna=True), pdf.all(skipna=True))
self.assert_eq(psdf.all(), pdf.all())
self.assert_eq(
- ps.DataFrame([np.nan]).all(skipna=False), pd.DataFrame([np.nan]).all(skipna=False)
+ ps.DataFrame([np.nan]).all(skipna=False),
+ pd.DataFrame([np.nan]).all(skipna=False),
+ almost=True,
+ )
+ self.assert_eq(
+ ps.DataFrame([None]).all(skipna=True),
+ pd.DataFrame([None]).all(skipna=True),
+ almost=True,
)
- self.assert_eq(ps.DataFrame([None]).all(skipna=True), pd.DataFrame([None]).all(skipna=True))
def test_any(self):
pdf = pd.DataFrame(
diff --git a/python/pyspark/pandas/tests/computation/test_compute.py b/python/pyspark/pandas/tests/computation/test_compute.py
index d4b49f2ac8b..9a29cb236a8 100644
--- a/python/pyspark/pandas/tests/computation/test_compute.py
+++ b/python/pyspark/pandas/tests/computation/test_compute.py
@@ -283,10 +283,6 @@ class FrameComputeMixin:
self.assert_eq(psdf.nunique(), pdf.nunique())
self.assert_eq(psdf.nunique(dropna=False), pdf.nunique(dropna=False))
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43810): Enable DataFrameSlowTests.test_quantile for pandas 2.0.0.",
- )
def test_quantile(self):
pdf, psdf = self.df_pair
@@ -332,59 +328,57 @@ class FrameComputeMixin:
pdf = pd.DataFrame({"x": ["a", "b", "c"]})
psdf = ps.from_pandas(pdf)
- self.assert_eq(psdf.quantile(0.5), pdf.quantile(0.5))
- self.assert_eq(psdf.quantile([0.25, 0.5, 0.75]), pdf.quantile([0.25, 0.5, 0.75]))
+ self.assert_eq(psdf.quantile(0.5), pdf.quantile(0.5, numeric_only=True))
+ self.assert_eq(
+ psdf.quantile([0.25, 0.5, 0.75]), pdf.quantile([0.25, 0.5, 0.75], numeric_only=True)
+ )
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
psdf.quantile(0.5, numeric_only=False)
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
psdf.quantile([0.25, 0.5, 0.75], numeric_only=False)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43558): Enable DataFrameSlowTests.test_product for pandas 2.0.0.",
- )
def test_product(self):
pdf = pd.DataFrame(
{"A": [1, 2, 3, 4, 5], "B": [10, 20, 30, 40, 50], "C": ["a", "b", "c", "d", "e"]}
)
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index())
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index())
# Named columns
pdf.columns.name = "Koalas"
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index())
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index())
# MultiIndex columns
pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index())
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index())
# Named MultiIndex columns
pdf.columns.names = ["Hello", "Koalas"]
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index())
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index())
# No numeric columns
pdf = pd.DataFrame({"key": ["a", "b", "c"], "val": ["x", "y", "z"]})
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index())
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index())
# No numeric named columns
pdf.columns.name = "Koalas"
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), almost=True)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), almost=True)
# No numeric MultiIndex columns
pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y")])
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), almost=True)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), almost=True)
# No numeric named MultiIndex columns
pdf.columns.names = ["Hello", "Koalas"]
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), almost=True)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), almost=True)
# All NaN columns
pdf = pd.DataFrame(
@@ -395,22 +389,22 @@ class FrameComputeMixin:
}
)
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), check_exact=False)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), check_exact=False)
# All NaN named columns
pdf.columns.name = "Koalas"
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), check_exact=False)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), check_exact=False)
# All NaN MultiIndex columns
pdf.columns = pd.MultiIndex.from_tuples([("a", "x"), ("b", "y"), ("c", "z")])
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), check_exact=False)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), check_exact=False)
# All NaN named MultiIndex columns
pdf.columns.names = ["Hello", "Koalas"]
psdf = ps.from_pandas(pdf)
- self.assert_eq(pdf.prod(), psdf.prod().sort_index(), check_exact=False)
+ self.assert_eq(pdf.prod(numeric_only=True), psdf.prod().sort_index(), check_exact=False)
class FrameComputeTests(FrameComputeMixin, ComparisonTestBase, SQLTestUtils):
diff --git a/python/pyspark/pandas/tests/groupby/test_stat.py b/python/pyspark/pandas/tests/groupby/test_stat.py
index 8a5096942e6..44bb3b00709 100644
--- a/python/pyspark/pandas/tests/groupby/test_stat.py
+++ b/python/pyspark/pandas/tests/groupby/test_stat.py
@@ -58,12 +58,10 @@ class GroupbyStatMixin:
check_exact=check_exact,
)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43554): Enable GroupByTests.test_basic_stat_funcs for pandas 2.0.0.",
- )
def test_basic_stat_funcs(self):
- self._test_stat_func(lambda groupby_obj: groupby_obj.var(), check_exact=False)
+ self._test_stat_func(
+ lambda groupby_obj: groupby_obj.var(numeric_only=True), check_exact=False
+ )
pdf, psdf = self.pdf, self.psdf
@@ -102,12 +100,12 @@ class GroupbyStatMixin:
self.assert_eq(
psdf.groupby("A").std().sort_index(),
- pdf.groupby("A").std().sort_index(),
+ pdf.groupby("A").std(numeric_only=True).sort_index(),
check_exact=False,
)
self.assert_eq(
psdf.groupby("A").sem().sort_index(),
- pdf.groupby("A").sem().sort_index(),
+ pdf.groupby("A").sem(numeric_only=True).sort_index(),
check_exact=False,
)
@@ -115,17 +113,11 @@ class GroupbyStatMixin:
# self._test_stat_func(lambda groupby_obj: groupby_obj.sum(), check_exact=False)
self.assert_eq(
psdf.groupby("A").sum().sort_index(),
- pdf.groupby("A").sum().sort_index(),
+ pdf.groupby("A").sum(numeric_only=True).sort_index(),
check_exact=False,
)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43706): Enable GroupByTests.test_mean " "for pandas 2.0.0.",
- )
def test_mean(self):
- self._test_stat_func(lambda groupby_obj: groupby_obj.mean())
- self._test_stat_func(lambda groupby_obj: groupby_obj.mean(numeric_only=None))
self._test_stat_func(lambda groupby_obj: groupby_obj.mean(numeric_only=True))
psdf = self.psdf
with self.assertRaises(TypeError):
@@ -267,10 +259,6 @@ class GroupbyStatMixin:
with self.assertRaisesRegex(TypeError, "Invalid index"):
self.psdf.groupby("B").nth("x")
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43551): Enable GroupByTests.test_prod for pandas 2.0.0.",
- )
def test_prod(self):
pdf = pd.DataFrame(
{
@@ -286,19 +274,12 @@ class GroupbyStatMixin:
psdf = ps.from_pandas(pdf)
for n in [0, 1, 2, 128, -1, -2, -128]:
- self._test_stat_func(
- lambda groupby_obj: groupby_obj.prod(min_count=n), check_exact=False
- )
- self._test_stat_func(
- lambda groupby_obj: groupby_obj.prod(numeric_only=None, min_count=n),
- check_exact=False,
- )
self._test_stat_func(
lambda groupby_obj: groupby_obj.prod(numeric_only=True, min_count=n),
check_exact=False,
)
self.assert_eq(
- pdf.groupby("A").prod(min_count=n).sort_index(),
+ pdf.groupby("A").prod(min_count=n, numeric_only=True).sort_index(),
psdf.groupby("A").prod(min_count=n).sort_index(),
almost=True,
)
diff --git a/python/pyspark/pandas/tests/series/test_stat.py b/python/pyspark/pandas/tests/series/test_stat.py
index 2c25e21954d..b54357bede4 100644
--- a/python/pyspark/pandas/tests/series/test_stat.py
+++ b/python/pyspark/pandas/tests/series/test_stat.py
@@ -321,10 +321,6 @@ class SeriesStatMixin:
with self.assertRaisesRegex(TypeError, "accuracy must be an integer; however"):
ps.Series([24.0, 21.0, 25.0, 33.0, 26.0]).median(accuracy="a")
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43463): Enable SeriesTests.test_rank for pandas 2.0.0.",
- )
def test_rank(self):
pser = pd.Series([1, 2, 3, 1], name="x")
psser = ps.from_pandas(pser)
@@ -338,10 +334,6 @@ class SeriesStatMixin:
non_numeric_pser = pd.Series(["a", "c", "b", "d"], name="x", index=[10, 11, 12, 13])
non_numeric_psser = ps.from_pandas(non_numeric_pser)
- self.assert_eq(
- non_numeric_pser.rank(numeric_only=True),
- non_numeric_psser.rank(numeric_only=True),
- )
self.assert_eq(
non_numeric_pser.rank(numeric_only=None),
non_numeric_psser.rank(numeric_only=None).sort_index(),
@@ -350,10 +342,14 @@ class SeriesStatMixin:
non_numeric_pser.rank(numeric_only=False),
non_numeric_psser.rank(numeric_only=False).sort_index(),
)
- self.assert_eq(
- (non_numeric_pser + "x").rank(numeric_only=True),
- (non_numeric_psser + "x").rank(numeric_only=True),
- )
+
+ msg = "Series.rank does not allow numeric_only=True with non-numeric dtype."
+ with self.assertRaisesRegex(TypeError, msg):
+ non_numeric_psser.rank(numeric_only=True)
+
+ msg = "Series.rank does not allow numeric_only=True with non-numeric dtype."
+ with self.assertRaisesRegex(TypeError, msg):
+ (non_numeric_psser + "x").rank(numeric_only=True)
msg = "method must be one of 'average', 'min', 'max', 'first', 'dense'"
with self.assertRaisesRegex(ValueError, msg):
@@ -378,10 +374,6 @@ class SeriesStatMixin:
with self.assertRaisesRegex(TypeError, msg):
psser.round(1.5)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43469): Enable SeriesTests.test_quantile for pandas 2.0.0.",
- )
def test_quantile(self):
pser = pd.Series([])
psser = ps.from_pandas(pser)
@@ -520,10 +512,6 @@ class SeriesStatMixin:
self.assert_eq(pser // 0, psser // 0)
self.assert_eq(pser.floordiv(np.nan), psser.floordiv(np.nan))
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43481): Enable SeriesTests.test_product for pandas 2.0.0.",
- )
def test_product(self):
pser = pd.Series([10, 20, 30, 40, 50])
psser = ps.from_pandas(pser)
@@ -539,23 +527,18 @@ class SeriesStatMixin:
psser = ps.from_pandas(pser)
self.assert_eq(pser.prod(), psser.prod())
- # Empty Series
- pser = pd.Series([])
- psser = ps.from_pandas(pser)
- self.assert_eq(pser.prod(), psser.prod())
-
# Boolean Series
pser = pd.Series([True, True, True])
psser = ps.from_pandas(pser)
- self.assert_eq(pser.prod(), psser.prod())
+ self.assert_eq(pser.prod(numeric_only=True), psser.prod())
pser = pd.Series([False, False, False])
psser = ps.from_pandas(pser)
- self.assert_eq(pser.prod(), psser.prod())
+ self.assert_eq(pser.prod(numeric_only=True), psser.prod())
pser = pd.Series([True, False, True])
psser = ps.from_pandas(pser)
- self.assert_eq(pser.prod(), psser.prod())
+ self.assert_eq(pser.prod(numeric_only=True), psser.prod())
# With `min_count` parameter
pser = pd.Series([10, 20, 30, 40, 50])
@@ -572,10 +555,10 @@ class SeriesStatMixin:
psser = ps.from_pandas(pser)
self.assert_eq(pser.prod(min_count=1), psser.prod(min_count=1))
- pser = pd.Series([])
- psser = ps.from_pandas(pser)
- self.assert_eq(pser.prod(min_count=1), psser.prod(min_count=1))
-
+ with self.assertRaisesRegex(TypeError, "Could not convert object \\(void\\) to numeric"):
+ ps.Series([]).prod(numeric_only=True)
+ with self.assertRaisesRegex(TypeError, "Could not convert object \\(void\\) to numeric"):
+ ps.Series([]).prod(min_count=1)
with self.assertRaisesRegex(TypeError, "Could not convert object \\(string\\) to numeric"):
ps.Series(["a", "b", "c"]).prod()
with self.assertRaisesRegex(
diff --git a/python/pyspark/pandas/tests/test_stats.py b/python/pyspark/pandas/tests/test_stats.py
index ec56fa7ef1a..be5340dafdc 100644
--- a/python/pyspark/pandas/tests/test_stats.py
+++ b/python/pyspark/pandas/tests/test_stats.py
@@ -33,15 +33,24 @@ from pyspark.testing.sqlutils import SQLTestUtils
class StatsTestsMixin:
def _test_stat_functions(self, pdf_or_pser, psdf_or_psser):
- functions = ["max", "min", "mean", "sum", "count"]
+ self.assert_eq(
+ psdf_or_psser.count(),
+ pdf_or_pser.count(),
+ almost=True,
+ )
+
+ functions = ["max", "min", "mean", "sum"]
for funcname in functions:
- self.assert_eq(getattr(psdf_or_psser, funcname)(), getattr(pdf_or_pser, funcname)())
+ self.assert_eq(
+ getattr(psdf_or_psser, funcname)(),
+ getattr(pdf_or_pser, funcname)(numeric_only=True),
+ )
functions = ["std", "var", "product", "sem"]
for funcname in functions:
self.assert_eq(
getattr(psdf_or_psser, funcname)(),
- getattr(pdf_or_pser, funcname)(),
+ getattr(pdf_or_pser, funcname)(numeric_only=True),
check_exact=False,
)
@@ -49,7 +58,7 @@ class StatsTestsMixin:
for funcname in functions:
self.assert_eq(
getattr(psdf_or_psser, funcname)(ddof=0),
- getattr(pdf_or_pser, funcname)(ddof=0),
+ getattr(pdf_or_pser, funcname)(ddof=0, numeric_only=True),
check_exact=False,
)
@@ -76,11 +85,6 @@ class StatsTestsMixin:
self._test_stat_functions(pdf.A, psdf.A)
self._test_stat_functions(pdf, psdf)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43499): Enable SeriesTests.test_stat_functions_with_no_numeric_columns "
- "for pandas 2.0.0.",
- )
def test_stat_functions_with_no_numeric_columns(self):
pdf = pd.DataFrame(
{
@@ -407,10 +411,6 @@ class StatsTestsMixin:
almost=True,
)
- @unittest.skipIf(
- LooseVersion(pd.__version__) >= LooseVersion("2.0.0"),
- "TODO(SPARK-43497): Enable SeriesTests.test_cov_corr_meta for pandas 2.0.0.",
- )
def test_cov_corr_meta(self):
# Disable arrow execution since corr() is using UDT internally which is not supported.
with self.sql_conf({SPARK_CONF_ARROW_ENABLED: False}):
@@ -428,7 +428,7 @@ class StatsTestsMixin:
index=pd.Index([1, 2, 3], name="myindex"),
)
psdf = ps.from_pandas(pdf)
- self.assert_eq(psdf.corr(), pdf.corr(), check_exact=False)
+ self.assert_eq(psdf.corr(), pdf.corr(numeric_only=True), check_exact=False)
def test_stats_on_boolean_dataframe(self):
pdf = pd.DataFrame({"A": [True, False, True], "B": [False, False, True]})
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org