You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2022/09/30 10:40:38 UTC
[spark] branch master updated: [SPARK-40621][PS] Implement `numeric_only` and `min_count` in `GroupBy.sum`
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 014165ff6ff [SPARK-40621][PS] Implement `numeric_only` and `min_count` in `GroupBy.sum`
014165ff6ff is described below
commit 014165ff6ffad87c2d6e970a14c1a13e15ba178f
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Fri Sep 30 19:40:05 2022 +0900
[SPARK-40621][PS] Implement `numeric_only` and `min_count` in `GroupBy.sum`
### What changes were proposed in this pull request?
Implement `numeric_only` and `min_count` in `GroupBy.sum`
### Why are the changes needed?
for API coverage
### Does this PR introduce _any_ user-facing change?
new parameter
```
In [2]: df = ps.DataFrame({"A": [1, 2, 1, 2], "B": [True, False, False, True], "C": [3, 4, 3, 4], "D": ["a", "a", "b", "a"]})
In [3]: df.groupby("A").sum(numeric_only=False).sort_index()
/Users/ruifeng.zheng/Dev/spark/python/pyspark/pandas/utils.py:975: PandasAPIOnSparkAdviceWarning: GroupBy.sum() can only support numeric and bool columns even ifnumeric_only=False, skip unsupported columns: ['D']
warnings.warn(message, PandasAPIOnSparkAdviceWarning)
B C
A
1 1 6
2 1 8
In [4]: df._to_pandas().groupby("A").sum(numeric_only=False).sort_index()
Out[4]:
B C D
A
1 1 6 ab
2 1 8 aa
In [5]: df.groupby("D").sum(min_count=3).sort_index()
Out[5]:
A B C
D
a 5.0 2.0 11.0
b NaN NaN NaN
In [6]: df._to_pandas().groupby("D").sum(min_count=3).sort_index()
Out[6]:
A B C
D
a 5.0 2.0 11.0
b NaN NaN NaN
```
### How was this patch tested?
added UT
Closes #38060 from zhengruifeng/ps_groupby_sum_numonly_mc.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/groupby.py | 72 +++++++++++++++++++++++++++--
python/pyspark/pandas/tests/test_groupby.py | 19 ++++++++
2 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index 60a791d3dea..9f459f094a9 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -806,28 +806,92 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
bool_to_numeric=True,
)
- def sum(self) -> FrameLike:
+ def sum(self, numeric_only: Optional[bool] = True, min_count: int = 0) -> FrameLike:
"""
Compute sum of group values
+ .. versionadded:: 3.3.0
+
+ Parameters
+ ----------
+ numeric_only : bool, default False
+ Include only float, int, boolean columns. If None, will attempt to use
+ everything, then use only numeric data.
+ It takes no effect since only numeric columns can be support here.
+
+ .. versionadded:: 3.4.0
+ min_count: int, default 0
+ The required number of valid values to perform the operation.
+ If fewer than min_count non-NA values are present the result will be NA.
+
+ .. versionadded:: 3.4.0
+
Examples
--------
>>> df = ps.DataFrame({"A": [1, 2, 1, 2], "B": [True, False, False, True],
- ... "C": [3, 4, 3, 4], "D": ["a", "b", "b", "a"]})
+ ... "C": [3, 4, 3, 4], "D": ["a", "a", "b", "a"]})
- >>> df.groupby("A").sum()
+ >>> df.groupby("A").sum().sort_index()
B C
A
1 1 6
2 1 8
+ >>> df.groupby("D").sum().sort_index()
+ A B C
+ D
+ a 5 2 11
+ b 1 0 3
+
+ >>> df.groupby("D").sum(min_count=3).sort_index()
+ A B C
+ D
+ a 5.0 2.0 11.0
+ b NaN NaN NaN
+
+ Notes
+ -----
+ There is a behavior difference between pandas-on-Spark and pandas:
+
+ * when there is a non-numeric aggregation column, it will be ignored
+ even if `numeric_only` is False.
+
See Also
--------
pyspark.pandas.Series.groupby
pyspark.pandas.DataFrame.groupby
"""
+ if numeric_only is not None and not isinstance(numeric_only, bool):
+ raise TypeError("numeric_only must be None or bool")
+ if not isinstance(min_count, int):
+ raise TypeError("min_count must be integer")
+
+ if numeric_only is not None and not numeric_only:
+ unsupported = [
+ col.name
+ for col in self._agg_columns
+ if not isinstance(col.spark.data_type, (NumericType, BooleanType))
+ ]
+ if len(unsupported) > 0:
+ log_advice(
+ "GroupBy.sum() can only support numeric and bool columns even if"
+ f"numeric_only=False, skip unsupported columns: {unsupported}"
+ )
+
+ if min_count > 0:
+
+ def sum(col: Column) -> Column:
+ return F.when(
+ F.count(F.when(~F.isnull(col), F.lit(0))) < min_count, F.lit(None)
+ ).otherwise(F.sum(col))
+
+ else:
+
+ def sum(col: Column) -> Column:
+ return F.sum(col)
+
return self._reduce_for_stat_function(
- F.sum, accepted_spark_types=(NumericType,), bool_to_numeric=True
+ sum, accepted_spark_types=(NumericType,), bool_to_numeric=True
)
# TODO: sync the doc.
diff --git a/python/pyspark/pandas/tests/test_groupby.py b/python/pyspark/pandas/tests/test_groupby.py
index 14e699793f3..ba283cc9a05 100644
--- a/python/pyspark/pandas/tests/test_groupby.py
+++ b/python/pyspark/pandas/tests/test_groupby.py
@@ -1413,6 +1413,25 @@ class GroupByTest(PandasOnSparkTestCase, TestUtils):
self._test_stat_func(lambda groupby_obj: groupby_obj.max(numeric_only=True))
self._test_stat_func(lambda groupby_obj: groupby_obj.max(numeric_only=True, min_count=2))
+ def test_sum(self):
+ pdf = pd.DataFrame(
+ {
+ "A": ["a", "a", "b", "a"],
+ "B": [1, 2, 1, 2],
+ "C": [-1.5, np.nan, -3.2, 0.1],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(pdf.groupby("A").sum().sort_index(), psdf.groupby("A").sum().sort_index())
+ self.assert_eq(
+ pdf.groupby("A").sum(min_count=2).sort_index(),
+ psdf.groupby("A").sum(min_count=2).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby("A").sum(min_count=3).sort_index(),
+ psdf.groupby("A").sum(min_count=3).sort_index(),
+ )
+
def test_mad(self):
self._test_stat_func(lambda groupby_obj: groupby_obj.mad())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org