You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2021/08/27 11:48:00 UTC
[spark] branch branch-3.2 updated:
[SPARK-36388][SPARK-36386][PYTHON][FOLLOWUP] Fix DataFrame groupby-rolling
and groupby-expanding to follow pandas 1.3
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 396b764 [SPARK-36388][SPARK-36386][PYTHON][FOLLOWUP] Fix DataFrame groupby-rolling and groupby-expanding to follow pandas 1.3
396b764 is described below
commit 396b76466b588644d7e3c420681561ef57184e6c
Author: itholic <ha...@databricks.com>
AuthorDate: Wed Aug 18 11:17:01 2021 -0700
[SPARK-36388][SPARK-36386][PYTHON][FOLLOWUP] Fix DataFrame groupby-rolling and groupby-expanding to follow pandas 1.3
This PR is followup for https://github.com/apache/spark/pull/33646 to add missing tests.
Some tests are missing
No
Unittest
Closes #33776 from itholic/SPARK-36388-followup.
Authored-by: itholic <ha...@databricks.com>
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
(cherry picked from commit c91ae544fdd44c67fe1e4c73825570dbe71a3206)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.../pandas/tests/test_ops_on_diff_frames_groupby_expanding.py | 9 ++++++---
.../pandas/tests/test_ops_on_diff_frames_groupby_rolling.py | 9 ++++++---
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
index 223adea..634cbd7 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py
@@ -52,14 +52,17 @@ class OpsOnDiffFramesGroupByExpandingTest(PandasOnSparkTestCase, TestUtils):
psdf = ps.from_pandas(pdf)
kkey = ps.from_pandas(pkey)
+ # The behavior of GroupBy.expanding is changed from pandas 1.3.
if LooseVersion(pd.__version__) >= LooseVersion("1.3"):
- # TODO(SPARK-36367): Fix the behavior to follow pandas >= 1.3
- pass
- else:
self.assert_eq(
getattr(psdf.groupby(kkey).expanding(2), f)().sort_index(),
getattr(pdf.groupby(pkey).expanding(2), f)().sort_index(),
)
+ else:
+ self.assert_eq(
+ getattr(psdf.groupby(kkey).expanding(2), f)().sort_index(),
+ getattr(pdf.groupby(pkey).expanding(2), f)().drop("a", axis=1).sort_index(),
+ )
self.assert_eq(
getattr(psdf.groupby(kkey)["b"].expanding(2), f)().sort_index(),
diff --git a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
index 4f97769..04ea448 100644
--- a/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
+++ b/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py
@@ -50,14 +50,17 @@ class OpsOnDiffFramesGroupByRollingTest(PandasOnSparkTestCase, TestUtils):
psdf = ps.from_pandas(pdf)
kkey = ps.from_pandas(pkey)
+ # The behavior of GroupBy.rolling is changed from pandas 1.3.
if LooseVersion(pd.__version__) >= LooseVersion("1.3"):
- # TODO(SPARK-36367): Fix the behavior to follow pandas >= 1.3
- pass
- else:
self.assert_eq(
getattr(psdf.groupby(kkey).rolling(2), f)().sort_index(),
getattr(pdf.groupby(pkey).rolling(2), f)().sort_index(),
)
+ else:
+ self.assert_eq(
+ getattr(psdf.groupby(kkey).rolling(2), f)().sort_index(),
+ getattr(pdf.groupby(pkey).rolling(2), f)().drop("a", axis=1).sort_index(),
+ )
self.assert_eq(
getattr(psdf.groupby(kkey)["b"].rolling(2), f)().sort_index(),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org