You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/03/26 07:18:05 UTC
(spark) branch master updated: [SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 932f9f70d2e6 [SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3
932f9f70d2e6 is described below
commit 932f9f70d2e6b5495c80a6cc53397aa1343d342b
Author: Takuya Ueshin <ue...@databricks.com>
AuthorDate: Thu Mar 26 16:17:53 2026 +0900
[SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3
### What changes were proposed in this pull request?
This PR updates pandas-on-Spark `GroupBy.idxmax` and `GroupBy.idxmin` so that `skipna=False` follows the version-specific pandas behavior more closely.
For pandas `< 3.0.0`, NA-containing groups return null results for the affected group/column outputs.
For pandas `>= 3.0.0`, NA-containing inputs raise for `skipna=False`, and no-NA inputs follow pandas 3’s all-missing-label behavior.
This PR also updates the related tests in `python/pyspark/pandas/tests/groupby/test_index.py` to cover both pandas 2 and pandas 3 behavior, including cases with non-default indexes.
### Why are the changes needed?
`GroupBy.idxmax(skipna=False)` and `GroupBy.idxmin(skipna=False)` differ between pandas 2 and pandas 3, and pandas-on-Spark was not matching those semantics.
As a result, pandas-on-Spark could return index labels where pandas would return null or raise, depending on the pandas version in use.
### Does this PR introduce _any_ user-facing change?
Yes.
For pandas-on-Spark groupby `idxmax(skipna=False)` and `idxmin(skipna=False)`, behavior now matches pandas more closely:
- with pandas 2, groups containing NA values return null results for the affected outputs
- with pandas 3, NA-containing inputs raise for `skipna=False`
- with pandas 3, no-NA `skipna=False` inputs follow pandas 3 behavior
### How was this patch tested?
Ran the related pandas-on-Spark groupby tests in both pandas 2 and pandas 3 environments, including:
- `GroupbyIndexTests.test_idxmax`
- `GroupbyIndexTests.test_idxmin`
- `GroupbyIndexTests.test_idxmax_idxmin_skipna_false_with_na`
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #55021 from ueshin/issuse/SPARK-56219/groupby_idxmin_idxmax.
Authored-by: Takuya Ueshin <ue...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/groupby.py | 77 +++++++++++----
python/pyspark/pandas/tests/groupby/test_index.py | 114 ++++++++++++++++++++++
2 files changed, 171 insertions(+), 20 deletions(-)
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index d995e74c12d1..256cee600318 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -2462,23 +2462,42 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
for s, name in zip(self._groupkeys, groupkey_names):
sdf = sdf.withColumn(name, s.spark.column)
index = self._psdf._internal.index_spark_column_names[0]
+ index_spark_type = self._psdf._internal.index_fields[0].spark_type
stat_exprs = []
for psser, scol in zip(self._agg_columns, self._agg_columns_scols):
name = psser._internal.data_spark_column_names[0]
- if skipna:
+ if LooseVersion(pd.__version__) < "3.0.0" or skipna:
order_column = scol.desc_nulls_last()
- else:
- order_column = scol.desc_nulls_first()
- window = Window.partitionBy(*groupkey_names).orderBy(
- order_column, NATURAL_ORDER_COLUMN_NAME
- )
- sdf = sdf.withColumn(
- name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
- )
- stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+ window = Window.partitionBy(*groupkey_names).orderBy(
+ order_column, NATURAL_ORDER_COLUMN_NAME
+ )
+
+ has_na_name = "__has_na_{}__".format(name)
+ sdf = sdf.withColumn(has_na_name, scol.isNull()).withColumn(
+ name,
+ F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None),
+ )
+ if skipna:
+ stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+ else:
+ stat_exprs.append(
+ F.when(F.max(scol_for(sdf, has_na_name)), None)
+ .otherwise(F.max(scol_for(sdf, name)))
+ .alias(name)
+ )
+ else:
+ # pandas 3 skipna=False: raise on any NA, otherwise return all-missing labels
+ stat_exprs.append(
+ F.last(
+ F.when(
+ scol.isNull(),
+ F.raise_error("idxmax with skipna=False encountered an NA value."),
+ ).otherwise(F.lit(None).cast(index_spark_type))
+ ).alias(name)
+ )
sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
@@ -2544,23 +2563,41 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
for s, name in zip(self._groupkeys, groupkey_names):
sdf = sdf.withColumn(name, s.spark.column)
index = self._psdf._internal.index_spark_column_names[0]
+ index_spark_type = self._psdf._internal.index_fields[0].spark_type
stat_exprs = []
for psser, scol in zip(self._agg_columns, self._agg_columns_scols):
name = psser._internal.data_spark_column_names[0]
- if skipna:
+ if LooseVersion(pd.__version__) < "3.0.0" or skipna:
order_column = scol.asc_nulls_last()
- else:
- order_column = scol.asc_nulls_first()
- window = Window.partitionBy(*groupkey_names).orderBy(
- order_column, NATURAL_ORDER_COLUMN_NAME
- )
- sdf = sdf.withColumn(
- name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
- )
- stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+ window = Window.partitionBy(*groupkey_names).orderBy(
+ order_column, NATURAL_ORDER_COLUMN_NAME
+ )
+ has_na_name = "__has_na_{}__".format(name)
+ sdf = sdf.withColumn(has_na_name, scol.isNull()).withColumn(
+ name,
+ F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None),
+ )
+ if skipna:
+ stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+ else:
+ stat_exprs.append(
+ F.when(F.max(scol_for(sdf, has_na_name)), None)
+ .otherwise(F.max(scol_for(sdf, name)))
+ .alias(name)
+ )
+ else:
+ # pandas 3 skipna=False: raise on any NA, otherwise return all-missing labels
+ stat_exprs.append(
+ F.last(
+ F.when(
+ scol.isNull(),
+ F.raise_error("idxmin with skipna=False encountered an NA value."),
+ ).otherwise(F.lit(None).cast(index_spark_type))
+ ).alias(name)
+ )
sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
diff --git a/python/pyspark/pandas/tests/groupby/test_index.py b/python/pyspark/pandas/tests/groupby/test_index.py
index aa7d7ab3069f..a9d4ec6c2bbe 100644
--- a/python/pyspark/pandas/tests/groupby/test_index.py
+++ b/python/pyspark/pandas/tests/groupby/test_index.py
@@ -17,6 +17,7 @@
import pandas as pd
+from pyspark.loose_version import LooseVersion
from pyspark import pandas as ps
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.testing.sqlutils import SQLTestUtils
@@ -115,6 +116,36 @@ class GroupbyIndexMixin:
psdf.groupby(("x", "a")).idxmax(skipna=False).sort_index(),
)
+ pdf = pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+ index=["r0", "r1", "r2", "r3"],
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ )
+
+ pdf = pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+ index=pd.Index(
+ pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+ ),
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ )
+
def test_idxmin(self):
pdf = pd.DataFrame(
{"a": [1, 1, 2, 2, 3] * 3, "b": [1, 2, 3, 4, 5] * 3, "c": [5, 4, 3, 2, 1] * 3}
@@ -162,6 +193,89 @@ class GroupbyIndexMixin:
psdf.groupby(("x", "a")).idxmin(skipna=False).sort_index(),
)
+ pdf = pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+ index=["r0", "r1", "r2", "r3"],
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ )
+
+ pdf = pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+ index=pd.Index(
+ pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+ ),
+ )
+ psdf = ps.from_pandas(pdf)
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ )
+
+ def test_idxmax_idxmin_skipna_false_with_na(self):
+ dfs = [
+ pd.DataFrame({"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]}),
+ pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]},
+ index=["r0", "r1", "r2", "r3"],
+ ),
+ pd.DataFrame(
+ {"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]},
+ index=pd.Index(
+ pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+ ),
+ ),
+ ]
+
+ for i, pdf in enumerate(dfs):
+ with self.subTest(i=i):
+ psdf = ps.from_pandas(pdf)
+ if LooseVersion(pd.__version__) < "3.0.0":
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+ )
+ self.assert_eq(
+ pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+ )
+ else:
+ with self.assertRaisesRegex(
+ Exception, "idxmax with skipna=False encountered an NA value"
+ ):
+ psdf.groupby(["a"]).idxmax(skipna=False).to_pandas()
+ with self.assertRaisesRegex(
+ Exception, "idxmin with skipna=False encountered an NA value"
+ ):
+ psdf.groupby(["a"]).idxmin(skipna=False).to_pandas()
+ with self.assertRaisesRegex(
+ Exception, "idxmax with skipna=False encountered an NA value"
+ ):
+ psdf.groupby(["a"])["b"].idxmax(skipna=False).to_pandas()
+ with self.assertRaisesRegex(
+ Exception, "idxmin with skipna=False encountered an NA value"
+ ):
+ psdf.groupby(["a"])["b"].idxmin(skipna=False).to_pandas()
+
class GroupbyIndexTests(
GroupbyIndexMixin,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org