You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/03/26 07:18:05 UTC

(spark) branch master updated: [SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 932f9f70d2e6 [SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3
932f9f70d2e6 is described below

commit 932f9f70d2e6b5495c80a6cc53397aa1343d342b
Author: Takuya Ueshin <ue...@databricks.com>
AuthorDate: Thu Mar 26 16:17:53 2026 +0900

    [SPARK-56219][PS] Align groupby idxmax and idxmin skipna=False behavior with pandas 2/3
    
    ### What changes were proposed in this pull request?
    
    This PR updates pandas-on-Spark `GroupBy.idxmax` and `GroupBy.idxmin` so that `skipna=False` follows the version-specific pandas behavior more closely.
    
    For pandas `< 3.0.0`, NA-containing groups return null results for the affected group/column outputs.
    
    For pandas `>= 3.0.0`, NA-containing inputs raise for `skipna=False`, and no-NA inputs follow pandas 3’s all-missing-label behavior.
    
    This PR also updates the related tests in `python/pyspark/pandas/tests/groupby/test_index.py` to cover both pandas 2 and pandas 3 behavior, including cases with non-default indexes.
    
    ### Why are the changes needed?
    
    `GroupBy.idxmax(skipna=False)` and `GroupBy.idxmin(skipna=False)` differ between pandas 2 and pandas 3, and pandas-on-Spark was not matching those semantics.
    
    As a result, pandas-on-Spark could return index labels where pandas would return null or raise, depending on the pandas version in use.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes.
    
    For pandas-on-Spark groupby `idxmax(skipna=False)` and `idxmin(skipna=False)`, behavior now matches pandas more closely:
    
    - with pandas 2, groups containing NA values return null results for the affected outputs
    - with pandas 3, NA-containing inputs raise for `skipna=False`
    - with pandas 3, no-NA `skipna=False` inputs follow pandas 3 behavior
    
    ### How was this patch tested?
    
    Ran the related pandas-on-Spark groupby tests in both pandas 2 and pandas 3 environments, including:
    
    - `GroupbyIndexTests.test_idxmax`
    - `GroupbyIndexTests.test_idxmin`
    - `GroupbyIndexTests.test_idxmax_idxmin_skipna_false_with_na`
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #55021 from ueshin/issuse/SPARK-56219/groupby_idxmin_idxmax.
    
    Authored-by: Takuya Ueshin <ue...@databricks.com>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/pandas/groupby.py                  |  77 +++++++++++----
 python/pyspark/pandas/tests/groupby/test_index.py | 114 ++++++++++++++++++++++
 2 files changed, 171 insertions(+), 20 deletions(-)

diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index d995e74c12d1..256cee600318 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -2462,23 +2462,42 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
         for s, name in zip(self._groupkeys, groupkey_names):
             sdf = sdf.withColumn(name, s.spark.column)
         index = self._psdf._internal.index_spark_column_names[0]
+        index_spark_type = self._psdf._internal.index_fields[0].spark_type
 
         stat_exprs = []
         for psser, scol in zip(self._agg_columns, self._agg_columns_scols):
             name = psser._internal.data_spark_column_names[0]
 
-            if skipna:
+            if LooseVersion(pd.__version__) < "3.0.0" or skipna:
                 order_column = scol.desc_nulls_last()
-            else:
-                order_column = scol.desc_nulls_first()
 
-            window = Window.partitionBy(*groupkey_names).orderBy(
-                order_column, NATURAL_ORDER_COLUMN_NAME
-            )
-            sdf = sdf.withColumn(
-                name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
-            )
-            stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+                window = Window.partitionBy(*groupkey_names).orderBy(
+                    order_column, NATURAL_ORDER_COLUMN_NAME
+                )
+
+                has_na_name = "__has_na_{}__".format(name)
+                sdf = sdf.withColumn(has_na_name, scol.isNull()).withColumn(
+                    name,
+                    F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None),
+                )
+                if skipna:
+                    stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+                else:
+                    stat_exprs.append(
+                        F.when(F.max(scol_for(sdf, has_na_name)), None)
+                        .otherwise(F.max(scol_for(sdf, name)))
+                        .alias(name)
+                    )
+            else:
+                # pandas 3 skipna=False: raise on any NA, otherwise return all-missing labels
+                stat_exprs.append(
+                    F.last(
+                        F.when(
+                            scol.isNull(),
+                            F.raise_error("idxmax with skipna=False encountered an NA value."),
+                        ).otherwise(F.lit(None).cast(index_spark_type))
+                    ).alias(name)
+                )
 
         sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
 
@@ -2544,23 +2563,41 @@ class GroupBy(Generic[FrameLike], metaclass=ABCMeta):
         for s, name in zip(self._groupkeys, groupkey_names):
             sdf = sdf.withColumn(name, s.spark.column)
         index = self._psdf._internal.index_spark_column_names[0]
+        index_spark_type = self._psdf._internal.index_fields[0].spark_type
 
         stat_exprs = []
         for psser, scol in zip(self._agg_columns, self._agg_columns_scols):
             name = psser._internal.data_spark_column_names[0]
 
-            if skipna:
+            if LooseVersion(pd.__version__) < "3.0.0" or skipna:
                 order_column = scol.asc_nulls_last()
-            else:
-                order_column = scol.asc_nulls_first()
 
-            window = Window.partitionBy(*groupkey_names).orderBy(
-                order_column, NATURAL_ORDER_COLUMN_NAME
-            )
-            sdf = sdf.withColumn(
-                name, F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None)
-            )
-            stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+                window = Window.partitionBy(*groupkey_names).orderBy(
+                    order_column, NATURAL_ORDER_COLUMN_NAME
+                )
+                has_na_name = "__has_na_{}__".format(name)
+                sdf = sdf.withColumn(has_na_name, scol.isNull()).withColumn(
+                    name,
+                    F.when(F.row_number().over(window) == 1, scol_for(sdf, index)).otherwise(None),
+                )
+                if skipna:
+                    stat_exprs.append(F.max(scol_for(sdf, name)).alias(name))
+                else:
+                    stat_exprs.append(
+                        F.when(F.max(scol_for(sdf, has_na_name)), None)
+                        .otherwise(F.max(scol_for(sdf, name)))
+                        .alias(name)
+                    )
+            else:
+                # pandas 3 skipna=False: raise on any NA, otherwise return all-missing labels
+                stat_exprs.append(
+                    F.last(
+                        F.when(
+                            scol.isNull(),
+                            F.raise_error("idxmin with skipna=False encountered an NA value."),
+                        ).otherwise(F.lit(None).cast(index_spark_type))
+                    ).alias(name)
+                )
 
         sdf = sdf.groupby(*groupkey_names).agg(*stat_exprs)
 
diff --git a/python/pyspark/pandas/tests/groupby/test_index.py b/python/pyspark/pandas/tests/groupby/test_index.py
index aa7d7ab3069f..a9d4ec6c2bbe 100644
--- a/python/pyspark/pandas/tests/groupby/test_index.py
+++ b/python/pyspark/pandas/tests/groupby/test_index.py
@@ -17,6 +17,7 @@
 
 import pandas as pd
 
+from pyspark.loose_version import LooseVersion
 from pyspark import pandas as ps
 from pyspark.testing.pandasutils import PandasOnSparkTestCase
 from pyspark.testing.sqlutils import SQLTestUtils
@@ -115,6 +116,36 @@ class GroupbyIndexMixin:
             psdf.groupby(("x", "a")).idxmax(skipna=False).sort_index(),
         )
 
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+            index=["r0", "r1", "r2", "r3"],
+        )
+        psdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+            psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+            psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+            index=pd.Index(
+                pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+            ),
+        )
+        psdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+            psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+            psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+        )
+
     def test_idxmin(self):
         pdf = pd.DataFrame(
             {"a": [1, 1, 2, 2, 3] * 3, "b": [1, 2, 3, 4, 5] * 3, "c": [5, 4, 3, 2, 1] * 3}
@@ -162,6 +193,89 @@ class GroupbyIndexMixin:
             psdf.groupby(("x", "a")).idxmin(skipna=False).sort_index(),
         )
 
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+            index=["r0", "r1", "r2", "r3"],
+        )
+        psdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+            psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+            psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+        )
+
+        pdf = pd.DataFrame(
+            {"a": [1, 1, 2, 2], "b": [1, 2, 3, 4], "c": [4, 3, 2, 1]},
+            index=pd.Index(
+                pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+            ),
+        )
+        psdf = ps.from_pandas(pdf)
+        self.assert_eq(
+            pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+            psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+        )
+        self.assert_eq(
+            pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+            psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+        )
+
+    def test_idxmax_idxmin_skipna_false_with_na(self):
+        dfs = [
+            pd.DataFrame({"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]}),
+            pd.DataFrame(
+                {"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]},
+                index=["r0", "r1", "r2", "r3"],
+            ),
+            pd.DataFrame(
+                {"a": [1, 1, 2, 2], "b": [1, None, 3, 4], "c": [4, 3, 2, 1]},
+                index=pd.Index(
+                    pd.to_datetime(["2020-01-01", "2020-01-02", "2020-01-03", "2020-01-04"])
+                ),
+            ),
+        ]
+
+        for i, pdf in enumerate(dfs):
+            with self.subTest(i=i):
+                psdf = ps.from_pandas(pdf)
+                if LooseVersion(pd.__version__) < "3.0.0":
+                    self.assert_eq(
+                        pdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+                        psdf.groupby(["a"]).idxmax(skipna=False).sort_index(),
+                    )
+                    self.assert_eq(
+                        pdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+                        psdf.groupby(["a"]).idxmin(skipna=False).sort_index(),
+                    )
+                    self.assert_eq(
+                        pdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+                        psdf.groupby(["a"])["b"].idxmax(skipna=False).sort_index(),
+                    )
+                    self.assert_eq(
+                        pdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+                        psdf.groupby(["a"])["b"].idxmin(skipna=False).sort_index(),
+                    )
+                else:
+                    with self.assertRaisesRegex(
+                        Exception, "idxmax with skipna=False encountered an NA value"
+                    ):
+                        psdf.groupby(["a"]).idxmax(skipna=False).to_pandas()
+                    with self.assertRaisesRegex(
+                        Exception, "idxmin with skipna=False encountered an NA value"
+                    ):
+                        psdf.groupby(["a"]).idxmin(skipna=False).to_pandas()
+                    with self.assertRaisesRegex(
+                        Exception, "idxmax with skipna=False encountered an NA value"
+                    ):
+                        psdf.groupby(["a"])["b"].idxmax(skipna=False).to_pandas()
+                    with self.assertRaisesRegex(
+                        Exception, "idxmin with skipna=False encountered an NA value"
+                    ):
+                        psdf.groupby(["a"])["b"].idxmin(skipna=False).to_pandas()
+
 
 class GroupbyIndexTests(
     GroupbyIndexMixin,


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org