You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/08/08 23:59:52 UTC

[spark] branch branch-3.5 updated: [SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.5 by this push:
     new 57044292063 [SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual
57044292063 is described below

commit 57044292063f2e9254b59059aee0e6ab8f06d0e8
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Wed Aug 9 08:59:29 2023 +0900

    [SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual
    
    ### What changes were proposed in this pull request?
    This PR adds support for pandas DataFrame in `assertDataFrameEqual` (in addition to pandas-on-Spark), while delaying all pandas imports until pandas environment dependency is verified.
    
    ### Why are the changes needed?
    The changes are needed to allow users to compare pandas DataFrame equality, while still ensuring compatibility for environments with no pandas installation.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, the PR affects the user-facing function `assertDataFrameEqual`
    
    ### How was this patch tested?
    Added tests to `python/pyspark/sql/tests/test_utils.py` and `python/pyspark/sql/tests/connect/test_utils.py`
    
    Closes #42332 from asl3/support-pandas-df.
    
    Authored-by: Amanda Liu <am...@databricks.com>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
    (cherry picked from commit 69bd8358827f7c39b620d5cb22126fffb1267384)
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/pandas/tests/test_utils.py |  16 +-
 python/pyspark/sql/tests/test_utils.py    | 191 ++++++++++++++++++++++--
 python/pyspark/testing/pandasutils.py     | 233 +++++++++++++++++++-----------
 python/pyspark/testing/utils.py           |  88 ++++++-----
 4 files changed, 378 insertions(+), 150 deletions(-)

diff --git a/python/pyspark/pandas/tests/test_utils.py b/python/pyspark/pandas/tests/test_utils.py
index 0bb03dd8749..60961dcf252 100644
--- a/python/pyspark/pandas/tests/test_utils.py
+++ b/python/pyspark/pandas/tests/test_utils.py
@@ -111,7 +111,7 @@ class UtilsTestsMixin:
         with self.assertRaisesRegex(IndexError, err_msg):
             validate_index_loc(psidx, -4)
 
-    def test_assert_df_assertPandasOnSparkEqual(self):
+    def test_assert_df_assert_pandas_on_spark_equal(self):
         import pyspark.pandas as ps
 
         psdf1 = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
@@ -120,15 +120,15 @@ class UtilsTestsMixin:
         assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=False)
         assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=True)
 
-    def test_assertPandasOnSparkEqual_ignoreOrder_default(self):
+    def test_assert_pandas_on_spark_equal_ignore_order(self):
         import pyspark.pandas as ps
 
         psdf1 = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
         psdf2 = ps.DataFrame({"a": [2, 1, 3], "b": [5, 4, 6], "c": [8, 7, 9]})
 
-        assertPandasOnSparkEqual(psdf1, psdf2)
+        assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=False)
 
-    def test_assert_series_assertPandasOnSparkEqual(self):
+    def test_assert_series_assert_pandas_on_spark_equal(self):
         import pyspark.pandas as ps
 
         s1 = ps.Series([212.32, 100.0001])
@@ -136,7 +136,7 @@ class UtilsTestsMixin:
 
         assertPandasOnSparkEqual(s1, s2, checkExact=False)
 
-    def test_assert_index_assertPandasOnSparkEqual(self):
+    def test_assert_index_assert_pandas_on_spark_equal(self):
         import pyspark.pandas as ps
 
         s1 = ps.Index([212.300001, 100.000])
@@ -144,7 +144,7 @@ class UtilsTestsMixin:
 
         assertPandasOnSparkEqual(s1, s2, almost=True)
 
-    def test_assert_error_assertPandasOnSparkEqual(self):
+    def test_assert_error_assert_pandas_on_spark_equal(self):
         import pyspark.pandas as ps
 
         list1 = [10, 20, 30]
@@ -165,13 +165,13 @@ class UtilsTestsMixin:
             },
         )
 
-    def test_assert_None_assertPandasOnSparkEqual(self):
+    def test_assert_None_assert_pandas_on_spark_equal(self):
         psdf1 = None
         psdf2 = None
 
         assertPandasOnSparkEqual(psdf1, psdf2)
 
-    def test_assert_empty_assertPandasOnSparkEqual(self):
+    def test_assert_empty_assert_pandas_on_spark_equal(self):
         import pyspark.pandas as ps
 
         psdf1 = ps.DataFrame()
diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index 93895465de7..e1b7f298d0a 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -15,6 +15,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import unittest
+import difflib
+from itertools import zip_longest
+
 from pyspark.sql.functions import sha2, to_timestamp
 from pyspark.errors import (
     AnalysisException,
@@ -23,7 +27,7 @@ from pyspark.errors import (
     IllegalArgumentException,
     SparkUpgradeException,
 )
-from pyspark.testing.utils import assertDataFrameEqual, assertSchemaEqual, _context_diff
+from pyspark.testing.utils import assertDataFrameEqual, assertSchemaEqual, _context_diff, have_numpy
 from pyspark.testing.sqlutils import ReusedSQLTestCase
 from pyspark.sql import Row
 import pyspark.sql.functions as F
@@ -40,12 +44,7 @@ from pyspark.sql.types import (
     IntegerType,
     BooleanType,
 )
-from pyspark.sql.dataframe import DataFrame
-import pyspark.pandas as ps
-
-import difflib
-from typing import List, Union
-from itertools import zip_longest
+from pyspark.testing.sqlutils import have_pandas
 
 
 class UtilsTestsMixin:
@@ -690,7 +689,7 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "actual",
                 "actual_type": None,
             },
@@ -703,7 +702,7 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "actual",
                 "actual_type": None,
             },
@@ -726,7 +725,7 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "expected",
                 "actual_type": None,
             },
@@ -739,33 +738,195 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "expected",
                 "actual_type": None,
             },
         )
 
+    @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
     def test_assert_equal_exact_pandas_df(self):
+        import pandas as pd
+        import numpy as np
+
+        df1 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+        )
+        df2 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+        )
+
+        assertDataFrameEqual(df1, df2, checkRowOrder=False)
+        assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+    @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+    def test_assert_approx_equal_pandas_df(self):
+        import pandas as pd
+        import numpy as np
+
+        # test that asserts close enough equality for pandas df
+        df1 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59)]), columns=["a", "b", "c"]
+        )
+        df2 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59.0001)]), columns=["a", "b", "c"]
+        )
+
+        assertDataFrameEqual(df1, df2, checkRowOrder=False)
+        assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+    @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+    def test_assert_approx_equal_fail_exact_pandas_df(self):
+        import pandas as pd
+        import numpy as np
+
+        # test that asserts close enough equality for pandas df
+        df1 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59)]), columns=["a", "b", "c"]
+        )
+        df2 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59.0001)]), columns=["a", "b", "c"]
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=False, rtol=0, atol=0)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=True, rtol=0, atol=0)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+    @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+    def test_assert_unequal_pandas_df(self):
+        import pandas as pd
+        import numpy as np
+
+        df1 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (6, 5, 4)]), columns=["a", "b", "c"]
+        )
+        df2 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=False)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+    @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+    def test_assert_type_error_pandas_df(self):
+        import pyspark.pandas as ps
+        import pandas as pd
+        import numpy as np
+
+        df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
+        df2 = pd.DataFrame(
+            data=np.array([(1, 2, 3), (4, 5, 6), (6, 5, 4)]), columns=["a", "b", "c"]
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=False)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_PANDAS_DATAFRAME",
+            message_parameters={
+                "left": df1.to_string(),
+                "left_dtype": str(df1.dtypes),
+                "right": df2.to_string(),
+                "right_dtype": str(df2.dtypes),
+            },
+        )
+
+    @unittest.skipIf(not have_pandas, "no pandas dependency")
+    def test_assert_equal_exact_pandas_on_spark_df(self):
+        import pyspark.pandas as ps
+
         df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
         df2 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
 
         assertDataFrameEqual(df1, df2, checkRowOrder=False)
         assertDataFrameEqual(df1, df2, checkRowOrder=True)
 
-    def test_assert_equal_exact_pandas_df(self):
+    @unittest.skipIf(not have_pandas, "no pandas dependency")
+    def test_assert_equal_exact_pandas_on_spark_df(self):
+        import pyspark.pandas as ps
+
         df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
         df2 = ps.DataFrame(data=[30, 20, 10], columns=["Numbers"])
 
         assertDataFrameEqual(df1, df2)
 
-    def test_assert_equal_approx_pandas_df(self):
+    @unittest.skipIf(not have_pandas, "no pandas dependency")
+    def test_assert_equal_approx_pandas_on_spark_df(self):
+        import pyspark.pandas as ps
+
         df1 = ps.DataFrame(data=[10.0001, 20.32, 30.1], columns=["Numbers"])
         df2 = ps.DataFrame(data=[10.0, 20.32, 30.1], columns=["Numbers"])
 
         assertDataFrameEqual(df1, df2, checkRowOrder=False)
         assertDataFrameEqual(df1, df2, checkRowOrder=True)
 
+    @unittest.skipIf(not have_pandas, "no pandas dependency")
     def test_assert_error_pandas_pyspark_df(self):
+        import pyspark.pandas as ps
         import pandas as pd
 
         df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
@@ -818,7 +979,7 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "actual",
                 "actual_type": type(dict1),
             },
@@ -831,7 +992,7 @@ class UtilsTestsMixin:
             exception=pe.exception,
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "actual",
                 "actual_type": type(dict1),
             },
diff --git a/python/pyspark/testing/pandasutils.py b/python/pyspark/testing/pandasutils.py
index 39196873482..1122944b2c0 100644
--- a/python/pyspark/testing/pandasutils.py
+++ b/python/pyspark/testing/pandasutils.py
@@ -23,7 +23,7 @@ import pandas as pd
 from contextlib import contextmanager
 from distutils.version import LooseVersion
 import decimal
-from typing import Union
+from typing import Any, Union
 
 import pyspark.pandas as ps
 from pyspark.pandas.frame import DataFrame
@@ -153,18 +153,29 @@ def _assert_pandas_equal(
 
 
 def _assert_pandas_almost_equal(
-    left: Union[pd.DataFrame, pd.Series, pd.Index], right: Union[pd.DataFrame, pd.Series, pd.Index]
+    left: Union[pd.DataFrame, pd.Series, pd.Index],
+    right: Union[pd.DataFrame, pd.Series, pd.Index],
+    rtol: float = 1e-5,
+    atol: float = 1e-8,
 ):
     """
     This function checks if given pandas objects approximately same,
     which means the conditions below:
       - Both objects are nullable
-      - Compare floats rounding to the number of decimal places, 7 after
-        dropping missing values (NaN, NaT, None)
+      - Compare decimals and floats, where two values a and b are approximately equal
+        if they satisfy the following formula:
+        absolute(a - b) <= (atol + rtol * absolute(b))
+        where rtol=1e-5 and atol=1e-8 by default
     """
-    # following pandas convention, rtol=1e-5 and atol=1e-8
-    rtol = 1e-5
-    atol = 1e-8
+
+    def compare_vals_approx(val1, val2):
+        # compare vals for approximate equality
+        if isinstance(lval, (float, decimal.Decimal)) or isinstance(rval, (float, decimal.Decimal)):
+            if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
+                return False
+        elif val1 != val2:
+            return False
+        return True
 
     if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
         if left.shape != right.shape:
@@ -200,19 +211,16 @@ def _assert_pandas_almost_equal(
                         },
                     )
             for lval, rval in zip(left[lcol].dropna(), right[rcol].dropna()):
-                if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
-                    isinstance(rval, float) or isinstance(rval, decimal.Decimal)
-                ):
-                    if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
-                        raise PySparkAssertionError(
-                            error_class="DIFFERENT_PANDAS_DATAFRAME",
-                            message_parameters={
-                                "left": left.to_string(),
-                                "left_dtype": str(left.dtypes),
-                                "right": right.to_string(),
-                                "right_dtype": str(right.dtypes),
-                            },
-                        )
+                if not compare_vals_approx(lval, rval):
+                    raise PySparkAssertionError(
+                        error_class="DIFFERENT_PANDAS_DATAFRAME",
+                        message_parameters={
+                            "left": left.to_string(),
+                            "left_dtype": str(left.dtypes),
+                            "right": right.to_string(),
+                            "right_dtype": str(right.dtypes),
+                        },
+                    )
         if left.columns.names != right.columns.names:
             raise PySparkAssertionError(
                 error_class="DIFFERENT_PANDAS_DATAFRAME",
@@ -246,19 +254,16 @@ def _assert_pandas_almost_equal(
                     },
                 )
         for lval, rval in zip(left.dropna(), right.dropna()):
-            if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
-                isinstance(rval, float) or isinstance(rval, decimal.Decimal)
-            ):
-                if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
-                    raise PySparkAssertionError(
-                        error_class="DIFFERENT_PANDAS_SERIES",
-                        message_parameters={
-                            "left": left.to_string(),
-                            "left_dtype": str(left.dtype),
-                            "right": right.to_string(),
-                            "right_dtype": str(right.dtype),
-                        },
-                    )
+            if not compare_vals_approx(lval, rval):
+                raise PySparkAssertionError(
+                    error_class="DIFFERENT_PANDAS_SERIES",
+                    message_parameters={
+                        "left": left.to_string(),
+                        "left_dtype": str(left.dtype),
+                        "right": right.to_string(),
+                        "right_dtype": str(right.dtype),
+                    },
+                )
     elif isinstance(left, pd.MultiIndex) and isinstance(right, pd.MultiIndex):
         if len(left) != len(right):
             raise PySparkAssertionError(
@@ -271,19 +276,16 @@ def _assert_pandas_almost_equal(
                 },
             )
         for lval, rval in zip(left, right):
-            if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
-                isinstance(rval, float) or isinstance(rval, decimal.Decimal)
-            ):
-                if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
-                    raise PySparkAssertionError(
-                        error_class="DIFFERENT_PANDAS_MULTIINDEX",
-                        message_parameters={
-                            "left": left,
-                            "left_dtype": str(left.dtype),
-                            "right": right,
-                            "right_dtype": str(right.dtype),
-                        },
-                    )
+            if not compare_vals_approx(lval, rval):
+                raise PySparkAssertionError(
+                    error_class="DIFFERENT_PANDAS_MULTIINDEX",
+                    message_parameters={
+                        "left": left,
+                        "left_dtype": str(left.dtype),
+                        "right": right,
+                        "right_dtype": str(right.dtype),
+                    },
+                )
     elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
         if len(left) != len(right):
             raise PySparkAssertionError(
@@ -307,21 +309,39 @@ def _assert_pandas_almost_equal(
                     },
                 )
         for lval, rval in zip(left.dropna(), right.dropna()):
-            if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
-                isinstance(rval, float) or isinstance(rval, decimal.Decimal)
-            ):
-                if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
-                    raise PySparkAssertionError(
-                        error_class="DIFFERENT_PANDAS_INDEX",
-                        message_parameters={
-                            "left": left,
-                            "left_dtype": str(left.dtype),
-                            "right": right,
-                            "right_dtype": str(right.dtype),
-                        },
-                    )
+            if not compare_vals_approx(lval, rval):
+                raise PySparkAssertionError(
+                    error_class="DIFFERENT_PANDAS_INDEX",
+                    message_parameters={
+                        "left": left,
+                        "left_dtype": str(left.dtype),
+                        "right": right,
+                        "right_dtype": str(right.dtype),
+                    },
+                )
     else:
-        raise ValueError("Unexpected values: (%s, %s)" % (left, right))
+        if not isinstance(left, (pd.DataFrame, pd.Series, pd.Index)):
+            raise PySparkAssertionError(
+                error_class="INVALID_TYPE_DF_EQUALITY_ARG",
+                message_parameters={
+                    "expected_type": f"{pd.DataFrame.__name__}, "
+                    f"{pd.Series.__name__}, "
+                    f"{pd.Index.__name__}, ",
+                    "arg_name": "left",
+                    "actual_type": type(left),
+                },
+            )
+        elif not isinstance(right, (pd.DataFrame, pd.Series, pd.Index)):
+            raise PySparkAssertionError(
+                error_class="INVALID_TYPE_DF_EQUALITY_ARG",
+                message_parameters={
+                    "expected_type": f"{pd.DataFrame.__name__}, "
+                    f"{pd.Series.__name__}, "
+                    f"{pd.Index.__name__}, ",
+                    "arg_name": "right",
+                    "actual_type": type(right),
+                },
+            )
 
 
 def assertPandasOnSparkEqual(
@@ -329,20 +349,22 @@ def assertPandasOnSparkEqual(
     expected: Union[DataFrame, pd.DataFrame, Series, pd.Series, Index, pd.Index],
     checkExact: bool = True,
     almost: bool = False,
-    checkRowOrder: bool = False,
+    rtol: float = 1e-5,
+    atol: float = 1e-8,
+    checkRowOrder: bool = True,
 ):
     r"""
-    A util function to assert equality between actual (pandas-on-Spark DataFrame) and expected
-    (pandas-on-Spark or pandas DataFrame).
+    A util function to assert equality between actual (pandas-on-Spark object) and expected
+    (pandas-on-Spark or pandas object).
 
     .. versionadded:: 3.5.0
 
     Parameters
     ----------
-    actual: pyspark.pandas.frame.DataFrame
-        The DataFrame that is being compared or tested.
-    expected: pyspark.pandas.frame.DataFrame or pd.DataFrame
-        The expected DataFrame, for comparison with the actual result.
+    actual: pandas-on-Spark DataFrame, Series, or Index
+        The object that is being compared or tested.
+    expected: pandas-on-Spark or pandas DataFrame, Series, or Index
+        The expected object, for comparison with the actual result.
     checkExact: bool, optional
         A flag indicating whether to compare exact equality.
         If set to 'True' (default), the data is compared exactly.
@@ -354,10 +376,16 @@ def assertPandasOnSparkEqual(
         (see documentation for more details).
         If set to 'False' (default), the data is compared exactly with `unittest`'s
         `assertEqual`.
+    rtol : float, optional
+        The relative tolerance, used in asserting almost equality for float values in actual
+        and expected. Set to 1e-5 by default. (See Notes)
+    atol : float, optional
+        The absolute tolerance, used in asserting almost equality for float values in actual
+        and expected. Set to 1e-8 by default. (See Notes)
     checkRowOrder : bool, optional
         A flag indicating whether the order of rows should be considered in the comparison.
-        If set to `False` (default), the row order is not taken into account.
-        If set to `True`, the order of rows is important and will be checked during comparison.
+        If set to `False`, the row order is not taken into account.
+        If set to `True` (default), the order of rows will be checked during comparison.
         (See Notes)
 
     Notes
@@ -365,6 +393,11 @@ def assertPandasOnSparkEqual(
     For `checkRowOrder`, note that pandas-on-Spark DataFrame ordering is non-deterministic, unless
     explicitly sorted.
 
+    When `almost` is set to True, approximate equality will be asserted, where two values
+    a and b are approximately equal if they satisfy the following formula:
+
+    ``absolute(a - b) <= (atol + rtol * absolute(b))``.
+
     Examples
     --------
     >>> import pyspark.pandas as ps
@@ -407,8 +440,9 @@ def assertPandasOnSparkEqual(
             },
         )
     else:
-        actual = actual.to_pandas()
-        if not isinstance(expected, pd.DataFrame):
+        if not isinstance(actual, (pd.DataFrame, pd.Index, pd.Series)):
+            actual = actual.to_pandas()
+        if not isinstance(expected, (pd.DataFrame, pd.Index, pd.Series)):
             expected = expected.to_pandas()
 
         if not checkRowOrder:
@@ -418,25 +452,40 @@ def assertPandasOnSparkEqual(
                 expected = expected.sort_values(by=expected.columns[0], ignore_index=True)
 
         if almost:
-            _assert_pandas_almost_equal(actual, expected)
+            _assert_pandas_almost_equal(actual, expected, rtol=rtol, atol=atol)
         else:
             _assert_pandas_equal(actual, expected, checkExact=checkExact)
 
 
 class PandasOnSparkTestUtils:
-    def convert_str_to_lambda(self, func):
+    def convert_str_to_lambda(self, func: str):
         """
         This function converts `func` str to lambda call
         """
         return lambda x: getattr(x, func)()
 
-    def assertPandasEqual(self, left, right, check_exact=True):
+    def assertPandasEqual(self, left: Any, right: Any, check_exact: bool = True):
         _assert_pandas_equal(left, right, check_exact)
 
-    def assertPandasAlmostEqual(self, left, right):
-        _assert_pandas_almost_equal(left, right)
-
-    def assert_eq(self, left, right, check_exact=True, almost=False):
+    def assertPandasAlmostEqual(
+        self,
+        left: Any,
+        right: Any,
+        rtol: float = 1e-5,
+        atol: float = 1e-8,
+    ):
+        _assert_pandas_almost_equal(left, right, rtol=rtol, atol=atol)
+
+    def assert_eq(
+        self,
+        left: Any,
+        right: Any,
+        check_exact: bool = True,
+        almost: bool = False,
+        rtol: float = 1e-5,
+        atol: float = 1e-8,
+        check_row_order: bool = True,
+    ):
         """
         Asserts if two arbitrary objects are equal or not. If given objects are Koalas DataFrame
         or Series, they are converted into pandas' and compared.
@@ -444,17 +493,37 @@ class PandasOnSparkTestUtils:
         :param left: object to compare
         :param right: object to compare
         :param check_exact: if this is False, the comparison is done less precisely.
-        :param almost: if this is enabled, the comparison is delegated to `unittest`'s
-                       `assertAlmostEqual`. See its documentation for more details.
+        :param almost: if this is enabled, the comparison asserts approximate equality
+            for float and decimal values, where two values a and b are approximately equal
+            if they satisfy the following formula:
+            absolute(a - b) <= (atol + rtol * absolute(b))
+        :param rtol: The relative tolerance, used in asserting approximate equality for
+            float values. Set to 1e-5 by default.
+        :param atol: The absolute tolerance, used in asserting approximate equality for
+            float values in actual and expected. Set to 1e-8 by default.
+        :param check_row_order: A flag indicating whether the order of rows should be considered
+            in the comparison. If set to False, row order will be ignored.
         """
         import pandas as pd
         from pandas.api.types import is_list_like
 
+        # for pandas-on-Spark DataFrames, allow choice to ignore row order
+        if isinstance(left, (ps.DataFrame, ps.Series, ps.Index)):
+            return assertPandasOnSparkEqual(
+                left,
+                right,
+                checkExact=check_exact,
+                almost=almost,
+                rtol=rtol,
+                atol=atol,
+                checkRowOrder=check_row_order,
+            )
+
         lobj = self._to_pandas(left)
         robj = self._to_pandas(right)
         if isinstance(lobj, (pd.DataFrame, pd.Series, pd.Index)):
             if almost:
-                _assert_pandas_almost_equal(lobj, robj)
+                _assert_pandas_almost_equal(lobj, robj, rtol=rtol, atol=atol)
             else:
                 _assert_pandas_equal(lobj, robj, checkExact=check_exact)
         elif is_list_like(lobj) and is_list_like(robj):
@@ -470,7 +539,7 @@ class PandasOnSparkTestUtils:
                 self.assertEqual(lobj, robj)
 
     @staticmethod
-    def _to_pandas(obj):
+    def _to_pandas(obj: Any):
         if isinstance(obj, (DataFrame, Series, Index)):
             return obj.to_pandas()
         else:
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index 8e02803efe5..7dd723634e2 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -39,7 +39,6 @@ from pyspark.find_spark_home import _find_spark_home
 from pyspark.sql.dataframe import DataFrame
 from pyspark.sql import Row
 from pyspark.sql.types import StructType, AtomicType, StructField
-import pyspark.pandas as ps
 
 have_scipy = False
 have_numpy = False
@@ -360,9 +359,16 @@ def assertSchemaEqual(actual: StructType, expected: StructType):
         )
 
 
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import pandas
+    import pyspark.pandas
+
+
 def assertDataFrameEqual(
-    actual: Union[DataFrame, ps.DataFrame, List[Row]],
-    expected: Union[DataFrame, ps.DataFrame, List[Row]],
+    actual: Union[DataFrame, "pandas.DataFrame", "pyspark.pandas.DataFrame", List[Row]],
+    expected: Union[DataFrame, "pandas.DataFrame", "pyspark.pandas.DataFrame", List[Row]],
     checkRowOrder: bool = False,
     rtol: float = 1e-5,
     atol: float = 1e-8,
@@ -371,7 +377,7 @@ def assertDataFrameEqual(
     A util function to assert equality between `actual` and `expected`
     (DataFrames or lists of Rows), with optional parameters `checkRowOrder`, `rtol`, and `atol`.
 
-    Supports Spark, Spark Connect, and pandas-on-Spark DataFrames.
+    Supports Spark, Spark Connect, pandas, and pandas-on-Spark DataFrames.
     For more information about pandas-on-Spark DataFrame equality, see the docs for
     `assertPandasOnSparkEqual`.
 
@@ -379,9 +385,9 @@ def assertDataFrameEqual(
 
     Parameters
     ----------
-    actual : DataFrame (Spark, Spark Connect, or pandas-on-Spark) or list of Rows
+    actual : DataFrame (Spark, Spark Connect, pandas, or pandas-on-Spark) or list of Rows
         The DataFrame that is being compared or tested.
-    expected : DataFrame (Spark, Spark Connect, or pandas-on-Spark) or list of Rows
+    expected : DataFrame (Spark, Spark Connect, pandas, or pandas-on-Spark) or list of Rows
         The expected result of the operation, for comparison with the actual result.
     checkRowOrder : bool, optional
         A flag indicating whether the order of rows should be considered in the comparison.
@@ -446,16 +452,13 @@ def assertDataFrameEqual(
     Row(id='2', amount=3000.0)
     ! Row(id='3', amount=2003.0)
     """
-    import pyspark.pandas as ps
-    from pyspark.testing.pandasutils import assertPandasOnSparkEqual
-
     if actual is None and expected is None:
         return True
     elif actual is None:
         raise PySparkAssertionError(
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "actual",
                 "actual_type": None,
             },
@@ -464,61 +467,56 @@ def assertDataFrameEqual(
         raise PySparkAssertionError(
             error_class="INVALID_TYPE_DF_EQUALITY_ARG",
             message_parameters={
-                "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                 "arg_name": "expected",
                 "actual_type": None,
             },
         )
 
+    has_pandas = False
     try:
-        # If Spark Connect dependencies are available, allow Spark Connect DataFrame
-        from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
+        # If pandas dependencies are available, allow pandas or pandas-on-Spark DataFrame
+        import pyspark.pandas as ps
+        import pandas as pd
+        from pyspark.testing.pandasutils import PandasOnSparkTestUtils
+
+        has_pandas = True
+    except ImportError:
+        # no pandas, so we won't call pandasutils functions
+        pass
 
-        if isinstance(actual, ps.DataFrame) or isinstance(expected, ps.DataFrame):
-            # handle pandas DataFrames
-            # assert approximate equality for float data
-            return assertPandasOnSparkEqual(
-                actual, expected, checkExact=False, checkRowOrder=checkRowOrder
-            )
-        elif not isinstance(actual, (DataFrame, ConnectDataFrame, list)):
-            raise PySparkAssertionError(
-                error_class="INVALID_TYPE_DF_EQUALITY_ARG",
-                message_parameters={
-                    "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
-                    "arg_name": "actual",
-                    "actual_type": type(actual),
-                },
-            )
-        elif not isinstance(expected, (DataFrame, ConnectDataFrame, list)):
-            raise PySparkAssertionError(
-                error_class="INVALID_TYPE_DF_EQUALITY_ARG",
-                message_parameters={
-                    "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
-                    "arg_name": "expected",
-                    "actual_type": type(expected),
-                },
-            )
-    except Exception:
-        if isinstance(actual, ps.DataFrame) or isinstance(expected, ps.DataFrame):
+    if has_pandas:
+        if (
+            isinstance(actual, pd.DataFrame)
+            or isinstance(expected, pd.DataFrame)
+            or isinstance(actual, ps.DataFrame)
+            or isinstance(expected, ps.DataFrame)
+        ):
             # handle pandas DataFrames
             # assert approximate equality for float data
-            return assertPandasOnSparkEqual(
-                actual, expected, checkExact=False, checkRowOrder=checkRowOrder
+            return PandasOnSparkTestUtils().assert_eq(
+                actual, expected, almost=True, rtol=rtol, atol=atol, check_row_order=checkRowOrder
             )
-        elif not isinstance(actual, (DataFrame, list)):
+
+        from pyspark.sql.utils import get_dataframe_class
+
+        # if is_remote(), allow Connect DataFrame
+        SparkDataFrame = get_dataframe_class()
+
+        if not isinstance(actual, (DataFrame, SparkDataFrame, list)):
             raise PySparkAssertionError(
                 error_class="INVALID_TYPE_DF_EQUALITY_ARG",
                 message_parameters={
-                    "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                    "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                     "arg_name": "actual",
                     "actual_type": type(actual),
                 },
             )
-        elif not isinstance(expected, (DataFrame, list)):
+        elif not isinstance(expected, (DataFrame, SparkDataFrame, list)):
             raise PySparkAssertionError(
                 error_class="INVALID_TYPE_DF_EQUALITY_ARG",
                 message_parameters={
-                    "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+                    "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
                     "arg_name": "expected",
                     "actual_type": type(expected),
                 },


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org