You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/08/08 23:59:52 UTC
[spark] branch branch-3.5 updated: [SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 57044292063 [SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual
57044292063 is described below
commit 57044292063f2e9254b59059aee0e6ab8f06d0e8
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Wed Aug 9 08:59:29 2023 +0900
[SPARK-44665][PYTHON] Add support for pandas DataFrame assertDataFrameEqual
### What changes were proposed in this pull request?
This PR adds support for pandas DataFrame in `assertDataFrameEqual` (in addition to pandas-on-Spark), while delaying all pandas imports until pandas environment dependency is verified.
### Why are the changes needed?
The changes are needed to allow users to compare pandas DataFrame equality, while still ensuring compatibility for environments with no pandas installation.
### Does this PR introduce _any_ user-facing change?
Yes, the PR affects the user-facing function `assertDataFrameEqual`
### How was this patch tested?
Added tests to `python/pyspark/sql/tests/test_utils.py` and `python/pyspark/sql/tests/connect/test_utils.py`
Closes #42332 from asl3/support-pandas-df.
Authored-by: Amanda Liu <am...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 69bd8358827f7c39b620d5cb22126fffb1267384)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/tests/test_utils.py | 16 +-
python/pyspark/sql/tests/test_utils.py | 191 ++++++++++++++++++++++--
python/pyspark/testing/pandasutils.py | 233 +++++++++++++++++++-----------
python/pyspark/testing/utils.py | 88 ++++++-----
4 files changed, 378 insertions(+), 150 deletions(-)
diff --git a/python/pyspark/pandas/tests/test_utils.py b/python/pyspark/pandas/tests/test_utils.py
index 0bb03dd8749..60961dcf252 100644
--- a/python/pyspark/pandas/tests/test_utils.py
+++ b/python/pyspark/pandas/tests/test_utils.py
@@ -111,7 +111,7 @@ class UtilsTestsMixin:
with self.assertRaisesRegex(IndexError, err_msg):
validate_index_loc(psidx, -4)
- def test_assert_df_assertPandasOnSparkEqual(self):
+ def test_assert_df_assert_pandas_on_spark_equal(self):
import pyspark.pandas as ps
psdf1 = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
@@ -120,15 +120,15 @@ class UtilsTestsMixin:
assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=False)
assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=True)
- def test_assertPandasOnSparkEqual_ignoreOrder_default(self):
+ def test_assert_pandas_on_spark_equal_ignore_order(self):
import pyspark.pandas as ps
psdf1 = ps.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6], "c": [7, 8, 9]})
psdf2 = ps.DataFrame({"a": [2, 1, 3], "b": [5, 4, 6], "c": [8, 7, 9]})
- assertPandasOnSparkEqual(psdf1, psdf2)
+ assertPandasOnSparkEqual(psdf1, psdf2, checkRowOrder=False)
- def test_assert_series_assertPandasOnSparkEqual(self):
+ def test_assert_series_assert_pandas_on_spark_equal(self):
import pyspark.pandas as ps
s1 = ps.Series([212.32, 100.0001])
@@ -136,7 +136,7 @@ class UtilsTestsMixin:
assertPandasOnSparkEqual(s1, s2, checkExact=False)
- def test_assert_index_assertPandasOnSparkEqual(self):
+ def test_assert_index_assert_pandas_on_spark_equal(self):
import pyspark.pandas as ps
s1 = ps.Index([212.300001, 100.000])
@@ -144,7 +144,7 @@ class UtilsTestsMixin:
assertPandasOnSparkEqual(s1, s2, almost=True)
- def test_assert_error_assertPandasOnSparkEqual(self):
+ def test_assert_error_assert_pandas_on_spark_equal(self):
import pyspark.pandas as ps
list1 = [10, 20, 30]
@@ -165,13 +165,13 @@ class UtilsTestsMixin:
},
)
- def test_assert_None_assertPandasOnSparkEqual(self):
+ def test_assert_None_assert_pandas_on_spark_equal(self):
psdf1 = None
psdf2 = None
assertPandasOnSparkEqual(psdf1, psdf2)
- def test_assert_empty_assertPandasOnSparkEqual(self):
+ def test_assert_empty_assert_pandas_on_spark_equal(self):
import pyspark.pandas as ps
psdf1 = ps.DataFrame()
diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index 93895465de7..e1b7f298d0a 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -15,6 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import unittest
+import difflib
+from itertools import zip_longest
+
from pyspark.sql.functions import sha2, to_timestamp
from pyspark.errors import (
AnalysisException,
@@ -23,7 +27,7 @@ from pyspark.errors import (
IllegalArgumentException,
SparkUpgradeException,
)
-from pyspark.testing.utils import assertDataFrameEqual, assertSchemaEqual, _context_diff
+from pyspark.testing.utils import assertDataFrameEqual, assertSchemaEqual, _context_diff, have_numpy
from pyspark.testing.sqlutils import ReusedSQLTestCase
from pyspark.sql import Row
import pyspark.sql.functions as F
@@ -40,12 +44,7 @@ from pyspark.sql.types import (
IntegerType,
BooleanType,
)
-from pyspark.sql.dataframe import DataFrame
-import pyspark.pandas as ps
-
-import difflib
-from typing import List, Union
-from itertools import zip_longest
+from pyspark.testing.sqlutils import have_pandas
class UtilsTestsMixin:
@@ -690,7 +689,7 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": None,
},
@@ -703,7 +702,7 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": None,
},
@@ -726,7 +725,7 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "expected",
"actual_type": None,
},
@@ -739,33 +738,195 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "expected",
"actual_type": None,
},
)
+ @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
def test_assert_equal_exact_pandas_df(self):
+ import pandas as pd
+ import numpy as np
+
+ df1 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+ )
+ df2 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+ )
+
+ assertDataFrameEqual(df1, df2, checkRowOrder=False)
+ assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+ @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+ def test_assert_approx_equal_pandas_df(self):
+ import pandas as pd
+ import numpy as np
+
+ # test that asserts close enough equality for pandas df
+ df1 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59)]), columns=["a", "b", "c"]
+ )
+ df2 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59.0001)]), columns=["a", "b", "c"]
+ )
+
+ assertDataFrameEqual(df1, df2, checkRowOrder=False)
+ assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+ @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+ def test_assert_approx_equal_fail_exact_pandas_df(self):
+ import pandas as pd
+ import numpy as np
+
+ # test that asserts close enough equality for pandas df
+ df1 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59)]), columns=["a", "b", "c"]
+ )
+ df2 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 59.0001)]), columns=["a", "b", "c"]
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=False, rtol=0, atol=0)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=True, rtol=0, atol=0)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+ def test_assert_unequal_pandas_df(self):
+ import pandas as pd
+ import numpy as np
+
+ df1 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (6, 5, 4)]), columns=["a", "b", "c"]
+ )
+ df2 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (7, 8, 9)]), columns=["a", "b", "c"]
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=False)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ @unittest.skipIf(not have_pandas or not have_numpy, "no pandas or numpy dependency")
+ def test_assert_type_error_pandas_df(self):
+ import pyspark.pandas as ps
+ import pandas as pd
+ import numpy as np
+
+ df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
+ df2 = pd.DataFrame(
+ data=np.array([(1, 2, 3), (4, 5, 6), (6, 5, 4)]), columns=["a", "b", "c"]
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=False)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": df1.to_string(),
+ "left_dtype": str(df1.dtypes),
+ "right": df2.to_string(),
+ "right_dtype": str(df2.dtypes),
+ },
+ )
+
+ @unittest.skipIf(not have_pandas, "no pandas dependency")
+ def test_assert_equal_exact_pandas_on_spark_df(self):
+ import pyspark.pandas as ps
+
df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
df2 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
assertDataFrameEqual(df1, df2, checkRowOrder=False)
assertDataFrameEqual(df1, df2, checkRowOrder=True)
- def test_assert_equal_exact_pandas_df(self):
+ @unittest.skipIf(not have_pandas, "no pandas dependency")
+ def test_assert_equal_exact_pandas_on_spark_df(self):
+ import pyspark.pandas as ps
+
df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
df2 = ps.DataFrame(data=[30, 20, 10], columns=["Numbers"])
assertDataFrameEqual(df1, df2)
- def test_assert_equal_approx_pandas_df(self):
+ @unittest.skipIf(not have_pandas, "no pandas dependency")
+ def test_assert_equal_approx_pandas_on_spark_df(self):
+ import pyspark.pandas as ps
+
df1 = ps.DataFrame(data=[10.0001, 20.32, 30.1], columns=["Numbers"])
df2 = ps.DataFrame(data=[10.0, 20.32, 30.1], columns=["Numbers"])
assertDataFrameEqual(df1, df2, checkRowOrder=False)
assertDataFrameEqual(df1, df2, checkRowOrder=True)
+ @unittest.skipIf(not have_pandas, "no pandas dependency")
def test_assert_error_pandas_pyspark_df(self):
+ import pyspark.pandas as ps
import pandas as pd
df1 = ps.DataFrame(data=[10, 20, 30], columns=["Numbers"])
@@ -818,7 +979,7 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": type(dict1),
},
@@ -831,7 +992,7 @@ class UtilsTestsMixin:
exception=pe.exception,
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": type(dict1),
},
diff --git a/python/pyspark/testing/pandasutils.py b/python/pyspark/testing/pandasutils.py
index 39196873482..1122944b2c0 100644
--- a/python/pyspark/testing/pandasutils.py
+++ b/python/pyspark/testing/pandasutils.py
@@ -23,7 +23,7 @@ import pandas as pd
from contextlib import contextmanager
from distutils.version import LooseVersion
import decimal
-from typing import Union
+from typing import Any, Union
import pyspark.pandas as ps
from pyspark.pandas.frame import DataFrame
@@ -153,18 +153,29 @@ def _assert_pandas_equal(
def _assert_pandas_almost_equal(
- left: Union[pd.DataFrame, pd.Series, pd.Index], right: Union[pd.DataFrame, pd.Series, pd.Index]
+ left: Union[pd.DataFrame, pd.Series, pd.Index],
+ right: Union[pd.DataFrame, pd.Series, pd.Index],
+ rtol: float = 1e-5,
+ atol: float = 1e-8,
):
"""
This function checks if given pandas objects approximately same,
which means the conditions below:
- Both objects are nullable
- - Compare floats rounding to the number of decimal places, 7 after
- dropping missing values (NaN, NaT, None)
+ - Compare decimals and floats, where two values a and b are approximately equal
+ if they satisfy the following formula:
+ absolute(a - b) <= (atol + rtol * absolute(b))
+ where rtol=1e-5 and atol=1e-8 by default
"""
- # following pandas convention, rtol=1e-5 and atol=1e-8
- rtol = 1e-5
- atol = 1e-8
+
+ def compare_vals_approx(val1, val2):
+ # compare vals for approximate equality
+ if isinstance(lval, (float, decimal.Decimal)) or isinstance(rval, (float, decimal.Decimal)):
+ if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
+ return False
+ elif val1 != val2:
+ return False
+ return True
if isinstance(left, pd.DataFrame) and isinstance(right, pd.DataFrame):
if left.shape != right.shape:
@@ -200,19 +211,16 @@ def _assert_pandas_almost_equal(
},
)
for lval, rval in zip(left[lcol].dropna(), right[rcol].dropna()):
- if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
- isinstance(rval, float) or isinstance(rval, decimal.Decimal)
- ):
- if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
- raise PySparkAssertionError(
- error_class="DIFFERENT_PANDAS_DATAFRAME",
- message_parameters={
- "left": left.to_string(),
- "left_dtype": str(left.dtypes),
- "right": right.to_string(),
- "right_dtype": str(right.dtypes),
- },
- )
+ if not compare_vals_approx(lval, rval):
+ raise PySparkAssertionError(
+ error_class="DIFFERENT_PANDAS_DATAFRAME",
+ message_parameters={
+ "left": left.to_string(),
+ "left_dtype": str(left.dtypes),
+ "right": right.to_string(),
+ "right_dtype": str(right.dtypes),
+ },
+ )
if left.columns.names != right.columns.names:
raise PySparkAssertionError(
error_class="DIFFERENT_PANDAS_DATAFRAME",
@@ -246,19 +254,16 @@ def _assert_pandas_almost_equal(
},
)
for lval, rval in zip(left.dropna(), right.dropna()):
- if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
- isinstance(rval, float) or isinstance(rval, decimal.Decimal)
- ):
- if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
- raise PySparkAssertionError(
- error_class="DIFFERENT_PANDAS_SERIES",
- message_parameters={
- "left": left.to_string(),
- "left_dtype": str(left.dtype),
- "right": right.to_string(),
- "right_dtype": str(right.dtype),
- },
- )
+ if not compare_vals_approx(lval, rval):
+ raise PySparkAssertionError(
+ error_class="DIFFERENT_PANDAS_SERIES",
+ message_parameters={
+ "left": left.to_string(),
+ "left_dtype": str(left.dtype),
+ "right": right.to_string(),
+ "right_dtype": str(right.dtype),
+ },
+ )
elif isinstance(left, pd.MultiIndex) and isinstance(right, pd.MultiIndex):
if len(left) != len(right):
raise PySparkAssertionError(
@@ -271,19 +276,16 @@ def _assert_pandas_almost_equal(
},
)
for lval, rval in zip(left, right):
- if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
- isinstance(rval, float) or isinstance(rval, decimal.Decimal)
- ):
- if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
- raise PySparkAssertionError(
- error_class="DIFFERENT_PANDAS_MULTIINDEX",
- message_parameters={
- "left": left,
- "left_dtype": str(left.dtype),
- "right": right,
- "right_dtype": str(right.dtype),
- },
- )
+ if not compare_vals_approx(lval, rval):
+ raise PySparkAssertionError(
+ error_class="DIFFERENT_PANDAS_MULTIINDEX",
+ message_parameters={
+ "left": left,
+ "left_dtype": str(left.dtype),
+ "right": right,
+ "right_dtype": str(right.dtype),
+ },
+ )
elif isinstance(left, pd.Index) and isinstance(right, pd.Index):
if len(left) != len(right):
raise PySparkAssertionError(
@@ -307,21 +309,39 @@ def _assert_pandas_almost_equal(
},
)
for lval, rval in zip(left.dropna(), right.dropna()):
- if (isinstance(lval, float) or isinstance(lval, decimal.Decimal)) and (
- isinstance(rval, float) or isinstance(rval, decimal.Decimal)
- ):
- if abs(float(lval) - float(rval)) > (atol + rtol * abs(float(rval))):
- raise PySparkAssertionError(
- error_class="DIFFERENT_PANDAS_INDEX",
- message_parameters={
- "left": left,
- "left_dtype": str(left.dtype),
- "right": right,
- "right_dtype": str(right.dtype),
- },
- )
+ if not compare_vals_approx(lval, rval):
+ raise PySparkAssertionError(
+ error_class="DIFFERENT_PANDAS_INDEX",
+ message_parameters={
+ "left": left,
+ "left_dtype": str(left.dtype),
+ "right": right,
+ "right_dtype": str(right.dtype),
+ },
+ )
else:
- raise ValueError("Unexpected values: (%s, %s)" % (left, right))
+ if not isinstance(left, (pd.DataFrame, pd.Series, pd.Index)):
+ raise PySparkAssertionError(
+ error_class="INVALID_TYPE_DF_EQUALITY_ARG",
+ message_parameters={
+ "expected_type": f"{pd.DataFrame.__name__}, "
+ f"{pd.Series.__name__}, "
+ f"{pd.Index.__name__}, ",
+ "arg_name": "left",
+ "actual_type": type(left),
+ },
+ )
+ elif not isinstance(right, (pd.DataFrame, pd.Series, pd.Index)):
+ raise PySparkAssertionError(
+ error_class="INVALID_TYPE_DF_EQUALITY_ARG",
+ message_parameters={
+ "expected_type": f"{pd.DataFrame.__name__}, "
+ f"{pd.Series.__name__}, "
+ f"{pd.Index.__name__}, ",
+ "arg_name": "right",
+ "actual_type": type(right),
+ },
+ )
def assertPandasOnSparkEqual(
@@ -329,20 +349,22 @@ def assertPandasOnSparkEqual(
expected: Union[DataFrame, pd.DataFrame, Series, pd.Series, Index, pd.Index],
checkExact: bool = True,
almost: bool = False,
- checkRowOrder: bool = False,
+ rtol: float = 1e-5,
+ atol: float = 1e-8,
+ checkRowOrder: bool = True,
):
r"""
- A util function to assert equality between actual (pandas-on-Spark DataFrame) and expected
- (pandas-on-Spark or pandas DataFrame).
+ A util function to assert equality between actual (pandas-on-Spark object) and expected
+ (pandas-on-Spark or pandas object).
.. versionadded:: 3.5.0
Parameters
----------
- actual: pyspark.pandas.frame.DataFrame
- The DataFrame that is being compared or tested.
- expected: pyspark.pandas.frame.DataFrame or pd.DataFrame
- The expected DataFrame, for comparison with the actual result.
+ actual: pandas-on-Spark DataFrame, Series, or Index
+ The object that is being compared or tested.
+ expected: pandas-on-Spark or pandas DataFrame, Series, or Index
+ The expected object, for comparison with the actual result.
checkExact: bool, optional
A flag indicating whether to compare exact equality.
If set to 'True' (default), the data is compared exactly.
@@ -354,10 +376,16 @@ def assertPandasOnSparkEqual(
(see documentation for more details).
If set to 'False' (default), the data is compared exactly with `unittest`'s
`assertEqual`.
+ rtol : float, optional
+ The relative tolerance, used in asserting almost equality for float values in actual
+ and expected. Set to 1e-5 by default. (See Notes)
+ atol : float, optional
+ The absolute tolerance, used in asserting almost equality for float values in actual
+ and expected. Set to 1e-8 by default. (See Notes)
checkRowOrder : bool, optional
A flag indicating whether the order of rows should be considered in the comparison.
- If set to `False` (default), the row order is not taken into account.
- If set to `True`, the order of rows is important and will be checked during comparison.
+ If set to `False`, the row order is not taken into account.
+ If set to `True` (default), the order of rows will be checked during comparison.
(See Notes)
Notes
@@ -365,6 +393,11 @@ def assertPandasOnSparkEqual(
For `checkRowOrder`, note that pandas-on-Spark DataFrame ordering is non-deterministic, unless
explicitly sorted.
+ When `almost` is set to True, approximate equality will be asserted, where two values
+ a and b are approximately equal if they satisfy the following formula:
+
+ ``absolute(a - b) <= (atol + rtol * absolute(b))``.
+
Examples
--------
>>> import pyspark.pandas as ps
@@ -407,8 +440,9 @@ def assertPandasOnSparkEqual(
},
)
else:
- actual = actual.to_pandas()
- if not isinstance(expected, pd.DataFrame):
+ if not isinstance(actual, (pd.DataFrame, pd.Index, pd.Series)):
+ actual = actual.to_pandas()
+ if not isinstance(expected, (pd.DataFrame, pd.Index, pd.Series)):
expected = expected.to_pandas()
if not checkRowOrder:
@@ -418,25 +452,40 @@ def assertPandasOnSparkEqual(
expected = expected.sort_values(by=expected.columns[0], ignore_index=True)
if almost:
- _assert_pandas_almost_equal(actual, expected)
+ _assert_pandas_almost_equal(actual, expected, rtol=rtol, atol=atol)
else:
_assert_pandas_equal(actual, expected, checkExact=checkExact)
class PandasOnSparkTestUtils:
- def convert_str_to_lambda(self, func):
+ def convert_str_to_lambda(self, func: str):
"""
This function converts `func` str to lambda call
"""
return lambda x: getattr(x, func)()
- def assertPandasEqual(self, left, right, check_exact=True):
+ def assertPandasEqual(self, left: Any, right: Any, check_exact: bool = True):
_assert_pandas_equal(left, right, check_exact)
- def assertPandasAlmostEqual(self, left, right):
- _assert_pandas_almost_equal(left, right)
-
- def assert_eq(self, left, right, check_exact=True, almost=False):
+ def assertPandasAlmostEqual(
+ self,
+ left: Any,
+ right: Any,
+ rtol: float = 1e-5,
+ atol: float = 1e-8,
+ ):
+ _assert_pandas_almost_equal(left, right, rtol=rtol, atol=atol)
+
+ def assert_eq(
+ self,
+ left: Any,
+ right: Any,
+ check_exact: bool = True,
+ almost: bool = False,
+ rtol: float = 1e-5,
+ atol: float = 1e-8,
+ check_row_order: bool = True,
+ ):
"""
Asserts if two arbitrary objects are equal or not. If given objects are Koalas DataFrame
or Series, they are converted into pandas' and compared.
@@ -444,17 +493,37 @@ class PandasOnSparkTestUtils:
:param left: object to compare
:param right: object to compare
:param check_exact: if this is False, the comparison is done less precisely.
- :param almost: if this is enabled, the comparison is delegated to `unittest`'s
- `assertAlmostEqual`. See its documentation for more details.
+ :param almost: if this is enabled, the comparison asserts approximate equality
+ for float and decimal values, where two values a and b are approximately equal
+ if they satisfy the following formula:
+ absolute(a - b) <= (atol + rtol * absolute(b))
+ :param rtol: The relative tolerance, used in asserting approximate equality for
+ float values. Set to 1e-5 by default.
+ :param atol: The absolute tolerance, used in asserting approximate equality for
+ float values in actual and expected. Set to 1e-8 by default.
+ :param check_row_order: A flag indicating whether the order of rows should be considered
+ in the comparison. If set to False, row order will be ignored.
"""
import pandas as pd
from pandas.api.types import is_list_like
+ # for pandas-on-Spark DataFrames, allow choice to ignore row order
+ if isinstance(left, (ps.DataFrame, ps.Series, ps.Index)):
+ return assertPandasOnSparkEqual(
+ left,
+ right,
+ checkExact=check_exact,
+ almost=almost,
+ rtol=rtol,
+ atol=atol,
+ checkRowOrder=check_row_order,
+ )
+
lobj = self._to_pandas(left)
robj = self._to_pandas(right)
if isinstance(lobj, (pd.DataFrame, pd.Series, pd.Index)):
if almost:
- _assert_pandas_almost_equal(lobj, robj)
+ _assert_pandas_almost_equal(lobj, robj, rtol=rtol, atol=atol)
else:
_assert_pandas_equal(lobj, robj, checkExact=check_exact)
elif is_list_like(lobj) and is_list_like(robj):
@@ -470,7 +539,7 @@ class PandasOnSparkTestUtils:
self.assertEqual(lobj, robj)
@staticmethod
- def _to_pandas(obj):
+ def _to_pandas(obj: Any):
if isinstance(obj, (DataFrame, Series, Index)):
return obj.to_pandas()
else:
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index 8e02803efe5..7dd723634e2 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -39,7 +39,6 @@ from pyspark.find_spark_home import _find_spark_home
from pyspark.sql.dataframe import DataFrame
from pyspark.sql import Row
from pyspark.sql.types import StructType, AtomicType, StructField
-import pyspark.pandas as ps
have_scipy = False
have_numpy = False
@@ -360,9 +359,16 @@ def assertSchemaEqual(actual: StructType, expected: StructType):
)
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+ import pandas
+ import pyspark.pandas
+
+
def assertDataFrameEqual(
- actual: Union[DataFrame, ps.DataFrame, List[Row]],
- expected: Union[DataFrame, ps.DataFrame, List[Row]],
+ actual: Union[DataFrame, "pandas.DataFrame", "pyspark.pandas.DataFrame", List[Row]],
+ expected: Union[DataFrame, "pandas.DataFrame", "pyspark.pandas.DataFrame", List[Row]],
checkRowOrder: bool = False,
rtol: float = 1e-5,
atol: float = 1e-8,
@@ -371,7 +377,7 @@ def assertDataFrameEqual(
A util function to assert equality between `actual` and `expected`
(DataFrames or lists of Rows), with optional parameters `checkRowOrder`, `rtol`, and `atol`.
- Supports Spark, Spark Connect, and pandas-on-Spark DataFrames.
+ Supports Spark, Spark Connect, pandas, and pandas-on-Spark DataFrames.
For more information about pandas-on-Spark DataFrame equality, see the docs for
`assertPandasOnSparkEqual`.
@@ -379,9 +385,9 @@ def assertDataFrameEqual(
Parameters
----------
- actual : DataFrame (Spark, Spark Connect, or pandas-on-Spark) or list of Rows
+ actual : DataFrame (Spark, Spark Connect, pandas, or pandas-on-Spark) or list of Rows
The DataFrame that is being compared or tested.
- expected : DataFrame (Spark, Spark Connect, or pandas-on-Spark) or list of Rows
+ expected : DataFrame (Spark, Spark Connect, pandas, or pandas-on-Spark) or list of Rows
The expected result of the operation, for comparison with the actual result.
checkRowOrder : bool, optional
A flag indicating whether the order of rows should be considered in the comparison.
@@ -446,16 +452,13 @@ def assertDataFrameEqual(
Row(id='2', amount=3000.0)
! Row(id='3', amount=2003.0)
"""
- import pyspark.pandas as ps
- from pyspark.testing.pandasutils import assertPandasOnSparkEqual
-
if actual is None and expected is None:
return True
elif actual is None:
raise PySparkAssertionError(
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": None,
},
@@ -464,61 +467,56 @@ def assertDataFrameEqual(
raise PySparkAssertionError(
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "expected",
"actual_type": None,
},
)
+ has_pandas = False
try:
- # If Spark Connect dependencies are available, allow Spark Connect DataFrame
- from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
+ # If pandas dependencies are available, allow pandas or pandas-on-Spark DataFrame
+ import pyspark.pandas as ps
+ import pandas as pd
+ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
+
+ has_pandas = True
+ except ImportError:
+ # no pandas, so we won't call pandasutils functions
+ pass
- if isinstance(actual, ps.DataFrame) or isinstance(expected, ps.DataFrame):
- # handle pandas DataFrames
- # assert approximate equality for float data
- return assertPandasOnSparkEqual(
- actual, expected, checkExact=False, checkRowOrder=checkRowOrder
- )
- elif not isinstance(actual, (DataFrame, ConnectDataFrame, list)):
- raise PySparkAssertionError(
- error_class="INVALID_TYPE_DF_EQUALITY_ARG",
- message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
- "arg_name": "actual",
- "actual_type": type(actual),
- },
- )
- elif not isinstance(expected, (DataFrame, ConnectDataFrame, list)):
- raise PySparkAssertionError(
- error_class="INVALID_TYPE_DF_EQUALITY_ARG",
- message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
- "arg_name": "expected",
- "actual_type": type(expected),
- },
- )
- except Exception:
- if isinstance(actual, ps.DataFrame) or isinstance(expected, ps.DataFrame):
+ if has_pandas:
+ if (
+ isinstance(actual, pd.DataFrame)
+ or isinstance(expected, pd.DataFrame)
+ or isinstance(actual, ps.DataFrame)
+ or isinstance(expected, ps.DataFrame)
+ ):
# handle pandas DataFrames
# assert approximate equality for float data
- return assertPandasOnSparkEqual(
- actual, expected, checkExact=False, checkRowOrder=checkRowOrder
+ return PandasOnSparkTestUtils().assert_eq(
+ actual, expected, almost=True, rtol=rtol, atol=atol, check_row_order=checkRowOrder
)
- elif not isinstance(actual, (DataFrame, list)):
+
+ from pyspark.sql.utils import get_dataframe_class
+
+ # if is_remote(), allow Connect DataFrame
+ SparkDataFrame = get_dataframe_class()
+
+ if not isinstance(actual, (DataFrame, SparkDataFrame, list)):
raise PySparkAssertionError(
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "actual",
"actual_type": type(actual),
},
)
- elif not isinstance(expected, (DataFrame, list)):
+ elif not isinstance(expected, (DataFrame, SparkDataFrame, list)):
raise PySparkAssertionError(
error_class="INVALID_TYPE_DF_EQUALITY_ARG",
message_parameters={
- "expected_type": Union[DataFrame, ps.DataFrame, List[Row]],
+ "expected_type": "Union[DataFrame, ps.DataFrame, List[Row]]",
"arg_name": "expected",
"actual_type": type(expected),
},
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org