You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/07/14 06:11:39 UTC
[spark] branch master updated: [SPARK-44217][PYTHON] Allow custom precision for fp approx equality
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1b8e9c6a957 [SPARK-44217][PYTHON] Allow custom precision for fp approx equality
1b8e9c6a957 is described below
commit 1b8e9c6a957de9089f4722842fb07bc2710a69c0
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Fri Jul 14 15:11:07 2023 +0900
[SPARK-44217][PYTHON] Allow custom precision for fp approx equality
### What changes were proposed in this pull request?
This PR adds optional arguments, `rtol` and `atol` , to `assertDataFrameEqual` to allow users to customize precision level when asserting approximate equality for float values in DataFrames.
### Why are the changes needed?
The change adds additional flexibility and user customization to the `assertDataFrameEqual` util.
### Does this PR introduce _any_ user-facing change?
Yes, the PR changes the args accepted for the user-facing util, `assertDataFrameEqual`.
### How was this patch tested?
Added tests to `runtime/python/pyspark/sql/tests/test_utils.py` and `runtime/python/pyspark/sql/tests/connect/test_utils.py`
Closes #41947 from asl3/custom-fp-precision.
Authored-by: Amanda Liu <am...@databricks.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/sql/tests/test_utils.py | 184 +++++++++++++++++++++++++++++++++
python/pyspark/testing/utils.py | 65 +++++++-----
2 files changed, 225 insertions(+), 24 deletions(-)
diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index ce8d83e6cb9..46692450cbd 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -118,6 +118,150 @@ class UtilsTestsMixin:
assertDataFrameEqual(df1, df2, checkRowOrder=False)
assertDataFrameEqual(df1, df2, checkRowOrder=True)
+ def test_assert_approx_equal_arraytype_float_default_rtol_fail(self):
+ # fails with default rtol, 1e-5
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.34]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.341]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+
+ expected_error_message = "Results do not match: "
+ percent_diff = (1 / 2) * 100
+ expected_error_message += "( %.5f %% )" % percent_diff
+ diff_msg = (
+ "[df]"
+ + "\n"
+ + str(df1.collect()[1])
+ + "\n\n"
+ + "[expected]"
+ + "\n"
+ + str(df2.collect()[1])
+ + "\n\n"
+ + "********************"
+ + "\n\n"
+ )
+ expected_error_message += "\n" + diff_msg
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_ROWS",
+ message_parameters={"error_msg": expected_error_message},
+ )
+
+ def test_assert_approx_equal_arraytype_float_custom_rtol_pass(self):
+ # passes with custom rtol, 1e-2
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.34]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.341]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+
+ assertDataFrameEqual(df1, df2, rtol=1e-2)
+
+ def test_assert_approx_equal_doubletype_custom_rtol_pass(self):
+ # passes with custom rtol, 1e-2
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", 97.01),
+ ("student2", 84.34),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grade", DoubleType(), True),
+ ]
+ ),
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("student1", 97.01),
+ ("student2", 84.341),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grade", DoubleType(), True),
+ ]
+ ),
+ )
+
+ assertDataFrameEqual(df1, df2, rtol=1e-2)
+
+ def test_assert_approx_equal_decimaltype_custom_rtol_pass(self):
+ # passes with custom rtol, 1e-2
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", 83.14),
+ ("student2", 97.12),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grade", DoubleType(), True),
+ ]
+ ),
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("student1", 83.14),
+ ("student2", 97.111),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grade", DoubleType(), True),
+ ]
+ ),
+ )
+
+ # cast to DecimalType
+ df1 = df1.withColumn("col_1", F.col("grade").cast("decimal(4,3)"))
+ df2 = df2.withColumn("col_1", F.col("grade").cast("decimal(4,3)"))
+
+ assertDataFrameEqual(df1, df2, rtol=1e-1)
+
def test_assert_notequal_arraytype(self):
df1 = self.spark.createDataFrame(
data=[
@@ -643,6 +787,24 @@ class UtilsTestsMixin:
assertDataFrameEqual(df1, df2, checkRowOrder=False)
assertDataFrameEqual(df1, df2, checkRowOrder=True)
+ def test_assert_pyspark_approx_equal_custom_rtol(self):
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("1", 1000.00),
+ ("2", 3000.00),
+ ],
+ schema=["id", "amount"],
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("1", 1000.01),
+ ("2", 3000.00),
+ ],
+ schema=["id", "amount"],
+ )
+
+ assertDataFrameEqual(df1, df2, rtol=1e-2)
+
def test_assert_pyspark_df_not_equal(self):
df1 = self.spark.createDataFrame(
data=[
@@ -839,6 +1001,28 @@ class UtilsTestsMixin:
assertDataFrameEqual(df1, df2, checkRowOrder=False)
assertDataFrameEqual(df1, df2, checkRowOrder=True)
+ def test_special_vals(self):
+ df1 = self.spark.createDataFrame(
+ data=[
+ (1, float("nan")),
+ (2, float("inf")),
+ (2, float("-inf")),
+ ],
+ schema=["id", "amount"],
+ )
+
+ df2 = self.spark.createDataFrame(
+ data=[
+ (1, float("nan")),
+ (2, float("inf")),
+ (2, float("-inf")),
+ ],
+ schema=["id", "amount"],
+ )
+
+ assertDataFrameEqual(df1, df2, checkRowOrder=False)
+ assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
def test_list_row_equal(self):
from pyspark.sql import Row
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index cb3f72b75ed..476470d7490 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -223,7 +223,11 @@ class PySparkErrorTestUtils:
def assertDataFrameEqual(
- df: DataFrame, expected: Union[DataFrame, List[Row]], checkRowOrder: bool = False
+ df: DataFrame,
+ expected: Union[DataFrame, List[Row]],
+ checkRowOrder: bool = False,
+ rtol: float = 1e-5,
+ atol: float = 1e-8,
):
"""
A util function to assert equality between `df` (DataFrame) and `expected`
@@ -231,54 +235,67 @@ def assertDataFrameEqual(
.. versionadded:: 3.5.0
- For float values, assert approximate equality (1e-5 by default).
-
Parameters
----------
df : DataFrame
The DataFrame that is being compared or tested.
-
expected : DataFrame or list of Rows
The expected result of the operation, for comparison with the actual result.
-
checkRowOrder : bool, optional
A flag indicating whether the order of rows should be considered in the comparison.
If set to `False` (default), the row order is not taken into account.
If set to `True`, the order of rows is important and will be checked during comparison.
+ (See Notes)
+ rtol : float, optional
+ The relative tolerance, used in asserting approximate equality for float values in df
+ and expected. Set to 1e-5 by default. (See Notes)
+ atol : float, optional
+ The absolute tolerance, used in asserting approximate equality for float values in df
+ and expected. Set to 1e-8 by default. (See Notes)
+
+ Notes
+ -----
+ For checkRowOrder, note that PySpark DataFrame ordering is non-deterministic, unless
+ explicitly sorted.
+
+ For DataFrames with float values, assertDataFrame asserts approximate equality.
+ Two float values a and b are approximately equal if the following equation is True:
+
+ ``absolute(a - b) <= (atol + rtol * absolute(b))``.
Examples
--------
- >>> from pyspark.sql import SparkSession
- >>> spark = SparkSession.builder.appName("assertDataFrameEqual example")\
- .config("spark.some.config.option", "some-value").getOrCreate()
>>> df1 = spark.createDataFrame(data=[("1", 1000), ("2", 3000)], schema=["id", "amount"])
>>> df2 = spark.createDataFrame(data=[("1", 1000), ("2", 3000)], schema=["id", "amount"])
- >>> assertDataFrameEqual(df1, df2) # pass
- >>> df1 = spark.createDataFrame(data=[("1", 1000.00), ("2", 3000.00), ("3", 2000.00)], \
- schema=["id", "amount"])
- >>> df2 = spark.createDataFrame(data=[("1", 1001.00), ("2", 3000.00), ("3", 2003.00)], \
- schema=["id", "amount"])
- >>> assertDataFrameEqual(df1, df2) # fail # doctest: +IGNORE_EXCEPTION_DETAIL
+ >>> assertDataFrameEqual(df1, df2)
+
+ Pass, DataFrames are identical
+
+ >>> df1 = spark.createDataFrame(data=[("1", 0.1), ("2", 3.23)], schema=["id", "amount"])
+ >>> df2 = spark.createDataFrame(data=[("1", 0.109), ("2", 3.23)], schema=["id", "amount"])
+ >>> assertDataFrameEqual(df1, df2, rtol=1e-1)
+
+ Pass, DataFrames are approx equal by rtol
+
+ >>> df1 = spark.createDataFrame(data=[("1", 1000.00), ("2", 3000.00), ("3", 2000.00)],
+ ... schema=["id", "amount"])
+ >>> df2 = spark.createDataFrame(data=[("1", 1001.00), ("2", 3000.00), ("3", 2003.00)],
+ ... schema=["id", "amount"])
+ >>> assertDataFrameEqual(df1, df2) # doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
PySparkAssertionError: [DIFFERENT_ROWS] Results do not match: ( 66.667 % )
[df]
Row(id='1', amount=1000.0)
- <BLANKLINE>
+
[expected]
Row(id='1', amount=1001.0)
- <BLANKLINE>
- ********************
- <BLANKLINE>
+
[df]
Row(id='3', amount=2000.0)
- <BLANKLINE>
+
[expected]
Row(id='3', amount=2003.0)
- <BLANKLINE>
- ********************
- <BLANKLINE>
- <BLANKLINE>
"""
if df is None and expected is None:
return True
@@ -336,7 +353,7 @@ def assertDataFrameEqual(
and all(compare_vals(val1[k], val2[k]) for k in val1.keys())
)
elif isinstance(val1, float) and isinstance(val2, float):
- if abs(val1 - val2) > 1e-5:
+ if abs(val1 - val2) > (atol + rtol * abs(val2)):
return False
else:
if val1 != val2:
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org