You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2023/07/14 06:11:39 UTC

[spark] branch master updated: [SPARK-44217][PYTHON] Allow custom precision for fp approx equality

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 1b8e9c6a957 [SPARK-44217][PYTHON] Allow custom precision for fp approx equality
1b8e9c6a957 is described below

commit 1b8e9c6a957de9089f4722842fb07bc2710a69c0
Author: Amanda Liu <am...@databricks.com>
AuthorDate: Fri Jul 14 15:11:07 2023 +0900

    [SPARK-44217][PYTHON] Allow custom precision for fp approx equality
    
    ### What changes were proposed in this pull request?
    This PR adds optional arguments, `rtol` and `atol` , to `assertDataFrameEqual` to allow users to customize precision level when asserting approximate equality for float values in DataFrames.
    
    ### Why are the changes needed?
    The change adds additional flexibility and user customization to the `assertDataFrameEqual` util.
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, the PR changes the args accepted for the user-facing util, `assertDataFrameEqual`.
    
    ### How was this patch tested?
    Added tests to `runtime/python/pyspark/sql/tests/test_utils.py` and `runtime/python/pyspark/sql/tests/connect/test_utils.py`
    
    Closes #41947 from asl3/custom-fp-precision.
    
    Authored-by: Amanda Liu <am...@databricks.com>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/sql/tests/test_utils.py | 184 +++++++++++++++++++++++++++++++++
 python/pyspark/testing/utils.py        |  65 +++++++-----
 2 files changed, 225 insertions(+), 24 deletions(-)

diff --git a/python/pyspark/sql/tests/test_utils.py b/python/pyspark/sql/tests/test_utils.py
index ce8d83e6cb9..46692450cbd 100644
--- a/python/pyspark/sql/tests/test_utils.py
+++ b/python/pyspark/sql/tests/test_utils.py
@@ -118,6 +118,150 @@ class UtilsTestsMixin:
         assertDataFrameEqual(df1, df2, checkRowOrder=False)
         assertDataFrameEqual(df1, df2, checkRowOrder=True)
 
+    def test_assert_approx_equal_arraytype_float_default_rtol_fail(self):
+        # fails with default rtol, 1e-5
+        df1 = self.spark.createDataFrame(
+            data=[
+                ("student1", [97.01, 89.23]),
+                ("student2", [91.86, 84.34]),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grades", ArrayType(FloatType()), True),
+                ]
+            ),
+        )
+        df2 = self.spark.createDataFrame(
+            data=[
+                ("student1", [97.01, 89.23]),
+                ("student2", [91.86, 84.341]),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grades", ArrayType(FloatType()), True),
+                ]
+            ),
+        )
+
+        expected_error_message = "Results do not match: "
+        percent_diff = (1 / 2) * 100
+        expected_error_message += "( %.5f %% )" % percent_diff
+        diff_msg = (
+            "[df]"
+            + "\n"
+            + str(df1.collect()[1])
+            + "\n\n"
+            + "[expected]"
+            + "\n"
+            + str(df2.collect()[1])
+            + "\n\n"
+            + "********************"
+            + "\n\n"
+        )
+        expected_error_message += "\n" + diff_msg
+
+        with self.assertRaises(PySparkAssertionError) as pe:
+            assertDataFrameEqual(df1, df2)
+
+        self.check_error(
+            exception=pe.exception,
+            error_class="DIFFERENT_ROWS",
+            message_parameters={"error_msg": expected_error_message},
+        )
+
+    def test_assert_approx_equal_arraytype_float_custom_rtol_pass(self):
+        # passes with custom rtol, 1e-2
+        df1 = self.spark.createDataFrame(
+            data=[
+                ("student1", [97.01, 89.23]),
+                ("student2", [91.86, 84.34]),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grades", ArrayType(FloatType()), True),
+                ]
+            ),
+        )
+        df2 = self.spark.createDataFrame(
+            data=[
+                ("student1", [97.01, 89.23]),
+                ("student2", [91.86, 84.341]),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grades", ArrayType(FloatType()), True),
+                ]
+            ),
+        )
+
+        assertDataFrameEqual(df1, df2, rtol=1e-2)
+
+    def test_assert_approx_equal_doubletype_custom_rtol_pass(self):
+        # passes with custom rtol, 1e-2
+        df1 = self.spark.createDataFrame(
+            data=[
+                ("student1", 97.01),
+                ("student2", 84.34),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grade", DoubleType(), True),
+                ]
+            ),
+        )
+        df2 = self.spark.createDataFrame(
+            data=[
+                ("student1", 97.01),
+                ("student2", 84.341),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grade", DoubleType(), True),
+                ]
+            ),
+        )
+
+        assertDataFrameEqual(df1, df2, rtol=1e-2)
+
+    def test_assert_approx_equal_decimaltype_custom_rtol_pass(self):
+        # passes with custom rtol, 1e-2
+        df1 = self.spark.createDataFrame(
+            data=[
+                ("student1", 83.14),
+                ("student2", 97.12),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grade", DoubleType(), True),
+                ]
+            ),
+        )
+        df2 = self.spark.createDataFrame(
+            data=[
+                ("student1", 83.14),
+                ("student2", 97.111),
+            ],
+            schema=StructType(
+                [
+                    StructField("student", StringType(), True),
+                    StructField("grade", DoubleType(), True),
+                ]
+            ),
+        )
+
+        # cast to DecimalType
+        df1 = df1.withColumn("col_1", F.col("grade").cast("decimal(4,3)"))
+        df2 = df2.withColumn("col_1", F.col("grade").cast("decimal(4,3)"))
+
+        assertDataFrameEqual(df1, df2, rtol=1e-1)
+
     def test_assert_notequal_arraytype(self):
         df1 = self.spark.createDataFrame(
             data=[
@@ -643,6 +787,24 @@ class UtilsTestsMixin:
         assertDataFrameEqual(df1, df2, checkRowOrder=False)
         assertDataFrameEqual(df1, df2, checkRowOrder=True)
 
+    def test_assert_pyspark_approx_equal_custom_rtol(self):
+        df1 = self.spark.createDataFrame(
+            data=[
+                ("1", 1000.00),
+                ("2", 3000.00),
+            ],
+            schema=["id", "amount"],
+        )
+        df2 = self.spark.createDataFrame(
+            data=[
+                ("1", 1000.01),
+                ("2", 3000.00),
+            ],
+            schema=["id", "amount"],
+        )
+
+        assertDataFrameEqual(df1, df2, rtol=1e-2)
+
     def test_assert_pyspark_df_not_equal(self):
         df1 = self.spark.createDataFrame(
             data=[
@@ -839,6 +1001,28 @@ class UtilsTestsMixin:
         assertDataFrameEqual(df1, df2, checkRowOrder=False)
         assertDataFrameEqual(df1, df2, checkRowOrder=True)
 
+    def test_special_vals(self):
+        df1 = self.spark.createDataFrame(
+            data=[
+                (1, float("nan")),
+                (2, float("inf")),
+                (2, float("-inf")),
+            ],
+            schema=["id", "amount"],
+        )
+
+        df2 = self.spark.createDataFrame(
+            data=[
+                (1, float("nan")),
+                (2, float("inf")),
+                (2, float("-inf")),
+            ],
+            schema=["id", "amount"],
+        )
+
+        assertDataFrameEqual(df1, df2, checkRowOrder=False)
+        assertDataFrameEqual(df1, df2, checkRowOrder=True)
+
     def test_list_row_equal(self):
         from pyspark.sql import Row
 
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index cb3f72b75ed..476470d7490 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -223,7 +223,11 @@ class PySparkErrorTestUtils:
 
 
 def assertDataFrameEqual(
-    df: DataFrame, expected: Union[DataFrame, List[Row]], checkRowOrder: bool = False
+    df: DataFrame,
+    expected: Union[DataFrame, List[Row]],
+    checkRowOrder: bool = False,
+    rtol: float = 1e-5,
+    atol: float = 1e-8,
 ):
     """
     A util function to assert equality between `df` (DataFrame) and `expected`
@@ -231,54 +235,67 @@ def assertDataFrameEqual(
 
     .. versionadded:: 3.5.0
 
-    For float values, assert approximate equality (1e-5 by default).
-
     Parameters
     ----------
     df : DataFrame
         The DataFrame that is being compared or tested.
-
     expected : DataFrame or list of Rows
         The expected result of the operation, for comparison with the actual result.
-
     checkRowOrder : bool, optional
         A flag indicating whether the order of rows should be considered in the comparison.
         If set to `False` (default), the row order is not taken into account.
         If set to `True`, the order of rows is important and will be checked during comparison.
+        (See Notes)
+    rtol : float, optional
+        The relative tolerance, used in asserting approximate equality for float values in df
+        and expected. Set to 1e-5 by default. (See Notes)
+    atol : float, optional
+        The absolute tolerance, used in asserting approximate equality for float values in df
+        and expected. Set to 1e-8 by default. (See Notes)
+
+    Notes
+    -----
+    For checkRowOrder, note that PySpark DataFrame ordering is non-deterministic, unless
+    explicitly sorted.
+
+    For DataFrames with float values, assertDataFrame asserts approximate equality.
+    Two float values a and b are approximately equal if the following equation is True:
+
+    ``absolute(a - b) <= (atol + rtol * absolute(b))``.
 
     Examples
     --------
-    >>> from pyspark.sql import SparkSession
-    >>> spark = SparkSession.builder.appName("assertDataFrameEqual example")\
-        .config("spark.some.config.option", "some-value").getOrCreate()
     >>> df1 = spark.createDataFrame(data=[("1", 1000), ("2", 3000)], schema=["id", "amount"])
     >>> df2 = spark.createDataFrame(data=[("1", 1000), ("2", 3000)], schema=["id", "amount"])
-    >>> assertDataFrameEqual(df1, df2) # pass
-    >>> df1 = spark.createDataFrame(data=[("1", 1000.00), ("2", 3000.00), ("3", 2000.00)], \
-        schema=["id", "amount"])
-    >>> df2 = spark.createDataFrame(data=[("1", 1001.00), ("2", 3000.00), ("3", 2003.00)], \
-        schema=["id", "amount"])
-    >>> assertDataFrameEqual(df1, df2) # fail  # doctest: +IGNORE_EXCEPTION_DETAIL
+    >>> assertDataFrameEqual(df1, df2)
+
+    Pass, DataFrames are identical
+
+    >>> df1 = spark.createDataFrame(data=[("1", 0.1), ("2", 3.23)], schema=["id", "amount"])
+    >>> df2 = spark.createDataFrame(data=[("1", 0.109), ("2", 3.23)], schema=["id", "amount"])
+    >>> assertDataFrameEqual(df1, df2, rtol=1e-1)
+
+    Pass, DataFrames are approx equal by rtol
+
+    >>> df1 = spark.createDataFrame(data=[("1", 1000.00), ("2", 3000.00), ("3", 2000.00)],
+    ... schema=["id", "amount"])
+    >>> df2 = spark.createDataFrame(data=[("1", 1001.00), ("2", 3000.00), ("3", 2003.00)],
+    ... schema=["id", "amount"])
+    >>> assertDataFrameEqual(df1, df2)  # doctest: +IGNORE_EXCEPTION_DETAIL
     Traceback (most recent call last):
     ...
     PySparkAssertionError: [DIFFERENT_ROWS] Results do not match: ( 66.667 % )
     [df]
     Row(id='1', amount=1000.0)
-    <BLANKLINE>
+
     [expected]
     Row(id='1', amount=1001.0)
-    <BLANKLINE>
-    ********************
-    <BLANKLINE>
+
     [df]
     Row(id='3', amount=2000.0)
-    <BLANKLINE>
+
     [expected]
     Row(id='3', amount=2003.0)
-    <BLANKLINE>
-    ********************
-    <BLANKLINE>
-    <BLANKLINE>
     """
     if df is None and expected is None:
         return True
@@ -336,7 +353,7 @@ def assertDataFrameEqual(
                     and all(compare_vals(val1[k], val2[k]) for k in val1.keys())
                 )
             elif isinstance(val1, float) and isinstance(val2, float):
-                if abs(val1 - val2) > 1e-5:
+                if abs(val1 - val2) > (atol + rtol * abs(val2)):
                     return False
             else:
                 if val1 != val2:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org