You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by "allisonwang-db (via GitHub)" <gi...@apache.org> on 2023/07/12 18:05:38 UTC
[GitHub] [spark] allisonwang-db commented on a diff in pull request #41947: [SPARK-44217][PYTHON] Allow custom precision for fp approx equality
allisonwang-db commented on code in PR #41947:
URL: https://github.com/apache/spark/pull/41947#discussion_r1261546994
##########
python/pyspark/testing/utils.py:
##########
@@ -234,14 +240,24 @@ def assertDataFrameEqual(df: DataFrame, expected: DataFrame, check_row_order: bo
----------
df : DataFrame
The DataFrame that is being compared or tested.
-
expected : DataFrame
The expected result of the operation, for comparison with the actual result.
-
check_row_order : bool, optional
A flag indicating whether the order of rows should be considered in the comparison.
If set to `False` (default), the row order is not taken into account.
If set to `True`, the order of rows is important and will be checked during comparison.
+ rtol : float, optional
+ The relative tolerance, used in asserting approximate equality for float values in df
+ and expected. Set to 1e-5 by default. (See Notes)
+ atol : float, optional
Review Comment:
Let's also add an example for these?
##########
python/pyspark/sql/tests/test_utils.py:
##########
@@ -115,6 +115,88 @@ def test_assert_approx_equal_arraytype_float(self):
assertDataFrameEqual(df1, df2)
+ def test_assert_approx_equal_arraytype_float_default_rtol_fail(self):
+ # fails with default rtol, 1e-5
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.34]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+ df2 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.341]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
+ ]
+ ),
+ )
+
+ expected_error_message = "Results do not match: "
+ percent_diff = (1 / 2) * 100
+ expected_error_message += "( %.5f %% )" % percent_diff
+ diff_msg = (
+ "[df]"
+ + "\n"
+ + str(df1.collect()[1])
+ + "\n\n"
+ + "[expected]"
+ + "\n"
+ + str(df2.collect()[1])
+ + "\n\n"
+ + "********************"
+ + "\n\n"
+ )
+ expected_error_message += "\n" + diff_msg
+
+ with self.assertRaises(PySparkAssertionError) as pe:
+ assertDataFrameEqual(df1, df2)
+
+ self.check_error(
+ exception=pe.exception,
+ error_class="DIFFERENT_ROWS",
+ message_parameters={"error_msg": expected_error_message},
+ )
+
+ def test_assert_approx_equal_arraytype_float_custom_rtol_pass(self):
+ # passes with custom rtol, 1e-2
+ df1 = self.spark.createDataFrame(
+ data=[
+ ("student1", [97.01, 89.23]),
+ ("student2", [91.86, 84.34]),
+ ],
+ schema=StructType(
+ [
+ StructField("student", StringType(), True),
+ StructField("grades", ArrayType(FloatType()), True),
Review Comment:
Can we also test 1) DoubleType 2) DecimalType?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org