You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by xi...@apache.org on 2025/08/28 22:38:29 UTC
(spark) branch master updated: [SPARK-53403][PS] Improve add/sub tests under ANSI
This is an automated email from the ASF dual-hosted git repository.
xinrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 0f5204c67d96 [SPARK-53403][PS] Improve add/sub tests under ANSI
0f5204c67d96 is described below
commit 0f5204c67d967ecb51bef460f96f5cd8cd5d9e20
Author: Xinrong Meng <xi...@apache.org>
AuthorDate: Thu Aug 28 15:38:14 2025 -0700
[SPARK-53403][PS] Improve add/sub tests under ANSI
### What changes were proposed in this pull request?
- Enable add/sub float tests under ANSI
- Code cleanup
### Why are the changes needed?
Part of https://issues.apache.org/jira/browse/SPARK-53389
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Test changes
Commands below pass:
```
1047 SPARK_ANSI_SQL_MODE=false ./python/run-tests --python-executables=python3.11 --testnames "pyspark.pandas.tests.data_type_ops.test_num_arithmetic ArithmeticTests.test_add"
1048 SPARK_ANSI_SQL_MODE=true ./python/run-tests --python-executables=python3.11 --testnames "pyspark.pandas.tests.data_type_ops.test_num_arithmetic ArithmeticTests.test_add"
1049 SPARK_ANSI_SQL_MODE=true ./python/run-tests --python-executables=python3.11 --testnames "pyspark.pandas.tests.data_type_ops.test_num_arithmetic ArithmeticTests.test_sub"
1050 SPARK_ANSI_SQL_MODE=false ./python/run-tests --python-executables=python3.11 --testnames "pyspark.pandas.tests.data_type_ops.test_num_arithmetic ArithmeticTests.test_sub"
```
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #52151 from xinrong-meng/ari_test.
Authored-by: Xinrong Meng <xi...@apache.org>
Signed-off-by: Xinrong Meng <xi...@apache.org>
---
.../tests/data_type_ops/test_num_arithmetic.py | 9 ++++++--
.../pandas/tests/data_type_ops/test_string_ops.py | 24 +++++++++++-----------
python/pyspark/pandas/tests/test_categorical.py | 1 -
3 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py b/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py
index a982c5e148b4..798a49ae121e 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py
@@ -20,6 +20,7 @@ import unittest
import pandas as pd
from pyspark import pandas as ps
+from pyspark.testing.utils import is_ansi_mode_test
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.pandas.tests.data_type_ops.testing_utils import OpsTestBase
@@ -46,7 +47,6 @@ class ArithmeticTestsMixin:
pser, psser = pdf[col], psdf[col]
self.assert_eq(pser + pser, psser + psser, check_exact=False)
self.assert_eq(pser + 1, psser + 1, check_exact=False)
- # self.assert_eq(pser + 0.1, psser + 0.1)
self.assert_eq(pser + pser.astype(bool), psser + psser.astype(bool), check_exact=False)
self.assert_eq(pser + True, psser + True, check_exact=False)
self.assert_eq(pser + False, psser + False, check_exact=False)
@@ -57,13 +57,15 @@ class ArithmeticTestsMixin:
else:
self.assertRaises(TypeError, lambda: psser + psdf[n_col])
+ if is_ansi_mode_test and not col.startswith("decimal"):
+ self.assert_eq(pser + 0.1, psser + 0.1)
+
def test_sub(self):
pdf, psdf = self.pdf, self.psdf
for col in self.numeric_df_cols:
pser, psser = pdf[col], psdf[col]
self.assert_eq(pser - pser, psser - psser, check_exact=False)
self.assert_eq(pser - 1, psser - 1, check_exact=False)
- # self.assert_eq(pser - 0.1, psser - 0.1)
self.assert_eq(pser - pser.astype(bool), psser - psser.astype(bool), check_exact=False)
self.assert_eq(pser - True, psser - True, check_exact=False)
self.assert_eq(pser - False, psser - False, check_exact=False)
@@ -74,6 +76,9 @@ class ArithmeticTestsMixin:
else:
self.assertRaises(TypeError, lambda: psser - psdf[n_col])
+ if is_ansi_mode_test and not col.startswith("decimal"):
+ self.assert_eq(pser - 0.1, psser - 0.1)
+
class ArithmeticTests(
ArithmeticTestsMixin,
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py b/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
index 9648ad9ab2c0..9c3f4cf1dec4 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py
@@ -33,19 +33,19 @@ if extension_object_dtypes_available:
class StringOpsTestsMixin:
@property
- def bool_pdf(self):
+ def str_pdf(self):
return pd.DataFrame({"this": ["x", "y", "z"], "that": ["z", "y", "x"]})
@property
- def bool_non_numeric_pdf(self):
- return pd.concat([self.bool_pdf, self.non_numeric_pdf], axis=1)
+ def str_non_numeric_pdf(self):
+ return pd.concat([self.str_pdf, self.non_numeric_pdf], axis=1)
@property
- def bool_non_numeric_psdf(self):
- return ps.from_pandas(self.bool_non_numeric_pdf)
+ def str_non_numeric_psdf(self):
+ return ps.from_pandas(self.str_non_numeric_pdf)
def test_add(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser + "x", psser + "x")
@@ -188,42 +188,42 @@ class StringOpsTestsMixin:
self.assertRaises(TypeError, lambda: ~self.psdf["string"])
def test_eq(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser == other_pser, psser == other_psser)
self.assert_eq(pser == pser, psser == psser)
def test_ne(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser != other_pser, psser != other_psser)
self.assert_eq(pser != pser, psser != psser)
def test_lt(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser < other_pser, psser < other_psser)
self.assert_eq(pser < pser, psser < psser)
def test_le(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser <= other_pser, psser <= other_psser)
self.assert_eq(pser <= pser, psser <= psser)
def test_gt(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser > other_pser, psser > other_psser)
self.assert_eq(pser > pser, psser > psser)
def test_ge(self):
- pdf, psdf = self.bool_non_numeric_pdf, self.bool_non_numeric_psdf
+ pdf, psdf = self.str_non_numeric_pdf, self.str_non_numeric_psdf
pser, psser = pdf["this"], psdf["this"]
other_pser, other_psser = pdf["that"], psdf["that"]
self.assert_eq(pser >= other_pser, psser >= other_psser)
diff --git a/python/pyspark/pandas/tests/test_categorical.py b/python/pyspark/pandas/tests/test_categorical.py
index 2730723b27e3..49cbcee47b80 100644
--- a/python/pyspark/pandas/tests/test_categorical.py
+++ b/python/pyspark/pandas/tests/test_categorical.py
@@ -91,7 +91,6 @@ class CategoricalTestsMixin:
self.assert_eq(pser.cat.add_categories([4, 5]), psser.cat.add_categories([4, 5]))
self.assert_eq(pser.cat.add_categories([]), psser.cat.add_categories([]))
- pser = pser.cat.add_categories(4)
psser = psser.cat.add_categories(4)
self.assertRaises(ValueError, lambda: psser.cat.add_categories(4))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org