You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2023/08/17 01:42:54 UTC
[spark] branch master updated: [SPARK-43618][SPARK-43658][CONNECT][PS][TESTS] Enabling more tests
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 2a5e75818ff [SPARK-43618][SPARK-43658][CONNECT][PS][TESTS] Enabling more tests
2a5e75818ff is described below
commit 2a5e75818ffa6d4f4a0b9c904029656596125c96
Author: itholic <ha...@databricks.com>
AuthorDate: Thu Aug 17 09:42:29 2023 +0800
[SPARK-43618][SPARK-43658][CONNECT][PS][TESTS] Enabling more tests
### What changes were proposed in this pull request?
This PR proposes to enable tests for pandas API on Spark with Spark Connect
### Why are the changes needed?
To increate test coverage
### Does this PR introduce _any_ user-facing change?
No, it's test-only.
### How was this patch tested?
Enabling the existing UTs.
Closes #42388 from itholic/enabling-tests.
Authored-by: itholic <ha...@databricks.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../connect/computation/test_parity_compute.py | 27 +++++++++++++----
.../diff_frames_ops/test_parity_setitem_frame.py | 6 +---
.../diff_frames_ops/test_parity_setitem_series.py | 4 +--
.../connect/groupby/test_parity_apply_func.py | 2 +-
.../tests/connect/series/test_parity_cumulative.py | 1 -
.../tests/connect/series/test_parity_index.py | 1 -
.../connect/series/test_parity_missing_data.py | 1 -
.../tests/connect/series/test_parity_stat.py | 1 -
.../tests/diff_frames_ops/test_setitem_series.py | 35 ----------------------
9 files changed, 25 insertions(+), 53 deletions(-)
diff --git a/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py b/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py
index e2b92190b6e..ba19f8a611b 100644
--- a/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py
+++ b/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py
@@ -15,6 +15,8 @@
# limitations under the License.
#
import unittest
+import pandas as pd
+import numpy as np
from pyspark import pandas as ps
from pyspark.pandas.tests.computation.test_compute import FrameComputeMixin
@@ -27,13 +29,28 @@ class FrameParityComputeTests(FrameComputeMixin, PandasOnSparkTestUtils, ReusedC
def psdf(self):
return ps.from_pandas(self.pdf)
- @unittest.skip("Spark Connect does not support RDD but the tests depend on them.")
def test_mode(self):
- super().test_mode()
+ pdf = pd.DataFrame(
+ {
+ "A": [1, 2, None, 4, 5, 4, 2],
+ "B": [-0.1, 0.2, -0.3, np.nan, 0.5, -0.1, -0.1],
+ "C": ["d", "b", "c", "c", "e", "a", "a"],
+ "D": [np.nan, np.nan, np.nan, np.nan, 0.1, -0.1, -0.1],
+ "E": [np.nan, np.nan, np.nan, np.nan, np.nan, np.nan, np.nan],
+ }
+ )
+ psdf = ps.from_pandas(pdf)
- @unittest.skip("TODO(SPARK-43618): Fix pyspark.sq.column._unary_op to work with Spark Connect.")
- def test_rank(self):
- super().test_rank()
+ self.assert_eq(psdf.mode(), pdf.mode())
+ self.assert_eq(psdf.mode(numeric_only=True), pdf.mode(numeric_only=True))
+ self.assert_eq(psdf.mode(dropna=False), pdf.mode(dropna=False))
+
+ # dataframe with single column
+ for c in ["A", "B", "C", "D", "E"]:
+ self.assert_eq(psdf[[c]].mode(), pdf[[c]].mode())
+
+ with self.assertRaises(ValueError):
+ psdf.mode(axis=2)
if __name__ == "__main__":
diff --git a/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py b/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py
index 397d4b79700..5d9135980dc 100644
--- a/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py
+++ b/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py
@@ -24,11 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class DiffFramesParitySetItemFrameTests(
DiffFramesSetItemFrameMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip(
- "TODO(SPARK-43658): Fix unexpected `SparkConnectGrpcException` from Spark Connect client."
- )
- def test_frame_iloc_setitem(self):
- super().test_frame_iloc_setitem()
+ pass
if __name__ == "__main__":
diff --git a/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py b/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py
index 07aff29d0e5..d7856fda098 100644
--- a/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py
+++ b/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py
@@ -24,9 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class DiffFramesParitySetItemSeriesTests(
DiffFramesSetItemSeriesMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip(
- "TODO(SPARK-43658): Fix unexpected `SparkConnectGrpcException` from Spark Connect client."
- )
+ @unittest.skip("TODO(SPARK-44826): Resolve testing timeout issue from Spark Connect.")
def test_series_iloc_setitem(self):
super().test_series_iloc_setitem()
diff --git a/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py b/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
index 8eeec768d2d..2c0d49ebb5c 100644
--- a/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
+++ b/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py
@@ -24,7 +24,7 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class GroupbyParityApplyFuncTests(
GroupbyApplyFuncMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
- @unittest.skip("TODO(SPARK-43628): Enable SparkContext with Spark Connect.")
+ @unittest.skip("TODO(SPARK-43629): Enable RDD dependent tests with Spark Connect.")
def test_apply_with_side_effect(self):
super().test_apply_with_side_effect()
diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py b/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py
index c5c61e1f33b..a4978dca6fa 100644
--- a/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py
+++ b/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py
@@ -24,7 +24,6 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class SeriesParityCumulativeTests(
SeriesCumulativeMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
-
pass
diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_index.py b/python/pyspark/pandas/tests/connect/series/test_parity_index.py
index 2b92cce61fb..06b8051e256 100644
--- a/python/pyspark/pandas/tests/connect/series/test_parity_index.py
+++ b/python/pyspark/pandas/tests/connect/series/test_parity_index.py
@@ -22,7 +22,6 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class SeriesParityIndexTests(SeriesIndexMixin, PandasOnSparkTestUtils, ReusedConnectTestCase):
-
pass
diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py b/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py
index a95b312bce6..e756b2e887d 100644
--- a/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py
+++ b/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py
@@ -24,7 +24,6 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class SeriesParityMissingDataTests(
SeriesMissingDataMixin, PandasOnSparkTestUtils, ReusedConnectTestCase
):
-
pass
diff --git a/python/pyspark/pandas/tests/connect/series/test_parity_stat.py b/python/pyspark/pandas/tests/connect/series/test_parity_stat.py
index 916e120f99d..f598fe0255b 100644
--- a/python/pyspark/pandas/tests/connect/series/test_parity_stat.py
+++ b/python/pyspark/pandas/tests/connect/series/test_parity_stat.py
@@ -22,7 +22,6 @@ from pyspark.testing.pandasutils import PandasOnSparkTestUtils
class SeriesParityStatTests(SeriesStatMixin, PandasOnSparkTestUtils, ReusedConnectTestCase):
-
pass
diff --git a/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py b/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py
index 3473118269c..66180bda87e 100644
--- a/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py
+++ b/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py
@@ -162,41 +162,6 @@ class DiffFramesSetItemSeriesMixin:
):
psser1.iloc[[1, 2]] = -psser_another
- pdf = pd.DataFrame({"x": [1, 2, 3], "y": [4, 5, 6]}, index=["cobra", "viper", "sidewinder"])
- psdf = ps.from_pandas(pdf)
-
- pser = pdf.x
- psery = pdf.y
- psser = psdf.x
- pssery = psdf.y
-
- piloc = pser.iloc
- kiloc = psser.iloc
-
- kiloc[[0, 1, 2]] = -psser_another
- piloc[[0, 1, 2]] = -pser_another
- self.assert_eq(psser, pser)
- self.assert_eq(psdf, pdf)
- self.assert_eq(pssery, psery)
-
- with self.assertRaisesRegex(
- ValueError,
- "cannot set using a list-like indexer with a different length than the value",
- ):
- kiloc[[1, 2]] = -psser_another
-
- kiloc[[0, 1, 2]] = 10 * psser_another
- piloc[[0, 1, 2]] = 10 * pser_another
- self.assert_eq(psser, pser)
- self.assert_eq(psdf, pdf)
- self.assert_eq(pssery, psery)
-
- with self.assertRaisesRegex(
- ValueError,
- "cannot set using a list-like indexer with a different length than the value",
- ):
- kiloc[[0]] = 10 * psser_another
-
class DiffFramesSetItemSeriesTests(
DiffFramesSetItemSeriesMixin, PandasOnSparkTestCase, SQLTestUtils
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org