You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/11/25 05:09:31 UTC
(spark) branch branch-4.0 updated: [SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new c00ba2729edd [SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job
c00ba2729edd is described below
commit c00ba2729eddc16f5edac799034906a84b9a56bf
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Tue Nov 25 14:08:29 2025 +0900
[SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job
This PR proposes to skip the tests that fails in 4.0 client <> master server job. It marks them with JIRA IDs.
To enable the build https://github.com/apache/spark/pull/53188
No, dev-only.
[Manually ran the tests in my fork.](https://github.com/HyukjinKwon/spark/actions/runs/19655913777)
No.
Closes #53204 from HyukjinKwon/SPARK-54477.
Authored-by: Hyukjin Kwon <gu...@apache.org>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
(cherry picked from commit 8f747eef9a38b62d63cb5983bc364f1952aee0d7)
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
python/pyspark/pandas/tests/test_typedef.py | 4 ++++
python/pyspark/sql/streaming/readwriter.py | 7 ++++---
.../test_parity_pandas_grouped_map_with_state.py | 4 ++++
.../connect/streaming/test_parity_foreach_batch.py | 4 ++++
.../sql/tests/pandas/test_pandas_cogrouped_map.py | 4 ++++
.../sql/tests/pandas/test_pandas_grouped_map.py | 19 +++++++++++++++++++
python/pyspark/sql/tests/pandas/test_pandas_map.py | 3 +++
.../sql/tests/test_python_streaming_datasource.py | 4 ++++
8 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/python/pyspark/pandas/tests/test_typedef.py b/python/pyspark/pandas/tests/test_typedef.py
index cac9aaf193a0..62ba2672de19 100644
--- a/python/pyspark/pandas/tests/test_typedef.py
+++ b/python/pyspark/pandas/tests/test_typedef.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
+import os
import sys
import unittest
import datetime
@@ -310,6 +311,9 @@ class TypeHintTestsMixin:
self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54495: To be reenabled"
+ )
def test_as_spark_type_pandas_on_spark_dtype(self):
type_mapper = {
# binary
diff --git a/python/pyspark/sql/streaming/readwriter.py b/python/pyspark/sql/streaming/readwriter.py
index 34af8cd9b070..aae66994ce64 100644
--- a/python/pyspark/sql/streaming/readwriter.py
+++ b/python/pyspark/sql/streaming/readwriter.py
@@ -1568,6 +1568,7 @@ class DataStreamWriter:
self._jwrite.foreach(jForeachWriter)
return self
+ # SPARK-54478: Reenable doctest
def foreachBatch(self, func: Callable[["DataFrame", int], None]) -> "DataStreamWriter":
"""
Sets the output of the streaming query to be processed using the provided
@@ -1600,9 +1601,9 @@ class DataStreamWriter:
... my_value = 100
... batch_df.collect()
...
- >>> q = df.writeStream.foreachBatch(func).start()
- >>> time.sleep(3)
- >>> q.stop()
+ >>> q = df.writeStream.foreachBatch(func).start() # doctest: +SKIP
+ >>> time.sleep(3) # doctest: +SKIP
+ >>> q.stop() # doctest: +SKIP
>>> # if in Spark Connect, my_value = -1, else my_value = 100
"""
from py4j.java_gateway import java_import
diff --git a/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
index 2da8b4aa3be8..f18a531329ba 100644
--- a/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
@@ -15,6 +15,7 @@
# limitations under the License.
#
import unittest
+import os
from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import (
GroupedApplyInPandasWithStateTestsMixin,
@@ -22,6 +23,9 @@ from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import (
from pyspark.testing.connectutils import ReusedConnectTestCase
+@unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54479: To be reenabled"
+)
class GroupedApplyInPandasWithStateTests(
GroupedApplyInPandasWithStateTestsMixin, ReusedConnectTestCase
):
diff --git a/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py b/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
index 632fa4628d1b..c2a747dd57f3 100644
--- a/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
+++ b/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
@@ -16,6 +16,7 @@
#
import unittest
+import os
from pyspark.sql.tests.streaming.test_streaming_foreach_batch import StreamingTestsForeachBatchMixin
from pyspark.testing.connectutils import ReusedConnectTestCase, should_test_connect
@@ -25,6 +26,9 @@ if should_test_connect:
from pyspark.errors.exceptions.connect import StreamingPythonRunnerInitializationException
+@unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54480: To be reenabled"
+)
class StreamingForeachBatchParityTests(StreamingTestsForeachBatchMixin, ReusedConnectTestCase):
def test_streaming_foreach_batch_propagates_python_errors(self):
super().test_streaming_foreach_batch_propagates_python_errors()
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
index 1f9532352679..ab937cb68f74 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
@@ -17,6 +17,7 @@
import unittest
from typing import cast
+import os
from pyspark.sql.functions import array, explode, col, lit, udf, pandas_udf, sum
from pyspark.sql.types import (
@@ -239,6 +240,9 @@ class CogroupedApplyInPandasTestsMixin:
self._test_merge_empty(fn=merge_pandas)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54481: To be reenabled"
+ )
def test_apply_in_pandas_returning_incompatible_type(self):
with self.quiet():
self.check_apply_in_pandas_returning_incompatible_type()
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
index 4ef334549ef5..eaab71b48559 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
@@ -17,6 +17,7 @@
import datetime
import unittest
+import os
from collections import OrderedDict
from decimal import Decimal
@@ -298,12 +299,21 @@ class GroupedApplyInPandasTestsMixin:
# columns must be in order of applyInPandas schema when no columns given
return pd.DataFrame([key + (pdf.v.mean(),)])
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def test_apply_in_pandas_returning_column_names(self):
self._test_apply_in_pandas(GroupedApplyInPandasTestsMixin.stats_with_column_names)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def test_apply_in_pandas_returning_no_column_names(self):
self._test_apply_in_pandas(GroupedApplyInPandasTestsMixin.stats_with_no_column_names)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def test_apply_in_pandas_returning_column_names_sometimes(self):
def stats(key, pdf):
if key[0] % 2:
@@ -343,9 +353,15 @@ class GroupedApplyInPandasTestsMixin:
lambda key, pdf: pd.DataFrame([key + (pdf.v.mean(), pdf.v.std())])
)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def test_apply_in_pandas_returning_empty_dataframe(self):
self._test_apply_in_pandas_returning_empty_dataframe(pd.DataFrame())
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def test_apply_in_pandas_returning_incompatible_type(self):
with self.quiet():
self.check_apply_in_pandas_returning_incompatible_type()
@@ -863,6 +879,9 @@ class GroupedApplyInPandasTestsMixin:
for row in result:
self.assertEqual(24.5, row[1])
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+ )
def _test_apply_in_pandas_returning_empty_dataframe_error(self, empty_df, error):
with self.quiet():
with self.assertRaisesRegex(PythonException, error):
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_map.py b/python/pyspark/sql/tests/pandas/test_pandas_map.py
index 692f9705411e..a9f66b161403 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_map.py
@@ -246,6 +246,9 @@ class MapInPandasTestsMixin:
actual = df.repartition(1).mapInPandas(f, "id long, value long").collect()
self.assertEqual(actual, expected)
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54483: To be reenabled"
+ )
def test_dataframes_with_incompatible_types(self):
with self.quiet():
self.check_dataframes_with_incompatible_types()
diff --git a/python/pyspark/sql/tests/test_python_streaming_datasource.py b/python/pyspark/sql/tests/test_python_streaming_datasource.py
index fa14b37b57e6..dd3669e021e7 100644
--- a/python/pyspark/sql/tests/test_python_streaming_datasource.py
+++ b/python/pyspark/sql/tests/test_python_streaming_datasource.py
@@ -138,6 +138,9 @@ class BasePythonStreamingDataSourceTestsMixin:
return TestDataSource
+ @unittest.skipIf(
+ os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54484: To be reenabled"
+ )
def test_stream_reader(self):
self.spark.dataSource.register(self._get_test_data_source())
df = self.spark.readStream.format("TestDataSource").load()
@@ -212,6 +215,7 @@ class BasePythonStreamingDataSourceTestsMixin:
assertDataFrameEqual(df, expected_data)
+ @unittest.skipIf(os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "To be reenabled")
def test_simple_stream_reader(self):
class SimpleStreamReader(SimpleDataSourceStreamReader):
def initialOffset(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org