You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2025/11/25 05:09:31 UTC

(spark) branch branch-4.0 updated: [SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job

This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new c00ba2729edd [SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job
c00ba2729edd is described below

commit c00ba2729eddc16f5edac799034906a84b9a56bf
Author: Hyukjin Kwon <gu...@apache.org>
AuthorDate: Tue Nov 25 14:08:29 2025 +0900

    [SPARK-54477][CONNECT][PYTHON][TESTS] Mark tests to reeanble for 4.0 client <> master server job
    
    This PR proposes to skip the tests that fails in 4.0 client <> master server job. It marks them with JIRA IDs.
    
    To enable the build https://github.com/apache/spark/pull/53188
    
    No, dev-only.
    
    [Manually ran the tests in my fork.](https://github.com/HyukjinKwon/spark/actions/runs/19655913777)
    
    No.
    
    Closes #53204 from HyukjinKwon/SPARK-54477.
    
    Authored-by: Hyukjin Kwon <gu...@apache.org>
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
    (cherry picked from commit 8f747eef9a38b62d63cb5983bc364f1952aee0d7)
    Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
 python/pyspark/pandas/tests/test_typedef.py           |  4 ++++
 python/pyspark/sql/streaming/readwriter.py            |  7 ++++---
 .../test_parity_pandas_grouped_map_with_state.py      |  4 ++++
 .../connect/streaming/test_parity_foreach_batch.py    |  4 ++++
 .../sql/tests/pandas/test_pandas_cogrouped_map.py     |  4 ++++
 .../sql/tests/pandas/test_pandas_grouped_map.py       | 19 +++++++++++++++++++
 python/pyspark/sql/tests/pandas/test_pandas_map.py    |  3 +++
 .../sql/tests/test_python_streaming_datasource.py     |  4 ++++
 8 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/pandas/tests/test_typedef.py b/python/pyspark/pandas/tests/test_typedef.py
index cac9aaf193a0..62ba2672de19 100644
--- a/python/pyspark/pandas/tests/test_typedef.py
+++ b/python/pyspark/pandas/tests/test_typedef.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 
+import os
 import sys
 import unittest
 import datetime
@@ -310,6 +311,9 @@ class TypeHintTestsMixin:
 
         self.assertRaisesRegex(TypeError, "object.*not understood", try_infer_return_type)
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54495: To be reenabled"
+    )
     def test_as_spark_type_pandas_on_spark_dtype(self):
         type_mapper = {
             # binary
diff --git a/python/pyspark/sql/streaming/readwriter.py b/python/pyspark/sql/streaming/readwriter.py
index 34af8cd9b070..aae66994ce64 100644
--- a/python/pyspark/sql/streaming/readwriter.py
+++ b/python/pyspark/sql/streaming/readwriter.py
@@ -1568,6 +1568,7 @@ class DataStreamWriter:
         self._jwrite.foreach(jForeachWriter)
         return self
 
+    # SPARK-54478: Reenable doctest
     def foreachBatch(self, func: Callable[["DataFrame", int], None]) -> "DataStreamWriter":
         """
         Sets the output of the streaming query to be processed using the provided
@@ -1600,9 +1601,9 @@ class DataStreamWriter:
         ...     my_value = 100
         ...     batch_df.collect()
         ...
-        >>> q = df.writeStream.foreachBatch(func).start()
-        >>> time.sleep(3)
-        >>> q.stop()
+        >>> q = df.writeStream.foreachBatch(func).start()  # doctest: +SKIP
+        >>> time.sleep(3)  # doctest: +SKIP
+        >>> q.stop()  # doctest: +SKIP
         >>> # if in Spark Connect, my_value = -1, else my_value = 100
         """
         from py4j.java_gateway import java_import
diff --git a/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
index 2da8b4aa3be8..f18a531329ba 100644
--- a/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
+++ b/python/pyspark/sql/tests/connect/pandas/test_parity_pandas_grouped_map_with_state.py
@@ -15,6 +15,7 @@
 # limitations under the License.
 #
 import unittest
+import os
 
 from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import (
     GroupedApplyInPandasWithStateTestsMixin,
@@ -22,6 +23,9 @@ from pyspark.sql.tests.pandas.test_pandas_grouped_map_with_state import (
 from pyspark.testing.connectutils import ReusedConnectTestCase
 
 
+@unittest.skipIf(
+    os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54479: To be reenabled"
+)
 class GroupedApplyInPandasWithStateTests(
     GroupedApplyInPandasWithStateTestsMixin, ReusedConnectTestCase
 ):
diff --git a/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py b/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
index 632fa4628d1b..c2a747dd57f3 100644
--- a/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
+++ b/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py
@@ -16,6 +16,7 @@
 #
 
 import unittest
+import os
 
 from pyspark.sql.tests.streaming.test_streaming_foreach_batch import StreamingTestsForeachBatchMixin
 from pyspark.testing.connectutils import ReusedConnectTestCase, should_test_connect
@@ -25,6 +26,9 @@ if should_test_connect:
     from pyspark.errors.exceptions.connect import StreamingPythonRunnerInitializationException
 
 
+@unittest.skipIf(
+    os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54480: To be reenabled"
+)
 class StreamingForeachBatchParityTests(StreamingTestsForeachBatchMixin, ReusedConnectTestCase):
     def test_streaming_foreach_batch_propagates_python_errors(self):
         super().test_streaming_foreach_batch_propagates_python_errors()
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
index 1f9532352679..ab937cb68f74 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py
@@ -17,6 +17,7 @@
 
 import unittest
 from typing import cast
+import os
 
 from pyspark.sql.functions import array, explode, col, lit, udf, pandas_udf, sum
 from pyspark.sql.types import (
@@ -239,6 +240,9 @@ class CogroupedApplyInPandasTestsMixin:
 
         self._test_merge_empty(fn=merge_pandas)
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54481: To be reenabled"
+    )
     def test_apply_in_pandas_returning_incompatible_type(self):
         with self.quiet():
             self.check_apply_in_pandas_returning_incompatible_type()
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
index 4ef334549ef5..eaab71b48559 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
@@ -17,6 +17,7 @@
 
 import datetime
 import unittest
+import os
 
 from collections import OrderedDict
 from decimal import Decimal
@@ -298,12 +299,21 @@ class GroupedApplyInPandasTestsMixin:
         # columns must be in order of applyInPandas schema when no columns given
         return pd.DataFrame([key + (pdf.v.mean(),)])
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def test_apply_in_pandas_returning_column_names(self):
         self._test_apply_in_pandas(GroupedApplyInPandasTestsMixin.stats_with_column_names)
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def test_apply_in_pandas_returning_no_column_names(self):
         self._test_apply_in_pandas(GroupedApplyInPandasTestsMixin.stats_with_no_column_names)
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def test_apply_in_pandas_returning_column_names_sometimes(self):
         def stats(key, pdf):
             if key[0] % 2:
@@ -343,9 +353,15 @@ class GroupedApplyInPandasTestsMixin:
                 lambda key, pdf: pd.DataFrame([key + (pdf.v.mean(), pdf.v.std())])
             )
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def test_apply_in_pandas_returning_empty_dataframe(self):
         self._test_apply_in_pandas_returning_empty_dataframe(pd.DataFrame())
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def test_apply_in_pandas_returning_incompatible_type(self):
         with self.quiet():
             self.check_apply_in_pandas_returning_incompatible_type()
@@ -863,6 +879,9 @@ class GroupedApplyInPandasTestsMixin:
         for row in result:
             self.assertEqual(24.5, row[1])
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54482: To be reenabled"
+    )
     def _test_apply_in_pandas_returning_empty_dataframe_error(self, empty_df, error):
         with self.quiet():
             with self.assertRaisesRegex(PythonException, error):
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_map.py b/python/pyspark/sql/tests/pandas/test_pandas_map.py
index 692f9705411e..a9f66b161403 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_map.py
@@ -246,6 +246,9 @@ class MapInPandasTestsMixin:
         actual = df.repartition(1).mapInPandas(f, "id long, value long").collect()
         self.assertEqual(actual, expected)
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54483: To be reenabled"
+    )
     def test_dataframes_with_incompatible_types(self):
         with self.quiet():
             self.check_dataframes_with_incompatible_types()
diff --git a/python/pyspark/sql/tests/test_python_streaming_datasource.py b/python/pyspark/sql/tests/test_python_streaming_datasource.py
index fa14b37b57e6..dd3669e021e7 100644
--- a/python/pyspark/sql/tests/test_python_streaming_datasource.py
+++ b/python/pyspark/sql/tests/test_python_streaming_datasource.py
@@ -138,6 +138,9 @@ class BasePythonStreamingDataSourceTestsMixin:
 
         return TestDataSource
 
+    @unittest.skipIf(
+        os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "SPARK-54484: To be reenabled"
+    )
     def test_stream_reader(self):
         self.spark.dataSource.register(self._get_test_data_source())
         df = self.spark.readStream.format("TestDataSource").load()
@@ -212,6 +215,7 @@ class BasePythonStreamingDataSourceTestsMixin:
 
         assertDataFrameEqual(df, expected_data)
 
+    @unittest.skipIf(os.environ.get("SPARK_SKIP_CONNECT_COMPAT_TESTS") == "1", "To be reenabled")
     def test_simple_stream_reader(self):
         class SimpleStreamReader(SimpleDataSourceStreamReader):
             def initialOffset(self):


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org