You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/01/17 05:24:32 UTC

(spark) branch branch-4.0 updated: [SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new e8a7f9428058 [SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests
e8a7f9428058 is described below

commit e8a7f9428058e4eb56b10fec0d258237956e3ccf
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Jan 16 21:24:10 2025 -0800

    [SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests
    
    ### What changes were proposed in this pull request?
    Skip slow pytorch tests
    
    ### Why are the changes needed?
    ```
    Finished test(python3.11): pyspark.ml.tests.connect.test_parity_torch_distributor (2902s)
    Finished test(python3.11): pyspark.ml.torch.tests.test_distributor (2904s)
    ```
    
    These test were disabled before, and were enabled in https://github.com/apache/spark/commit/dc84e529ba96ba8afc24216e5fc28d95ce8ce290
    
    > So I think the root cause is still related to the resources, since TorchDistributor works on barrier mode, when there is not enough resources in Github Action, the tests just keep waiting.
    
    I think the reason still stand.
    
    ### Does this PR introduce _any_ user-facing change?
    no, test-only
    
    ### How was this patch tested?
    ci
    
    ### Was this patch authored or co-authored using generative AI tooling?
    no
    
    Closes #49539 from zhengruifeng/py_skip_pytorch_test.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit c81fa332099ce00fea86c2588628098b02e0ef3a)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../tests/connect/test_parity_torch_distributor.py | 24 ++++++++++++++--------
 python/pyspark/ml/torch/tests/test_distributor.py  | 12 ++++++++---
 2 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
index 3cd8abfc6e4e..6c747be97773 100644
--- a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
+++ b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
@@ -48,9 +48,11 @@ if not is_remote_only():
         def tearDownClass(cls):
             cls.spark.stop()
 
-    @unittest.skipIf(
-        not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
-    )
+    # @unittest.skipIf(
+    #     not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+    # )
+    # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+    @unittest.skip("Disabled due to slowness")
     class TorchDistributorLocalUnitTestsOnConnect(
         TorchDistributorLocalUnitTestsMixin, unittest.TestCase
     ):
@@ -79,9 +81,11 @@ if not is_remote_only():
                 (None, 3, False, "NONE"),
             ]
 
-    @unittest.skipIf(
-        not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
-    )
+    # @unittest.skipIf(
+    #     not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+    # )
+    # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+    @unittest.skip("Disabled due to slowness")
     class TorchDistributorLocalUnitTestsIIOnConnect(
         TorchDistributorLocalUnitTestsMixin, unittest.TestCase
     ):
@@ -111,9 +115,11 @@ if not is_remote_only():
                 (None, 3, False, "NONE"),
             ]
 
-    @unittest.skipIf(
-        not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
-    )
+    # @unittest.skipIf(
+    #     not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+    # )
+    # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+    @unittest.skip("Disabled due to slowness")
     class TorchDistributorDistributedUnitTestsOnConnect(
         TorchDistributorDistributedUnitTestsMixin, unittest.TestCase
     ):
diff --git a/python/pyspark/ml/torch/tests/test_distributor.py b/python/pyspark/ml/torch/tests/test_distributor.py
index 7b6a93afbff7..34800710395c 100644
--- a/python/pyspark/ml/torch/tests/test_distributor.py
+++ b/python/pyspark/ml/torch/tests/test_distributor.py
@@ -403,7 +403,9 @@ class TorchDistributorLocalUnitTestsMixin:
         self.assertEqual(output, "success" * 4096)
 
 
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
 class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -425,7 +427,9 @@ class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unitte
         cls.spark.stop()
 
 
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
 class TorchDistributorLocalUnitTestsII(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
     @classmethod
     def setUpClass(cls):
@@ -496,7 +500,9 @@ class TorchDistributorDistributedUnitTestsMixin:
         self.assertEqual(output, "success" * 4096)
 
 
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
 class TorchDistributorDistributedUnitTests(
     TorchDistributorDistributedUnitTestsMixin, unittest.TestCase
 ):


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org