You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/01/17 05:24:32 UTC
(spark) branch branch-4.0 updated: [SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new e8a7f9428058 [SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests
e8a7f9428058 is described below
commit e8a7f9428058e4eb56b10fec0d258237956e3ccf
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Thu Jan 16 21:24:10 2025 -0800
[SPARK-50863][PYTHON][ML][TESTS] Skip slow pytorch tests
### What changes were proposed in this pull request?
Skip slow pytorch tests
### Why are the changes needed?
```
Finished test(python3.11): pyspark.ml.tests.connect.test_parity_torch_distributor (2902s)
Finished test(python3.11): pyspark.ml.torch.tests.test_distributor (2904s)
```
These test were disabled before, and were enabled in https://github.com/apache/spark/commit/dc84e529ba96ba8afc24216e5fc28d95ce8ce290
> So I think the root cause is still related to the resources, since TorchDistributor works on barrier mode, when there is not enough resources in Github Action, the tests just keep waiting.
I think the reason still stand.
### Does this PR introduce _any_ user-facing change?
no, test-only
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #49539 from zhengruifeng/py_skip_pytorch_test.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit c81fa332099ce00fea86c2588628098b02e0ef3a)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../tests/connect/test_parity_torch_distributor.py | 24 ++++++++++++++--------
python/pyspark/ml/torch/tests/test_distributor.py | 12 ++++++++---
2 files changed, 24 insertions(+), 12 deletions(-)
diff --git a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
index 3cd8abfc6e4e..6c747be97773 100644
--- a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
+++ b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
@@ -48,9 +48,11 @@ if not is_remote_only():
def tearDownClass(cls):
cls.spark.stop()
- @unittest.skipIf(
- not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
- )
+ # @unittest.skipIf(
+ # not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+ # )
+ # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+ @unittest.skip("Disabled due to slowness")
class TorchDistributorLocalUnitTestsOnConnect(
TorchDistributorLocalUnitTestsMixin, unittest.TestCase
):
@@ -79,9 +81,11 @@ if not is_remote_only():
(None, 3, False, "NONE"),
]
- @unittest.skipIf(
- not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
- )
+ # @unittest.skipIf(
+ # not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+ # )
+ # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+ @unittest.skip("Disabled due to slowness")
class TorchDistributorLocalUnitTestsIIOnConnect(
TorchDistributorLocalUnitTestsMixin, unittest.TestCase
):
@@ -111,9 +115,11 @@ if not is_remote_only():
(None, 3, False, "NONE"),
]
- @unittest.skipIf(
- not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
- )
+ # @unittest.skipIf(
+ # not have_torch or is_remote_only(), torch_requirement_message or "Requires JVM access"
+ # )
+ # TODO(SPARK-50864): Re-enable this test after fixing the slowness
+ @unittest.skip("Disabled due to slowness")
class TorchDistributorDistributedUnitTestsOnConnect(
TorchDistributorDistributedUnitTestsMixin, unittest.TestCase
):
diff --git a/python/pyspark/ml/torch/tests/test_distributor.py b/python/pyspark/ml/torch/tests/test_distributor.py
index 7b6a93afbff7..34800710395c 100644
--- a/python/pyspark/ml/torch/tests/test_distributor.py
+++ b/python/pyspark/ml/torch/tests/test_distributor.py
@@ -403,7 +403,9 @@ class TorchDistributorLocalUnitTestsMixin:
self.assertEqual(output, "success" * 4096)
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
@@ -425,7 +427,9 @@ class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unitte
cls.spark.stop()
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
class TorchDistributorLocalUnitTestsII(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
@@ -496,7 +500,9 @@ class TorchDistributorDistributedUnitTestsMixin:
self.assertEqual(output, "success" * 4096)
-@unittest.skipIf(not have_torch, torch_requirement_message)
+# @unittest.skipIf(not have_torch, torch_requirement_message)
+# TODO(SPARK-50864): Re-enable this test after fixing the slowness
+@unittest.skip("Disabled due to slowness")
class TorchDistributorDistributedUnitTests(
TorchDistributorDistributedUnitTestsMixin, unittest.TestCase
):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org