You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/09/01 16:35:19 UTC
[spark] branch branch-3.3 updated: [SPARK-40304][K8S][TESTS] Add `decomTestTag` to K8s Integration Test
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 399c397e703 [SPARK-40304][K8S][TESTS] Add `decomTestTag` to K8s Integration Test
399c397e703 is described below
commit 399c397e7035665c928b1d439a860f9e7b1ce3b3
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Sep 1 09:34:55 2022 -0700
[SPARK-40304][K8S][TESTS] Add `decomTestTag` to K8s Integration Test
### What changes were proposed in this pull request?
This PR aims to add a new test tag, `decomTestTag`, to K8s Integration Test.
### Why are the changes needed?
Decommission-related tests took over 6 minutes (`363s`). It would be helpful we can run them selectively.
```
[info] - Test basic decommissioning (44 seconds, 51 milliseconds)
[info] - Test basic decommissioning with shuffle cleanup (44 seconds, 450 milliseconds)
[info] - Test decommissioning with dynamic allocation & shuffle cleanups (2 minutes, 43 seconds)
[info] - Test decommissioning timeouts (44 seconds, 389 milliseconds)
[info] - SPARK-37576: Rolling decommissioning (1 minute, 8 seconds)
```
### Does this PR introduce _any_ user-facing change?
No, this is a test-only change.
### How was this patch tested?
Pass the CIs and test manually.
```
$ build/sbt -Psparkr -Pkubernetes -Pkubernetes-integration-tests \
-Dspark.kubernetes.test.deployMode=docker-desktop "kubernetes-integration-tests/test" \
-Dtest.exclude.tags=minikube,local,decom
...
[info] KubernetesSuite:
[info] - Run SparkPi with no resources (12 seconds, 441 milliseconds)
[info] - Run SparkPi with no resources & statefulset allocation (11 seconds, 949 milliseconds)
[info] - Run SparkPi with a very long application name. (11 seconds, 999 milliseconds)
[info] - Use SparkLauncher.NO_RESOURCE (11 seconds, 846 milliseconds)
[info] - Run SparkPi with a master URL without a scheme. (11 seconds, 176 milliseconds)
[info] - Run SparkPi with an argument. (11 seconds, 868 milliseconds)
[info] - Run SparkPi with custom labels, annotations, and environment variables. (11 seconds, 858 milliseconds)
[info] - All pods have the same service account by default (11 seconds, 5 milliseconds)
[info] - Run extraJVMOptions check on driver (5 seconds, 757 milliseconds)
[info] - Verify logging configuration is picked from the provided SPARK_CONF_DIR/log4j2.properties (12 seconds, 467 milliseconds)
[info] - Run SparkPi with env and mount secrets. (21 seconds, 119 milliseconds)
[info] - Run PySpark on simple pi.py example (13 seconds, 129 milliseconds)
[info] - Run PySpark to test a pyfiles example (14 seconds, 937 milliseconds)
[info] - Run PySpark with memory customization (12 seconds, 195 milliseconds)
[info] - Run in client mode. (11 seconds, 343 milliseconds)
[info] - Start pod creation from template (11 seconds, 975 milliseconds)
[info] - SPARK-38398: Schedule pod creation from template (11 seconds, 901 milliseconds)
[info] - Run SparkR on simple dataframe.R example (14 seconds, 305 milliseconds)
...
```
Closes #37755 from dongjoon-hyun/SPARK-40304.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit fd0498f81df72c196f19a5b26053660f6f3f4d70)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../deploy/k8s/integrationtest/DecommissionSuite.scala | 13 +++++++------
.../spark/deploy/k8s/integrationtest/KubernetesSuite.scala | 1 +
2 files changed, 8 insertions(+), 6 deletions(-)
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
index 5d1a57fb46e..81f4660afe9 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DecommissionSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.internal.config.PLUGINS
private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
import DecommissionSuite._
- import KubernetesSuite.k8sTestTag
+ import KubernetesSuite.{decomTestTag, k8sTestTag}
def runDecommissionTest(f: () => Unit): Unit = {
val logConfFilePath = s"${sparkHomeDir.toFile}/conf/log4j2.properties"
@@ -61,7 +61,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
}
}
- test("Test basic decommissioning", k8sTestTag) {
+ test("Test basic decommissioning", k8sTestTag, decomTestTag) {
runDecommissionTest(() => {
sparkAppConf
.set(config.DECOMMISSION_ENABLED.key, "true")
@@ -91,7 +91,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
})
}
- test("Test basic decommissioning with shuffle cleanup", k8sTestTag) {
+ test("Test basic decommissioning with shuffle cleanup", k8sTestTag, decomTestTag) {
runDecommissionTest(() => {
sparkAppConf
.set(config.DECOMMISSION_ENABLED.key, "true")
@@ -122,7 +122,8 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
})
}
- test("Test decommissioning with dynamic allocation & shuffle cleanups", k8sTestTag) {
+ test("Test decommissioning with dynamic allocation & shuffle cleanups",
+ k8sTestTag, decomTestTag) {
runDecommissionTest(() => {
sparkAppConf
.set(config.DECOMMISSION_ENABLED.key, "true")
@@ -183,7 +184,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
})
}
- test("Test decommissioning timeouts", k8sTestTag) {
+ test("Test decommissioning timeouts", k8sTestTag, decomTestTag) {
runDecommissionTest(() => {
sparkAppConf
.set(config.DECOMMISSION_ENABLED.key, "true")
@@ -216,7 +217,7 @@ private[spark] trait DecommissionSuite { k8sSuite: KubernetesSuite =>
})
}
- test("SPARK-37576: Rolling decommissioning", k8sTestTag) {
+ test("SPARK-37576: Rolling decommissioning", k8sTestTag, decomTestTag) {
runDecommissionTest(() => {
sparkAppConf
.set("spark.kubernetes.container.image", pyImage)
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 3db51b28600..3d7a9313031 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -612,6 +612,7 @@ private[spark] object KubernetesSuite {
val k8sTestTag = Tag("k8s")
val localTestTag = Tag("local")
val schedulingTestTag = Tag("schedule")
+ val decomTestTag = Tag("decom")
val rTestTag = Tag("r")
val MinikubeTag = Tag("minikube")
val SPARK_PI_MAIN_CLASS: String = "org.apache.spark.examples.SparkPi"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org