You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/03/11 03:02:10 UTC
[spark] branch master updated: [SPARK-38511][K8S] Remove `priorityClassName` propagation in favor of explicit settings
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new aec70e8 [SPARK-38511][K8S] Remove `priorityClassName` propagation in favor of explicit settings
aec70e8 is described below
commit aec70e802e0ff530c63f2f62eb19b29ec2c4ed35
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Mar 10 19:01:09 2022 -0800
[SPARK-38511][K8S] Remove `priorityClassName` propagation in favor of explicit settings
### What changes were proposed in this pull request?
This PR aims to remove implicit `priorityClassName` propagation from driver pod to podgroup in favor of explicit settings via Driver Pod templates and PodGroup templates.
### Why are the changes needed?
Currently, Apache Spark supports different `priorityClassName` for driver and executors. In Apache Spark, the most complex case is that the users use three priorityClassNames for (1) driver pod, (2) executor pod, and (3) podgroup.
This PR allows users define and use different sets for pod priority and pod group in case of https://github.com/apache/spark/pull/35786#discussion_r823745191 .
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CI and K8s ITs.
```
[info] KubernetesSuite:
[info] VolcanoSuite:
[info] - Run SparkPi with volcano scheduler (9 seconds, 975 milliseconds)
[info] - SPARK-38187: Run SparkPi Jobs with minCPU (28 seconds, 508 milliseconds)
[info] - SPARK-38187: Run SparkPi Jobs with minMemory (27 seconds, 459 milliseconds)
[info] - SPARK-38188: Run SparkPi jobs with 2 queues (only 1 enabled) (13 seconds, 285 milliseconds)
[info] - SPARK-38188: Run SparkPi jobs with 2 queues (all enabled) (21 seconds, 344 milliseconds)
[info] - SPARK-38423: Run driver job to validate priority order (15 seconds, 382 milliseconds)
[info] Run completed in 3 minutes, 8 seconds.
[info] Total number of tests run: 6
[info] Suites: completed 2, aborted 0
[info] Tests: succeeded 6, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```
Closes #35807 from dongjoon-hyun/SPARK-38511.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../deploy/k8s/features/VolcanoFeatureStep.scala | 5 ----
.../k8s/features/VolcanoFeatureStepSuite.scala | 20 ----------------
.../high-priority-driver-podgroup-template.yml | 21 +++++++++++++++++
.../low-priority-driver-podgroup-template.yml | 21 +++++++++++++++++
.../medium-priority-driver-podgroup-template.yml | 21 +++++++++++++++++
.../k8s/integrationtest/VolcanoTestsSuite.scala | 27 ++++------------------
6 files changed, 67 insertions(+), 48 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
index 9b37789..091923a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStep.scala
@@ -30,7 +30,6 @@ private[spark] class VolcanoFeatureStep extends KubernetesDriverCustomFeatureCon
private lazy val podGroupName = s"${kubernetesConf.appId}-podgroup"
private lazy val namespace = kubernetesConf.namespace
- private var priorityClassName: Option[String] = None
override def init(config: KubernetesDriverConf): Unit = {
kubernetesConf = config
@@ -52,16 +51,12 @@ private[spark] class VolcanoFeatureStep extends KubernetesDriverCustomFeatureCon
var spec = pg.getSpec
if (spec == null) spec = new PodGroupSpec
- priorityClassName.foreach(spec.setPriorityClassName(_))
pg.setSpec(spec)
Seq(pg)
}
override def configurePod(pod: SparkPod): SparkPod = {
-
- priorityClassName = Option(pod.pod.getSpec.getPriorityClassName)
-
val k8sPodBuilder = new PodBuilder(pod.pod)
.editMetadata()
.addToAnnotations(POD_GROUP_ANNOTATION, podGroupName)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
index 1b24e86..12daffc 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/VolcanoFeatureStepSuite.scala
@@ -18,7 +18,6 @@ package org.apache.spark.deploy.k8s.features
import java.io.File
-import io.fabric8.kubernetes.api.model.{ContainerBuilder, PodBuilder}
import io.fabric8.volcano.scheduling.v1beta1.PodGroup
import org.apache.spark.{SparkConf, SparkFunSuite}
@@ -50,25 +49,6 @@ class VolcanoFeatureStepSuite extends SparkFunSuite {
assert(annotations.get("scheduling.k8s.io/group-name") === s"${kubernetesConf.appId}-podgroup")
}
- test("SPARK-38423: Support priorityClassName") {
- // test null priority
- val podWithNullPriority = SparkPod.initialPod()
- assert(podWithNullPriority.pod.getSpec.getPriorityClassName === null)
- verifyPriority(SparkPod.initialPod())
- // test normal priority
- val podWithPriority = SparkPod(
- new PodBuilder()
- .withNewMetadata()
- .endMetadata()
- .withNewSpec()
- .withPriorityClassName("priority")
- .endSpec()
- .build(),
- new ContainerBuilder().build())
- assert(podWithPriority.pod.getSpec.getPriorityClassName === "priority")
- verifyPriority(podWithPriority)
- }
-
test("SPARK-38455: Support driver podgroup template") {
val templatePath = new File(
getClass.getResource("/driver-podgroup-template.yml").getFile).getAbsolutePath
diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/high-priority-driver-podgroup-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/high-priority-driver-podgroup-template.yml
new file mode 100644
index 0000000..a64431d
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/high-priority-driver-podgroup-template.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: scheduling.volcano.sh/v1beta1
+kind: PodGroup
+spec:
+ priorityClassName: high
+ queue: queue
diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/low-priority-driver-podgroup-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/low-priority-driver-podgroup-template.yml
new file mode 100644
index 0000000..5e89630
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/low-priority-driver-podgroup-template.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: scheduling.volcano.sh/v1beta1
+kind: PodGroup
+spec:
+ priorityClassName: low
+ queue: queue
diff --git a/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/medium-priority-driver-podgroup-template.yml b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/medium-priority-driver-podgroup-template.yml
new file mode 100644
index 0000000..5773e8b
--- /dev/null
+++ b/resource-managers/kubernetes/integration-tests/src/test/resources/volcano/medium-priority-driver-podgroup-template.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: scheduling.volcano.sh/v1beta1
+kind: PodGroup
+spec:
+ priorityClassName: medium
+ queue: queue
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
index eab7993..8d50544 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/VolcanoTestsSuite.scala
@@ -336,29 +336,6 @@ private[spark] trait VolcanoTestsSuite extends BeforeAndAfterEach { k8sSuite: Ku
}
}
- test("SPARK-38423: Run SparkPi Jobs with priorityClassName", k8sTestTag, volcanoTag) {
- // Prepare the priority resource
- createOrReplaceYAMLResource(VOLCANO_PRIORITY_YAML)
- val priorities = Seq("low", "medium", "high")
- val groupName = generateGroupName("priority")
- priorities.foreach { p =>
- Future {
- val templatePath = new File(
- getClass.getResource(s"/volcano/$p-priority-driver-template.yml").getFile
- ).getAbsolutePath
- runJobAndVerify(
- p, groupLoc = Option(groupName),
- driverTemplate = Option(templatePath)
- )
- }
- }
- // Make sure all jobs are Succeeded
- Eventually.eventually(TIMEOUT, INTERVAL) {
- val pods = getPods(role = "driver", groupName, statusPhase = "Succeeded")
- assert(pods.size === priorities.size)
- }
- }
-
test("SPARK-38423: Run driver job to validate priority order", k8sTestTag, volcanoTag) {
// Prepare the priority resource and queue
createOrReplaceYAMLResource(DISABLE_QUEUE)
@@ -370,11 +347,15 @@ private[spark] trait VolcanoTestsSuite extends BeforeAndAfterEach { k8sSuite: Ku
val templatePath = new File(
getClass.getResource(s"/volcano/$p-priority-driver-template.yml").getFile
).getAbsolutePath
+ val pgTemplatePath = new File(
+ getClass.getResource(s"/volcano/$p-priority-driver-podgroup-template.yml").getFile
+ ).getAbsolutePath
val groupName = generateGroupName(p)
runJobAndVerify(
p, groupLoc = Option(groupName),
queue = Option("queue"),
driverTemplate = Option(templatePath),
+ driverPodGroupTemplate = Option(pgTemplatePath),
isDriverJob = true
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org