You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/02/09 05:48:21 UTC
[spark] branch master updated: [SPARK-34407][K8S]
KubernetesClusterSchedulerBackend.stop should clean up K8s resources
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new ea339c3 [SPARK-34407][K8S] KubernetesClusterSchedulerBackend.stop should clean up K8s resources
ea339c3 is described below
commit ea339c38b43c59931257386efdd490507f7de64d
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Mon Feb 8 21:47:23 2021 -0800
[SPARK-34407][K8S] KubernetesClusterSchedulerBackend.stop should clean up K8s resources
### What changes were proposed in this pull request?
This PR aims to fix `KubernetesClusterSchedulerBackend.stop` to wrap `super.stop` with `Utils.tryLogNonFatalError`.
### Why are the changes needed?
[CoarseGrainedSchedulerBackend.stop](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/cluster/CoarseGrainedSchedulerBackend.scala#L559) may throw `SparkException` and this causes K8s resource (pod and configmap) leakage.
### Does this PR introduce _any_ user-facing change?
No. This is a bug fix.
### How was this patch tested?
Pass the CI with the newly added test case.
Closes #31533 from dongjoon-hyun/SPARK-34407.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../cluster/k8s/KubernetesClusterSchedulerBackend.scala | 6 +++++-
.../cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala | 12 +++++++++++-
2 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
index cb3378c..cd3b36c 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackend.scala
@@ -101,7 +101,11 @@ private[spark] class KubernetesClusterSchedulerBackend(
}
override def stop(): Unit = {
- super.stop()
+ // When `CoarseGrainedSchedulerBackend.stop` throws `SparkException`,
+ // K8s cluster scheduler should log and proceed in order to delete the K8s cluster resources.
+ Utils.tryLogNonFatalError {
+ super.stop()
+ }
Utils.tryLogNonFatalError {
snapshotsStore.stop()
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
index fdc3633..bbae9dd 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterSchedulerBackendSuite.scala
@@ -34,7 +34,7 @@ import org.apache.spark.deploy.k8s.Fabric8Aliases._
import org.apache.spark.resource.{ResourceProfile, ResourceProfileManager}
import org.apache.spark.rpc.{RpcEndpoint, RpcEndpointRef, RpcEnv}
import org.apache.spark.scheduler.{ExecutorKilled, LiveListenerBus, TaskSchedulerImpl}
-import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.RemoveExecutor
+import org.apache.spark.scheduler.cluster.CoarseGrainedClusterMessages.{RemoveExecutor, StopDriver}
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
import org.apache.spark.scheduler.cluster.k8s.ExecutorLifecycleTestUtils.TEST_SPARK_APP_ID
@@ -189,4 +189,14 @@ class KubernetesClusterSchedulerBackendSuite extends SparkFunSuite with BeforeAn
TimeUnit.MILLISECONDS)
verify(labeledPods).delete()
}
+
+ test("SPARK-34407: CoarseGrainedSchedulerBackend.stop may throw SparkException") {
+ schedulerBackendUnderTest.start()
+
+ when(driverEndpointRef.askSync[Boolean](StopDriver)).thenThrow(new RuntimeException)
+ schedulerBackendUnderTest.stop()
+
+ // Verify the last operation of `schedulerBackendUnderTest.stop`.
+ verify(kubernetesClient).close()
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org