You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/03/13 05:24:34 UTC
[spark] branch master updated: [SPARK-38537][K8S] Unify `Statefulset*` to `StatefulSet*`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 786a70e [SPARK-38537][K8S] Unify `Statefulset*` to `StatefulSet*`
786a70e is described below
commit 786a70e710369b195d7c117b33fe9983044014d6
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sat Mar 12 21:23:46 2022 -0800
[SPARK-38537][K8S] Unify `Statefulset*` to `StatefulSet*`
### What changes were proposed in this pull request?
K8s has [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) and Apache Spark is currently using both `Statefulset*` and `StatefulSet*`. The worst case is a mixed case like `class StatefulSetAllocatorSuite` in `StatefulsetAllocatorSuite.scala`. This PR aims to unify them to a K8s original name, `StatefulSet`.
https://github.com/apache/spark/blob/c032928515e74367137c668ce692d8fd53696485/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala#L39
To sum up, two files are renamed and five files are changed.
```
$ git diff master --stat
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala | 5 ++---
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +-
resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetPodsAllocator.scala => StatefulSetPodsAllocator.scala} | 2 +-
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala | 4 ++--
resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/{StatefulsetAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} | 4 ++--
5 files changed, 8 insertions(+), 9 deletions(-)
```
### Why are the changes needed?
To be consistent not only inside Apache Spark, but also with K8s.
### Does this PR introduce _any_ user-facing change?
No. This is a new code in Apache Spark 3.3.
### How was this patch tested?
Pass the CIs.
Closes #35827 from dongjoon-hyun/SPARK-38537.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala | 5 ++---
.../spark/scheduler/cluster/k8s/KubernetesClusterManager.scala | 2 +-
...StatefulsetPodsAllocator.scala => StatefulSetPodsAllocator.scala} | 2 +-
.../spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala | 4 ++--
...atefulsetAllocatorSuite.scala => StatefulSetAllocatorSuite.scala} | 4 ++--
5 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala
index 2e0d4fa..cc08120 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/AbstractPodsAllocator.scala
@@ -26,9 +26,8 @@ import org.apache.spark.resource.ResourceProfile
* :: DeveloperApi ::
* A abstract interface for allowing different types of pods allocation.
*
- * The internal Spark implementations are [[StatefulsetPodsAllocator]]
- * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers
- * such as Volcano, Yunikorn, etc.
+ * The internal Spark implementations are [[StatefulSetPodsAllocator]]
+ * and [[ExecutorPodsAllocator]]. This may be useful for folks integrating with custom schedulers.
*
* This API may change or be removed at anytime.
*
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
index 9497349..10ea3a8 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManager.scala
@@ -137,7 +137,7 @@ private[spark] class KubernetesClusterManager extends ExternalClusterManager wit
snapshotsStore: ExecutorPodsSnapshotsStore) = {
val executorPodsAllocatorName = sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR) match {
case "statefulset" =>
- classOf[StatefulsetPodsAllocator].getName
+ classOf[StatefulSetPodsAllocator].getName
case "direct" =>
classOf[ExecutorPodsAllocator].getName
case fullClass =>
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala
similarity index 99%
rename from resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala
rename to resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala
index 0d00d96..294ee70 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetPodsAllocator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetPodsAllocator.scala
@@ -34,7 +34,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.resource.ResourceProfile
import org.apache.spark.util.{Clock, Utils}
-class StatefulsetPodsAllocator(
+class StatefulSetPodsAllocator(
conf: SparkConf,
secMgr: SecurityManager,
executorBuilder: KubernetesExecutorBuilder,
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala
index ae1477e..2b6bfe8 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/KubernetesClusterManagerSuite.scala
@@ -47,8 +47,8 @@ class KubernetesClusterManagerSuite extends SparkFunSuite with BeforeAndAfter {
test("constructing a AbstractPodsAllocator works") {
val validConfigs = List("statefulset", "direct",
- "org.apache.spark.scheduler.cluster.k8s.StatefulsetPodsAllocator",
- "org.apache.spark.scheduler.cluster.k8s.ExecutorPodsAllocator")
+ classOf[StatefulSetPodsAllocator].getName,
+ classOf[ExecutorPodsAllocator].getName)
validConfigs.foreach { c =>
val manager = new KubernetesClusterManager()
when(sc.conf.get(KUBERNETES_ALLOCATION_PODS_ALLOCATOR)).thenReturn(c)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala
similarity index 98%
rename from resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala
rename to resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala
index 5f8ceb2..748f509 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulsetAllocatorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/StatefulSetAllocatorSuite.scala
@@ -80,7 +80,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
@Mock
private var driverPodOperations: PodResource[Pod] = _
- private var podsAllocatorUnderTest: StatefulsetPodsAllocator = _
+ private var podsAllocatorUnderTest: StatefulSetPodsAllocator = _
private var snapshotsStore: DeterministicExecutorPodsSnapshotsStore = _
@@ -111,7 +111,7 @@ class StatefulSetAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
when(executorBuilder.buildFromFeatures(any(classOf[KubernetesExecutorConf]), meq(secMgr),
meq(kubernetesClient), any(classOf[ResourceProfile]))).thenAnswer(executorPodAnswer())
snapshotsStore = new DeterministicExecutorPodsSnapshotsStore()
- podsAllocatorUnderTest = new StatefulsetPodsAllocator(
+ podsAllocatorUnderTest = new StatefulSetPodsAllocator(
conf, secMgr, executorBuilder, kubernetesClient, snapshotsStore, null)
when(schedulerBackend.getExecutorIds).thenReturn(Seq.empty)
podsAllocatorUnderTest.start(TEST_SPARK_APP_ID, schedulerBackend)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org