You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/06/12 06:09:04 UTC
[spark] branch master updated: [SPARK-39450][K8S] Reuse PVCs by default
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new a6c724ed816 [SPARK-39450][K8S] Reuse PVCs by default
a6c724ed816 is described below
commit a6c724ed816bd83cc6b3f5178b462e63b8e85972
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sat Jun 11 23:08:46 2022 -0700
[SPARK-39450][K8S] Reuse PVCs by default
### What changes were proposed in this pull request?
This PR aims to reuse PVCs by default via setting the followings.
```
spark.kubernetes.driver.ownPersistentVolumeClaim=true
spark.kubernetes.driver.reusePersistentVolumeClaim=true
```
### Why are the changes needed?
By reusing PVCs (and PVs), we can remove the overhead of PV creation and removal.
### Does this PR introduce _any_ user-facing change?
Yes. The migration guide is updated with this change.
### How was this patch tested?
Pass the CIs.
Closes #36849 from dongjoon-hyun/SPARK-39450.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
docs/core-migration-guide.md | 4 ++++
docs/running-on-kubernetes.md | 4 ++--
.../core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala | 4 ++--
.../spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala | 7 ++++---
4 files changed, 12 insertions(+), 7 deletions(-)
diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md
index 745b80d6eec..1a16b8f112a 100644
--- a/docs/core-migration-guide.md
+++ b/docs/core-migration-guide.md
@@ -22,6 +22,10 @@ license: |
* Table of contents
{:toc}
+## Upgrading from Core 3.3 to 3.4
+
+- Since Spark 3.4, Spark driver will own `PersistentVolumnClaim`s and try to reuse if they are not assigned to live executors. To restore the behavior before Spark 3.4, you can set `spark.kubernetes.driver.ownPersistentVolumeClaim` to `false` and `spark.kubernetes.driver.reusePersistentVolumeClaim` to `false`.
+
## Upgrading from Core 3.2 to 3.3
- Since Spark 3.3, Spark migrates its log4j dependency from 1.x to 2.x because log4j 1.x has reached end of life and is no longer supported by the community. Vulnerabilities reported after August 2015 against log4j 1.x were not checked and will not be fixed. Users should rewrite original log4j properties files using log4j2 syntax (XML, JSON, YAML, or properties format). Spark rewrites the `conf/log4j.properties.template` which is included in Spark distribution, to `conf/log4j2.properties [...]
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 3445e22f434..d141fbd804a 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -1395,7 +1395,7 @@ See the [configuration page](configuration.html) for information on Spark config
</tr>
<tr>
<td><code>spark.kubernetes.driver.ownPersistentVolumeClaim</code></td>
- <td><code>false</code></td>
+ <td><code>true</code></td>
<td>
If true, driver pod becomes the owner of on-demand persistent volume claims instead of the executor pods
</td>
@@ -1403,7 +1403,7 @@ See the [configuration page](configuration.html) for information on Spark config
</tr>
<tr>
<td><code>spark.kubernetes.driver.reusePersistentVolumeClaim</code></td>
- <td><code>false</code></td>
+ <td><code>true</code></td>
<td>
If true, driver pod tries to reuse driver-owned on-demand persistent volume claims
of the deleted executor pods if exists. This can be useful to reduce executor pod
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
index 7930cd0ce15..168c86ecb3a 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala
@@ -69,7 +69,7 @@ private[spark] object Config extends Logging {
"instead of the executor pods")
.version("3.2.0")
.booleanConf
- .createWithDefault(false)
+ .createWithDefault(true)
val KUBERNETES_DRIVER_REUSE_PVC =
ConfigBuilder("spark.kubernetes.driver.reusePersistentVolumeClaim")
@@ -83,7 +83,7 @@ private[spark] object Config extends Logging {
s"sometimes. This config requires ${KUBERNETES_DRIVER_OWN_PVC.key}=true.")
.version("3.2.0")
.booleanConf
- .createWithDefault(false)
+ .createWithDefault(true)
val KUBERNETES_NAMESPACE =
ConfigBuilder("spark.kubernetes.namespace")
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
index d263bd00731..87bd8ef3d9d 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodsAllocatorSuite.scala
@@ -117,6 +117,10 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
conf, secMgr, executorBuilder, kubernetesClient, snapshotsStore, waitForExecutorPodsClock)
when(schedulerBackend.getExecutorIds).thenReturn(Seq.empty)
podsAllocatorUnderTest.start(TEST_SPARK_APP_ID, schedulerBackend)
+ when(kubernetesClient.persistentVolumeClaims()).thenReturn(persistentVolumeClaims)
+ when(persistentVolumeClaims.withLabel(any(), any())).thenReturn(labeledPersistentVolumeClaims)
+ when(labeledPersistentVolumeClaims.list()).thenReturn(persistentVolumeClaimList)
+ when(persistentVolumeClaimList.getItems).thenReturn(Seq.empty[PersistentVolumeClaim].asJava)
}
test("SPARK-36052: test splitSlots") {
@@ -694,9 +698,6 @@ class ExecutorPodsAllocatorSuite extends SparkFunSuite with BeforeAndAfter {
.set(s"$prefix.option.sizeLimit", "200Gi")
.set(s"$prefix.option.storageClass", "gp2")
- when(kubernetesClient.persistentVolumeClaims()).thenReturn(persistentVolumeClaims)
- when(persistentVolumeClaims.withLabel(any(), any())).thenReturn(labeledPersistentVolumeClaims)
- when(labeledPersistentVolumeClaims.list()).thenReturn(persistentVolumeClaimList)
when(persistentVolumeClaimList.getItems)
.thenReturn(Seq(persistentVolumeClaim("pvc-0", "gp2", "200Gi")).asJava)
when(executorBuilder.buildFromFeatures(any(classOf[KubernetesExecutorConf]), meq(secMgr),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org