You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pr...@apache.org on 2020/04/30 03:46:16 UTC
[spark] branch master updated: [SPARK-31601][K8S] Fix
spark.kubernetes.executor.podNamePrefix to work
This is an automated email from the ASF dual-hosted git repository.
prashant pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 85dad37 [SPARK-31601][K8S] Fix spark.kubernetes.executor.podNamePrefix to work
85dad37 is described below
commit 85dad37f69ebb617c8ac015dbbbda11054170298
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Apr 30 09:15:12 2020 +0530
[SPARK-31601][K8S] Fix spark.kubernetes.executor.podNamePrefix to work
### What changes were proposed in this pull request?
This PR aims to fix `spark.kubernetes.executor.podNamePrefix` to work.
### Why are the changes needed?
Currently, the configuration is broken like the following.
```
bin/spark-submit \
--master k8s://$K8S_MASTER \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
-c spark.kubernetes.container.image=spark:pr \
-c spark.kubernetes.driver.pod.name=mypod \
-c spark.kubernetes.executor.podNamePrefix=mypod \
local:///opt/spark/examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar
```
**BEFORE SPARK-31601**
```
pod/mypod 1/1 Running 0 9s
pod/spark-pi-7469dd71c499fafb-exec-1 1/1 Running 0 4s
pod/spark-pi-7469dd71c499fafb-exec-2 1/1 Running 0 4s
```
**AFTER SPARK-31601**
```
pod/mypod 1/1 Running 0 8s
pod/mypod-exec-1 1/1 Running 0 3s
pod/mypod-exec-2 1/1 Running 0 3s
```
### Does this PR introduce any user-facing change?
Yes. This is a bug fix. The conf will work as described in the documentation.
### How was this patch tested?
Pass the Jenkins and run the above comment manually.
Closes #28401 from dongjoon-hyun/SPARK-31601.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Prashant Sharma <pr...@in.ibm.com>
---
.../org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala | 1 -
.../apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala | 1 -
2 files changed, 2 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index eec275e..63f1812 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -154,7 +154,6 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
val additionalProps = mutable.Map(
KUBERNETES_DRIVER_POD_NAME.key -> driverPodName,
"spark.app.id" -> conf.appId,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> conf.resourceNamePrefix,
KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true",
MEMORY_OVERHEAD_FACTOR.key -> overheadFactor.toString)
// try upload local, resolvable files to a hadoop compatible file system
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index ef69600..c8c934b 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -124,7 +124,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
val expectedSparkConf = Map(
KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
"spark.app.id" -> KubernetesTestConf.APP_ID,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> kubernetesConf.resourceNamePrefix,
"spark.kubernetes.submitInDriver" -> "true",
MEMORY_OVERHEAD_FACTOR.key -> MEMORY_OVERHEAD_FACTOR.defaultValue.get.toString)
assert(featureStep.getAdditionalPodSystemProperties() === expectedSparkConf)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org