You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pr...@apache.org on 2020/04/30 03:47:09 UTC
[spark] branch branch-3.0 updated: [SPARK-31601][K8S] Fix
spark.kubernetes.executor.podNamePrefix to work
This is an automated email from the ASF dual-hosted git repository.
prashant pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 82b8f7f [SPARK-31601][K8S] Fix spark.kubernetes.executor.podNamePrefix to work
82b8f7f is described below
commit 82b8f7fc9d21f4ac506d8cd613158f0511f5cb1d
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Apr 30 09:15:12 2020 +0530
[SPARK-31601][K8S] Fix spark.kubernetes.executor.podNamePrefix to work
### What changes were proposed in this pull request?
This PR aims to fix `spark.kubernetes.executor.podNamePrefix` to work.
### Why are the changes needed?
Currently, the configuration is broken like the following.
```
bin/spark-submit \
--master k8s://$K8S_MASTER \
--deploy-mode cluster \
--name spark-pi \
--class org.apache.spark.examples.SparkPi \
-c spark.kubernetes.container.image=spark:pr \
-c spark.kubernetes.driver.pod.name=mypod \
-c spark.kubernetes.executor.podNamePrefix=mypod \
local:///opt/spark/examples/jars/spark-examples_2.12-3.1.0-SNAPSHOT.jar
```
**BEFORE SPARK-31601**
```
pod/mypod 1/1 Running 0 9s
pod/spark-pi-7469dd71c499fafb-exec-1 1/1 Running 0 4s
pod/spark-pi-7469dd71c499fafb-exec-2 1/1 Running 0 4s
```
**AFTER SPARK-31601**
```
pod/mypod 1/1 Running 0 8s
pod/mypod-exec-1 1/1 Running 0 3s
pod/mypod-exec-2 1/1 Running 0 3s
```
### Does this PR introduce any user-facing change?
Yes. This is a bug fix. The conf will work as described in the documentation.
### How was this patch tested?
Pass the Jenkins and run the above comment manually.
Closes #28401 from dongjoon-hyun/SPARK-31601.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Prashant Sharma <pr...@in.ibm.com>
(cherry picked from commit 85dad37f69ebb617c8ac015dbbbda11054170298)
Signed-off-by: Prashant Sharma <pr...@in.ibm.com>
---
.../org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala | 1 -
.../apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala | 1 -
2 files changed, 2 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
index e2d70d7..1edd3f7 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStep.scala
@@ -158,7 +158,6 @@ private[spark] class BasicDriverFeatureStep(conf: KubernetesDriverConf)
val additionalProps = mutable.Map(
KUBERNETES_DRIVER_POD_NAME.key -> driverPodName,
"spark.app.id" -> conf.appId,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> conf.resourceNamePrefix,
KUBERNETES_DRIVER_SUBMIT_CHECK.key -> "true",
MEMORY_OVERHEAD_FACTOR.key -> overheadFactor.toString)
// try upload local, resolvable files to a hadoop compatible file system
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
index ce66afd..89e7ff9 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicDriverFeatureStepSuite.scala
@@ -124,7 +124,6 @@ class BasicDriverFeatureStepSuite extends SparkFunSuite {
val expectedSparkConf = Map(
KUBERNETES_DRIVER_POD_NAME.key -> "spark-driver-pod",
"spark.app.id" -> KubernetesTestConf.APP_ID,
- KUBERNETES_EXECUTOR_POD_NAME_PREFIX.key -> kubernetesConf.resourceNamePrefix,
"spark.kubernetes.submitInDriver" -> "true",
MEMORY_OVERHEAD_FACTOR.key -> MEMORY_OVERHEAD_FACTOR.defaultValue.get.toString)
assert(featureStep.getAdditionalPodSystemProperties() === expectedSparkConf)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org