You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by mc...@apache.org on 2018/04/02 19:00:44 UTC
spark git commit: [SPARK-23825][K8S] Requesting memory + memory
overhead for pod memory
Repository: spark
Updated Branches:
refs/heads/master 44a9f8e6e -> 6151f29f9
[SPARK-23825][K8S] Requesting memory + memory overhead for pod memory
## What changes were proposed in this pull request?
Kubernetes driver and executor pods should request `memory + memoryOverhead` as their resources instead of just `memory`, see https://issues.apache.org/jira/browse/SPARK-23825
## How was this patch tested?
Existing unit tests were adapted.
Author: David Vogelbacher <dv...@palantir.com>
Closes #20943 from dvogelbacher/spark-23825.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6151f29f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6151f29f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6151f29f
Branch: refs/heads/master
Commit: 6151f29f9f589301159482044fc32717f430db6e
Parents: 44a9f8e
Author: David Vogelbacher <dv...@palantir.com>
Authored: Mon Apr 2 12:00:37 2018 -0700
Committer: mcheah <mc...@palantir.com>
Committed: Mon Apr 2 12:00:37 2018 -0700
----------------------------------------------------------------------
.../deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala | 5 +----
.../spark/scheduler/cluster/k8s/ExecutorPodFactory.scala | 5 +----
.../k8s/submit/steps/BasicDriverConfigurationStepSuite.scala | 2 +-
.../spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala | 6 ++++--
4 files changed, 7 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6151f29f/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala
index 347c4d2..b811db3 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStep.scala
@@ -93,9 +93,6 @@ private[spark] class BasicDriverConfigurationStep(
.withAmount(driverCpuCores)
.build()
val driverMemoryQuantity = new QuantityBuilder(false)
- .withAmount(s"${driverMemoryMiB}Mi")
- .build()
- val driverMemoryLimitQuantity = new QuantityBuilder(false)
.withAmount(s"${driverMemoryWithOverheadMiB}Mi")
.build()
val maybeCpuLimitQuantity = driverLimitCores.map { limitCores =>
@@ -117,7 +114,7 @@ private[spark] class BasicDriverConfigurationStep(
.withNewResources()
.addToRequests("cpu", driverCpuQuantity)
.addToRequests("memory", driverMemoryQuantity)
- .addToLimits("memory", driverMemoryLimitQuantity)
+ .addToLimits("memory", driverMemoryQuantity)
.addToLimits(maybeCpuLimitQuantity.toMap.asJava)
.endResources()
.addToArgs("driver")
http://git-wip-us.apache.org/repos/asf/spark/blob/6151f29f/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala
index 98cbd56..ac42385 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactory.scala
@@ -108,9 +108,6 @@ private[spark] class ExecutorPodFactory(
SPARK_ROLE_LABEL -> SPARK_POD_EXECUTOR_ROLE) ++
executorLabels
val executorMemoryQuantity = new QuantityBuilder(false)
- .withAmount(s"${executorMemoryMiB}Mi")
- .build()
- val executorMemoryLimitQuantity = new QuantityBuilder(false)
.withAmount(s"${executorMemoryWithOverhead}Mi")
.build()
val executorCpuQuantity = new QuantityBuilder(false)
@@ -167,7 +164,7 @@ private[spark] class ExecutorPodFactory(
.withImagePullPolicy(imagePullPolicy)
.withNewResources()
.addToRequests("memory", executorMemoryQuantity)
- .addToLimits("memory", executorMemoryLimitQuantity)
+ .addToLimits("memory", executorMemoryQuantity)
.addToRequests("cpu", executorCpuQuantity)
.endResources()
.addAllToEnv(executorEnv.asJava)
http://git-wip-us.apache.org/repos/asf/spark/blob/6151f29f/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStepSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStepSuite.scala
index ce06853..e59c6d2 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/steps/BasicDriverConfigurationStepSuite.scala
@@ -91,7 +91,7 @@ class BasicDriverConfigurationStepSuite extends SparkFunSuite {
val resourceRequirements = preparedDriverSpec.driverContainer.getResources
val requests = resourceRequirements.getRequests.asScala
assert(requests("cpu").getAmount === "2")
- assert(requests("memory").getAmount === "256Mi")
+ assert(requests("memory").getAmount === "456Mi")
val limits = resourceRequirements.getLimits.asScala
assert(limits("memory").getAmount === "456Mi")
assert(limits("cpu").getAmount === "4")
http://git-wip-us.apache.org/repos/asf/spark/blob/6151f29f/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala
index 7755b93..cee8fe2 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/scheduler/cluster/k8s/ExecutorPodFactorySuite.scala
@@ -66,13 +66,15 @@ class ExecutorPodFactorySuite extends SparkFunSuite with BeforeAndAfter with Bef
assert(executor.getMetadata.getLabels.size() === 3)
assert(executor.getMetadata.getLabels.get(SPARK_EXECUTOR_ID_LABEL) === "1")
- // There is exactly 1 container with no volume mounts and default memory limits.
- // Default memory limit is 1024M + 384M (minimum overhead constant).
+ // There is exactly 1 container with no volume mounts and default memory limits and requests.
+ // Default memory limit/request is 1024M + 384M (minimum overhead constant).
assert(executor.getSpec.getContainers.size() === 1)
assert(executor.getSpec.getContainers.get(0).getImage === executorImage)
assert(executor.getSpec.getContainers.get(0).getVolumeMounts.isEmpty)
assert(executor.getSpec.getContainers.get(0).getResources.getLimits.size() === 1)
assert(executor.getSpec.getContainers.get(0).getResources
+ .getRequests.get("memory").getAmount === "1408Mi")
+ assert(executor.getSpec.getContainers.get(0).getResources
.getLimits.get("memory").getAmount === "1408Mi")
// The pod has no node selector, volumes.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org