You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/09/20 05:02:24 UTC
[spark] branch branch-3.3 updated: [SPARK-38802][K8S][TESTS] Add Support for `spark.kubernetes.test.(driver|executor)RequestCores`
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 883a481e44a [SPARK-38802][K8S][TESTS] Add Support for `spark.kubernetes.test.(driver|executor)RequestCores`
883a481e44a is described below
commit 883a481e44a1f91ef3fc3aea2838a598cbd6cf0f
Author: Yikun Jiang <yi...@gmail.com>
AuthorDate: Thu Apr 7 15:31:17 2022 -0700
[SPARK-38802][K8S][TESTS] Add Support for `spark.kubernetes.test.(driver|executor)RequestCores`
### What changes were proposed in this pull request?
This patch adds support for `spark.kubernetes.test.(driver|executor)RequestCores`, this help devs set specific cores info for (driver|executor)RequestCores.
### Why are the changes needed?
In some cases (such as resource limited case), we want to set specifc `RequestCores`.
See also: https://github.com/apache/spark/pull/35830#pullrequestreview-929597027
### Does this PR introduce _any_ user-facing change?
No, test only
### How was this patch tested?
IT passed
Closes #36087 from Yikun/SPARK-38802.
Authored-by: Yikun Jiang <yi...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit 83963828b54bffe99527a004057272bc584cbc26)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
resource-managers/kubernetes/integration-tests/README.md | 16 ++++++++++++++++
.../deploy/k8s/integrationtest/KubernetesSuite.scala | 6 ++++++
.../spark/deploy/k8s/integrationtest/TestConstants.scala | 2 ++
3 files changed, 24 insertions(+)
diff --git a/resource-managers/kubernetes/integration-tests/README.md b/resource-managers/kubernetes/integration-tests/README.md
index 748664cf41b..af0b1ec3dc7 100644
--- a/resource-managers/kubernetes/integration-tests/README.md
+++ b/resource-managers/kubernetes/integration-tests/README.md
@@ -268,6 +268,22 @@ to the wrapper scripts and using the wrapper scripts will simply set these appro
</td>
<td></td>
</tr>
+ <tr>
+ <td><code>spark.kubernetes.test.driverRequestCores</code></td>
+ <td>
+ Set cpu resource for each driver pod in test, this is currently only for test on cpu resource limited cluster,
+ it's not recommended for other scenarios.
+ </td>
+ <td></td>
+ </tr>
+ <tr>
+ <td><code>spark.kubernetes.test.executorRequestCores</code></td>
+ <td>
+ Set cpu resource for each executor pod in test, this is currently only for test on cpu resource limited cluster,
+ it's not recommended for other scenarios.
+ </td>
+ <td></td>
+ </tr>
</table>
# Running the Kubernetes Integration Tests with SBT
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
index 3d7a9313031..041b2886c41 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/KubernetesSuite.scala
@@ -192,6 +192,12 @@ class KubernetesSuite extends SparkFunSuite
.set("spark.kubernetes.driver.label.spark-app-locator", appLocator)
.set("spark.kubernetes.executor.label.spark-app-locator", appLocator)
.set(NETWORK_AUTH_ENABLED.key, "true")
+ sys.props.get(CONFIG_DRIVER_REQUEST_CORES).map { cpu =>
+ sparkAppConf.set("spark.kubernetes.driver.request.cores", cpu)
+ }
+ sys.props.get(CONFIG_EXECUTOR_REQUEST_CORES).map { cpu =>
+ sparkAppConf.set("spark.kubernetes.executor.request.cores", cpu)
+ }
if (!kubernetesTestComponents.hasUserSpecifiedNamespace) {
kubernetesTestComponents.createNamespace()
}
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala
index c46839f1dff..2175d23d449 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/TestConstants.scala
@@ -34,4 +34,6 @@ object TestConstants {
val CONFIG_KEY_IMAGE_TAG_FILE = "spark.kubernetes.test.imageTagFile"
val CONFIG_KEY_IMAGE_REPO = "spark.kubernetes.test.imageRepo"
val CONFIG_KEY_UNPACK_DIR = "spark.kubernetes.test.unpackSparkDir"
+ val CONFIG_DRIVER_REQUEST_CORES = "spark.kubernetes.test.driverRequestCores"
+ val CONFIG_EXECUTOR_REQUEST_CORES = "spark.kubernetes.test.executorRequestCores"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org