You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/26 16:27:26 UTC
(spark) branch branch-4.2 updated: [SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new e5dd1ca380fe [SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
e5dd1ca380fe is described below
commit e5dd1ca380fe5469f3d6279d094f73aae76fbe9b
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sat Sep 26 09:26:47 2026 -0700
[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
### What changes were proposed in this pull request?
This PR aims to support `spark.executor.extraLibraryPath` for K8s executors.
- `BasicExecutorFeatureStep` passes it to the executor container as `SPARK_EXECUTOR_LIBRARY_PATH`.
- `entrypoint.sh` prepends `SPARK_EXECUTOR_LIBRARY_PATH` to `LD_LIBRARY_PATH` before launching the executor JVM.
### Why are the changes needed?
Currently, only Standalone and YARN apply `spark.executor.extraLibraryPath`, and K8s silently ignores it. Like Standalone and YARN, this PR prepends the paths to the existing `LD_LIBRARY_PATH` so that the value from the image or `spark.executorEnv.LD_LIBRARY_PATH` is preserved.
### Does this PR introduce _any_ user-facing change?
Yes. `spark.executor.extraLibraryPath` takes effect for K8s executors when the image uses Spark's `entrypoint.sh`. The official `apache/spark` images will be updated separately in the `apache/spark-docker` repository.
### How was this patch tested?
Pass the CIs with the newly added unit test and K8s integration test.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59070 from dongjoon-hyun/SPARK-59798.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit f5498a9dab976099d83f5230f7600b52960b63ce)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../scala/org/apache/spark/deploy/k8s/Constants.scala | 1 +
.../deploy/k8s/features/BasicExecutorFeatureStep.scala | 1 +
.../k8s/features/BasicExecutorFeatureStepSuite.scala | 9 +++++++++
.../docker/src/main/dockerfiles/spark/entrypoint.sh | 4 ++++
.../deploy/k8s/integrationtest/BasicTestsSuite.scala | 17 +++++++++++++++++
5 files changed, 32 insertions(+)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 4ffd617c4e4c..6d131cc0c2f3 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -78,6 +78,7 @@ object Constants {
val ENV_EXECUTOR_ATTRIBUTE_EXECUTOR_ID = "SPARK_EXECUTOR_ATTRIBUTE_EXECUTOR_ID"
val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
val ENV_CLASSPATH = "SPARK_CLASSPATH"
+ val ENV_EXECUTOR_LIBRARY_PATH = "SPARK_EXECUTOR_LIBRARY_PATH"
val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
val ENV_SPARK_USER = "SPARK_USER"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index 655df6c5acad..adabe5edc42b 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -181,6 +181,7 @@ private[spark] class BasicExecutorFeatureStep(
++ kubernetesConf.environment
++ sparkAuthSecret
++ Seq(ENV_CLASSPATH -> kubernetesConf.get(EXECUTOR_CLASS_PATH).orNull)
+ ++ Seq(ENV_EXECUTOR_LIBRARY_PATH -> kubernetesConf.get(EXECUTOR_LIBRARY_PATH).orNull)
++ allOpts) ++
KubernetesUtils.buildEnvVarsWithFieldRef(
Seq(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index 77dcb11ba43b..e4d75d387f11 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -303,6 +303,15 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
checkOwnerReferences(executor.pod, DRIVER_POD_UID)
}
+ test("SPARK-59798: extra library path gets translated into an environment variable") {
+ baseConf.set(config.EXECUTOR_LIBRARY_PATH, "/opt/lib1:/opt/lib2")
+ initDefaultProfile(baseConf)
+ val step = new BasicExecutorFeatureStep(newExecutorConf(), defaultProfile)
+ val executor = step.configurePod(SparkPod.initialPod())
+
+ checkEnv(executor, baseConf, Map(ENV_EXECUTOR_LIBRARY_PATH -> "/opt/lib1:/opt/lib2"))
+ }
+
test("SPARK-32655 Support appId/execId placeholder in SPARK_EXECUTOR_DIRS") {
val kconf = newExecutorConf(environment = Map(ENV_EXECUTOR_DIRS ->
"/p1/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID,/p2/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID"))
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index a2ae7e356694..c0dee3165eaf 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -89,6 +89,10 @@ case "$1" in
;;
executor)
shift 1
+ # Prepend spark.executor.extraLibraryPath to the existing library path
+ if [ -n "$SPARK_EXECUTOR_LIBRARY_PATH" ]; then
+ export LD_LIBRARY_PATH="$SPARK_EXECUTOR_LIBRARY_PATH${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ fi
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 22295dcaa2af..e9c821f03683 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -185,6 +185,23 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
}
})
}
+
+ test("SPARK-59798: Support spark.executor.extraLibraryPath", k8sTestTag) {
+ sparkAppConf
+ .set("spark.executor.extraLibraryPath", "/opt/lib1:/opt/lib2")
+ .set("spark.executor.extraJavaOptions", "-XshowSettings:properties")
+ .set("spark.kubernetes.executor.deleteOnTermination", "false")
+ runSparkApplicationAndVerifyCompletion(
+ appResource = containerLocalSparkDistroExamplesJar,
+ mainClass = SPARK_PI_MAIN_CLASS,
+ expectedDriverLogOnCompletion = Seq("Pi is roughly 3"),
+ // The executor JVM prepends LD_LIBRARY_PATH to java.library.path
+ expectedExecutorLogOnCompletion = Seq("java.library.path = /opt/lib1"),
+ appArgs = Array.empty[String],
+ driverPodChecker = doBasicDriverPodCheck,
+ executorPodChecker = doBasicExecutorPodCheck,
+ isJVM = true)
+ }
}
private[spark] object BasicTestsSuite extends SparkFunSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org