You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/28 01:10:17 UTC
(spark) branch branch-4.3 updated: [SPARK-59798][4.3][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 1cee0c27a67e [SPARK-59798][4.3][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
1cee0c27a67e is described below
commit 1cee0c27a67ee6d9568a882a14d9afe61ec2d5e9
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Sep 27 18:10:04 2026 -0700
[SPARK-59798][4.3][K8S] Support `spark.executor.extraLibraryPath` for K8s executors
### What changes were proposed in this pull request?
This PR is a backport of #59070 to `branch-4.3`.
This PR aims to support `spark.executor.extraLibraryPath` for K8s executors.
- `BasicExecutorFeatureStep` passes it to the executor container as `SPARK_EXECUTOR_LIBRARY_PATH`.
- `entrypoint.sh` prepends `SPARK_EXECUTOR_LIBRARY_PATH` to `LD_LIBRARY_PATH` before launching the executor JVM.
The previous cherry-pick to `branch-4.3` was reverted because the new unit test failed to compile. Unlike `master` and `branch-4.x`, `BasicExecutorFeatureStep` of `branch-4.3` still takes `SecurityManager` because SPARK-59479 (#58768) is not in `branch-4.3`. So, the new unit test passes `new SecurityManager(baseConf)` like the other test cases of `BasicExecutorFeatureStepSuite`. This is the only difference from the original commit.
### Why are the changes needed?
Currently, only Standalone and YARN apply `spark.executor.extraLibraryPath`, and K8s silently ignores it. Like Standalone and YARN, this PR prepends the paths to the existing `LD_LIBRARY_PATH` so that the value from the image or `spark.executorEnv.LD_LIBRARY_PATH` is preserved.
In addition, `apache/spark-docker` already supports `SPARK_EXECUTOR_LIBRARY_PATH` in `entrypoint.sh.template` via SPARK-59799 (apache/spark-docker#134). Since `add-dockerfiles.sh` generates `entrypoint.sh` of new Spark versions from this template, this PR makes `branch-4.3` consistent with its upcoming official docker images.
### Does this PR introduce _any_ user-facing change?
Yes. `spark.executor.extraLibraryPath` takes effect for K8s executors when the image uses Spark's `entrypoint.sh`. Previously, it was silently ignored.
### How was this patch tested?
Pass the CIs with the newly added unit test and K8s integration test.
In addition, the new unit test and the compilation of K8s integration tests are verified locally.
```
$ build/sbt -Pkubernetes -Pkubernetes-integration-tests 'kubernetes/testOnly *BasicExecutorFeatureStepSuite' 'kubernetes-integration-tests/Test/compile'
...
[info] - SPARK-59798: extra library path gets translated into an environment variable (11 milliseconds)
...
[info] Tests: succeeded 40, failed 0, canceled 0, ignored 0, pending 0
[info] All tests passed.
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 5.5
Closes #59084 from dongjoon-hyun/SPARK-59798-4.3.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../scala/org/apache/spark/deploy/k8s/Constants.scala | 1 +
.../deploy/k8s/features/BasicExecutorFeatureStep.scala | 1 +
.../k8s/features/BasicExecutorFeatureStepSuite.scala | 10 ++++++++++
.../docker/src/main/dockerfiles/spark/entrypoint.sh | 4 ++++
.../deploy/k8s/integrationtest/BasicTestsSuite.scala | 17 +++++++++++++++++
5 files changed, 33 insertions(+)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 4ffd617c4e4c..6d131cc0c2f3 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -78,6 +78,7 @@ object Constants {
val ENV_EXECUTOR_ATTRIBUTE_EXECUTOR_ID = "SPARK_EXECUTOR_ATTRIBUTE_EXECUTOR_ID"
val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
val ENV_CLASSPATH = "SPARK_CLASSPATH"
+ val ENV_EXECUTOR_LIBRARY_PATH = "SPARK_EXECUTOR_LIBRARY_PATH"
val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
val ENV_SPARK_USER = "SPARK_USER"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index 9eb9303fa0dd..f38dd2461248 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -187,6 +187,7 @@ private[spark] class BasicExecutorFeatureStep(
++ kubernetesConf.environment
++ sparkAuthSecret
++ Seq(ENV_CLASSPATH -> kubernetesConf.get(EXECUTOR_CLASS_PATH).orNull)
+ ++ Seq(ENV_EXECUTOR_LIBRARY_PATH -> kubernetesConf.get(EXECUTOR_LIBRARY_PATH).orNull)
++ allOpts) ++
KubernetesUtils.buildEnvVarsWithFieldRef(
Seq(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index ba5aa35985da..5662e410e551 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -329,6 +329,16 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
checkOwnerReferences(executor.pod, DRIVER_POD_UID)
}
+ test("SPARK-59798: extra library path gets translated into an environment variable") {
+ baseConf.set(config.EXECUTOR_LIBRARY_PATH, "/opt/lib1:/opt/lib2")
+ initDefaultProfile(baseConf)
+ val step = new BasicExecutorFeatureStep(newExecutorConf(), new SecurityManager(baseConf),
+ defaultProfile)
+ val executor = step.configurePod(SparkPod.initialPod())
+
+ checkEnv(executor, baseConf, Map(ENV_EXECUTOR_LIBRARY_PATH -> "/opt/lib1:/opt/lib2"))
+ }
+
test("SPARK-32655 Support appId/execId placeholder in SPARK_EXECUTOR_DIRS") {
val kconf = newExecutorConf(environment = Map(ENV_EXECUTOR_DIRS ->
"/p1/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID,/p2/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID"))
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index a2ae7e356694..c0dee3165eaf 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -89,6 +89,10 @@ case "$1" in
;;
executor)
shift 1
+ # Prepend spark.executor.extraLibraryPath to the existing library path
+ if [ -n "$SPARK_EXECUTOR_LIBRARY_PATH" ]; then
+ export LD_LIBRARY_PATH="$SPARK_EXECUTOR_LIBRARY_PATH${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+ fi
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 22295dcaa2af..e9c821f03683 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -185,6 +185,23 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
}
})
}
+
+ test("SPARK-59798: Support spark.executor.extraLibraryPath", k8sTestTag) {
+ sparkAppConf
+ .set("spark.executor.extraLibraryPath", "/opt/lib1:/opt/lib2")
+ .set("spark.executor.extraJavaOptions", "-XshowSettings:properties")
+ .set("spark.kubernetes.executor.deleteOnTermination", "false")
+ runSparkApplicationAndVerifyCompletion(
+ appResource = containerLocalSparkDistroExamplesJar,
+ mainClass = SPARK_PI_MAIN_CLASS,
+ expectedDriverLogOnCompletion = Seq("Pi is roughly 3"),
+ // The executor JVM prepends LD_LIBRARY_PATH to java.library.path
+ expectedExecutorLogOnCompletion = Seq("java.library.path = /opt/lib1"),
+ appArgs = Array.empty[String],
+ driverPodChecker = doBasicDriverPodCheck,
+ executorPodChecker = doBasicExecutorPodCheck,
+ isJVM = true)
+ }
}
private[spark] object BasicTestsSuite extends SparkFunSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org