You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/26 17:56:35 UTC
(spark) branch branch-4.0 updated: Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new d827a742f520 Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"
d827a742f520 is described below
commit d827a742f5200bb57ac623053992549e7cc1feb1
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sat Sep 26 10:56:22 2026 -0700
Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"
This reverts commit 94d5185c3d0d2b921d7a2645e8a86b8961180483.
---
.../scala/org/apache/spark/deploy/k8s/Constants.scala | 1 -
.../deploy/k8s/features/BasicExecutorFeatureStep.scala | 1 -
.../k8s/features/BasicExecutorFeatureStepSuite.scala | 9 ---------
.../docker/src/main/dockerfiles/spark/entrypoint.sh | 4 ----
.../deploy/k8s/integrationtest/BasicTestsSuite.scala | 17 -----------------
5 files changed, 32 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 02ab9789db5e..aee07c096fe5 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -76,7 +76,6 @@ object Constants {
val ENV_EXECUTOR_ATTRIBUTE_EXECUTOR_ID = "SPARK_EXECUTOR_ATTRIBUTE_EXECUTOR_ID"
val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
val ENV_CLASSPATH = "SPARK_CLASSPATH"
- val ENV_EXECUTOR_LIBRARY_PATH = "SPARK_EXECUTOR_LIBRARY_PATH"
val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
val ENV_SPARK_USER = "SPARK_USER"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index 02f7b3d7d87d..20050de69f89 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -165,7 +165,6 @@ private[spark] class BasicExecutorFeatureStep(
++ kubernetesConf.environment
++ sparkAuthSecret
++ Seq(ENV_CLASSPATH -> kubernetesConf.get(EXECUTOR_CLASS_PATH).orNull)
- ++ Seq(ENV_EXECUTOR_LIBRARY_PATH -> kubernetesConf.get(EXECUTOR_LIBRARY_PATH).orNull)
++ allOpts) ++
KubernetesUtils.buildEnvVarsWithFieldRef(
Seq(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index e41a10bb143d..3ed6d50f689d 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -241,15 +241,6 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
checkOwnerReferences(executor.pod, DRIVER_POD_UID)
}
- test("SPARK-59798: extra library path gets translated into an environment variable") {
- baseConf.set(config.EXECUTOR_LIBRARY_PATH, "/opt/lib1:/opt/lib2")
- initDefaultProfile(baseConf)
- val step = new BasicExecutorFeatureStep(newExecutorConf(), defaultProfile)
- val executor = step.configurePod(SparkPod.initialPod())
-
- checkEnv(executor, baseConf, Map(ENV_EXECUTOR_LIBRARY_PATH -> "/opt/lib1:/opt/lib2"))
- }
-
test("SPARK-32655 Support appId/execId placeholder in SPARK_EXECUTOR_DIRS") {
val kconf = newExecutorConf(environment = Map(ENV_EXECUTOR_DIRS ->
"/p1/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID,/p2/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID"))
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index a2a7d67909a6..f9561b9aa4ed 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -91,10 +91,6 @@ case "$1" in
;;
executor)
shift 1
- # Prepend spark.executor.extraLibraryPath to the existing library path
- if [ -n "$SPARK_EXECUTOR_LIBRARY_PATH" ]; then
- export LD_LIBRARY_PATH="$SPARK_EXECUTOR_LIBRARY_PATH${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
- fi
CMD=(
${JAVA_HOME}/bin/java
"${SPARK_EXECUTOR_JAVA_OPTS[@]}"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 801972858bb5..0dafe30c364a 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -177,23 +177,6 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
}
})
}
-
- test("SPARK-59798: Support spark.executor.extraLibraryPath", k8sTestTag) {
- sparkAppConf
- .set("spark.executor.extraLibraryPath", "/opt/lib1:/opt/lib2")
- .set("spark.executor.extraJavaOptions", "-XshowSettings:properties")
- .set("spark.kubernetes.executor.deleteOnTermination", "false")
- runSparkApplicationAndVerifyCompletion(
- appResource = containerLocalSparkDistroExamplesJar,
- mainClass = SPARK_PI_MAIN_CLASS,
- expectedDriverLogOnCompletion = Seq("Pi is roughly 3"),
- // The executor JVM prepends LD_LIBRARY_PATH to java.library.path
- expectedExecutorLogOnCompletion = Seq("java.library.path = /opt/lib1"),
- appArgs = Array.empty[String],
- driverPodChecker = doBasicDriverPodCheck,
- executorPodChecker = doBasicExecutorPodCheck,
- isJVM = true)
- }
}
private[spark] object BasicTestsSuite extends SparkFunSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org