You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/26 17:56:35 UTC

(spark) branch branch-4.0 updated: Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new d827a742f520 Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"
d827a742f520 is described below

commit d827a742f5200bb57ac623053992549e7cc1feb1
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sat Sep 26 10:56:22 2026 -0700

    Revert "[SPARK-59798][K8S] Support `spark.executor.extraLibraryPath` for K8s executors"
    
    This reverts commit 94d5185c3d0d2b921d7a2645e8a86b8961180483.
---
 .../scala/org/apache/spark/deploy/k8s/Constants.scala   |  1 -
 .../deploy/k8s/features/BasicExecutorFeatureStep.scala  |  1 -
 .../k8s/features/BasicExecutorFeatureStepSuite.scala    |  9 ---------
 .../docker/src/main/dockerfiles/spark/entrypoint.sh     |  4 ----
 .../deploy/k8s/integrationtest/BasicTestsSuite.scala    | 17 -----------------
 5 files changed, 32 deletions(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
index 02ab9789db5e..aee07c096fe5 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Constants.scala
@@ -76,7 +76,6 @@ object Constants {
   val ENV_EXECUTOR_ATTRIBUTE_EXECUTOR_ID = "SPARK_EXECUTOR_ATTRIBUTE_EXECUTOR_ID"
   val ENV_JAVA_OPT_PREFIX = "SPARK_JAVA_OPT_"
   val ENV_CLASSPATH = "SPARK_CLASSPATH"
-  val ENV_EXECUTOR_LIBRARY_PATH = "SPARK_EXECUTOR_LIBRARY_PATH"
   val ENV_DRIVER_BIND_ADDRESS = "SPARK_DRIVER_BIND_ADDRESS"
   val ENV_SPARK_CONF_DIR = "SPARK_CONF_DIR"
   val ENV_SPARK_USER = "SPARK_USER"
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index 02f7b3d7d87d..20050de69f89 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -165,7 +165,6 @@ private[spark] class BasicExecutorFeatureStep(
           ++ kubernetesConf.environment
           ++ sparkAuthSecret
           ++ Seq(ENV_CLASSPATH -> kubernetesConf.get(EXECUTOR_CLASS_PATH).orNull)
-          ++ Seq(ENV_EXECUTOR_LIBRARY_PATH -> kubernetesConf.get(EXECUTOR_LIBRARY_PATH).orNull)
           ++ allOpts) ++
       KubernetesUtils.buildEnvVarsWithFieldRef(
         Seq(
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index e41a10bb143d..3ed6d50f689d 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -241,15 +241,6 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
     checkOwnerReferences(executor.pod, DRIVER_POD_UID)
   }
 
-  test("SPARK-59798: extra library path gets translated into an environment variable") {
-    baseConf.set(config.EXECUTOR_LIBRARY_PATH, "/opt/lib1:/opt/lib2")
-    initDefaultProfile(baseConf)
-    val step = new BasicExecutorFeatureStep(newExecutorConf(), defaultProfile)
-    val executor = step.configurePod(SparkPod.initialPod())
-
-    checkEnv(executor, baseConf, Map(ENV_EXECUTOR_LIBRARY_PATH -> "/opt/lib1:/opt/lib2"))
-  }
-
   test("SPARK-32655 Support appId/execId placeholder in SPARK_EXECUTOR_DIRS") {
     val kconf = newExecutorConf(environment = Map(ENV_EXECUTOR_DIRS ->
       "/p1/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID,/p2/SPARK_APPLICATION_ID/SPARK_EXECUTOR_ID"))
diff --git a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
index a2a7d67909a6..f9561b9aa4ed 100755
--- a/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
+++ b/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/entrypoint.sh
@@ -91,10 +91,6 @@ case "$1" in
     ;;
   executor)
     shift 1
-    # Prepend spark.executor.extraLibraryPath to the existing library path
-    if [ -n "$SPARK_EXECUTOR_LIBRARY_PATH" ]; then
-      export LD_LIBRARY_PATH="$SPARK_EXECUTOR_LIBRARY_PATH${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
-    fi
     CMD=(
       ${JAVA_HOME}/bin/java
       "${SPARK_EXECUTOR_JAVA_OPTS[@]}"
diff --git a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
index 801972858bb5..0dafe30c364a 100644
--- a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
+++ b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/BasicTestsSuite.scala
@@ -177,23 +177,6 @@ private[spark] trait BasicTestsSuite { k8sSuite: KubernetesSuite =>
         }
       })
   }
-
-  test("SPARK-59798: Support spark.executor.extraLibraryPath", k8sTestTag) {
-    sparkAppConf
-      .set("spark.executor.extraLibraryPath", "/opt/lib1:/opt/lib2")
-      .set("spark.executor.extraJavaOptions", "-XshowSettings:properties")
-      .set("spark.kubernetes.executor.deleteOnTermination", "false")
-    runSparkApplicationAndVerifyCompletion(
-      appResource = containerLocalSparkDistroExamplesJar,
-      mainClass = SPARK_PI_MAIN_CLASS,
-      expectedDriverLogOnCompletion = Seq("Pi is roughly 3"),
-      // The executor JVM prepends LD_LIBRARY_PATH to java.library.path
-      expectedExecutorLogOnCompletion = Seq("java.library.path = /opt/lib1"),
-      appArgs = Array.empty[String],
-      driverPodChecker = doBasicDriverPodCheck,
-      executorPodChecker = doBasicExecutorPodCheck,
-      isJVM = true)
-  }
 }
 
 private[spark] object BasicTestsSuite extends SparkFunSuite {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org