You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/14 18:24:53 UTC
(spark) branch branch-4.x updated: [SPARK-59500][K8S] Promote `KubernetesExecutorConf` to `DeveloperApi`
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new fa05c685cd86 [SPARK-59500][K8S] Promote `KubernetesExecutorConf` to `DeveloperApi`
fa05c685cd86 is described below
commit fa05c685cd86ade20285e337844819413e9ede06
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Sep 14 11:24:35 2026 -0700
[SPARK-59500][K8S] Promote `KubernetesExecutorConf` to `DeveloperApi`
### What changes were proposed in this pull request?
This PR aims to promote `KubernetesExecutorConf` to `DeveloperApi` and `Unstable` with a Java-friendly constructor, symmetric to `KubernetesDriverConf`.
### Why are the changes needed?
`KubernetesDriverConf` was promoted to `DeveloperApi` at Apache Spark 4.0.0 via SPARK-48121, and its Java-friendly constructor was added at 4.3.0 via SPARK-58333.
https://github.com/apache/spark/blob/389de941f002a5c92e22dc3ed0f65af602a174db/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala#L93-L96
`KubernetesExecutorConf` is still `private[spark]` although it has been exposed to users since 3.3.0 via `KubernetesExecutorCustomFeatureConfigStep.init(config: KubernetesExecutorConf)`. This PR promotes it in the same way to provide a Java-friendly `DeveloperApi` for external projects such as the Apache Spark Kubernetes Operator.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs with a newly added test case in `KubernetesConfSuite`.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5.1
Closes #58783 from dongjoon-hyun/SPARK-59500.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
(cherry picked from commit eb4c1880f6347e51b0f485ee12e6c1f7087deb9f)
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../apache/spark/deploy/k8s/KubernetesConf.scala | 25 ++++++++++++++++++-
.../spark/deploy/k8s/KubernetesConfSuite.scala | 29 +++++++++++++++++++++-
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
index df85ef5e5856..7e5a86e024e6 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/KubernetesConf.scala
@@ -202,7 +202,15 @@ class KubernetesDriverConf(
}
}
-private[spark] class KubernetesExecutorConf(
+/**
+ * :: DeveloperApi ::
+ *
+ * Used for K8s operations internally and Spark K8s operator.
+ */
+@Unstable
+@DeveloperApi
+@Since("4.4.0")
+class KubernetesExecutorConf(
sparkConf: SparkConf,
val appId: String,
val executorId: String,
@@ -211,6 +219,21 @@ private[spark] class KubernetesExecutorConf(
customAuthSecret: Option[String] = None)
extends KubernetesConf(sparkConf) with Logging {
+ /**
+ * Java-friendly constructor that accepts a nullable Pod for driverPod and
+ * a nullable String for customAuthSecret instead of Option.
+ */
+ @Since("4.4.0")
+ def this(
+ sparkConf: SparkConf,
+ appId: String,
+ executorId: String,
+ driverPod: Pod,
+ resourceProfileId: Int,
+ customAuthSecret: String) =
+ this(sparkConf, appId, executorId, Option(driverPod), resourceProfileId,
+ Option(customAuthSecret))
+
def authSecret: Option[String] = {
customAuthSecret
.orElse(get(AUTH_SECRET))
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
index 7b57ff8d65f9..8a70927b30e6 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/KubernetesConfSuite.scala
@@ -17,7 +17,7 @@
package org.apache.spark.deploy.k8s
-import io.fabric8.kubernetes.api.model.{LocalObjectReferenceBuilder, PodBuilder}
+import io.fabric8.kubernetes.api.model.{LocalObjectReferenceBuilder, Pod, PodBuilder}
import org.apache.spark.{SPARK_VERSION, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.k8s.Config._
@@ -163,6 +163,33 @@ class KubernetesConfSuite extends SparkFunSuite {
assert(conf.authSecret.isEmpty)
}
+ test("Java-friendly KubernetesExecutorConf constructor with nullable driverPod and secret") {
+ val sparkConf = new SparkConf(false)
+ val conf = new KubernetesExecutorConf(
+ sparkConf,
+ KubernetesTestConf.APP_ID,
+ EXECUTOR_ID,
+ null: Pod,
+ DEFAULT_RESOURCE_PROFILE_ID,
+ null: String)
+ assert(conf.appId === KubernetesTestConf.APP_ID)
+ assert(conf.executorId === EXECUTOR_ID)
+ assert(conf.driverPod === None)
+ assert(conf.resourceProfileId === DEFAULT_RESOURCE_PROFILE_ID)
+ assert(conf.authSecret === None)
+
+ val confWithValues = new KubernetesExecutorConf(
+ sparkConf,
+ KubernetesTestConf.APP_ID,
+ EXECUTOR_ID,
+ DRIVER_POD,
+ 3,
+ "custom-secret")
+ assert(confWithValues.driverPod === Some(DRIVER_POD))
+ assert(confWithValues.resourceProfileId === 3)
+ assert(confWithValues.authSecret === Some("custom-secret"))
+ }
+
test("KubernetesExecutorConf authSecret resolution.") {
val confWithoutSecret = KubernetesConf.createExecutorConf(
new SparkConf(false),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org