(spark) branch branch-4.x updated: [SPARK-59666][K8S] Avoid creating a K8s client when there is no driver pod

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new e11d61b6dafe [SPARK-59666][K8S] Avoid creating a K8s client when there is no driver pod
e11d61b6dafe is described below

commit e11d61b6dafedc2ad14649ba0e5da65ae6b6690e
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Mon Sep 21 07:41:59 2026 -0700

    [SPARK-59666][K8S] Avoid creating a K8s client when there is no driver pod
    
    ### What changes were proposed in this pull request?
    
    This PR aims to check `spark.kubernetes.driver.pod.name` before creating a Kubernetes client in `SparkKubernetesDiagnosticsSetter`, instead of after.
    
    ### Why are the changes needed?
    
    The driver pod name is required to patch the `spark.exit-exception` annotation, and it is unset in a client deploy mode submission. In that case, the current code creates a Kubernetes client and closes it without patching anything.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass the CIs with the newly added test case.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 5
    
    Closes #58930 from dongjoon-hyun/SPARK-59666.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit a9cf2c11b6f3df7d3294c67f6334b46c7f8395ab)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../spark/deploy/k8s/SparkKubernetesDiagnosticsSetter.scala   | 10 +++++-----
 .../deploy/k8s/SparkKubernetesDiagnosticsSetterSuite.scala    | 11 +++++++++++
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetter.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetter.scala
index 433b0228c813..87c455c1c13d 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetter.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetter.scala
@@ -64,11 +64,11 @@ private[spark] class SparkKubernetesDiagnosticsSetter(clientProvider: Kubernetes
     // Skip it so that a submission failure (e.g. pod creation conflict) does not overwrite
     // the annotation of an existing driver pod with the same name.
     if (conf.get(SUBMIT_DEPLOY_MODE) != "cluster") {
-      val diagnostics = SparkStringUtils.abbreviate(
-        Utils.redact(conf.get(STRING_REDACTION_PATTERN), Utils.stringifyException(throwable)),
-        KUBERNETES_EXIT_EXCEPTION_MESSAGE_LIMIT_BYTES)
-      Utils.tryWithResource(clientProvider.create(conf)) { client =>
-        conf.get(KUBERNETES_DRIVER_POD_NAME).foreach { podName =>
+      conf.get(KUBERNETES_DRIVER_POD_NAME).foreach { podName =>
+        val diagnostics = SparkStringUtils.abbreviate(
+          Utils.redact(conf.get(STRING_REDACTION_PATTERN), Utils.stringifyException(throwable)),
+          KUBERNETES_EXIT_EXCEPTION_MESSAGE_LIMIT_BYTES)
+        Utils.tryWithResource(clientProvider.create(conf)) { client =>
           client.pods()
             .inNamespace(conf.get(KUBERNETES_NAMESPACE))
             .withName(podName)
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetterSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetterSuite.scala
index 8a01c3814bbb..1166f2d73c74 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetterSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/SparkKubernetesDiagnosticsSetterSuite.scala
@@ -113,4 +113,15 @@ class SparkKubernetesDiagnosticsSetterSuite extends SparkFunSuite
     verify(clientProvider, never()).create(any(classOf[SparkConf]))
     verifyNoInteractions(driverPodOperations)
   }
+
+  test("SPARK-59666: setDiagnostics should not create a client without a driver pod name") {
+    val diagnostics = new Throwable("Fake diagnostics stack trace")
+    val conf = new SparkConf()
+      .set(KUBERNETES_DRIVER_MASTER_URL, k8sClusterManagerUrl)
+      .set(KUBERNETES_NAMESPACE, namespace)
+
+    setter.setDiagnostics(diagnostics, conf)
+
+    verify(clientProvider, never()).create(any(classOf[SparkConf]))
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org