(spark-kubernetes-operator) branch main updated: [SPARK-56752] Enable `spark.kubernetes.driver.annotateExitException` by default

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new 82485a5  [SPARK-56752] Enable `spark.kubernetes.driver.annotateExitException` by default
82485a5 is described below

commit 82485a5c4ca6379a5c11045358713f2301105c8b
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Wed May 6 09:19:59 2026 -0700

    [SPARK-56752] Enable `spark.kubernetes.driver.annotateExitException` by default
    
    ### What changes were proposed in this pull request?
    
    This PR aims to default `spark.kubernetes.driver.annotateExitException` to `true` in `SparkAppSubmissionWorker.buildDriverConf()` when the user has not specified a value.
    
    ### Why are the changes needed?
    
    To improve out-of-the-box observability by annotating driver pods with exit exceptions, which is valuable for post-mortem debugging.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass the CIs with a new unit test.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Claude Opus 4.7
    
    Closes #659 from dongjoon-hyun/SPARK-56752.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../k8s/operator/SparkAppSubmissionWorker.java      |  1 +
 .../k8s/operator/SparkAppSubmissionWorkerTest.java  | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java
index ab3802e..8c048c9 100644
--- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java
+++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorker.java
@@ -167,6 +167,7 @@ public class SparkAppSubmissionWorker {
     effectiveSparkConf.setIfMissing("spark.app.id", appId);
     effectiveSparkConf.setIfMissing("spark.authenticate", "true");
     effectiveSparkConf.setIfMissing("spark.io.encryption.enabled", "true");
+    effectiveSparkConf.setIfMissing("spark.kubernetes.driver.annotateExitException", "true");
     // In case of static allocation, use K8s Garbage Collection instead of explicit API invocations
     if (!"true".equalsIgnoreCase(effectiveSparkConf.get("spark.dynamicAllocation.enabled", "false"))
         && applicationSpec.getApplicationTolerations().getResourceRetainPolicy() !=
diff --git a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java
index af71b4c..a6c89bb 100644
--- a/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java
+++ b/spark-submission-worker/src/test/java/org/apache/spark/k8s/operator/SparkAppSubmissionWorkerTest.java
@@ -320,4 +320,25 @@ class SparkAppSubmissionWorkerTest {
     SparkAppDriverConf conf3 = submissionWorker.buildDriverConf(mockApp, Map.of());
     assertEquals("false", conf3.get("spark.kubernetes.executor.deleteOnTermination", "true"));
   }
+
+  @Test
+  void annotateExitExceptionShouldDefaultToTrueWhenUnset() {
+    SparkApplication mockApp = mock(SparkApplication.class);
+    ApplicationSpec mockSpec = mock(ApplicationSpec.class);
+    ObjectMeta appMeta = new ObjectMetaBuilder().withName("app1").withNamespace("ns1").build();
+    when(mockSpec.getApplicationTolerations()).thenReturn(new ApplicationTolerations());
+    when(mockApp.getSpec()).thenReturn(mockSpec);
+    when(mockApp.getMetadata()).thenReturn(appMeta);
+
+    SparkAppSubmissionWorker submissionWorker = new SparkAppSubmissionWorker();
+
+    when(mockSpec.getSparkConf()).thenReturn(Map.of());
+    SparkAppDriverConf conf1 = submissionWorker.buildDriverConf(mockApp, Map.of());
+    assertEquals("true", conf1.get("spark.kubernetes.driver.annotateExitException", "false"));
+
+    when(mockSpec.getSparkConf())
+        .thenReturn(Map.of("spark.kubernetes.driver.annotateExitException", "false"));
+    SparkAppDriverConf conf2 = submissionWorker.buildDriverConf(mockApp, Map.of());
+    assertEquals("false", conf2.get("spark.kubernetes.driver.annotateExitException", "true"));
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org