(spark-kubernetes-operator) branch main updated: [SPARK-54092] Use Java-friendly `KubernetesClientUtils` APIs

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-kubernetes-operator.git


The following commit(s) were added to refs/heads/main by this push:
     new c70a95f  [SPARK-54092] Use Java-friendly `KubernetesClientUtils` APIs
c70a95f is described below

commit c70a95f83dc8381b6c7096d1041d061dcb63f461
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Thu Oct 30 09:07:33 2025 -0700

    [SPARK-54092] Use Java-friendly `KubernetesClientUtils` APIs
    
    ### What changes were proposed in this pull request?
    
    This PR aims to use Java-friendly `KubernetesClientUtils` APIs.
    
    ### Why are the changes needed?
    
    Apache Spark 4.1.0-preview3 introduced new Java friendly APIs. We had better take advantage of them.
    - https://github.com/apache/spark/pull/52542
    
    | Scala Version | New Java-friendly Version |
    | - | - |
    | `buildConfigMap` (Since 3.1.0) | `buildConfigMapJava` (Since 4.1.0) |
    | `buildKeyToPathObjects` (Since 3.1.0) | `buildKeyToPathObjectsJava` (Since 4.1.0) |
    | `buildSparkConfDirFilesMap` (Since 3.1.1) | `buildSparkConfDirFilesMapJava` (Since 4.1.0) |
    
    ### Does this PR introduce _any_ user-facing change?
    
    No behavior change.
    
    ### How was this patch tested?
    
    Pass the CIs.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #410 from dongjoon-hyun/SPARK-54092.
    
    Authored-by: Dongjoon Hyun <do...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../spark/k8s/operator/SparkAppResourceSpec.java   | 37 ++++++++--------------
 1 file changed, 13 insertions(+), 24 deletions(-)

diff --git a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
index 39ed3f0..51b305f 100644
--- a/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
+++ b/spark-submission-worker/src/main/java/org/apache/spark/k8s/operator/SparkAppResourceSpec.java
@@ -19,15 +19,10 @@
 
 package org.apache.spark.k8s.operator;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.stream.Collectors;
+import static scala.jdk.javaapi.CollectionConverters.asJava;
 
-import scala.Tuple2;
-import scala.collection.immutable.HashMap;
-import scala.collection.immutable.Map;
-import scala.jdk.CollectionConverters;
+import java.util.*;
+import java.util.stream.Collectors;
 
 import io.fabric8.kubernetes.api.model.Container;
 import io.fabric8.kubernetes.api.model.ContainerBuilder;
@@ -79,12 +74,13 @@ public class SparkAppResourceSpec {
       List<ConfigMapSpec> configMapSpecs) {
     this.kubernetesDriverConf = kubernetesDriverConf;
     String namespace = kubernetesDriverConf.sparkConf().get(Config.KUBERNETES_NAMESPACE().key());
-    Map<String, String> confFilesMap =
-        KubernetesClientUtils.buildSparkConfDirFilesMap(
+    Map<String, String> originalConfFilesMap =
+        KubernetesClientUtils.buildSparkConfDirFilesMapJava(
                 kubernetesDriverConf.configMapNameDriver(),
                 kubernetesDriverConf.sparkConf(),
-                kubernetesDriverSpec.systemProperties())
-            .$plus(new Tuple2<>(Config.KUBERNETES_NAMESPACE().key(), namespace));
+                asJava(kubernetesDriverSpec.systemProperties()));
+    Map<String, String> confFilesMap = new HashMap<>(originalConfFilesMap);
+    confFilesMap.put(Config.KUBERNETES_NAMESPACE().key(), namespace);
     SparkPod sparkPod = addConfigMap(kubernetesDriverSpec.pod(), confFilesMap);
     this.configuredPod =
         new PodBuilder(sparkPod.pod())
@@ -93,16 +89,12 @@ public class SparkAppResourceSpec {
             .endSpec()
             .build();
     this.driverPreResources =
-        new ArrayList<>(
-            CollectionConverters.SeqHasAsJava(kubernetesDriverSpec.driverPreKubernetesResources())
-                .asJava());
+        new ArrayList<>(asJava(kubernetesDriverSpec.driverPreKubernetesResources()));
     this.driverResources =
-        new ArrayList<>(
-            CollectionConverters.SeqHasAsJava(kubernetesDriverSpec.driverKubernetesResources())
-                .asJava());
+        new ArrayList<>(asJava(kubernetesDriverSpec.driverKubernetesResources()));
     this.driverResources.add(
-        KubernetesClientUtils.buildConfigMap(
-            kubernetesDriverConf.configMapNameDriver(), confFilesMap, new HashMap<>()));
+        KubernetesClientUtils.buildConfigMapJava(
+            kubernetesDriverConf.configMapNameDriver(), confFilesMap, Map.of()));
     this.driverPreResources.addAll(ConfigMapSpecUtils.buildConfigMaps(configMapSpecs));
     this.driverResources.addAll(configureDriverServerIngress(sparkPod, driverServiceIngressList));
     this.driverPreResources.forEach(r -> setNamespaceIfMissing(r, namespace));
@@ -147,10 +139,7 @@ public class SparkAppResourceSpec {
             .addNewVolume()
             .withName(Constants.SPARK_CONF_VOLUME_DRIVER())
             .withNewConfigMap()
-            .withItems(
-                CollectionConverters.SeqHasAsJava(
-                        KubernetesClientUtils.buildKeyToPathObjects(confFilesMap))
-                    .asJava())
+            .withItems(KubernetesClientUtils.buildKeyToPathObjectsJava(confFilesMap))
             .withName(kubernetesDriverConf.configMapNameDriver())
             .endConfigMap()
             .endVolume()


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org