[incubator-kyuubi] branch branch-1.6 updated: [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark application

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/branch-1.6 by this push:
     new 7c5ca112a [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark application
7c5ca112a is described below

commit 7c5ca112a6f2a10a923a232f7de0d5048d27eadd
Author: zwangsheng <22...@qq.com>
AuthorDate: Thu Oct 20 13:29:21 2022 +0800

    [KYUUBI #3670] [SPARK][K8S] Fix Spark master match rule on tagging Spark application
    
    ### _Why are the changes needed?_
    
    Fix kyuubi application manager tag spark kubernetes application with yarn tag.
    
    The reason for this is the need to determine whether resourceManager start with K8S instead of being equal to K8S.
    
    Remove duplicate code from spark batch process builder.
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #3670 from zwangsheng/bugfix/kyuubi_application_manager_tag_application.
    
    Closes #3670
    
    d4e8aa30 [zwangsheng] fix
    7b5419eb [zwangsheng] back
    a976f994 [zwangsheng] fix
    
    Authored-by: zwangsheng <22...@qq.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
    (cherry picked from commit cebbd230b080db32a8ae995e0cc8826775024ee4)
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../apache/kyuubi/engine/KyuubiApplicationManager.scala  |  2 +-
 .../kyuubi/engine/KyuubiApplicationManagerSuite.scala    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
index 659e9d23a..b26c8be3f 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
@@ -160,7 +160,7 @@ object KyuubiApplicationManager {
       conf: KyuubiConf): Unit = {
     (applicationType.toUpperCase, resourceManager.map(_.toUpperCase())) match {
       case ("SPARK", Some("YARN")) => setupSparkYarnTag(applicationTag, conf)
-      case ("SPARK", Some("K8S")) => setupSparkK8sTag(applicationTag, conf)
+      case ("SPARK", Some(rm)) if rm.startsWith("K8S") => setupSparkK8sTag(applicationTag, conf)
       case ("SPARK", _) =>
         // if the master is not identified ahead, add all tags
         setupSparkYarnTag(applicationTag, conf)
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
index 93ef02cdc..b01f82a24 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/KyuubiApplicationManagerSuite.scala
@@ -19,6 +19,7 @@ package org.apache.kyuubi.engine
 
 import org.apache.kyuubi.{KyuubiException, KyuubiFunSuite}
 import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.engine.KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY
 
 class KyuubiApplicationManagerSuite extends KyuubiFunSuite {
   test("application access path") {
@@ -70,4 +71,19 @@ class KyuubiApplicationManagerSuite extends KyuubiFunSuite {
         localDirLimitConf)
     }
   }
+
+  test("Test kyuubi application Manager tag spark on kubernetes application") {
+    val conf: KyuubiConf = KyuubiConf()
+    val tag = "kyuubi-test-tag"
+    KyuubiApplicationManager.tagApplication(
+      tag,
+      "SPARK",
+      Some("k8s://https://kyuubi-test:8443"),
+      conf)
+
+    val kubernetesTag = conf.getOption("spark.kubernetes.driver.label." + LABEL_KYUUBI_UNIQUE_KEY)
+    val yarnTag = conf.getOption("spark.yarn.tags")
+    assert(kubernetesTag.nonEmpty && tag.equals(kubernetesTag.get))
+    assert(yarnTag.isEmpty)
+  }
 }