You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/12/12 07:55:20 UTC

[incubator-kyuubi] branch master updated: [KYUUBI #3914] [K8S][SPARK] Set spark driver host to ip instead of pod name when kyuubi on k8s submit spark with client deploy-mode

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new c9eb3cc58 [KYUUBI #3914] [K8S][SPARK] Set spark driver host to ip instead of pod name when kyuubi on k8s submit spark with client deploy-mode
c9eb3cc58 is described below

commit c9eb3cc5894d058ceadfe2465941f48dedf67749
Author: zwangsheng <22...@qq.com>
AuthorDate: Mon Dec 12 15:55:07 2022 +0800

    [KYUUBI #3914] [K8S][SPARK] Set spark driver host to ip instead of pod name when kyuubi on k8s submit spark with client deploy-mode
    
    ### _Why are the changes needed?_
    
    Set `spark.driver.host` to ip instead of pod name when kyuubi on k8s submit spark with client deploy-mode.
    
    When Kyuubi On Kubernetes submit spark with client deploy mode, spark driver will using kyuubi pod name as `spark.driver.host`, which can't be recognized by executors(Exclude kyuubi deployed as Statefulset + serivce case).
    
    We have done this in #1596
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #3914 from zwangsheng/k8s/set_driver_host_if_client.
    
    Closes #3914
    
    8fb89657 [zwangsheng] fix style
    fc38143f [Binjie Yang] Merge branch 'master' into k8s/set_driver_host_if_client
    4a313655 [zwangsheng] fix style
    c61b7b8f [zwangsheng] fix
    
    Lead-authored-by: zwangsheng <22...@qq.com>
    Co-authored-by: Binjie Yang <52...@users.noreply.github.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 .../main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala  | 6 ++++++
 .../kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala   | 1 -
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
index c7cb29661..aed0cb68c 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala
@@ -17,6 +17,7 @@
 
 package org.apache.kyuubi.engine.spark
 
+import java.net.InetAddress
 import java.time.Instant
 import java.util.{Locale, UUID}
 import java.util.concurrent.{CountDownLatch, ScheduledExecutorService, ThreadPoolExecutor, TimeUnit}
@@ -201,6 +202,11 @@ object SparkSQLEngine extends Logging {
       _sparkConf.setIfMissing(
         "spark.kubernetes.executor.podNamePrefix",
         generateExecutorPodNamePrefixForK8s(user))
+
+      if (!isOnK8sClusterMode) {
+        // set driver host to ip instead of kyuubi pod name
+        _sparkConf.set("spark.driver.host", InetAddress.getLocalHost.getHostAddress)
+      }
     }
 
     // Set web ui port 0 to avoid port conflicts during non-k8s cluster mode
diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
index 2047b4759..c8894679d 100644
--- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
+++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala
@@ -78,7 +78,6 @@ class KyuubiOnKubernetesWithClientSparkTestsSuite
   override protected def connectionConf: Map[String, String] = {
     super.connectionConf ++ Map(
       "spark.submit.deployMode" -> "client",
-      "spark.driver.host" -> kyuubiServerIp,
       "kyuubi.frontend.connection.url.use.hostname" -> "false",
       "spark.kubernetes.executor.label.kyuubi-it-test" -> "client")
   }