[incubator-kyuubi] branch master updated: [KYUUBI #1594] SparkProcessBuild submit add --conf spark.driver.host=

[ya...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git


The following commit(s) were added to refs/heads/master by this push:
     new 872551d  [KYUUBI #1594] SparkProcessBuild submit add --conf spark.driver.host=
872551d is described below

commit 872551d3aaf851edfd59b2bcfa4a5ce2be4710ef
Author: zwangsheng <22...@qq.com>
AuthorDate: Fri Dec 31 19:25:18 2021 +0800

    [KYUUBI #1594] SparkProcessBuild submit add --conf spark.driver.host=
    
    <!--
    Thanks for sending a pull request!
    
    Here are some tips for you:
      1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
      2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
      3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
    -->
    
    ### _Why are the changes needed?_
    <!--
    Please clarify why the changes are needed. For instance,
      1. If you add a feature, you can talk about the use case of it.
      2. If you fix a bug, you can clarify why it is a bug.
    -->
    detail #1594
    When Kyuubi On Kubernetes submit spark engine by client mode, executor pod can not connect to driver by kyuubi pod name, which is default set.
    
    ### _How was this patch tested?_
    - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
    
    Closes #1596 from zwangsheng/kyuubi/1594.
    
    Closes #1594
    
    d4b41005 [zwangsheng] add comment
    f7bcffb6 [zwangsheng] fix test name
    581eb508 [zwangsheng] fix test name
    b19a2c21 [zwangsheng] simple & add testsuit
    83979935 [zwangsheng] fix
    1a9ae9b1 [zwangsheng] fix doc
    d857eac7 [zwangsheng] Merge branch 'kyuubi/1594' of https://github.com/zwangsheng/incubator-kyuubi into kyuubi/1594
    b723f4ed [zwangsheng] retest
    8210de56 [zwangsheng] make import normal
    92ec9cf4 [zwangsheng] 1594
    
    Authored-by: zwangsheng <22...@qq.com>
    Signed-off-by: Kent Yao <ya...@apache.org>
---
 .../kyuubi/engine/spark/SparkProcessBuilder.scala  | 16 ++++++++++++++-
 .../engine/spark/SparkProcessBuilderSuite.scala    | 23 ++++++++++++++++++++--
 2 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
index b78a5ad..e3c491b 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
@@ -27,7 +27,7 @@ import org.apache.hadoop.security.UserGroupInformation
 
 import org.apache.kyuubi._
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
 import org.apache.kyuubi.engine.ProcBuilder
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
@@ -157,6 +157,20 @@ class SparkProcessBuilder(
       buffer += s"$newKey=$v"
     }
 
+    /**
+     * Kyuubi respect user setting config, if user set `spark.driver.host`, will pass it on.
+     * If user don't set this, will use thrift binary bind host to set.
+     * Kyuubi wants the Engine to bind hostName or IP with Kyuubi.
+     * Spark driver will pass this configuration as the driver-url to the executors
+     * to build RPC communication.
+     */
+    if (!allConf.contains("spark.driver.host")) {
+      conf.get(FRONTEND_THRIFT_BINARY_BIND_HOST).foreach(host => {
+        buffer += CONF
+        buffer += s"spark.driver.host=${host}"
+      })
+    }
+
     // iff the keytab is specified, PROXY_USER is not supported
     if (!useKeytab()) {
       buffer += PROXY_USER
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
index d50b75a..79fb21f 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
@@ -26,8 +26,7 @@ import org.scalatest.time.SpanSugar._
 
 import org.apache.kyuubi.{KerberizedTestHelper, KyuubiSQLException, Utils}
 import org.apache.kyuubi.config.KyuubiConf
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_LOG_TIMEOUT
-import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAIN_RESOURCE
+import org.apache.kyuubi.config.KyuubiConf.{ENGINE_LOG_TIMEOUT, ENGINE_SPARK_MAIN_RESOURCE, FRONTEND_THRIFT_BINARY_BIND_HOST}
 import org.apache.kyuubi.ha.HighAvailabilityConf
 import org.apache.kyuubi.ha.client.ZooKeeperAuthTypes
 import org.apache.kyuubi.service.ServiceUtils
@@ -272,6 +271,26 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper {
     assert(b1.toString.contains(s"--conf spark.files=$testKeytab"))
 
   }
+
+  test("engine bind on host name or IP with Kyuubi") {
+    val conf = KyuubiConf()
+    conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
+
+    val builder = new SparkProcessBuilder("test", conf)
+    assert(builder.toString.contains("--conf spark.driver.host=kyuubi-example"))
+  }
+
+  test("respect to user set config") {
+    val conf = KyuubiConf()
+    conf.set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "kyuubi-example")
+    conf.set("spark.driver.host", "spark-example")
+
+    val builder = new SparkProcessBuilder("test", conf)
+    assertResult(false) {
+      builder.toString.contains("--conf spark.driver.host=kyuubi-example")
+    }
+    assert(builder.toString.contains("--conf spark.driver.host=spark-example"))
+  }
 }
 
 class FakeSparkProcessBuilder(config: KyuubiConf)