You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/05/08 07:14:01 UTC
[kyuubi] branch branch-1.7 updated: [KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn (#4802)
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch branch-1.7
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.7 by this push:
new dee4bf91c [KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn (#4802)
dee4bf91c is described below
commit dee4bf91c62e6fef801f68877439595cc8edb946
Author: Fei Wang <fw...@ebay.com>
AuthorDate: Mon May 8 15:13:56 2023 +0800
[KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn (#4802)
We shall use different engine submit timeout for different resource manager.
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4801 from turboFei/engine_submit_timeout.
Closes #4801
e34852a64 [fwang12] nit
ad69008e7 [fwang12] 1.7.2
db11330c5 [fwang12] save
Authored-by: fwang12 <fw...@ebay.com>
(cherry picked from commit 6ae0c8b1417995ec3d0036dc79214db83368b0db)
Signed-off-by: fwang12 <fw...@ebay.com>
---
docs/deployment/settings.md | 2 ++
.../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 12 ++++++++++++
.../kyuubi/engine/KubernetesApplicationOperation.scala | 2 +-
.../org/apache/kyuubi/engine/YarnApplicationOperation.scala | 2 +-
4 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index c3368751d..f079b5b44 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -145,6 +145,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.jdbc.java.options | <undefined> | The extra Java options for the JDBC query engine [...]
| kyuubi.engine.jdbc.memory | 1g | The heap memory for the JDBC query engine [...]
| kyuubi.engine.jdbc.type | <undefined> | The short name of JDBC type [...]
+| kyuubi.engine.kubernetes.submit.timeout | PT30S | The engine submit timeout for Kubernetes application. [...]
| kyuubi.engine.operation.convert.catalog.database.enabled | true | When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines [...]
| kyuubi.engine.operation.log.dir.root | engine_operation_logs | Root directory for query operation log at engine-side. [...]
| kyuubi.engine.pool.name | engine-pool | The name of the engine pool. [...]
@@ -172,6 +173,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.user.isolated.spark.session | true | When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including the temporary views, function registries, SQL configuration, and the current database. Note that, it does not affect if the share level is connection or user. [...]
| kyuubi.engine.user.isolated.spark.session.idle.interval | PT1M | The interval to check if the user-isolated Spark session is timeout. [...]
| kyuubi.engine.user.isolated.spark.session.idle.timeout | PT6H | If kyuubi.engine.user.isolated.spark.session is false, we will release the Spark session if its corresponding user is inactive after this configured timeout. [...]
+| kyuubi.engine.yarn.submit.timeout | PT30S | The engine submit timeout for YARN application. [...]
### Event
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 8b7fdeebc..3140f708d 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -2537,6 +2537,18 @@ object KyuubiConf {
.timeConf
.createWithDefaultString("PT30S")
+ val ENGINE_KUBERNETES_SUBMIT_TIMEOUT: ConfigEntry[Long] =
+ buildConf("kyuubi.engine.kubernetes.submit.timeout")
+ .doc("The engine submit timeout for Kubernetes application.")
+ .version("1.7.2")
+ .fallbackConf(ENGINE_SUBMIT_TIMEOUT)
+
+ val ENGINE_YARN_SUBMIT_TIMEOUT: ConfigEntry[Long] =
+ buildConf("kyuubi.engine.yarn.submit.timeout")
+ .doc("The engine submit timeout for YARN application.")
+ .version("1.7.2")
+ .fallbackConf(ENGINE_SUBMIT_TIMEOUT)
+
/**
* Holds information about keys that have been deprecated.
*
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index 6a7ea46ec..399399a03 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -49,7 +49,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
kubernetesClient = KubernetesUtils.buildKubernetesClient(conf) match {
case Some(client) =>
info(s"Initialized Kubernetes Client connect to: ${client.getMasterUrl}")
- submitTimeout = conf.get(KyuubiConf.ENGINE_SUBMIT_TIMEOUT)
+ submitTimeout = conf.get(KyuubiConf.ENGINE_KUBERNETES_SUBMIT_TIMEOUT)
// Disable resync, see https://github.com/fabric8io/kubernetes-client/discussions/5015
enginePodInformer = client.pods()
.withLabel(LABEL_KYUUBI_UNIQUE_KEY)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
index ea2bf6dcd..1f06484fc 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
@@ -37,7 +37,7 @@ class YarnApplicationOperation extends ApplicationOperation with Logging {
private var submitTimeout: Long = _
override def initialize(conf: KyuubiConf): Unit = {
- submitTimeout = conf.get(KyuubiConf.ENGINE_SUBMIT_TIMEOUT)
+ submitTimeout = conf.get(KyuubiConf.ENGINE_YARN_SUBMIT_TIMEOUT)
val yarnConf = KyuubiHadoopUtils.newYarnConfiguration(conf)
// YarnClient is thread-safe
val c = YarnClient.createYarnClient()