You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by fe...@apache.org on 2023/05/08 07:05:56 UTC
[kyuubi] branch master updated: [KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn
This is an automated email from the ASF dual-hosted git repository.
feiwang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 6ae0c8b14 [KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn
6ae0c8b14 is described below
commit 6ae0c8b1417995ec3d0036dc79214db83368b0db
Author: fwang12 <fw...@ebay.com>
AuthorDate: Mon May 8 15:05:47 2023 +0800
[KYUUBI #4801] Using different engine submit timeout config for kubernetes and yarn
### _Why are the changes needed?_
We shall use different engine submit timeout for different resource manager.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #4801 from turboFei/engine_submit_timeout.
Closes #4801
e34852a64 [fwang12] nit
ad69008e7 [fwang12] 1.7.2
db11330c5 [fwang12] save
Authored-by: fwang12 <fw...@ebay.com>
Signed-off-by: fwang12 <fw...@ebay.com>
---
docs/deployment/settings.md | 2 ++
.../src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 12 ++++++++++++
.../kyuubi/engine/KubernetesApplicationOperation.scala | 2 +-
.../org/apache/kyuubi/engine/YarnApplicationOperation.scala | 2 +-
4 files changed, 16 insertions(+), 2 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index b12185c3c..0c7cdfc89 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -155,6 +155,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.jdbc.java.options | <undefined> | The extra Java options for the JDBC query engine [...]
| kyuubi.engine.jdbc.memory | 1g | The heap memory for the JDBC query engine [...]
| kyuubi.engine.jdbc.type | <undefined> | The short name of JDBC type [...]
+| kyuubi.engine.kubernetes.submit.timeout | PT30S | The engine submit timeout for Kubernetes application. [...]
| kyuubi.engine.operation.convert.catalog.database.enabled | true | When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines [...]
| kyuubi.engine.operation.log.dir.root | engine_operation_logs | Root directory for query operation log at engine-side. [...]
| kyuubi.engine.pool.name | engine-pool | The name of the engine pool. [...]
@@ -182,6 +183,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
| kyuubi.engine.user.isolated.spark.session | true | When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including the temporary views, function registries, SQL configuration, and the current database. Note that, it does not affect if the share level is connection or user. [...]
| kyuubi.engine.user.isolated.spark.session.idle.interval | PT1M | The interval to check if the user-isolated Spark session is timeout. [...]
| kyuubi.engine.user.isolated.spark.session.idle.timeout | PT6H | If kyuubi.engine.user.isolated.spark.session is false, we will release the Spark session if its corresponding user is inactive after this configured timeout. [...]
+| kyuubi.engine.yarn.submit.timeout | PT30S | The engine submit timeout for YARN application. [...]
### Event
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 63a9ea648..a6a594063 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -2577,6 +2577,18 @@ object KyuubiConf {
.timeConf
.createWithDefaultString("PT30S")
+ val ENGINE_KUBERNETES_SUBMIT_TIMEOUT: ConfigEntry[Long] =
+ buildConf("kyuubi.engine.kubernetes.submit.timeout")
+ .doc("The engine submit timeout for Kubernetes application.")
+ .version("1.7.2")
+ .fallbackConf(ENGINE_SUBMIT_TIMEOUT)
+
+ val ENGINE_YARN_SUBMIT_TIMEOUT: ConfigEntry[Long] =
+ buildConf("kyuubi.engine.yarn.submit.timeout")
+ .doc("The engine submit timeout for YARN application.")
+ .version("1.7.2")
+ .fallbackConf(ENGINE_SUBMIT_TIMEOUT)
+
/**
* Holds information about keys that have been deprecated.
*
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
index c569dc9dc..a6fe28674 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KubernetesApplicationOperation.scala
@@ -49,7 +49,7 @@ class KubernetesApplicationOperation extends ApplicationOperation with Logging {
kubernetesClient = KubernetesUtils.buildKubernetesClient(conf) match {
case Some(client) =>
info(s"Initialized Kubernetes Client connect to: ${client.getMasterUrl}")
- submitTimeout = conf.get(KyuubiConf.ENGINE_SUBMIT_TIMEOUT)
+ submitTimeout = conf.get(KyuubiConf.ENGINE_KUBERNETES_SUBMIT_TIMEOUT)
// Disable resync, see https://github.com/fabric8io/kubernetes-client/discussions/5015
enginePodInformer = client.pods()
.withLabel(LABEL_KYUUBI_UNIQUE_KEY)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
index ea2bf6dcd..1f06484fc 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/YarnApplicationOperation.scala
@@ -37,7 +37,7 @@ class YarnApplicationOperation extends ApplicationOperation with Logging {
private var submitTimeout: Long = _
override def initialize(conf: KyuubiConf): Unit = {
- submitTimeout = conf.get(KyuubiConf.ENGINE_SUBMIT_TIMEOUT)
+ submitTimeout = conf.get(KyuubiConf.ENGINE_YARN_SUBMIT_TIMEOUT)
val yarnConf = KyuubiHadoopUtils.newYarnConfiguration(conf)
// YarnClient is thread-safe
val c = YarnClient.createYarnClient()