You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ul...@apache.org on 2021/12/01 02:53:53 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1472] allow the engine not to self-terminate.
This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new cc40c87 [KYUUBI #1472] allow the engine not to self-terminate.
cc40c87 is described below
commit cc40c87743f20fb1a324bf3579210d53474e7ef5
Author: Wang Zhen <wa...@qiyi.com>
AuthorDate: Wed Dec 1 10:53:37 2021 +0800
[KYUUBI #1472] allow the engine not to self-terminate.
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Allow the engine not to self-terminate. For details: #1472.
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1475 from wForget/KYUUBI-1472.
Closes #1472
97412e54 [Wang Zhen] Merge remote-tracking branch 'remotes/origin/master'
8ff0cf72 [Wang Zhen] [KYUUBI-1472] allow the engine not to self-terminate. Use zero or negative to represent no limit.
7cfa03c1 [Wang Zhen] [KYUUBI-1472] allow the engine not to self-terminate.
Authored-by: Wang Zhen <wa...@qiyi.com>
Signed-off-by: ulysses-you <ul...@apache.org>
---
docs/deployment/settings.md | 2 +-
.../main/scala/org/apache/kyuubi/config/KyuubiConf.scala | 3 ++-
.../scala/org/apache/kyuubi/session/SessionManager.scala | 16 +++++++++-------
3 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index e6e9eb5..4d17304 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -298,7 +298,7 @@ kyuubi\.session\.check<br>\.interval|<div style='width: 65pt;word-wrap: break-wo
kyuubi\.session\.conf<br>\.ignore\.list|<div style='width: 65pt;word-wrap: break-word;white-space: normal'></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from [...]
kyuubi\.session\.conf<br>\.restrict\.list|<div style='width: 65pt;word-wrap: break-word;white-space: normal'></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but [...]
kyuubi\.session\.engine<br>\.check\.interval|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT5M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The check interval for engine timeout</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
-kyuubi\.session\.engine<br>\.idle\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT30M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>engine timeout, the engine will self-terminate when it's not accessed for this duration</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
+kyuubi\.session\.engine<br>\.idle\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT30M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>engine timeout, the engine will self-terminate when it's not accessed for this duration. 0 or negative means not to self-terminate.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
kyuubi\.session\.engine<br>\.initialize\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT3M</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Timeout for starting the background engine, e.g. SparkSQLEngine.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
kyuubi\.session\.engine<br>\.launch\.async|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When opening kyuubi session, whether to launch backend engine asynchronously. When true, the Kyuubi server will set up the connection with the client without delay as the backend engine will be created asynchronously.</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.4.0</div>
kyuubi\.session\.engine<br>\.log\.timeout|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT24H</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>If we use Spark as the engine then the session submit log is the console output of spark-submit. We will retain the session submit log until over the config value.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.1.0</div>
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index 408808b..b27c096 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -623,7 +623,8 @@ object KyuubiConf {
.createWithDefault(Duration.ofMinutes(5).toMillis)
val ENGINE_IDLE_TIMEOUT: ConfigEntry[Long] = buildConf("session.engine.idle.timeout")
- .doc("engine timeout, the engine will self-terminate when it's not accessed for this duration")
+ .doc("engine timeout, the engine will self-terminate when it's not accessed for this " +
+ "duration. 0 or negative means not to self-terminate.")
.version("1.0.0")
.timeConf
.createWithDefault(Duration.ofMinutes(30L).toMillis)
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala
index 2ea57df..7c5c192 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala
@@ -270,15 +270,17 @@ abstract class SessionManager(name: String) extends CompositeService(name) {
_latestLogoutTime = System.currentTimeMillis()
val interval = conf.get(ENGINE_CHECK_INTERVAL)
val idleTimeout = conf.get(ENGINE_IDLE_TIMEOUT)
- val checkTask = new Runnable {
- override def run(): Unit = {
- if (!shutdown &&
- System.currentTimeMillis() - latestLogoutTime > idleTimeout && getOpenSessionCount <= 0) {
- info(s"Idled for more than $idleTimeout ms, terminating")
- stop()
+ if (idleTimeout > 0) {
+ val checkTask = new Runnable {
+ override def run(): Unit = {
+ if (!shutdown && System.currentTimeMillis() - latestLogoutTime > idleTimeout &&
+ getOpenSessionCount <= 0) {
+ info(s"Idled for more than $idleTimeout ms, terminating")
+ stop()
+ }
}
}
+ timeoutChecker.scheduleWithFixedDelay(checkTask, interval, interval, TimeUnit.MILLISECONDS)
}
- timeoutChecker.scheduleWithFixedDelay(checkTask, interval, interval, TimeUnit.MILLISECONDS)
}
}