You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ya...@apache.org on 2021/09/02 03:46:20 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #1010] Rename
engine.share.level.sub.domain to engine.share.level.subdomain
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new d375eb0 [KYUUBI #1010] Rename engine.share.level.sub.domain to engine.share.level.subdomain
d375eb0 is described below
commit d375eb0cfc9b26815ad8f2ee1f37d801b3289468
Author: Kent Yao <ya...@apache.org>
AuthorDate: Thu Sep 2 11:46:07 2021 +0800
[KYUUBI #1010] Rename engine.share.level.sub.domain to engine.share.level.subdomain
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
`subdomain` is a widely used term in computer science
### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1010 from yaooqinn/subdomain.
Closes #1010
72ed19a3 [Kent Yao] address comments
4dcae414 [Kent Yao] fix test
c3bb788d [Kent Yao] Rename engine.share.level.sub.domain to engine.share.level.subdomain
b7762357 [Kent Yao] Rename engine.share.level.sub.domain to engine.share.level.subdomain
Authored-by: Kent Yao <ya...@apache.org>
Signed-off-by: Kent Yao <ya...@apache.org>
---
docs/deployment/settings.md | 5 ++-
.../org/apache/kyuubi/config/KyuubiConf.scala | 19 ++++++---
.../scala/org/apache/kyuubi/engine/EngineRef.scala | 12 +++---
.../org/apache/kyuubi/engine/EngineRefSuite.scala | 48 +++++++++++++---------
.../kyuubi/engine/spark/SparkSqlEngineSuite.scala | 8 ++--
.../operation/KyuubiOperationPerUserSuite.scala | 2 +-
6 files changed, 55 insertions(+), 39 deletions(-)
diff --git a/docs/deployment/settings.md b/docs/deployment/settings.md
index 6237227..be9a37f 100644
--- a/docs/deployment/settings.md
+++ b/docs/deployment/settings.md
@@ -172,8 +172,9 @@ kyuubi\.engine<br>\.initialize\.sql|<div style='width: 65pt;word-wrap: break-wor
kyuubi\.engine\.pool<br>\.size|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>-1</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>The size of engine pool. Note that, if the size is less than 1, the engine pool will not be enabled; otherwise, the size of the engine pool will be min(this, kyuubi.engine.pool.size.threshold).</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
kyuubi\.engine\.pool<br>\.size\.threshold|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>9</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>This parameter is introduced as a server-side parameter, and controls the upper limit of the engine pool.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
kyuubi\.engine\.session<br>\.initialize\.sql|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>SHOW DATABASES</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver.</div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.3.0</div>
-kyuubi\.engine\.share<br>\.level|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Engines will be shared in different levels, available configs are: <ul> <li>CONNECTION: engine will not be shared but only used by the current client connection</li> <li>USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.sub.domain</li> <li>SERVER: the Ap [...]
-kyuubi\.engine\.share<br>\.level\.sub\.domain|<div style='width: 65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Allow end-users to create a sub-domain for the share level of an engine. A sub-domain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form. For example, for `USER` share level, an end-user can share a certain engine within a sub-domain, not for all of its clients. End-users [...]
+kyuubi\.engine\.share<br>\.level|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>USER</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Engines will be shared in different levels, available configs are: <ul> <li>CONNECTION: engine will not be shared but only used by the current client connection</li> <li>USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain</li> <li>SERVER: the App [...]
+kyuubi\.engine\.share<br>\.level\.sub\.domain|<div style='width: 65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) - Using kyuubi.engine.share.level.subdomain instead</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.2.0</div>
+kyuubi\.engine\.share<br>\.level\.subdomain|<div style='width: 65pt;word-wrap: break-word;white-space: normal'><undefined></div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form. For example, for `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are f [...]
kyuubi\.engine\.single<br>\.spark\.session|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database.</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.3.0</div>
kyuubi\.engine\.ui\.stop<br>\.enabled|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>true</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When true, allows Kyuubi engine to be killed from the Spark Web UI.</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.3.0</div>
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
index ae5cd22..48da7ea 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala
@@ -574,13 +574,10 @@ object KyuubiConf {
private val validEngineSubDomain: Pattern = "^[a-zA-Z_-]{1,14}$".r.pattern
+ @deprecated(s"using kyuubi.engine.share.level.subdomain instead", "1.4.0")
val ENGINE_SHARE_LEVEL_SUB_DOMAIN: OptionalConfigEntry[String] =
buildConf("engine.share.level.sub.domain")
- .doc("Allow end-users to create a sub-domain for the share level of an engine. A" +
- " sub-domain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form." +
- " For example, for `USER` share level, an end-user can share a certain engine within" +
- " a sub-domain, not for all of its clients. End-users are free to create multiple" +
- " engines in the `USER` share level")
+ .doc("(deprecated) - Using kyuubi.engine.share.level.subdomain instead")
.version("1.2.0")
.stringConf
.transform(_.toLowerCase(Locale.ROOT))
@@ -588,6 +585,16 @@ object KyuubiConf {
"must be [1, 14] length alphabet string, e.g. 'abc', 'apache'")
.createOptional
+ val ENGINE_SHARE_LEVEL_SUBDOMAIN: ConfigEntry[Option[String]] =
+ buildConf("engine.share.level.subdomain")
+ .doc("Allow end-users to create a subdomain for the share level of an engine. A" +
+ " subdomain is a case-insensitive string values in `^[a-zA-Z_-]{1,14}$` form." +
+ " For example, for `USER` share level, an end-user can share a certain engine within" +
+ " a subdomain, not for all of its clients. End-users are free to create multiple" +
+ " engines in the `USER` share level")
+ .version("1.4.0")
+ .fallbackConf(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+
val ENGINE_CONNECTION_URL_USE_HOSTNAME: ConfigEntry[Boolean] =
buildConf("engine.connection.url.use.hostname")
.doc("When true, engine register with hostname to zookeeper. When spark run on k8s" +
@@ -601,7 +608,7 @@ object KyuubiConf {
" <li>CONNECTION: engine will not be shared but only used by the current client" +
" connection</li>" +
" <li>USER: engine will be shared by all sessions created by a unique username," +
- s" see also ${ENGINE_SHARE_LEVEL_SUB_DOMAIN.key}</li>" +
+ s" see also ${ENGINE_SHARE_LEVEL_SUBDOMAIN.key}</li>" +
" <li>SERVER: the App will be shared by Kyuubi servers</li></ul>")
.version("1.2.0")
.fallbackConf(LEGACY_ENGINE_SHARE_LEVEL)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
index 0eecccb..219dcf8 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/EngineRef.scala
@@ -62,7 +62,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf, user: String, sessionI
private val poolThreshold: Int = conf.get(ENGINE_POOL_SIZE_THRESHOLD)
@VisibleForTesting
- private[kyuubi] val subDomain: Option[String] = conf.get(ENGINE_SHARE_LEVEL_SUB_DOMAIN).orElse {
+ private[kyuubi] val subdomain: Option[String] = conf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN).orElse {
val clientPoolSize: Int = conf.get(ENGINE_POOL_SIZE)
if (clientPoolSize > 0) {
@@ -94,7 +94,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf, user: String, sessionI
@VisibleForTesting
private[kyuubi] val defaultEngineName: String = shareLevel match {
case CONNECTION => s"kyuubi_${shareLevel}_${appUser}_$sessionId"
- case _ => subDomain match {
+ case _ => subdomain match {
case Some(domain) => s"kyuubi_${shareLevel}_${appUser}_${domain}_$sessionId"
case _ => s"kyuubi_${shareLevel}_${appUser}_$sessionId"
}
@@ -104,15 +104,15 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf, user: String, sessionI
* The EngineSpace used to expose itself to the KyuubiServers in `serverSpace`
*
* For `CONNECTION` share level:
- * /$serverSpace_CONNECTION/$user/$sessionId
+ * /`serverSpace_CONNECTION`/`user`/`sessionId`
* For `USER` share level:
- * /$serverSpace_USER/$user[/$subDomain]
+ * /`serverSpace_USER`/`user`[/`subdomain`]
*
*/
@VisibleForTesting
private[kyuubi] lazy val engineSpace: String = shareLevel match {
case CONNECTION => ZKPaths.makePath(s"${serverSpace}_$shareLevel", appUser, sessionId)
- case _ => subDomain match {
+ case _ => subdomain match {
case Some(domain) => ZKPaths.makePath(s"${serverSpace}_$shareLevel", appUser, domain)
case None => ZKPaths.makePath(s"${serverSpace}_$shareLevel", appUser)
}
@@ -126,7 +126,7 @@ private[kyuubi] class EngineRef private(conf: KyuubiConf, user: String, sessionI
case CONNECTION => f
case _ =>
val lockPath =
- ZKPaths.makePath(s"${serverSpace}_$shareLevel", "lock", appUser, subDomain.orNull)
+ ZKPaths.makePath(s"${serverSpace}_$shareLevel", "lock", appUser, subdomain.orNull)
var lock: InterProcessSemaphoreMutex = null
try {
try {
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
index 78d0127..066d496 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/EngineRefSuite.scala
@@ -51,11 +51,17 @@ class EngineRefSuite extends KyuubiFunSuite {
super.afterAll()
}
- test(s"${CONNECTION} shared level engine name") {
+ override def beforeEach(): Unit = {
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN)
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ super.beforeEach()
+ }
+
+ test("CONNECTION shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
Seq(None, Some("suffix")).foreach { domain =>
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, CONNECTION.toString)
- domain.foreach(conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, _))
+ domain.foreach(conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key, _))
val engine = EngineRef(conf, user, id)
assert(engine.engineSpace ===
ZKPaths.makePath(s"kyuubi_$CONNECTION", user, id.identifier.toString))
@@ -63,31 +69,33 @@ class EngineRefSuite extends KyuubiFunSuite {
}
}
- test(s"${USER} shared level engine name") {
+ test("USER shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, USER.toString)
- conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
val appName = EngineRef(conf, user, id)
assert(appName.engineSpace === ZKPaths.makePath(s"kyuubi_$USER", user))
assert(appName.defaultEngineName === s"kyuubi_${USER}_${user}_${id.identifier}")
- conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, "abc")
- val appName2 = EngineRef(conf, user, id)
- assert(appName2.engineSpace ===
- ZKPaths.makePath(s"kyuubi_$USER", user, "abc"))
- assert(appName2.defaultEngineName === s"kyuubi_${USER}_${user}_abc_${id.identifier}")
+ Seq(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN,
+ KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN).foreach { k =>
+ conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN)
+ conf.set(k.key, "abc")
+ val appName2 = EngineRef(conf, user, id)
+ assert(appName2.engineSpace ===
+ ZKPaths.makePath(s"kyuubi_$USER", user, "abc"))
+ assert(appName2.defaultEngineName === s"kyuubi_${USER}_${user}_abc_${id.identifier}")
+ }
}
- test(s"${SERVER} shared level engine name") {
+ test("SERVER shared level engine name") {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
conf.set(KyuubiConf.ENGINE_SHARE_LEVEL, SERVER.toString)
- conf.unset(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN)
val appName = EngineRef(conf, user, id)
assert(appName.engineSpace ===
ZKPaths.makePath(s"kyuubi_$SERVER", user))
assert(appName.defaultEngineName === s"kyuubi_${SERVER}_${user}_${id.identifier}")
- conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key, "abc")
+ conf.set(KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "abc")
val appName2 = EngineRef(conf, user, id)
assert(appName2.engineSpace ===
ZKPaths.makePath(s"kyuubi_$SERVER", user, "abc"))
@@ -98,26 +106,26 @@ class EngineRefSuite extends KyuubiFunSuite {
val id = SessionHandle(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10)
// test subdomain
- conf.set(ENGINE_SHARE_LEVEL_SUB_DOMAIN, "abc")
+ conf.set(ENGINE_SHARE_LEVEL_SUBDOMAIN.key, "abc")
val engine1 = EngineRef(conf, user, id)
- assert(engine1.subDomain === Some("abc"))
+ assert(engine1.subdomain === Some("abc"))
// unset domain
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
val engine2 = EngineRef(conf, user, id)
- assert(engine2.subDomain === None)
+ assert(engine2.subdomain === None)
// 1 <= engine pool size < threshold
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
conf.set(ENGINE_POOL_SIZE, 3)
val engine3 = EngineRef(conf, user, id)
- assert(engine3.subDomain.get.startsWith("engine-pool-"))
+ assert(engine3.subdomain.get.startsWith("engine-pool-"))
// engine pool size > threshold
- conf.unset(ENGINE_SHARE_LEVEL_SUB_DOMAIN)
+ conf.unset(ENGINE_SHARE_LEVEL_SUBDOMAIN)
conf.set(ENGINE_POOL_SIZE, 100)
val engine4 = EngineRef(conf, user, id)
- val engineNumber = Integer.parseInt(engine4.subDomain.get.substring(12))
+ val engineNumber = Integer.parseInt(engine4.subdomain.get.substring(12))
val threshold = ENGINE_POOL_SIZE_THRESHOLD.defaultVal.get
assert(engineNumber <= threshold)
}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
index 57a4554..7cd1004 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkSqlEngineSuite.scala
@@ -75,7 +75,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with JDBCTestUtils {
Seq("1", ",", "", "a" * 15, "abc.xyz").foreach { invalid =>
val sparkHiveConfigs = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> invalid)
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> invalid)
withSessionConf(Map.empty)(sparkHiveConfigs)(Map.empty) {
assertJDBCConnectionFail()
}
@@ -85,7 +85,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with JDBCTestUtils {
test("Engine isolation with sub domain configurations") {
val sparkHiveConfigs = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "spark",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "spark",
"spark.driver.memory" -> "1000M")
var mem: String = null
withSessionConf(Map.empty)(sparkHiveConfigs)(Map.empty) {
@@ -99,7 +99,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with JDBCTestUtils {
val sparkHiveConfigs2 = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "spark",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "spark",
"spark.driver.memory" -> "1001M")
withSessionConf(Map.empty)(sparkHiveConfigs2)(Map.empty) {
withJdbcStatement() { statement =>
@@ -112,7 +112,7 @@ class SparkSqlEngineSuite extends WithKyuubiServer with JDBCTestUtils {
val sparkHiveConfigs3 = Map(
ENGINE_SHARE_LEVEL.key -> "USER",
- ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "kyuubi",
+ ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "kyuubi",
"spark.driver.memory" -> "1002M")
withSessionConf(Map.empty)(sparkHiveConfigs3)(Map.empty) {
withJdbcStatement() { statement =>
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
index 4e6de55..bdd41d2 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/KyuubiOperationPerUserSuite.scala
@@ -59,7 +59,7 @@ class KyuubiOperationPerUserSuite extends WithKyuubiServer with JDBCTests {
test("ensure two connections share the same engine when specifying subDomain.") {
withSessionConf()(
Map(
- KyuubiConf.ENGINE_SHARE_LEVEL_SUB_DOMAIN.key -> "abc"
+ KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "abc"
))(Map.empty) {
var r1: String = null