You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2023/10/27 02:48:45 UTC

[kyuubi] branch branch-1.8 updated: [KYUUBI #5412] Resolve the relative zk configuration dir based on KYUUBI_HOME

This is an automated email from the ASF dual-hosted git repository.

chengpan pushed a commit to branch branch-1.8
in repository https://gitbox.apache.org/repos/asf/kyuubi.git


The following commit(s) were added to refs/heads/branch-1.8 by this push:
     new 837a5411d [KYUUBI #5412] Resolve the relative zk configuration dir based on KYUUBI_HOME
837a5411d is described below

commit 837a5411d70f552b21cc61a88afc4e40e917b6ba
Author: lawulu <bi...@gmail.com>
AuthorDate: Fri Oct 27 10:48:22 2023 +0800

    [KYUUBI #5412] Resolve the relative zk configuration dir based on KYUUBI_HOME
    
    ### _Why are the changes needed?_
    Both   the default  value of `kyuubi.zookeeper.embedded.data.log.dir` and `kyuubi.zookeeper.embedded.data.dir` are `embedded_zookeeper`. It is used based on the current dir.
    If the configuration dir is a relative path , it is resolved relative to `KYUUBI_HOME`.
    
    ### _How was this patch tested?_
    - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
    
    - [ ] Add screenshots for manual tests if appropriate
    
    - [x] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request
    
    ### _Was this patch authored or co-authored using generative AI tooling?_
    
    No
    
    Closes #5412 from biangjuang/master.
    
    Closes #5412
    
    17e502c03 [Cheng Pan] Update kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala
    e0aa01c82 [Cheng Pan] Update kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala
    bb6af93df [Cheng Pan] Update docs/deployment/migration-guide.md
    53bd61bc3 [lawulu] Resolve the relative zk configuration dir based on KYUUBI_HOME
    
    Lead-authored-by: lawulu <bi...@gmail.com>
    Co-authored-by: Cheng Pan <pa...@gmail.com>
    Signed-off-by: Cheng Pan <ch...@apache.org>
    (cherry picked from commit 30a127a1baf69f0b05063042b05f28e457c7fad1)
    Signed-off-by: Cheng Pan <ch...@apache.org>
---
 docs/configuration/settings.md                     | 26 +++++++++++-----------
 docs/deployment/migration-guide.md                 |  4 +++-
 .../kyuubi/zookeeper/EmbeddedZookeeper.scala       | 14 +++++++++---
 .../apache/kyuubi/zookeeper/ZookeeperConf.scala    |  9 +++++---
 .../kyuubi/zookeeper/EmbeddedZookeeperSuite.scala  | 15 ++++++++++++-
 5 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/docs/configuration/settings.md b/docs/configuration/settings.md
index d7cea9789..3ef24d51d 100644
--- a/docs/configuration/settings.md
+++ b/docs/configuration/settings.md
@@ -469,19 +469,19 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co
 
 ### Zookeeper
 
-|                       Key                        |      Default       |                                                                                 Meaning                                                                                  |  Type   | Since |
-|--------------------------------------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------|
-| kyuubi.zookeeper.embedded.client.port            | 2181               | clientPort for the embedded ZooKeeper server to listen for client connections, a client here could be Kyuubi server, engine, and JDBC client                             | int     | 1.2.0 |
-| kyuubi.zookeeper.embedded.client.port.address    | &lt;undefined&gt;  | clientPortAddress for the embedded ZooKeeper server to                                                                                                                   | string  | 1.2.0 |
-| kyuubi.zookeeper.embedded.client.use.hostname    | false              | When true, embedded Zookeeper prefer to bind hostname, otherwise, ip address.                                                                                            | boolean | 1.7.2 |
-| kyuubi.zookeeper.embedded.data.dir               | embedded_zookeeper | dataDir for the embedded zookeeper server where stores the in-memory database snapshots and, unless specified otherwise, the transaction log of updates to the database. | string  | 1.2.0 |
-| kyuubi.zookeeper.embedded.data.log.dir           | embedded_zookeeper | dataLogDir for the embedded ZooKeeper server where writes the transaction log .                                                                                          | string  | 1.2.0 |
-| kyuubi.zookeeper.embedded.directory              | embedded_zookeeper | The temporary directory for the embedded ZooKeeper server                                                                                                                | string  | 1.0.0 |
-| kyuubi.zookeeper.embedded.max.client.connections | 120                | maxClientCnxns for the embedded ZooKeeper server to limit the number of concurrent connections of a single client identified by IP address                               | int     | 1.2.0 |
-| kyuubi.zookeeper.embedded.max.session.timeout    | 60000              | maxSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 20 times the tickTime                                | int     | 1.2.0 |
-| kyuubi.zookeeper.embedded.min.session.timeout    | 6000               | minSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 2 times the tickTime                                 | int     | 1.2.0 |
-| kyuubi.zookeeper.embedded.port                   | 2181               | The port of the embedded ZooKeeper server                                                                                                                                | int     | 1.0.0 |
-| kyuubi.zookeeper.embedded.tick.time              | 3000               | tickTime in milliseconds for the embedded ZooKeeper server                                                                                                               | int     | 1.2.0 |
+|                       Key                        |      Default       |                                                                                                                  Meaning                                                                                                                   |  Type   | Since |
+|--------------------------------------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------|
+| kyuubi.zookeeper.embedded.client.port            | 2181               | clientPort for the embedded ZooKeeper server to listen for client connections, a client here could be Kyuubi server, engine, and JDBC client                                                                                               | int     | 1.2.0 |
+| kyuubi.zookeeper.embedded.client.port.address    | &lt;undefined&gt;  | clientPortAddress for the embedded ZooKeeper server to                                                                                                                                                                                     | string  | 1.2.0 |
+| kyuubi.zookeeper.embedded.client.use.hostname    | false              | When true, embedded Zookeeper prefer to bind hostname, otherwise, ip address.                                                                                                                                                              | boolean | 1.7.2 |
+| kyuubi.zookeeper.embedded.data.dir               | embedded_zookeeper | dataDir for the embedded zookeeper server where stores the in-memory database snapshots and, unless specified otherwise, the transaction log of updates to the database. If it is a relative path, it is resolved relative to KYUUBI_HOME. | string  | 1.2.0 |
+| kyuubi.zookeeper.embedded.data.log.dir           | embedded_zookeeper | dataLogDir for the embedded ZooKeeper server where writes the transaction log. If it is a relative path, it is resolved relative to KYUUBI_HOME.                                                                                           | string  | 1.2.0 |
+| kyuubi.zookeeper.embedded.directory              | embedded_zookeeper | The temporary directory for the embedded ZooKeeper server. If it is a relative path, it is resolved relative to KYUUBI_HOME.                                                                                                               | string  | 1.0.0 |
+| kyuubi.zookeeper.embedded.max.client.connections | 120                | maxClientCnxns for the embedded ZooKeeper server to limit the number of concurrent connections of a single client identified by IP address                                                                                                 | int     | 1.2.0 |
+| kyuubi.zookeeper.embedded.max.session.timeout    | 60000              | maxSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 20 times the tickTime                                                                                                  | int     | 1.2.0 |
+| kyuubi.zookeeper.embedded.min.session.timeout    | 6000               | minSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 2 times the tickTime                                                                                                   | int     | 1.2.0 |
+| kyuubi.zookeeper.embedded.port                   | 2181               | The port of the embedded ZooKeeper server                                                                                                                                                                                                  | int     | 1.0.0 |
+| kyuubi.zookeeper.embedded.tick.time              | 3000               | tickTime in milliseconds for the embedded ZooKeeper server                                                                                                                                                                                 | int     | 1.2.0 |
 
 ## Spark Configurations
 
diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md
index 58df0fcc6..0430e8cff 100644
--- a/docs/deployment/migration-guide.md
+++ b/docs/deployment/migration-guide.md
@@ -23,7 +23,9 @@
   Both Derby and SQLite are mainly for testing purposes, and they're not supposed to be used in production.
   To restore previous behavior, set `kyuubi.metadata.store.jdbc.database.type=DERBY` and
   `kyuubi.metadata.store.jdbc.url=jdbc:derby:memory:kyuubi_state_store_db;create=true`.
-
+* Since Kyuubi 1.8, if the directory of the embedded zookeeper configuration (`kyuubi.zookeeper.embedded.directory`
+  & `kyuubi.zookeeper.embedded.data.dir` & `kyuubi.zookeeper.embedded.data.log.dir`) is a relative path, it is resolved
+  relative to `$KYUUBI_HOME` instead of `$PWD`.
 * Since Kyuubi 1.8, PROMETHEUS is changed as the default metrics reporter. To restore previous behavior,
   set `kyuubi.metrics.reporters=JSON`.
 
diff --git a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
index 17caffedf..1592d9063 100644
--- a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
+++ b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeper.scala
@@ -19,9 +19,10 @@ package org.apache.kyuubi.zookeeper
 
 import java.io.File
 import java.net.InetSocketAddress
+import java.nio.file.Paths
 
 import org.apache.kyuubi.Utils._
-import org.apache.kyuubi.config.KyuubiConf
+import org.apache.kyuubi.config.{ConfigEntry, KyuubiConf}
 import org.apache.kyuubi.service.{AbstractService, ServiceState}
 import org.apache.kyuubi.shaded.zookeeper.server.{NIOServerCnxnFactory, ZooKeeperServer}
 import org.apache.kyuubi.zookeeper.ZookeeperConf._
@@ -37,8 +38,9 @@ class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
   private var host: String = _
 
   override def initialize(conf: KyuubiConf): Unit = synchronized {
-    dataDirectory = new File(conf.get(ZK_DATA_DIR))
-    dataLogDirectory = new File(conf.get(ZK_DATA_LOG_DIR))
+    dataDirectory = resolvePathIfRelative(conf, ZK_DATA_DIR)
+    dataLogDirectory = resolvePathIfRelative(conf, ZK_DATA_LOG_DIR)
+
     val clientPort = conf.get(ZK_CLIENT_PORT)
     val tickTime = conf.get(ZK_TICK_TIME)
     val maxClientCnxns = conf.get(ZK_MAX_CLIENT_CONNECTIONS)
@@ -93,4 +95,10 @@ class EmbeddedZookeeper extends AbstractService("EmbeddedZookeeper") {
     assert(zks != null, s"$getName is in $getServiceState")
     s"$host:${serverFactory.getLocalPort}"
   }
+
+  def resolvePathIfRelative(conf: KyuubiConf, configEntry: ConfigEntry[String]): File = {
+    val dirFromConfig = conf.get(configEntry)
+    Paths.get(sys.env.getOrElse(KyuubiConf.KYUUBI_HOME, ".")).resolve(dirFromConfig).toFile
+  }
+
 }
diff --git a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala
index 6ef494896..9b0844e69 100644
--- a/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala
+++ b/kyuubi-zookeeper/src/main/scala/org/apache/kyuubi/zookeeper/ZookeeperConf.scala
@@ -31,7 +31,8 @@ object ZookeeperConf {
 
   @deprecated("using kyuubi.zookeeper.embedded.data.dir instead", since = "1.2.0")
   val EMBEDDED_ZK_TEMP_DIR: ConfigEntry[String] = buildConf("kyuubi.zookeeper.embedded.directory")
-    .doc("The temporary directory for the embedded ZooKeeper server")
+    .doc("The temporary directory for the embedded ZooKeeper server. " +
+      "If it is a relative path, it is resolved relative to KYUUBI_HOME. ")
     .version("1.0.0")
     .stringConf
     .createWithDefault("embedded_zookeeper")
@@ -58,12 +59,14 @@ object ZookeeperConf {
 
   val ZK_DATA_DIR: ConfigEntry[String] = buildConf("kyuubi.zookeeper.embedded.data.dir")
     .doc("dataDir for the embedded zookeeper server where stores the in-memory database" +
-      " snapshots and, unless specified otherwise, the transaction log of updates to the database.")
+      " snapshots and, unless specified otherwise, the transaction log of updates to the" +
+      " database. If it is a relative path, it is resolved relative to KYUUBI_HOME.")
     .version("1.2.0")
     .fallbackConf(EMBEDDED_ZK_TEMP_DIR)
 
   val ZK_DATA_LOG_DIR: ConfigEntry[String] = buildConf("kyuubi.zookeeper.embedded.data.log.dir")
-    .doc("dataLogDir for the embedded ZooKeeper server where writes the transaction log .")
+    .doc("dataLogDir for the embedded ZooKeeper server where writes the transaction log. " +
+      "If it is a relative path, it is resolved relative to KYUUBI_HOME.")
     .version("1.2.0")
     .fallbackConf(ZK_DATA_DIR)
 
diff --git a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
index 69e798ac5..8e1abda4f 100644
--- a/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
+++ b/kyuubi-zookeeper/src/test/scala/org/apache/kyuubi/zookeeper/EmbeddedZookeeperSuite.scala
@@ -22,7 +22,7 @@ import org.apache.kyuubi.config.KyuubiConf
 import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory
 import org.apache.kyuubi.shaded.curator.framework.imps.CuratorFrameworkState
 import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry
-import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS}
+import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS, ZK_DATA_DIR, ZK_DATA_LOG_DIR}
 
 class EmbeddedZookeeperSuite extends KyuubiFunSuite {
   private var zkServer: EmbeddedZookeeper = _
@@ -64,4 +64,17 @@ class EmbeddedZookeeperSuite extends KyuubiFunSuite {
     zkServer.initialize(conf)
     assert(zkServer.getConnectString.contains("127.0.0.1"))
   }
+
+  test("relative path from zookeeper config should be in kyuubi_home") {
+    zkServer = new EmbeddedZookeeper()
+    val conf = KyuubiConf()
+      .set(ZK_CLIENT_PORT, 0)
+      .set(ZK_DATA_LOG_DIR, "embedded_zookeeper_log")
+      .set(ZK_DATA_DIR, "/tmp/embedded_zookeeper_data")
+
+    val dataDir = zkServer.resolvePathIfRelative(conf, ZK_DATA_DIR)
+    val dataLogDir = zkServer.resolvePathIfRelative(conf, ZK_DATA_LOG_DIR)
+    assert(dataDir.getAbsolutePath.equals("/tmp/embedded_zookeeper_data"))
+    assert(dataLogDir.getAbsolutePath.contains("/embedded_zookeeper_log"))
+  }
 }