You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ya...@apache.org on 2021/12/07 08:06:10 UTC
[incubator-kyuubi] branch branch-1.4 updated: [KYUUBI #1509] Make KYUUBI_WORK_DIR_ROOT as the default root path.
This is an automated email from the ASF dual-hosted git repository.
yao pushed a commit to branch branch-1.4
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/branch-1.4 by this push:
new 7c0313f [KYUUBI #1509] Make KYUUBI_WORK_DIR_ROOT as the default root path.
7c0313f is described below
commit 7c0313f0df155168e509748c99e12ff53198ac35
Author: Wang Zhen <wa...@qiyi.com>
AuthorDate: Tue Dec 7 16:05:44 2021 +0800
[KYUUBI #1509] Make KYUUBI_WORK_DIR_ROOT as the default root path.
<!--
Thanks for sending a pull request!
Here are some tips for you:
1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html
2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
-->
### _Why are the changes needed?_
<!--
Please clarify why the changes are needed. For instance,
1. If you add a feature, you can talk about the use case of it.
2. If you fix a bug, you can clarify why it is a bug.
-->
Make KYUUBI_WORK_DIR_ROOT as the default root path. For details: #1509.
### _How was this patch tested?_
- [X] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [X] Add screenshots for manual tests if appropriate
- [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #1519 from wForget/KYUUBI-1509.
Closes #1509
ff078e5f [Wang Zhen] [KYUUBI-1509] Make KYUUBI_WORK_DIR_ROOT as the default root path.
Authored-by: Wang Zhen <wa...@qiyi.com>
Signed-off-by: Kent Yao <ya...@apache.org>
(cherry picked from commit 986d9830db875419eab3ef631534b431e65d3262)
Signed-off-by: Kent Yao <ya...@apache.org>
---
.../engine/spark/session/SparkSQLSessionManager.scala | 5 +++--
kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala | 13 +++++++++++++
.../src/test/scala/org/apache/kyuubi/UtilsSuite.scala | 12 +++++++++++-
.../org/apache/kyuubi/metrics/JsonReporterService.scala | 3 ++-
.../org/apache/kyuubi/session/KyuubiSessionManager.scala | 5 +++--
5 files changed, 32 insertions(+), 6 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala
index 046948c..133919d 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala
@@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.session
import org.apache.hive.service.rpc.thrift.TProtocolVersion
import org.apache.spark.sql.{AnalysisException, SparkSession}
-import org.apache.kyuubi.KyuubiSQLException
+import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.engine.ShareLevel
@@ -43,7 +43,8 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession)
def this(spark: SparkSession) = this(classOf[SparkSQLSessionManager].getSimpleName, spark)
override def initialize(conf: KyuubiConf): Unit = {
- _operationLogRoot = Some(conf.get(ENGINE_OPERATION_LOG_DIR_ROOT))
+ val absPath = Utils.getAbsolutePathFromWork(conf.get(ENGINE_OPERATION_LOG_DIR_ROOT))
+ _operationLogRoot = Some(absPath.toAbsolutePath.toString)
super.initialize(conf)
}
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
index 611934b..36e13cf 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
@@ -103,6 +103,19 @@ object Utils extends Logging {
error)
}
+ def getAbsolutePathFromWork(pathStr: String, env: Map[String, String] = sys.env): Path = {
+ val path = Paths.get(pathStr)
+ if (path.isAbsolute) {
+ path
+ } else {
+ val workDir = env.get("KYUUBI_WORK_DIR_ROOT") match {
+ case Some(dir) => dir
+ case _ => System.getProperty("user.dir")
+ }
+ Paths.get(workDir, pathStr)
+ }
+ }
+
/**
* Delete a directory recursively.
*/
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
index 3f61e5a..3b9906d 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
@@ -19,7 +19,7 @@ package org.apache.kyuubi
import java.io.{File, IOException}
import java.net.InetAddress
-import java.nio.file.Files
+import java.nio.file.{Files, Paths}
import java.security.PrivilegedExceptionAction
import java.util.Properties
@@ -130,4 +130,14 @@ class UtilsSuite extends KyuubiFunSuite {
assert(Utils.findLocalInetAddress !== InetAddress.getLocalHost)
}
}
+
+ test("getAbsolutePathFromWork") {
+ val workDir = System.getenv("KYUUBI_WORK_DIR_ROOT")
+ val path1 = "path1"
+ assert(Utils.getAbsolutePathFromWork(path1).toAbsolutePath.toString ===
+ Paths.get(workDir, path1).toAbsolutePath.toString)
+
+ val path2 = "/tmp/path2"
+ assert(Utils.getAbsolutePathFromWork(path2).toString === path2)
+ }
}
diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala
index 1ebbb30..7f82a8e 100644
--- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala
+++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala
@@ -30,6 +30,7 @@ import com.codahale.metrics.MetricRegistry
import com.codahale.metrics.json.MetricsModule
import com.fasterxml.jackson.databind.ObjectMapper
+import org.apache.kyuubi.Utils
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.metrics.MetricsConf._
import org.apache.kyuubi.service.AbstractService
@@ -43,7 +44,7 @@ class JsonReporterService(registry: MetricRegistry)
private var reportPath: Path = _
override def initialize(conf: KyuubiConf): Unit = synchronized {
- reportDir = Paths.get(conf.get(METRICS_JSON_LOCATION)).toAbsolutePath
+ reportDir = Utils.getAbsolutePathFromWork(conf.get(METRICS_JSON_LOCATION))
Files.createDirectories(reportDir)
reportPath = Paths.get(reportDir.toString, "report.json").toAbsolutePath
super.initialize(conf)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
index 83220ec..ca5521b 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/session/KyuubiSessionManager.scala
@@ -20,7 +20,7 @@ package org.apache.kyuubi.session
import com.codahale.metrics.MetricRegistry
import org.apache.hive.service.rpc.thrift.TProtocolVersion
-import org.apache.kyuubi.KyuubiSQLException
+import org.apache.kyuubi.{KyuubiSQLException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.credentials.HadoopCredentialsManager
@@ -37,7 +37,8 @@ class KyuubiSessionManager private (name: String) extends SessionManager(name) {
override def initialize(conf: KyuubiConf): Unit = {
addService(credentialsManager)
- _operationLogRoot = Some(conf.get(SERVER_OPERATION_LOG_DIR_ROOT))
+ val absPath = Utils.getAbsolutePathFromWork(conf.get(SERVER_OPERATION_LOG_DIR_ROOT))
+ _operationLogRoot = Some(absPath.toAbsolutePath.toString)
super.initialize(conf)
}