You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/08/23 05:50:21 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3298] Unify the approach of get classLoader in Kyuubi
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new a54daf27c [KYUUBI #3298] Unify the approach of get classLoader in Kyuubi
a54daf27c is described below
commit a54daf27c2798d99cc3fb9af7e4ac17b54d6a3e8
Author: yikf <yi...@gmail.com>
AuthorDate: Tue Aug 23 13:50:10 2022 +0800
[KYUUBI #3298] Unify the approach of get classLoader in Kyuubi
### _Why are the changes needed?_
fix https://github.com/apache/incubator-kyuubi/issues/3298
Curently, a serval snippets use `getClass.getClassLoader` to get the classLoader in Apache Kyuubi, this pr aims to unify this behavior in `Utils.getContextOrKyuubiClassLoader`
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [ ] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3299 from yikf/unify-calssloader.
Closes #3298
ae081ad9 [yikf] Unify the approach of get classLoader in Kyuubi
Authored-by: yikf <yi...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../org/apache/kyuubi/tpcds/benchmark/Benchmark.scala | 9 +--------
.../kyuubi/tpcds/benchmark/TPCDS_2_4_Queries.scala | 4 +---
.../apache/kyuubi/spark/connector/kudu/KuduMixin.scala | 5 +++--
.../kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala | 5 +++--
.../kyuubi/spark/connector/tpch/TPCHQuerySuite.scala | 4 ++--
.../kyuubi/engine/jdbc/doris/WithDorisContainer.scala | 4 +++-
.../jdbc/doris/WithKyuubiServerAndDorisContainer.scala | 2 +-
.../src/main/scala/org/apache/kyuubi/Utils.scala | 17 ++++++++++++++++-
.../src/test/scala/org/apache/kyuubi/UtilsSuite.scala | 2 +-
.../kyuubi/credentials/HadoopCredentialsManager.scala | 6 ++++--
.../apache/kyuubi/engine/KyuubiApplicationManager.scala | 4 ++--
.../kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala | 2 +-
12 files changed, 38 insertions(+), 26 deletions(-)
diff --git a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Benchmark.scala b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Benchmark.scala
index 8071bca1b..48eb45035 100644
--- a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Benchmark.scala
+++ b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/Benchmark.scala
@@ -17,13 +17,6 @@
package org.apache.kyuubi.tpcds.benchmark
-import scala.concurrent._
-import scala.concurrent.ExecutionContext.Implicits.global
-import scala.concurrent.duration.DurationInt
-import scala.language.implicitConversions
-import scala.util.{Failure => SFailure, Success, Try}
-import scala.util.control.NonFatal
-
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
// scalastyle:off
@@ -43,7 +36,7 @@ abstract class Benchmark(
implicit protected def toOption[A](a: A): Option[A] = Option(a)
val buildInfo: Map[String, String] =
- Try(getClass.getClassLoader.loadClass("org.apache.spark.BuildInfo")).map { cls =>
+ Try(Utils.getContextOrKyuubiClassLoader.loadClass("org.apache.spark.BuildInfo")).map { cls =>
cls.getMethods
.filter(_.getReturnType == classOf[String])
.filterNot(_.getName == "toString")
diff --git a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/TPCDS_2_4_Queries.scala b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/TPCDS_2_4_Queries.scala
index 220caafc2..92248537a 100644
--- a/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/TPCDS_2_4_Queries.scala
+++ b/dev/kyuubi-tpcds/src/main/scala/org/apache/kyuubi/tpcds/benchmark/TPCDS_2_4_Queries.scala
@@ -17,8 +17,6 @@
package org.apache.kyuubi.tpcds.benchmark
-import scala.io.{Codec, Source}
-
/**
* This implements the official TPCDS v2.4 queries with only cosmetic modifications.
*/
@@ -133,7 +131,7 @@ trait TPCDS_2_4_Queries extends Benchmark {
"ss_max")
val tpcds2_4Queries: Seq[Query] = queryNames.map { queryName =>
- val in = getClass.getClassLoader.getResourceAsStream(s"tpcds_2_4/$queryName.sql")
+ val in = Utils.getContextOrKyuubiClassLoader.getResourceAsStream(s"tpcds_2_4/$queryName.sql")
val queryContent: String = Source.fromInputStream(in)(Codec.UTF8).mkString
in.close()
diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
index 6d6f3beaa..dee09db38 100644
--- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
+++ b/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduMixin.scala
@@ -21,7 +21,7 @@ import java.io.File
import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService, ForAllTestContainer}
-import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.{KyuubiFunSuite, Utils}
trait KuduMixin extends KyuubiFunSuite with ForAllTestContainer {
@@ -30,7 +30,8 @@ trait KuduMixin extends KyuubiFunSuite with ForAllTestContainer {
override val container: DockerComposeContainer =
DockerComposeContainer
.Def(
- composeFiles = new File(getClass.getClassLoader.getResource("kudu-compose.yml").toURI),
+ composeFiles =
+ new File(Utils.getContextOrKyuubiClassLoader.getResource("kudu-compose.yml").toURI),
exposedServices = ExposedService("kudu-master", KUDU_MASTER_PORT) :: Nil)
.createContainer()
diff --git a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala
index efa87253b..83679989a 100644
--- a/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpcds/src/test/scala/org/apache/kyuubi/spark/connector/tpcds/TPCDSQuerySuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.scalatest.tags.Slow
-import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.{KyuubiFunSuite, Utils}
import org.apache.kyuubi.spark.connector.common.GoldenFileUtils._
import org.apache.kyuubi.spark.connector.common.LocalSparkSession.withSparkSession
@@ -65,7 +65,8 @@ class TPCDSQuerySuite extends KyuubiFunSuite {
withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark =>
spark.sql("USE tpcds.tiny")
queries.map { queryName =>
- val in = getClass.getClassLoader.getResourceAsStream(s"kyuubi/tpcds_3.2/$queryName.sql")
+ val in = Utils.getContextOrKyuubiClassLoader
+ .getResourceAsStream(s"kyuubi/tpcds_3.2/$queryName.sql")
val queryContent: String = Source.fromInputStream(in)(Codec.UTF8).mkString
in.close()
queryName -> queryContent
diff --git a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHQuerySuite.scala b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHQuerySuite.scala
index bc365128c..efeaeb36c 100644
--- a/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHQuerySuite.scala
+++ b/extensions/spark/kyuubi-spark-connector-tpch/src/test/scala/org/apache/kyuubi/spark/connector/tpch/TPCHQuerySuite.scala
@@ -24,7 +24,7 @@ import org.apache.spark.SparkConf
import org.apache.spark.sql.SparkSession
import org.scalatest.tags.Slow
-import org.apache.kyuubi.KyuubiFunSuite
+import org.apache.kyuubi.{KyuubiFunSuite, Utils}
import org.apache.kyuubi.spark.connector.common.GoldenFileUtils._
import org.apache.kyuubi.spark.connector.common.LocalSparkSession.withSparkSession
@@ -62,7 +62,7 @@ class TPCHQuerySuite extends KyuubiFunSuite {
withSparkSession(SparkSession.builder.config(sparkConf).getOrCreate()) { spark =>
spark.sql("USE tpch.tiny")
queries.map { queryName =>
- val in = getClass.getClassLoader.getResourceAsStream(
+ val in = Utils.getContextOrKyuubiClassLoader.getResourceAsStream(
s"kyuubi/tpch/$queryName.sql")
val queryContent: String = Source.fromInputStream(in)(Codec.UTF8).mkString
in.close()
diff --git a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
index be8a94788..8092e3299 100644
--- a/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
+++ b/externals/kyuubi-jdbc-engine/src/test/scala/org/apache/kyuubi/engine/jdbc/doris/WithDorisContainer.scala
@@ -22,6 +22,7 @@ import java.time.Duration
import com.dimafeng.testcontainers.{DockerComposeContainer, ExposedService}
import org.testcontainers.containers.wait.strategy.DockerHealthcheckWaitStrategy
+import org.apache.kyuubi.Utils
import org.apache.kyuubi.engine.jdbc.WithJdbcServerContainer
trait WithDorisContainer extends WithJdbcServerContainer {
@@ -37,7 +38,8 @@ trait WithDorisContainer extends WithJdbcServerContainer {
override val container: DockerComposeContainer =
DockerComposeContainer
.Def(
- composeFiles = new File(getClass.getClassLoader.getResource("doris-compose.yml").toURI),
+ composeFiles = new File(Utils.getContextOrKyuubiClassLoader
+ .getResource("doris-compose.yml").toURI),
exposedServices = Seq[ExposedService](
ExposedService(
DORIS_FE_SERVICE_NAME,
diff --git a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala
index ef41108ae..209681135 100644
--- a/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala
+++ b/integration-tests/kyuubi-jdbc-it/src/test/scala/org/apache/kyuubi/it/jdbc/doris/WithKyuubiServerAndDorisContainer.scala
@@ -32,7 +32,7 @@ trait WithKyuubiServerAndDorisContainer extends WithKyuubiServer with WithDorisE
.set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome)
.set(
ENGINE_JDBC_EXTRA_CLASSPATH,
- getClass.getClassLoader.getResource(
+ Utils.getContextOrKyuubiClassLoader.getResource(
"mysql/mysql-connector-java-8.0.30.jar").toURI.getPath)
}
diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
index 11025d54a..fa1ba21c5 100644
--- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
+++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala
@@ -55,7 +55,7 @@ object Utils extends Logging {
.map(d => new File(d + File.separator + KYUUBI_CONF_FILE_NAME))
.filter(_.exists())
.orElse {
- Option(getClass.getClassLoader.getResource(KYUUBI_CONF_FILE_NAME)).map { url =>
+ Option(Utils.getContextOrKyuubiClassLoader.getResource(KYUUBI_CONF_FILE_NAME)).map { url =>
new File(url.getFile)
}.filter(_.exists())
}
@@ -314,4 +314,19 @@ object Utils extends Logging {
}
def isCommandAvailable(cmd: String): Boolean = s"which $cmd".! == 0
+
+ /**
+ * Get the ClassLoader which loaded Kyuubi.
+ */
+ def getKyuubiClassLoader: ClassLoader = getClass.getClassLoader
+
+ /**
+ * Get the Context ClassLoader on this thread or, if not present, the ClassLoader that
+ * loaded Kyuubi.
+ *
+ * This should be used whenever passing a ClassLoader to Class.ForName or finding the currently
+ * active loader when setting up ClassLoader delegation chains.
+ */
+ def getContextOrKyuubiClassLoader: ClassLoader =
+ Option(Thread.currentThread().getContextClassLoader).getOrElse(getKyuubiClassLoader)
}
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
index c62e268cc..f75f299ae 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/UtilsSuite.scala
@@ -34,7 +34,7 @@ class UtilsSuite extends KyuubiFunSuite {
test("build information check") {
val buildFile = "kyuubi-version-info.properties"
- val str = this.getClass.getClassLoader.getResourceAsStream(buildFile)
+ val str = Utils.getContextOrKyuubiClassLoader.getResourceAsStream(buildFile)
val props = new Properties()
assert(str !== null)
props.load(str)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopCredentialsManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopCredentialsManager.scala
index 0d923b214..fe710e678 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopCredentialsManager.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/credentials/HadoopCredentialsManager.scala
@@ -30,7 +30,7 @@ import scala.util.{Failure, Success, Try}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.security.Credentials
-import org.apache.kyuubi.Logging
+import org.apache.kyuubi.{Logging, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.config.KyuubiConf._
import org.apache.kyuubi.service.AbstractService
@@ -316,7 +316,9 @@ object HadoopCredentialsManager extends Logging {
def loadProviders(kyuubiConf: KyuubiConf): Map[String, HadoopDelegationTokenProvider] = {
val loader =
- ServiceLoader.load(classOf[HadoopDelegationTokenProvider], getClass.getClassLoader)
+ ServiceLoader.load(
+ classOf[HadoopDelegationTokenProvider],
+ Utils.getContextOrKyuubiClassLoader)
val providers = mutable.ArrayBuffer[HadoopDelegationTokenProvider]()
val iterator = loader.iterator
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
index 659e9d23a..7f2501ec2 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/KyuubiApplicationManager.scala
@@ -24,7 +24,7 @@ import java.util.{Locale, ServiceLoader}
import scala.collection.JavaConverters._
import scala.util.control.NonFatal
-import org.apache.kyuubi.KyuubiException
+import org.apache.kyuubi.{KyuubiException, Utils}
import org.apache.kyuubi.config.KyuubiConf
import org.apache.kyuubi.engine.KubernetesApplicationOperation.LABEL_KYUUBI_UNIQUE_KEY
import org.apache.kyuubi.engine.flink.FlinkProcessBuilder
@@ -35,7 +35,7 @@ class KyuubiApplicationManager extends AbstractService("KyuubiApplicationManager
// TODO: maybe add a configuration is better
private val operations = {
- ServiceLoader.load(classOf[ApplicationOperation], getClass.getClassLoader)
+ ServiceLoader.load(classOf[ApplicationOperation], Utils.getContextOrKyuubiClassLoader)
.iterator().asScala.toSeq
}
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
index d7e2f06cf..0cffaeca3 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/server/metadata/jdbc/JDBCMetadataStore.scala
@@ -77,7 +77,7 @@ class JDBCMetadataStore(conf: KyuubiConf) extends MetadataStore with Logging {
}
private def initSchema(): Unit = {
- val classLoader = getClass.getClassLoader
+ val classLoader = Utils.getContextOrKyuubiClassLoader
val initSchemaStream: Option[InputStream] = dbType match {
case DERBY =>
Option(classLoader.getResourceAsStream("sql/derby/metadata-store-schema-derby.sql"))