You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2026/09/02 10:06:54 UTC
(kyuubi) branch branch-1.11 updated: [KYUUBI #7653] Cache reflection method lookups in Spark engine
This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new 594d05aab5 [KYUUBI #7653] Cache reflection method lookups in Spark engine
594d05aab5 is described below
commit 594d05aab5ba34f8e60e09c5f4f5771e77beeb1b
Author: Cheng Pan <pa...@gmail.com>
AuthorDate: Wed Sep 2 17:51:27 2026 +0800
[KYUUBI #7653] Cache reflection method lookups in Spark engine
Repeated `DynMethods` lookups occur on hot paths for Hive result serialization, Spark UI rendering, and URI construction. This caches the resolved methods so reflection lookup happens once instead of on every call.
- `dev/reformat`
- `build/mvn test -pl externals/kyuubi-spark-sql-engine -am -Dtest=none -DwildcardSuites=org.apache.kyuubi.engine.spark.schema.RowSetSuite`
- `build/mvn test -pl externals/kyuubi-spark-sql-engine -am -Dtest=none -DwildcardSuites=org.apache.kyuubi.engine.spark.KyuubiSparkUtilSuite`
- `build/mvn test -Pspark-4.0 -Pscala-2.13 -pl externals/kyuubi-spark-sql-engine -am -Dtest=none -DwildcardSuites=org.apache.kyuubi.engine.spark.KyuubiSparkUtilSuite`
Yes. Assisted-by: GLM 5.3
Closes #7653 from pan3793/kyuubi-rowset-bound-method.
Closes #7653
cc5bafc60 [Cheng Pan] Cache Spark reflection methods
adcae0cd8 [Cheng Pan] Cache HiveResult reflection methods
Authored-by: Cheng Pan <pa...@gmail.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
(cherry picked from commit 445efc825eecfa987f9b810789174e1f9b28e75d)
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../kyuubi/engine/spark/KyuubiSparkUtil.scala | 52 +++++++++----------
.../apache/kyuubi/engine/spark/schema/RowSet.scala | 20 +++++---
.../scala/org/apache/spark/ui/SparkUIUtils.scala | 58 ++++++++++++----------
3 files changed, 66 insertions(+), 64 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala
index 8c19ea5c25..6768244338 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala
@@ -131,37 +131,31 @@ object KyuubiSparkUtil extends Logging {
// jakarta.ws.rs, this is an equivalent implementation using reflection of the following
// plain invocation:
// {javax|jakarta}.ws.rs.core.UriBuilder.fromUri(uri).fragment(fragment).build()
- def buildURI(uri: URI, fragment: String): URI = {
+ private lazy val uriBuilderClassName: String =
if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
- var uriBuilder = DynMethods.builder("fromUri")
- .impl("jakarta.ws.rs.core.UriBuilder", classOf[URI])
- .buildStatic()
- .invoke[AnyRef](uri)
-
- uriBuilder = DynMethods.builder("fragment")
- .impl("jakarta.ws.rs.core.UriBuilder", classOf[String])
- .build(uriBuilder)
- .invoke[AnyRef](fragment)
-
- DynMethods.builder("build")
- .impl("jakarta.ws.rs.core.UriBuilder", classOf[Array[AnyRef]])
- .build(uriBuilder)
- .invoke[URI](Array.empty[AnyRef])
+ "jakarta.ws.rs.core.UriBuilder"
} else {
- var uriBuilder = DynMethods.builder("fromUri")
- .impl("javax.ws.rs.core.UriBuilder", classOf[URI])
- .buildStatic()
- .invoke[AnyRef](uri)
-
- uriBuilder = DynMethods.builder("fragment")
- .impl("javax.ws.rs.core.UriBuilder", classOf[String])
- .build(uriBuilder)
- .invoke[AnyRef](fragment)
-
- DynMethods.builder("build")
- .impl("javax.ws.rs.core.UriBuilder", classOf[Array[AnyRef]])
- .build(uriBuilder)
- .invoke[URI](Array.empty[AnyRef])
+ "javax.ws.rs.core.UriBuilder"
}
+
+ private lazy val fromUriMethod: DynMethods.StaticMethod =
+ DynMethods.builder("fromUri")
+ .impl(uriBuilderClassName, classOf[URI])
+ .buildStaticChecked()
+
+ private lazy val fragmentMethod: DynMethods.UnboundMethod =
+ DynMethods.builder("fragment")
+ .impl(uriBuilderClassName, classOf[String])
+ .buildChecked()
+
+ private lazy val buildMethod: DynMethods.UnboundMethod =
+ DynMethods.builder("build")
+ .impl(uriBuilderClassName, classOf[Array[AnyRef]])
+ .buildChecked()
+
+ def buildURI(uri: URI, fragment: String): URI = {
+ val uriBuilder = fromUriMethod.invoke[AnyRef](uri)
+ val fragmentedUriBuilder = fragmentMethod.invoke[AnyRef](uriBuilder, fragment)
+ buildMethod.invoke[URI](fragmentedUriBuilder, Array.empty[AnyRef])
}
}
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
index 47e6351b2d..250486e146 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala
@@ -30,18 +30,16 @@ object RowSet {
// SPARK-47911 (4.0.0) introduced it
type BinaryFormatter = Array[Byte] => String
- def getBinaryFormatter: BinaryFormatter =
+ private val getBinaryFormatterMethod: DynMethods.BoundMethod =
DynMethods.builder("getBinaryFormatter")
.impl(HiveResult.getClass) // for Spark 4.0 and later
.orNoop() // for Spark 3.5 and before
.buildChecked(HiveResult)
- .invokeChecked[BinaryFormatter]()
- def toHiveString(
- valueAndType: (Any, DataType),
- nested: JBoolean = false,
- timeFormatters: TimeFormatters,
- binaryFormatter: BinaryFormatter): String =
+ def getBinaryFormatter: BinaryFormatter =
+ getBinaryFormatterMethod.invokeChecked[BinaryFormatter]()
+
+ private val toHiveStringMethod: DynMethods.BoundMethod =
DynMethods.builder("toHiveString")
.impl( // for Spark 3.5 and before
HiveResult.getClass,
@@ -55,5 +53,11 @@ object RowSet {
classOf[TimeFormatters],
classOf[BinaryFormatter])
.buildChecked(HiveResult)
- .invokeChecked[String](valueAndType, nested, timeFormatters, binaryFormatter)
+
+ def toHiveString(
+ valueAndType: (Any, DataType),
+ nested: JBoolean = false,
+ timeFormatters: TimeFormatters,
+ binaryFormatter: BinaryFormatter): String =
+ toHiveStringMethod.invokeChecked[String](valueAndType, nested, timeFormatters, binaryFormatter)
}
diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
index 4862372a19..427ceb2788 100644
--- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
+++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/SparkUIUtils.scala
@@ -30,19 +30,8 @@ import org.apache.kyuubi.util.reflect.DynMethods
*/
object SparkUIUtils {
- def formatDuration(ms: Long): String = {
- UIUtils.formatDuration(ms)
- }
-
- def headerSparkPage(
- request: HttpServletRequestLike,
- title: String,
- content: Seq[Node],
- activeTab: SparkUITab,
- helpText: Option[String] = None,
- showVisualization: JBoolean = false,
- useDataTables: JBoolean = false): Seq[Node] = {
- val headerSparkPageMethod = if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
+ private val headerSparkPageMethod: DynMethods.BoundMethod =
+ if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
DynMethods.builder("headerSparkPage")
.impl(
UIUtils.getClass,
@@ -67,21 +56,9 @@ object SparkUIUtils {
classOf[Boolean])
.buildChecked(UIUtils)
}
- headerSparkPageMethod.invoke[Seq[Node]](
- request,
- title,
- () => content,
- activeTab,
- helpText,
- showVisualization,
- useDataTables)
- }
- def prependBaseUri(
- request: HttpServletRequestLike,
- basePath: String = "",
- resource: String = ""): String = {
- val prependBaseUriMethod = if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
+ private val prependBaseUriMethod: DynMethods.BoundMethod =
+ if (SPARK_ENGINE_RUNTIME_VERSION >= "4.0") {
DynMethods.builder("prependBaseUri")
.impl(
UIUtils.getClass,
@@ -98,6 +75,33 @@ object SparkUIUtils {
classOf[String])
.buildChecked(UIUtils)
}
+
+ def formatDuration(ms: Long): String = {
+ UIUtils.formatDuration(ms)
+ }
+
+ def headerSparkPage(
+ request: HttpServletRequestLike,
+ title: String,
+ content: Seq[Node],
+ activeTab: SparkUITab,
+ helpText: Option[String] = None,
+ showVisualization: JBoolean = false,
+ useDataTables: JBoolean = false): Seq[Node] = {
+ headerSparkPageMethod.invoke[Seq[Node]](
+ request,
+ title,
+ () => content,
+ activeTab,
+ helpText,
+ showVisualization,
+ useDataTables)
+ }
+
+ def prependBaseUri(
+ request: HttpServletRequestLike,
+ basePath: String = "",
+ resource: String = ""): String = {
prependBaseUriMethod.invoke[String](request, basePath, resource)
}
}