You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2026/05/17 16:13:14 UTC
(kyuubi) branch branch-1.10 updated: [KYUUBI #7443] Fix NullPointerException when configuring the wrong SPARK_HOME
This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch branch-1.10
in repository https://gitbox.apache.org/repos/asf/kyuubi.git
The following commit(s) were added to refs/heads/branch-1.10 by this push:
new b5641e55c0 [KYUUBI #7443] Fix NullPointerException when configuring the wrong SPARK_HOME
b5641e55c0 is described below
commit b5641e55c015f939e5b1a9066b73c88007b387b5
Author: lifumao <li...@tencent.com>
AuthorDate: Mon May 18 00:12:34 2026 +0800
[KYUUBI #7443] Fix NullPointerException when configuring the wrong SPARK_HOME
### Why are the changes needed?
Fix NullPointerException when configuring the wrong SPARK_HOME. For example:
```
bin/kyuubi-beeline \
-u 'jdbc:kyuubi://localhost:10009/default' \
--conf kyuubi.engineEnv.SPARK_HOME=/path/of/not/exist
```
### How was this patch tested?
1. Unit test
2. Manual test
### Was this patch authored or co-authored using generative AI tooling?
Partially assisted by Claude Code (Claude Opus 4.7) for unit test.
Closes #7450 from maomaodev/kyuubi-7443.
Closes #7443
cb2e3618c [lifumao] Fix NullPointerException when configuring the wrong SPARK_HOME
Authored-by: lifumao <li...@tencent.com>
Signed-off-by: Cheng Pan <ch...@apache.org>
(cherry picked from commit d5a62ab6c2c90b280cdf04b62f430f8e1e1a0505)
Signed-off-by: Cheng Pan <ch...@apache.org>
---
.../kyuubi/engine/spark/SparkProcessBuilder.scala | 11 +++-
.../engine/spark/SparkProcessBuilderSuite.scala | 64 ++++++++++++++++++++++
2 files changed, 73 insertions(+), 2 deletions(-)
diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
index aacdddef32..0371979bf9 100644
--- a/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
+++ b/kyuubi-server/src/main/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilder.scala
@@ -106,13 +106,20 @@ class SparkProcessBuilder(
}
private[kyuubi] def extractSparkCoreScalaVersion(fileNames: Iterable[String]): String = {
- fileNames.collectFirst { case SPARK_CORE_SCALA_VERSION_REGEX(scalaVersion) => scalaVersion }
+ Option(fileNames).getOrElse(Iterable.empty)
+ .collectFirst { case SPARK_CORE_SCALA_VERSION_REGEX(scalaVersion) => scalaVersion }
.getOrElse(throw new KyuubiException("Failed to extract Scala version from spark-core jar"))
}
override protected val engineScalaBinaryVersion: String = {
env.get("SPARK_SCALA_VERSION").filter(StringUtils.isNotBlank).getOrElse {
- extractSparkCoreScalaVersion(Paths.get(sparkHome, "jars").toFile.list())
+ val jarsDir = Paths.get(sparkHome, "jars").toFile
+ val fileNames = Option(jarsDir.list()).getOrElse {
+ throw new KyuubiException(
+ s"Failed to list jars under $sparkHome, please check if SPARK_HOME is configured " +
+ "correctly and the jars directory exists")
+ }
+ extractSparkCoreScalaVersion(fileNames)
}
}
diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
index 82bd99070c..78ad5cb11a 100644
--- a/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
+++ b/kyuubi-server/src/test/scala/org/apache/kyuubi/engine/spark/SparkProcessBuilderSuite.scala
@@ -421,6 +421,70 @@ class SparkProcessBuilderSuite extends KerberizedTestHelper with MockitoSugar {
}
}
+ test("Fix NullPointerException when SPARK_HOME is invalid") {
+ val notFoundMsg = "Failed to extract Scala version"
+ val listFailMsg = "Failed to list jars"
+
+ def newConf(sparkHome: String): KyuubiConf = {
+ KyuubiConf(false)
+ .set("kyuubi.engineEnv.SPARK_HOME", sparkHome)
+ .set("kyuubi.engineEnv.SPARK_SCALA_VERSION", "")
+ }
+
+ // case 1: SPARK_HOME does not exists
+ val nonExistentHome = Paths.get(
+ Utils.createTempDir("invalid-spark-home").toString,
+ s"not-exist-${UUID.randomUUID()}").toString
+ val nonExistentErr = intercept[KyuubiException] {
+ new SparkProcessBuilder("kentyao", true, newConf(nonExistentHome))
+ }
+ assert(!nonExistentErr.getMessage.contains("NullPointerException"))
+ assert(nonExistentErr.getMessage.contains(listFailMsg))
+
+ // case 2: SPARK_HOME exists but the `jars` subdirectory is missing
+ val homeWithoutJarsDir = Utils.createTempDir("spark-home-no-jars-dir")
+ val withoutJarsErr = intercept[KyuubiException] {
+ new SparkProcessBuilder("kentyao", true, newConf(homeWithoutJarsDir.toString))
+ }
+ assert(!withoutJarsErr.getMessage.contains("NullPointerException"))
+ assert(withoutJarsErr.getMessage.contains(listFailMsg))
+
+ // case 3: SPARK_HOME exists, but the `jars` is a regular file instead of a directory
+ val homeWithJarsAsFile = Utils.createTempDir("spark-home-jars-as-file")
+ Files.createFile(homeWithJarsAsFile.resolve("jars"))
+ val jarsAsFileErr = intercept[KyuubiException] {
+ new SparkProcessBuilder("kentyao", true, newConf(homeWithJarsAsFile.toString))
+ }
+ assert(!jarsAsFileErr.getMessage.contains("NullPointerException"))
+ assert(jarsAsFileErr.getMessage.contains(listFailMsg))
+
+ // case 4: SPARK_HOME exists, but the `jars` is an empty directory
+ val homeWithEmptyJars = Utils.createTempDir("spark-home-empty-jars")
+ Files.createDirectory(homeWithEmptyJars.resolve("jars"))
+ val emptyJarsHomeErr = intercept[KyuubiException] {
+ new SparkProcessBuilder("kentyao", true, newConf(homeWithEmptyJars.toString))
+ }
+ assert(!emptyJarsHomeErr.getMessage.contains("NullPointerException"))
+ assert(emptyJarsHomeErr.getMessage.contains(notFoundMsg))
+
+ // case 5: SPARK_HOME exists, but the `jars` does not contain any spark-core jar
+ val homeWithoutCoreJar = Utils.createTempDir("spark-home-no-core")
+ val noCoreJarsDir = Files.createDirectory(homeWithoutCoreJar.resolve("jars"))
+ Files.createFile(noCoreJarsDir.resolve("hadoop-common-3.3.4.jar"))
+ Files.createFile(noCoreJarsDir.resolve("scala-library-2.12.18.jar"))
+ val noCoreErr = intercept[KyuubiException] {
+ new SparkProcessBuilder("kentyao", true, newConf(homeWithoutCoreJar.toString))
+ }
+ assert(!noCoreErr.getMessage.contains("NullPointerException"))
+ assert(noCoreErr.getMessage.contains(notFoundMsg))
+
+ // case 6: SPARK_HOME exists, and the `jars` contains spark-core jar
+ val validSparkHome = Utils.createTempDir("spark-home-valid")
+ val validJarsDir = Files.createDirectory(validSparkHome.resolve("jars"))
+ Files.createFile(validJarsDir.resolve("spark-core_2.12-3.5.0.jar"))
+ new SparkProcessBuilder("kentyao", true, newConf(validSparkHome.toString))
+ }
+
test("match scala version of spark home") {
Seq(
"spark-3.2.4-bin-hadoop3.2",