You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2020/02/28 08:03:51 UTC
[spark] branch branch-3.0 updated: [SPARK-30970][K8S][CORE] Fix NPE
while resolving k8s master url
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new b8e9cdc [SPARK-30970][K8S][CORE] Fix NPE while resolving k8s master url
b8e9cdc is described below
commit b8e9cdcd14dcda68dde0c646f58d10880332691e
Author: Kent Yao <ya...@hotmail.com>
AuthorDate: Fri Feb 28 00:01:20 2020 -0800
[SPARK-30970][K8S][CORE] Fix NPE while resolving k8s master url
### What changes were proposed in this pull request?
```
bin/spark-sql --master k8s:///https://kubernetes.docker.internal:6443 --conf spark.kubernetes.container.image=yaooqinn/spark:v2.4.4
Exception in thread "main" java.lang.NullPointerException
at org.apache.spark.util.Utils$.checkAndGetK8sMasterUrl(Utils.scala:2739)
at org.apache.spark.deploy.SparkSubmit.prepareSubmitEnvironment(SparkSubmit.scala:261)
at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:774)
at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
```
Althrough `k8s:///https://kubernetes.docker.internal:6443` is a wrong master url but should not throw npe
The `case null` will never be touched.
https://github.com/apache/spark/blob/3f4060c340d6bac412e8819c4388ccba226efcf3/core/src/main/scala/org/apache/spark/util/Utils.scala#L2772-L2776
### Why are the changes needed?
bug fix
### Does this PR introduce any user-facing change?
no
### How was this patch tested?
add ut case
Closes #27721 from yaooqinn/SPARK-30970.
Authored-by: Kent Yao <ya...@hotmail.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
(cherry picked from commit 1383bd459a834fb075c5b570338fab0886110df9)
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
core/src/main/scala/org/apache/spark/util/Utils.scala | 15 ++++++---------
.../src/test/scala/org/apache/spark/util/UtilsSuite.scala | 4 ++++
2 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 297cc5e..dde4323 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -2772,19 +2772,16 @@ private[spark] object Utils extends Logging {
}
val masterScheme = new URI(masterWithoutK8sPrefix).getScheme
- val resolvedURL = masterScheme.toLowerCase(Locale.ROOT) match {
- case "https" =>
+
+ val resolvedURL = Option(masterScheme).map(_.toLowerCase(Locale.ROOT)) match {
+ case Some("https") =>
masterWithoutK8sPrefix
- case "http" =>
+ case Some("http") =>
logWarning("Kubernetes master URL uses HTTP instead of HTTPS.")
masterWithoutK8sPrefix
- case null =>
- val resolvedURL = s"https://$masterWithoutK8sPrefix"
- logInfo("No scheme specified for kubernetes master URL, so defaulting to https. Resolved " +
- s"URL is $resolvedURL.")
- resolvedURL
case _ =>
- throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme)
+ throw new IllegalArgumentException("Invalid Kubernetes master scheme: " + masterScheme
+ + " found in URL: " + masterWithoutK8sPrefix)
}
s"k8s://$resolvedURL"
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index 8f8902e..f5e438b 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1243,6 +1243,10 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
intercept[IllegalArgumentException] {
Utils.checkAndGetK8sMasterUrl("k8s://foo://host:port")
}
+
+ intercept[IllegalArgumentException] {
+ Utils.checkAndGetK8sMasterUrl("k8s:///https://host:port")
+ }
}
test("stringHalfWidth") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org