You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/01/18 22:44:28 UTC
spark git commit: [SPARK-22962][K8S] Fail fast if submission client
local files are used
Repository: spark
Updated Branches:
refs/heads/master e01919e83 -> 5d7c4ba4d
[SPARK-22962][K8S] Fail fast if submission client local files are used
## What changes were proposed in this pull request?
In the Kubernetes mode, fails fast in the submission process if any submission client local dependencies are used as the use case is not supported yet.
## How was this patch tested?
Unit tests, integration tests, and manual tests.
vanzin foxish
Author: Yinan Li <li...@gmail.com>
Closes #20320 from liyinan926/master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5d7c4ba4
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5d7c4ba4
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5d7c4ba4
Branch: refs/heads/master
Commit: 5d7c4ba4d73a72f26d591108db3c20b4a6c84f3f
Parents: e01919e
Author: Yinan Li <li...@gmail.com>
Authored: Thu Jan 18 14:44:22 2018 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Thu Jan 18 14:44:22 2018 -0800
----------------------------------------------------------------------
docs/running-on-kubernetes.md | 5 +++-
.../k8s/submit/DriverConfigOrchestrator.scala | 14 ++++++++-
.../submit/DriverConfigOrchestratorSuite.scala | 31 +++++++++++++++++++-
3 files changed, 47 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5d7c4ba4/docs/running-on-kubernetes.md
----------------------------------------------------------------------
diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md
index 08ec34c..d6b1735 100644
--- a/docs/running-on-kubernetes.md
+++ b/docs/running-on-kubernetes.md
@@ -117,7 +117,10 @@ This URI is the location of the example jar that is already in the Docker image.
If your application's dependencies are all hosted in remote locations like HDFS or HTTP servers, they may be referred to
by their appropriate remote URIs. Also, application dependencies can be pre-mounted into custom-built Docker images.
Those dependencies can be added to the classpath by referencing them with `local://` URIs and/or setting the
-`SPARK_EXTRA_CLASSPATH` environment variable in your Dockerfiles.
+`SPARK_EXTRA_CLASSPATH` environment variable in your Dockerfiles. The `local://` scheme is also required when referring to
+dependencies in custom-built Docker images in `spark-submit`. Note that using application dependencies from the submission
+client's local file system is currently not yet supported.
+
### Using Remote Dependencies
When there are application dependencies hosted in remote locations like HDFS or HTTP servers, the driver and executor pods
http://git-wip-us.apache.org/repos/asf/spark/blob/5d7c4ba4/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala
index c9cc300..ae70904 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestrator.scala
@@ -20,7 +20,7 @@ import java.util.UUID
import com.google.common.primitives.Longs
-import org.apache.spark.SparkConf
+import org.apache.spark.{SparkConf, SparkException}
import org.apache.spark.deploy.k8s.{KubernetesUtils, MountSecretsBootstrap}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
@@ -117,6 +117,12 @@ private[spark] class DriverConfigOrchestrator(
.map(_.split(","))
.getOrElse(Array.empty[String])
+ // TODO(SPARK-23153): remove once submission client local dependencies are supported.
+ if (existSubmissionLocalFiles(sparkJars) || existSubmissionLocalFiles(sparkFiles)) {
+ throw new SparkException("The Kubernetes mode does not yet support referencing application " +
+ "dependencies in the local file system.")
+ }
+
val dependencyResolutionStep = if (sparkJars.nonEmpty || sparkFiles.nonEmpty) {
Seq(new DependencyResolutionStep(
sparkJars,
@@ -162,6 +168,12 @@ private[spark] class DriverConfigOrchestrator(
initContainerBootstrapStep
}
+ private def existSubmissionLocalFiles(files: Seq[String]): Boolean = {
+ files.exists { uri =>
+ Utils.resolveURI(uri).getScheme == "file"
+ }
+ }
+
private def existNonContainerLocalFiles(files: Seq[String]): Boolean = {
files.exists { uri =>
Utils.resolveURI(uri).getScheme != "local"
http://git-wip-us.apache.org/repos/asf/spark/blob/5d7c4ba4/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestratorSuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestratorSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestratorSuite.scala
index 65274c6..033d303 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestratorSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/DriverConfigOrchestratorSuite.scala
@@ -16,7 +16,7 @@
*/
package org.apache.spark.deploy.k8s.submit
-import org.apache.spark.{SparkConf, SparkFunSuite}
+import org.apache.spark.{SparkConf, SparkException, SparkFunSuite}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.submit.steps._
@@ -117,6 +117,35 @@ class DriverConfigOrchestratorSuite extends SparkFunSuite {
classOf[DriverMountSecretsStep])
}
+ test("Submission using client local dependencies") {
+ val sparkConf = new SparkConf(false)
+ .set(CONTAINER_IMAGE, DRIVER_IMAGE)
+ var orchestrator = new DriverConfigOrchestrator(
+ APP_ID,
+ LAUNCH_TIME,
+ Some(JavaMainAppResource("file:///var/apps/jars/main.jar")),
+ APP_NAME,
+ MAIN_CLASS,
+ APP_ARGS,
+ sparkConf)
+ assertThrows[SparkException] {
+ orchestrator.getAllConfigurationSteps
+ }
+
+ sparkConf.set("spark.files", "/path/to/file1,/path/to/file2")
+ orchestrator = new DriverConfigOrchestrator(
+ APP_ID,
+ LAUNCH_TIME,
+ Some(JavaMainAppResource("local:///var/apps/jars/main.jar")),
+ APP_NAME,
+ MAIN_CLASS,
+ APP_ARGS,
+ sparkConf)
+ assertThrows[SparkException] {
+ orchestrator.getAllConfigurationSteps
+ }
+ }
+
private def validateStepTypes(
orchestrator: DriverConfigOrchestrator,
types: Class[_ <: DriverConfigurationStep]*): Unit = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org