You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2014/01/08 15:51:24 UTC
[1/3] git commit: Minor typo fix for yarn client
Updated Branches:
refs/heads/master bb6a39a68 -> 6eef78d76
Minor typo fix for yarn client
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/da4694a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/da4694a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/da4694a0
Branch: refs/heads/master
Commit: da4694a0d85433832fda1dd917fc698a08e65907
Parents: e4d6057
Author: Raymond Liu <ra...@intel.com>
Authored: Mon Jan 6 16:11:39 2014 +0800
Committer: Raymond Liu <ra...@intel.com>
Committed: Tue Jan 7 10:24:10 2014 +0800
----------------------------------------------------------------------
.../alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +-
.../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/da4694a0/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 6abb4d5..23781ea 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -102,7 +102,7 @@ class Client(args: ClientArguments, conf: Configuration, sparkConf: SparkConf)
(System.getenv("SPARK_JAR") == null) -> "Error: You must set SPARK_JAR environment variable!",
(args.userJar == null) -> "Error: You must specify a user jar!",
(args.userClass == null) -> "Error: You must specify a user class!",
- (args.numWorkers <= 0) -> "Error: You must specify atleast 1 worker!",
+ (args.numWorkers <= 0) -> "Error: You must specify at least 1 worker!",
(args.amMemory <= YarnAllocationHandler.MEMORY_OVERHEAD) -> ("Error: AM memory size must be " +
"greater than: " + YarnAllocationHandler.MEMORY_OVERHEAD),
(args.workerMemory <= YarnAllocationHandler.MEMORY_OVERHEAD) -> ("Error: Worker memory size " +
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/da4694a0/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 440ad5c..be323d7 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -122,7 +122,7 @@ class Client(args: ClientArguments, conf: Configuration, sparkConf: SparkConf)
(System.getenv("SPARK_JAR") == null) -> "Error: You must set SPARK_JAR environment variable!",
(args.userJar == null) -> "Error: You must specify a user jar!",
(args.userClass == null) -> "Error: You must specify a user class!",
- (args.numWorkers <= 0) -> "Error: You must specify atleast 1 worker!",
+ (args.numWorkers <= 0) -> "Error: You must specify at least 1 worker!",
(args.amMemory <= YarnAllocationHandler.MEMORY_OVERHEAD) -> ("Error: AM memory size must be" +
"greater than: " + YarnAllocationHandler.MEMORY_OVERHEAD),
(args.workerMemory <= YarnAllocationHandler.MEMORY_OVERHEAD) -> ("Error: Worker memory size" +
[2/3] git commit: Export --file for YarnClient mode to support
sending extra files to worker on yarn cluster
Posted by tg...@apache.org.
Export --file for YarnClient mode to support sending extra files to worker on yarn cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/67af8031
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/67af8031
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/67af8031
Branch: refs/heads/master
Commit: 67af8031368c4a3558aeb46c290b7b7a09516539
Parents: da4694a
Author: Raymond Liu <ra...@intel.com>
Authored: Mon Jan 6 16:12:25 2014 +0800
Committer: Raymond Liu <ra...@intel.com>
Committed: Tue Jan 7 10:24:11 2014 +0800
----------------------------------------------------------------------
docs/running-on-yarn.md | 2 ++
.../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 4 +++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/67af8031/docs/running-on-yarn.md
----------------------------------------------------------------------
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index a35e003..da5a502 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -114,6 +114,8 @@ For example:
SPARK_YARN_APP_JAR=examples/target/scala-{{site.SCALA_VERSION}}/spark-examples-assembly-{{site.SPARK_VERSION}}.jar \
MASTER=yarn-client ./bin/spark-shell
+You can also send extra files to yarn cluster for worker to use by exporting SPARK_YARN_DIST_FILES=file1,file2... etc.
+
# Building Spark for Hadoop/YARN 2.2.x
See [Building Spark with Maven](building-with-maven.html) for instructions on how to build Spark using the Maven process.
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/67af8031/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
index 324ef46..4b1b5da 100644
--- a/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/scheduler/cluster/YarnClientSchedulerBackend.scala
@@ -39,6 +39,7 @@ private[spark] class YarnClientSchedulerBackend(
val defaultWorkerNumber = "1"
val userJar = System.getenv("SPARK_YARN_APP_JAR")
+ val distFiles = System.getenv("SPARK_YARN_DIST_FILES")
var workerCores = System.getenv("SPARK_WORKER_CORES")
var workerMemory = System.getenv("SPARK_WORKER_MEMORY")
var workerNumber = System.getenv("SPARK_WORKER_INSTANCES")
@@ -64,7 +65,8 @@ private[spark] class YarnClientSchedulerBackend(
"--worker-memory", workerMemory,
"--worker-cores", workerCores,
"--num-workers", workerNumber,
- "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher"
+ "--master-class", "org.apache.spark.deploy.yarn.WorkerLauncher",
+ "--files", distFiles
)
val args = new ClientArguments(argsArray, conf)
[3/3] git commit: Merge pull request #345 from colorant/yarn
Posted by tg...@apache.org.
Merge pull request #345 from colorant/yarn
support distributing extra files to worker for yarn client mode
So that user doesn't need to package all dependency into one assemble jar as spark app jar
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/6eef78d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/6eef78d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/6eef78d7
Branch: refs/heads/master
Commit: 6eef78d769379b98a1fa6d4f2fa606216e5063a6
Parents: bb6a39a 67af803
Author: Thomas Graves <tg...@apache.org>
Authored: Wed Jan 8 08:49:20 2014 -0600
Committer: Thomas Graves <tg...@apache.org>
Committed: Wed Jan 8 08:49:20 2014 -0600
----------------------------------------------------------------------
docs/running-on-yarn.md | 2 ++
.../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +-
.../spark/scheduler/cluster/YarnClientSchedulerBackend.scala | 4 +++-
.../src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 +-
4 files changed, 7 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/6eef78d7/docs/running-on-yarn.md
----------------------------------------------------------------------