You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/01/08 20:46:23 UTC
spark git commit: [SPARK-5130][Deploy]Take yarn-cluster as cluster
mode in spark-submit
Repository: spark
Updated Branches:
refs/heads/master 0a597276d -> 0760787da
[SPARK-5130][Deploy]Take yarn-cluster as cluster mode in spark-submit
https://issues.apache.org/jira/browse/SPARK-5130
Author: WangTaoTheTonic <ba...@aliyun.com>
Closes #3929 from WangTaoTheTonic/SPARK-5130 and squashes the following commits:
c490648 [WangTaoTheTonic] take yarn-cluster as cluster mode in spark-submit
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0760787d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0760787d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0760787d
Branch: refs/heads/master
Commit: 0760787da885187b0c6dcd5c28753f0ab014d5ed
Parents: 0a59727
Author: WangTaoTheTonic <ba...@aliyun.com>
Authored: Thu Jan 8 11:45:42 2015 -0800
Committer: Andrew Or <an...@databricks.com>
Committed: Thu Jan 8 11:45:42 2015 -0800
----------------------------------------------------------------------
bin/spark-submit | 5 +++++
bin/spark-submit2.cmd | 6 ++++++
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0760787d/bin/spark-submit
----------------------------------------------------------------------
diff --git a/bin/spark-submit b/bin/spark-submit
index f92d90c..aefd38a 100755
--- a/bin/spark-submit
+++ b/bin/spark-submit
@@ -38,11 +38,16 @@ while (($#)); do
export SPARK_SUBMIT_CLASSPATH=$2
elif [ "$1" = "--driver-java-options" ]; then
export SPARK_SUBMIT_OPTS=$2
+ elif [ "$1" = "--master" ]; then
+ export MASTER=$2
fi
shift
done
DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf"
+if [ "$MASTER" == "yarn-cluster" ]; then
+ SPARK_SUBMIT_DEPLOY_MODE=cluster
+fi
export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"}
export SPARK_SUBMIT_PROPERTIES_FILE=${SPARK_SUBMIT_PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"}
http://git-wip-us.apache.org/repos/asf/spark/blob/0760787d/bin/spark-submit2.cmd
----------------------------------------------------------------------
diff --git a/bin/spark-submit2.cmd b/bin/spark-submit2.cmd
index cf6046d..daf0284 100644
--- a/bin/spark-submit2.cmd
+++ b/bin/spark-submit2.cmd
@@ -45,11 +45,17 @@ if [%1] == [] goto continue
set SPARK_SUBMIT_CLASSPATH=%2
) else if [%1] == [--driver-java-options] (
set SPARK_SUBMIT_OPTS=%2
+ ) else if [%1] == [--master] (
+ set MASTER=%2
)
shift
goto loop
:continue
+if [%MASTER%] == [yarn-cluster] (
+ set SPARK_SUBMIT_DEPLOY_MODE=cluster
+)
+
rem For client mode, the driver will be launched in the same JVM that launches
rem SparkSubmit, so we may need to read the properties file for any extra class
rem paths, library paths, java options and memory early on. Otherwise, it will
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org