You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2016/01/06 21:26:46 UTC
incubator-systemml git commit: Update sparkDML.sh parameters and path
handling
Repository: incubator-systemml
Updated Branches:
refs/heads/master b6819220d -> 38007a8d2
Update sparkDML.sh parameters and path handling
Update sparkDML.sh to remove some hard-coded parameters and fix a path of SystemML configuration in spark-submit command-line options.
Fix setting SYSTEMML_HOME and revert default parameters and location of SystemML-config.xml.
Change building spark configuration parameters and setting default parameters.
Closes #19.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/38007a8d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/38007a8d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/38007a8d
Branch: refs/heads/master
Commit: 38007a8d2a18a7d9ba0429dabd53fb2533f0e526
Parents: b681922
Author: Tatsuya.Nishiyama <ni...@gmail.com>
Authored: Fri Dec 11 17:26:42 2015 +0900
Committer: Deron Eriksson <de...@us.ibm.com>
Committed: Wed Jan 6 12:07:30 2016 -0800
----------------------------------------------------------------------
scripts/sparkDML.sh | 52 +++++++++++++++++++++++++++++-------------------
1 file changed, 31 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/38007a8d/scripts/sparkDML.sh
----------------------------------------------------------------------
diff --git a/scripts/sparkDML.sh b/scripts/sparkDML.sh
index 0d642b4..fe5bc91 100644
--- a/scripts/sparkDML.sh
+++ b/scripts/sparkDML.sh
@@ -22,20 +22,31 @@
#set -x
+
# Environment
-SPARK_HOME=/home/biadmin/spark-1.4.0/spark-1.4.0-SNAPSHOT-bin-hadoop2.4
-SYSTEMML_HOME="."
+# Following variables must be rewritten by your installation paths.
+DEFAULT_SPARK_HOME=/usr/local/spark-1.4.0/spark-1.4.0-SNAPSHOT-bin-hadoop2.4
+DEFAULT_SYSTEMML_HOME=.
+
+if [ -z ${SPARK_HOME} ]; then
+ SPARK_HOME=${DEFAULT_SPARK_HOME}
+fi
+
+if [ -z ${SYSTEMML_HOME} ]; then
+ SYSTEMML_HOME=${DEFAULT_SYSTEMML_HOME}
+fi
# Default Values
-master=yarn-client
-driver_memory=20G
-num_executors=5
-executor_memory=60G
-executor_cores=24
+master="--master yarn-client"
+driver_memory="--driver-memory 20G"
+num_executors="--num-executors 5"
+executor_memory="--executor-memory 60G"
+executor_cores="--executor-cores 24"
conf="--conf spark.driver.maxResultSize=0 --conf spark.akka.frameSize=128"
+
# error help print
printUsageExit()
@@ -53,7 +64,7 @@ Usage: $0 [-h] [SPARK-SUBMIT OPTIONS] -f <dml-filename> [SYSTEMML OPTIONS]
SPARK-SUBMIT OPTIONS:
--conf <property>=<value> Configuration settings:
- spark.driver.maxResultSize Default: 0
+ spark.driver.maxResultSize Default: 0
spark.akka.frameSize Default: 128
--driver-memory <num> Memory for driver (e.g. 512M)] Default: 20G
--master <string> local | yarn-client | yarn-cluster] Default: yarn-client
@@ -73,17 +84,16 @@ EOF
exit 1
}
-
# command line parameter processing
while true ; do
case "$1" in
-h) printUsageExit ; exit 1 ;;
- --master) master=$2 ; shift 2 ;;
- --driver-memory) driver_memory=$2 ; shift 2 ;;
- --num-executors) num_executors=$2 ; shift 2 ;;
- --executor-memory) executor_memory=$2 ; shift 2 ;;
- --executor-cores) executor_cores=$2 ; shift 2 ;;
+ --master) master="--master "$2 ; shift 2 ;;
+ --driver-memory) driver_memory="--driver-memory "$2 ; shift 2 ;;
+ --num-executors) num_executors="--num-executors "$2 ; shift 2 ;;
+ --executor-memory) executor_memory="--executor-memory "$2 ; shift 2 ;;
+ --executor-cores) executor_cores="--executor-cores "$2 ; shift 2 ;;
--conf) conf=${conf}' --conf '$2 ; shift 2 ;;
-f) f=$2 ; shift 2 ;;
--stats) stats="-stats" ; shift 1 ;;
@@ -98,15 +108,15 @@ done
# SystemML Spark invocation
$SPARK_HOME/bin/spark-submit \
- --master ${master} \
- --driver-memory ${driver_memory} \
- --num-executors ${num_executors} \
- --executor-memory ${executor_memory} \
- --executor-cores ${executor_cores} \
+ ${master} \
+ ${driver_memory} \
+ ${num_executors} \
+ ${executor_memory} \
+ ${executor_cores} \
${conf} \
- $SYSTEMML_HOME/SystemML.jar \
+ ${SYSTEMML_HOME}/SystemML.jar \
-f ${f} \
- -config=$SYSTEMML_HOME/SystemML-config.xml \
+ -config=${SYSTEMML_HOME}/SystemML-config.xml \
-exec hybrid_spark \
$explain \
$stats \