You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by na...@apache.org on 2017/10/11 16:22:29 UTC
systemml git commit: [SYSTEMML-1945] added --deploy-mode param to
python scripts
Repository: systemml
Updated Branches:
refs/heads/master 13a017583 -> 8f786aa22
[SYSTEMML-1945] added --deploy-mode param to python scripts
Closes #681
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8f786aa2
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8f786aa2
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8f786aa2
Branch: refs/heads/master
Commit: 8f786aa227d536558ed684060cef0e628bf3247f
Parents: 13a0175
Author: Krishna Kalyan <kr...@gmail.com>
Authored: Wed Oct 11 09:20:40 2017 -0700
Committer: Nakul Jindal <na...@gmail.com>
Committed: Wed Oct 11 09:20:40 2017 -0700
----------------------------------------------------------------------
bin/systemml-spark-submit.py | 9 +++++----
scripts/perftest/python/run_perftest.py | 3 ++-
scripts/perftest/python/utils_misc.py | 3 +++
3 files changed, 10 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/bin/systemml-spark-submit.py
----------------------------------------------------------------------
diff --git a/bin/systemml-spark-submit.py b/bin/systemml-spark-submit.py
index b4da801..2546236 100755
--- a/bin/systemml-spark-submit.py
+++ b/bin/systemml-spark-submit.py
@@ -41,8 +41,8 @@ def default_jars(systemml_home):
return target_jars, systemml_jar
-def spark_submit_entry(master, driver_memory, num_executors, executor_memory,
- executor_cores, conf,
+def spark_submit_entry(master, deploy_mode, driver_memory, num_executors,
+ executor_memory, executor_cores, conf,
nvargs, args, config, explain, debug, stats, gpu, f):
"""
This function is responsible for the execution of arguments via
@@ -100,7 +100,7 @@ def spark_submit_entry(master, driver_memory, num_executors, executor_memory,
# stats, explain, target_jars
cmd_spark = [spark_path, '--class', 'org.apache.sysml.api.DMLScript',
- '--master', master,
+ '--master', master, '--deploy-mode', deploy_mode,
'--driver-memory', driver_memory,
'--conf', default_conf,
'--jars', cuda_jars, systemml_jars]
@@ -129,7 +129,8 @@ if __name__ == '__main__':
cparser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description='System-ML Spark Submit Script')
# SPARK-SUBMIT Options
- cparser.add_argument('--master', default='local[*]', help='local, yarn-client, yarn-cluster', metavar='')
+ cparser.add_argument('--master', default='local[*]', help='local, yarn', metavar='')
+ cparser.add_argument('--deploy-mode', help='client, cluster', default='client', metavar='')
cparser.add_argument('--driver-memory', default='8G', help='Memory for driver (e.g. 512M, 1G)', metavar='')
cparser.add_argument('--num-executors', nargs=1, help='Number of executors to launch', metavar='')
cparser.add_argument('--executor-memory', nargs=1, help='Memory per executor', metavar='')
http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/scripts/perftest/python/run_perftest.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/run_perftest.py b/scripts/perftest/python/run_perftest.py
index 6c016a8..6e87261 100755
--- a/scripts/perftest/python/run_perftest.py
+++ b/scripts/perftest/python/run_perftest.py
@@ -355,7 +355,8 @@ if __name__ == '__main__':
'set <force> option to skip conservative memory estimates '
'and use GPU wherever possible', nargs='?', const='no_option')
# Spark Configuration Option
- cparser.add_argument('--master', help='local, yarn-client, yarn-cluster', metavar='')
+ cparser.add_argument('--master', help='local, yarn', metavar='')
+ cparser.add_argument('--deploy-mode', help='client, cluster', metavar='')
cparser.add_argument('--driver-memory', help='Memory for driver (e.g. 512M)', metavar='')
cparser.add_argument('--num-executors', help='Number of executors to launch', metavar='')
cparser.add_argument('--executor-memory', help='Memory per executor', metavar='')
http://git-wip-us.apache.org/repos/asf/systemml/blob/8f786aa2/scripts/perftest/python/utils_misc.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py
index ebc4376..63d8c1b 100755
--- a/scripts/perftest/python/utils_misc.py
+++ b/scripts/perftest/python/utils_misc.py
@@ -84,6 +84,9 @@ def split_config_args(args):
if args['master'] is not None:
backend_args_dict['--master'] = args['master']
+ if args['deploy_mode'] is not None:
+ backend_args_dict['--deploy-mode'] = args['master']
+
if args['num_executors'] is not None:
backend_args_dict['--num-executors'] = args['num_executors']