You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by na...@apache.org on 2017/08/12 19:32:36 UTC
systemml git commit: [HOTFIX] write stdout and stderr for perftests
Repository: systemml
Updated Branches:
refs/heads/master 95de23586 -> dc4bfd95e
[HOTFIX] write stdout and stderr for perftests
Closes #615
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/dc4bfd95
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/dc4bfd95
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/dc4bfd95
Branch: refs/heads/master
Commit: dc4bfd95e893f924d80ca3af25c101495134fa77
Parents: 95de235
Author: krishnakalyan3 <kr...@gmail.com>
Authored: Sat Aug 12 12:31:26 2017 -0700
Committer: Nakul Jindal <na...@gmail.com>
Committed: Sat Aug 12 12:32:16 2017 -0700
----------------------------------------------------------------------
scripts/perftest/python/run_perftest.py | 2 +-
scripts/perftest/python/utils_exec.py | 19 +++++++++++++++++--
scripts/perftest/python/utils_misc.py | 10 +++++-----
3 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/run_perftest.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/run_perftest.py b/scripts/perftest/python/run_perftest.py
index a15d7e6..d430569 100755
--- a/scripts/perftest/python/run_perftest.py
+++ b/scripts/perftest/python/run_perftest.py
@@ -134,7 +134,7 @@ def algorithm_workflow(algo, exec_type, config_path, dml_file_name, action_mode,
if exit_flag_success:
time = 'data_exists'
else:
- time = exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict)
+ time = exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict, config_path)
write_success(time, temp_cwd)
print('{},{},{},{},{},{}'.format(algo, action_mode, intercept, mat_type, mat_shape, time))
http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/utils_exec.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_exec.py b/scripts/perftest/python/utils_exec.py
index 0eb2873..87ae3cd 100755
--- a/scripts/perftest/python/utils_exec.py
+++ b/scripts/perftest/python/utils_exec.py
@@ -27,7 +27,7 @@ import re
# Subprocess and log parsing related functions
-def subprocess_exec(cmd_string, extract=None):
+def subprocess_exec(cmd_string, log_file_path=None, extract=None):
"""
Execute the input string as subprocess
@@ -38,18 +38,25 @@ def subprocess_exec(cmd_string, extract=None):
Based on extract as time/dir we extract this information from
the logs accordingly
+ log_file_path: String
+ Path to write the log file
+
return: String
Based on extract we return the relevant string
"""
# Debug
# print(cmd_string)
- proc1 = subprocess.Popen(shlex.split(cmd_string), stdout=subprocess.PIPE,
+ exec_command = shlex.split(cmd_string)
+ proc1 = subprocess.Popen(exec_command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
error_arr, out_arr = get_all_logs(proc1)
std_outs = out_arr + error_arr
return_code = proc1.returncode
+ if log_file_path is not None:
+ write_logs(std_outs, log_file_path + '.log')
+
if return_code == 0:
if extract == 'time':
return_data = parse_time(std_outs)
@@ -65,6 +72,14 @@ def subprocess_exec(cmd_string, extract=None):
return return_data
+def write_logs(std_outs, log_file_path):
+ """
+ Write all logs to the specified location
+ """
+ with open(log_file_path, 'w')as log:
+ log.write("\n".join(std_outs))
+
+
def get_all_logs(process):
"""
Based on the subprocess capture logs
http://git-wip-us.apache.org/repos/asf/systemml/blob/dc4bfd95/scripts/perftest/python/utils_misc.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py
index 0a765f6..e247ce8 100755
--- a/scripts/perftest/python/utils_misc.py
+++ b/scripts/perftest/python/utils_misc.py
@@ -166,7 +166,7 @@ def config_reader(read_path):
return conf_file
-def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict):
+def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup_args_dict, log_file_name=None):
"""
This function is responsible of execution of input arguments via python sub process,
We also extract time obtained from the output of this subprocess
@@ -186,6 +186,9 @@ def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup
sup_args_dict: Dictionary
Supplementary arguments required by the script
+ log_file_name: String
+ Path to write the logfile
+
return: String
The value of time parsed from the logs / error
"""
@@ -207,10 +210,7 @@ def exec_dml_and_parse_time(exec_type, dml_file_name, args, spark_args_dict, sup
cmd = [exec_script, spark_pre_args, '-f', algorithm, args, sup_args]
cmd_string = ' '.join(cmd)
- # Debug
- # print(cmd_string)
-
- time = subprocess_exec(cmd_string, 'time')
+ time = subprocess_exec(cmd_string, log_file_name, 'time')
return time