You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by na...@apache.org on 2017/10/03 00:54:55 UTC
systemml git commit: [MINOR] bug fix in perf test script
Repository: systemml
Updated Branches:
refs/heads/master c27c488be -> 50b3c80c6
[MINOR] bug fix in perf test script
Project: http://git-wip-us.apache.org/repos/asf/systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/50b3c80c
Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/50b3c80c
Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/50b3c80c
Branch: refs/heads/master
Commit: 50b3c80c61b9b6ab6cdb8b28a1544952fb8dd19d
Parents: c27c488
Author: Nakul Jindal <na...@gmail.com>
Authored: Mon Oct 2 17:54:17 2017 -0700
Committer: Nakul Jindal <na...@gmail.com>
Committed: Mon Oct 2 17:54:17 2017 -0700
----------------------------------------------------------------------
scripts/perftest/python/datagen.py | 4 +--
scripts/perftest/python/predict.py | 4 +--
scripts/perftest/python/train.py | 4 +--
scripts/perftest/python/utils_exec.py | 41 ++++++++++++++++++++----------
scripts/perftest/python/utils_fs.py | 2 +-
scripts/perftest/python/utils_misc.py | 4 +--
6 files changed, 37 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/datagen.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/datagen.py b/scripts/perftest/python/datagen.py
index 6794187..54f2eff 100755
--- a/scripts/perftest/python/datagen.py
+++ b/scripts/perftest/python/datagen.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
#
-#-------------------------------------------------------------
+# -------------------------------------------------------------
import itertools
from os.path import join
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/predict.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/predict.py b/scripts/perftest/python/predict.py
index 67467b1..56ba999 100755
--- a/scripts/perftest/python/predict.py
+++ b/scripts/perftest/python/predict.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
#
-#-------------------------------------------------------------
+# -------------------------------------------------------------
import sys
from os.path import join
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/train.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/train.py b/scripts/perftest/python/train.py
index a95950d..4428e8f 100755
--- a/scripts/perftest/python/train.py
+++ b/scripts/perftest/python/train.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
#
-#-------------------------------------------------------------
+# -------------------------------------------------------------
import sys
from os.path import join
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_exec.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_exec.py b/scripts/perftest/python/utils_exec.py
index 27f75f2..d51cf2d 100755
--- a/scripts/perftest/python/utils_exec.py
+++ b/scripts/perftest/python/utils_exec.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,12 +18,14 @@
# specific language governing permissions and limitations
# under the License.
#
-#-------------------------------------------------------------
+# -------------------------------------------------------------
import sys
import subprocess
import shlex
import re
+import tempfile
+import os
# Subprocess and log parsing related functions
@@ -49,24 +51,34 @@ def subprocess_exec(cmd_string, log_file_path=None, extract=None):
# print(cmd_string)
exec_command = shlex.split(cmd_string)
- proc1 = subprocess.Popen(exec_command, stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ log_file = None
+ is_temp_file = False
- error_arr, out_arr = get_all_logs(proc1)
- std_outs = out_arr + error_arr
- std_outs.insert(0, ' '.join(exec_command))
+ if log_file_path is not None:
+ log_file_path = log_file_path + '.log'
+ log_file = open(log_file_path, "w+")
+ else:
+ os_log_file, log_file_path = tempfile.mkstemp()
+ log_file = os.fdopen(os_log_file, 'w+')
+ is_temp_file = True
+
+ log_file.write(' '.join(exec_command))
+ log_file.write('\n')
+ proc1 = subprocess.Popen(exec_command, stdout=log_file,
+ stderr=subprocess.STDOUT)
+ proc1.wait()
return_code = proc1.returncode
- if log_file_path is not None:
- write_logs(std_outs, log_file_path + '.log')
+ log_file.close()
+ log_file = open(log_file_path, 'r+')
if return_code == 0:
if extract == 'time':
- return_data = parse_time(std_outs)
+ return_data = parse_time(log_file)
if extract == 'dir':
- return_data = parse_hdfs_paths(std_outs)
+ return_data = parse_hdfs_paths(log_file)
if extract == 'hdfs_base':
- return_data = parse_hdfs_base(std_outs)
+ return_data = parse_hdfs_base(log_file)
if extract is None:
return_data = 0
@@ -74,9 +86,12 @@ def subprocess_exec(cmd_string, log_file_path=None, extract=None):
return_data = 'proc_fail'
print('sub-process failed, return code {}'.format(return_code))
- return return_data
+ if is_temp_file:
+ os.remove(log_file_path)
+ return return_data
+
def parse_hdfs_base(std_outs):
"""
return: String
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_fs.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_fs.py b/scripts/perftest/python/utils_fs.py
index 0e3a7b6..a604ca1 100755
--- a/scripts/perftest/python/utils_fs.py
+++ b/scripts/perftest/python/utils_fs.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/systemml/blob/50b3c80c/scripts/perftest/python/utils_misc.py
----------------------------------------------------------------------
diff --git a/scripts/perftest/python/utils_misc.py b/scripts/perftest/python/utils_misc.py
index 87b870b..ebc4376 100755
--- a/scripts/perftest/python/utils_misc.py
+++ b/scripts/perftest/python/utils_misc.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-#-------------------------------------------------------------
+# -------------------------------------------------------------
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -18,7 +18,7 @@
# specific language governing permissions and limitations
# under the License.
#
-#-------------------------------------------------------------
+# -------------------------------------------------------------
from os.path import join
import os