incubator-systemml git commit: Added python script to launch systemml in standalone mode

[na...@apache.org]

Repository: incubator-systemml
Updated Branches:
  refs/heads/master ea6e2fe39 -> f73673d59


Added python script to launch systemml in standalone mode

Closes #461


Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f73673d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f73673d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f73673d5

Branch: refs/heads/master
Commit: f73673d59383ac947111cb84787cfa4df3ca7344
Parents: ea6e2fe
Author: Nakul Jindal <na...@gmail.com>
Authored: Fri Apr 21 14:25:50 2017 -0700
Committer: Nakul Jindal <na...@gmail.com>
Committed: Fri Apr 21 14:25:50 2017 -0700

----------------------------------------------------------------------
 bin/systemml-standalone.py | 199 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 199 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f73673d5/bin/systemml-standalone.py
----------------------------------------------------------------------
diff --git a/bin/systemml-standalone.py b/bin/systemml-standalone.py
new file mode 100755
index 0000000..367bcdf
--- /dev/null
+++ b/bin/systemml-standalone.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+import os
+import shutil
+import sys
+from os.path import join, exists
+
+
+# error help print
+def print_usage_and_exit():
+    this_script = sys.argv[0]
+    print('Usage: ' + this_script + ' <dml-filename> [arguments]')
+    sys.exit(1)
+
+
+# from http://stackoverflow.com/questions/1724693/find-a-file-in-python
+def find_file(name, path):
+    for root, dirs, files in os.walk(path):
+        if name in files:
+            return join(root, name)
+    return None
+
+
+if len(sys.argv) < 2:
+    print('Wrong usage')
+    print_usage_and_exit()
+
+
+# find the systemML root path which contains the bin folder, the script folder and the target folder
+# tolerate path with spaces
+script_dir = os.path.dirname(os.path.realpath(__file__))
+project_root_dir = os.path.dirname(script_dir)
+user_dir = os.getcwd()
+
+scripts_dir = join(project_root_dir, 'scripts')
+build_dir = join(project_root_dir, 'target')
+lib_dir = join(build_dir, 'lib')
+dml_script_class = join(build_dir, 'classes', 'org', 'apache', 'sysml', 'api', 'DMLScript.class')
+hadoop_home = join(lib_dir, 'hadoop')
+
+
+build_err_msg = 'You must build the project before running this script.'
+build_dir_err_msg = 'Could not find target directory ' + build_dir + '. ' + build_err_msg
+
+lib_dir_err_msg = 'Could not find required libraries.' + build_err_msg
+dml_script_err_msg = 'Could not find ' + dml_script_class + '. ' + build_err_msg
+
+# check if the project had been built and the jar files exist
+if not(exists(build_dir)):
+    print(build_dir_err_msg)
+    sys.exit(1)
+if not(exists(lib_dir)):
+    print(lib_dir_err_msg)
+    sys.exit(1)
+if not(exists(dml_script_class)):
+    print(dml_script_err_msg)
+    sys.exit(1)
+
+print('================================================================================')
+
+
+# if the present working directory is the project root or bin folder, then use the temp folder as user.dir
+if user_dir == project_root_dir or user_dir == join(project_root_dir, 'bin'):
+    user_dir = join(project_root_dir, 'temp')
+    print('Output dir: ' + user_dir)
+
+# if the SystemML-config.xml does not exist, create it from the template
+systemml_config_path = join(project_root_dir, 'conf', 'SystemML-config.xml')
+systemml_template_config_path = join(project_root_dir, 'conf', 'SystemML-config.xml.template')
+if not(exists(systemml_config_path)):
+    shutil.copyfile(systemml_template_config_path, systemml_config_path)
+    print('... created ' + systemml_config_path)
+
+# if the log4j.properties do not exist, create them from the template
+log4j_properties_path = join(project_root_dir, 'conf', 'log4j.properties')
+log4j_template_properties_path = join(project_root_dir, 'conf', 'log4j.properties.template')
+if not(exists(log4j_properties_path)):
+    shutil.copyfile(log4j_template_properties_path, log4j_properties_path)
+    print('... created ' + log4j_properties_path)
+
+
+script_file = sys.argv[1]
+
+# if the script file path was omitted, try to complete the script path
+if not(exists(script_file)):
+    script_file_name = os.path.abspath(script_file)
+    script_file_found = find_file(script_file, scripts_dir)
+    if script_file_found is None:
+        print('Could not find DML script: ' + script_file)
+        print_usage_and_exit()
+    else:
+        script_file = script_file_found
+        print('DML Script:' + script_file)
+
+# add libraries which were generated by the build to the classpath
+systemml_jar = join(build_dir, 'classes')
+
+
+# For the *nix and windows, os.pathsep works reliably
+# however for cygwin, the pathsep is set for *nix, which is ':'
+# but the underlying java, which is a windows program requires ';'
+# also all arguments passed to the JVM need to be converted to windows style
+# if in the cygwin environment
+lib_dir_star = join(lib_dir, '*')
+if sys.platform == 'cygwin':
+    classpath_sep = os.pathsep
+    classpath_sep = ';'
+    lib_dir = os.popen('cygpath -pw ' + lib_dir).read().strip()
+    lib_dir_star = '"' + lib_dir + "\*" + '"'
+    systemml_jar = '"' + os.popen('cygpath -pw ' + systemml_jar).read().strip() + '"'
+    hadoop_home = '"' + os.popen('cygpath -pw ' + hadoop_home).read().strip() + '"'
+    log4j_properties_path = '"' + os.popen('cygpath -pw ' + log4j_properties_path).read().strip() + '"'
+    user_dir = '"' + os.popen('cygpath -pw ' + user_dir).read().strip() + '"'
+    script_file = '"' + os.popen('cygpath -pw ' + script_file).read().strip() + '"'
+    systemml_config_path = '"' + os.popen('cygpath -pw ' + systemml_config_path).read().strip() + '"'
+    classpath = lib_dir_star + '\\' + classpath_sep + systemml_jar
+else:
+    #classpath = '"' + lib_dir_star + '"' + os.pathsep + '"' + systemml_jar + '"'
+    classpath = lib_dir_star + os.pathsep + systemml_jar
+
+
+# Set the HADOOP_HOME environment variable
+if 'HADOOP_HOME' not in os.environ:
+    os.environ['HADOOP_HOME'] = hadoop_home
+
+
+print('================================================================================')
+
+# Set default Java options
+systemml_default_java_opts = \
+    '-Xmx8g -Xms4g -Xmn1g ' + \
+    '-cp ' + classpath + ' ' + \
+    '-Dlog4j.configuration=file:' + log4j_properties_path + ' ' \
+    '-Duser.dir=' + user_dir
+
+
+# Reads in key-value pairs from the conf/systemml-env.sh file
+def parse_env_file(env_file_path):
+    env_vars = {}
+    with open(env_file_path) as f:
+        for l in f:
+            l = l.strip()
+            if l and not(l.startswith('#')) and '=' in l:
+                k, v = l.split('=', 1)
+                k = k.strip()
+                v = v.strip()
+                if len(v) > 0:
+                    # strip quotes
+                    if v[0] == v[len(v) - 1] and v[0] in ['"', "'"]:
+                        v = v[1:-1]
+
+                env_vars[k] = v
+    return env_vars
+
+
+# Add any custom Java options set by the user at command line, overriding defaults as necessary.
+if 'SYSTEMML_JAVA_OPTS' in os.environ:
+    systemml_java_opts = os.environ['SYSTEMML_JAVA_OPTS']
+    systemml_default_java_opts = systemml_default_java_opts + ' ' + systemml_java_opts
+    # del os.environ['SYSTEMML_JAVA_OPTS']
+
+# Add any custom Java options set by the user in the environment variables file,
+# overriding defaults as necessary.
+systemml_env_path = join(project_root_dir, 'conf', 'systemml-env.sh')
+if exists(systemml_env_path):
+    env_vars = parse_env_file(systemml_env_path)
+    os.environ.update(env_vars)
+
+
+# Invoke the jar with options and arguments
+cmd = ['java', systemml_default_java_opts, 'org.apache.sysml.api.DMLScript', '-f', script_file, '-exec singlenode', '-config', systemml_config_path] + sys.argv[2:]
+# For debugging
+# print(' '.join(cmd))
+
+return_code = os.system(' '.join(cmd))
+
+if return_code != 0:
+    print('Failed to run SystemML. Exit code :' + str(return_code))
+    print(' '.join(cmd))