You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ja...@apache.org on 2024/04/08 03:24:36 UTC
(systemds) branch py-logging-issue created (now e4b4546e6b)
This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a change to branch py-logging-issue
in repository https://gitbox.apache.org/repos/asf/systemds.git
at e4b4546e6b [SYSTEMDS-3687] Python API startup fixes
This branch includes the following new commits:
new 9db191f0fb [DOCS] update 3.2.0 version in the docs
new e4b4546e6b [SYSTEMDS-3687] Python API startup fixes
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(systemds) 02/02: [SYSTEMDS-3687] Python API startup fixes
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch py-logging-issue
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit e4b4546e6bcbf17b7612c35765f2fd47d19e90bc
Author: Sebastian Baunsgaard <ba...@apache.org>
AuthorDate: Thu Apr 4 19:05:19 2024 +0200
[SYSTEMDS-3687] Python API startup fixes
This PR change the startup of the python interface to properly use the
jar files, and fixes a release issue where if the SYSTEMDS_ROOT is not
set the python API did not properly hook into the released jar artifacts.
---
.gitignore | 1 -
src/main/python/.gitignore | 7 +++
.../python/{.gitignore => conf/log4j.properties} | 37 +++++--------
src/main/python/pre_setup.py | 10 ++++
.../python/systemds/context/systemds_context.py | 61 +++++++++++++++-------
5 files changed, 71 insertions(+), 45 deletions(-)
diff --git a/.gitignore b/.gitignore
index 1a83a3a80e..5e40b37fd9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -64,7 +64,6 @@ buildNumber.properties
_site/
# Tutorial data mnist
-src/main/python/systemds/examples/tutorials/*/
scripts/nn/examples/data/*
# User configuration files
diff --git a/src/main/python/.gitignore b/src/main/python/.gitignore
index ea43a07f92..0ac4a56899 100644
--- a/src/main/python/.gitignore
+++ b/src/main/python/.gitignore
@@ -44,3 +44,10 @@ tests/examples/tutorials/model
tests/lineage/temp
python_venv/
+venv
+
+# Main Jar location for API communiation.
+systemds/SystemDS.jar
+
+# tutorial
+systemds/examples/tutorials/*
diff --git a/src/main/python/.gitignore b/src/main/python/conf/log4j.properties
similarity index 64%
copy from src/main/python/.gitignore
copy to src/main/python/conf/log4j.properties
index ea43a07f92..9a381e00aa 100644
--- a/src/main/python/.gitignore
+++ b/src/main/python/conf/log4j.properties
@@ -7,9 +7,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -19,28 +19,15 @@
#
#-------------------------------------------------------------
-# Git ignore for python files.
-systemds/lib/
-systemds.egg-info/
-systemds/conf/
-build/
-LICENSE
-NOTICE
-generator.log
-dist
-docs/build
-docs/source/_build
-tests/onnx_systemds/output_test
-tests/onnx_systemds/dml_output
-tests/onnx_systemds/test_models/*.onnx
+log4j.rootLogger=ERROR,console
-# git ignore tmp test files
-tests/federated/output
-tests/federated/worker
-tests/federated/tmp
-tests/list/tmp
-tests/algorithms/readwrite/
-tests/examples/tutorials/model
-tests/lineage/temp
+log4j.logger.org.apache.sysds=ERROR
+log4j.logger.org.apache.sysds.utils.SettingsChecker=WARN
+log4j.logger.org.apache.spark=ERROR
+log4j.logger.org.apache.hadoop=OFF
+log4j.logger.org.apache.hadoop.util.NativeCodeLoader=INFO
-python_venv/
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
diff --git a/src/main/python/pre_setup.py b/src/main/python/pre_setup.py
index 5c7314a4e7..e4bb99ae3f 100755
--- a/src/main/python/pre_setup.py
+++ b/src/main/python/pre_setup.py
@@ -80,6 +80,16 @@ if not os.path.exists(CONF_DIR):
shutil.copy(os.path.join(SYSTEMDS_ROOT,'conf', 'log4j.properties'), os.path.join(this_path, PYTHON_DIR, 'conf', 'log4j.properties'))
shutil.copy(os.path.join(SYSTEMDS_ROOT,'conf', 'SystemDS-config-defaults.xml'), os.path.join(this_path, PYTHON_DIR, 'conf', 'SystemDS-config-defaults.xml'))
+
+# Take SystemDS file.
+shutil.copy(os.path.join(SYSTEMDS_ROOT, 'target', 'SystemDS.jar'), os.path.join(PYTHON_DIR, 'SystemDS.jar'))
+
+# remove redundant SystemDS file inside lib.
+for file in os.listdir(os.path.join(PYTHON_DIR, 'lib')):
+ if "systemds" in file:
+ if "extra" not in file:
+ os.remove(os.path.join(PYTHON_DIR, 'lib', file))
+
SYSTEMDS_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.getcwd())))
shutil.copyfile(os.path.join(SYSTEMDS_ROOT, 'LICENSE'), 'LICENSE')
shutil.copyfile(os.path.join(SYSTEMDS_ROOT, 'NOTICE'), 'NOTICE')
diff --git a/src/main/python/systemds/context/systemds_context.py b/src/main/python/systemds/context/systemds_context.py
index 5f34086807..85d2588c64 100644
--- a/src/main/python/systemds/context/systemds_context.py
+++ b/src/main/python/systemds/context/systemds_context.py
@@ -155,34 +155,47 @@ class SystemDSContext(object):
"""Build the command line argument for the startup of the JVM
:param port: The port address to use if -1 chose random port."""
+ # Base command
command = ["java", "-cp"]
+
+ # Find the operating system specifc separator, nt means its Windows
+ cp_separator = ";" if os.name == "nt" else ":"
root = os.environ.get("SYSTEMDS_ROOT")
+
if root == None:
- # If there is no systemds install default to use the PIP packaged java files.
+ # If there is no systemds install default to use the pip packaged java files.
root = os.path.join(get_module_dir())
- # nt means its Windows
- cp_separator = ";" if os.name == "nt" else ":"
-
- if os.environ.get("SYSTEMDS_ROOT") != None:
+ # Find the SystemDS jar file.
+ if root != None: # root path was set
+ self._log.debug("SYSTEMDS_ROOT was set, searching for jar file")
lib_release = os.path.join(root, "lib")
- lib_cp = os.path.join(root, "target", "lib")
- if os.path.exists(lib_release):
- classpath = cp_separator.join([os.path.join(lib_release, '*')])
- elif os.path.exists(lib_cp):
- systemds_cp = os.path.join(root, "target", "SystemDS.jar")
- classpath = cp_separator.join(
- [os.path.join(lib_cp, '*'), systemds_cp])
+ systemds_cp = os.path.join(root, "target", "SystemDS.jar")
+ if os.path.exists(lib_release): # It looks like it was a release path for root.
+ classpath = os.path.join(root, "SystemDS.jar")
+ if not os.path.exists(classpath):
+ for f in os.listdir(root):
+ if "systemds" in f:
+ if os.path.exists(classpath):
+ raise(ValueError("Invalid setup there were multiple conflicting systemds jar fines in" + root))
+ else:
+ classpath = os.path.join(root, f)
+ if not os.path.exists(classpath):
+ raise ValueError(
+ "Invalid setup did not find SystemDS jar file in " + root)
+ elif os.path.exists(systemds_cp):
+ classpath = cp_separator.join([systemds_cp])
else:
raise ValueError(
- "Invalid setup at SYSTEMDS_ROOT env variable path " + lib_cp)
- else:
- lib1 = os.path.join(root, "lib", "*")
- lib2 = os.path.join(root, "lib")
- classpath = cp_separator.join([lib1, lib2])
+ "Invalid setup at SYSTEMDS_ROOT env variable path " + root)
+ else: # root path was not set use the pip installed SystemDS
+ self._log.warning("SYSTEMDS_ROOT was unset, defaulting to python packaged jar files")
+ systemds_cp = os.path.join(root,"SystemDS.jar")
+ classpath = cp_separator.join([systemds_cp])
command.append(classpath)
+ # Find the logging configuration file.
if os.environ.get("LOG4JPROP") == None:
files = glob(os.path.join(root, "conf", "log4j*.properties"))
if len(files) > 1:
@@ -195,16 +208,23 @@ class SystemDSContext(object):
else:
command.append("-Dlog4j.configuration=file:" + files[0])
else:
- command.append("-Dlog4j.configuration=file:" +os.environ.get("LOG4JPROP"))
+ logging_file = os.environ.get("LOG4JPROP")
+ if os.path.exists(logging_file):
+ command.append("-Dlog4j.configuration=file:" +os.environ.get("LOG4JPROP"))
+ else:
+ self._log.warning("LOG4JPROP is set but path is invalid: " + str(logging_file))
+ # Specify the main function inside SystemDS to launch in java.
command.append("org.apache.sysds.api.PythonDMLScript")
+ # Find the configuration file for systemds.
+ # TODO: refine the choise of configuration file
files = glob(os.path.join(root, "conf", "SystemDS*.xml"))
if len(files) > 1:
self._log.warning(
"Multiple config files found selecting: " + files[0])
if len(files) == 0:
- self._log.warning("No log4j file found at: "
+ self._log.warning("No xml config file found at: "
+ os.path.join(root, "conf")
+ " therefore using default settings")
else:
@@ -219,6 +239,9 @@ class SystemDSContext(object):
command.append("--python")
command.append(str(actual_port))
+ self._log.info("Command " + str(command))
+ self._log.info("Port used for communication: " + str(actual_port))
+
return command, actual_port
def __start(self, port: int, capture_stdout: bool, retry: int = 0):
(systemds) 01/02: [DOCS] update 3.2.0 version in the docs
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
janardhan pushed a commit to branch py-logging-issue
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 9db191f0fbdc5ecc584f94d9bf9379eb93c279e9
Author: Janardhan Pulivarthi <ja...@apache.org>
AuthorDate: Sat Mar 23 08:33:30 2024 +0000
[DOCS] update 3.2.0 version in the docs
---
docs/_config.yml | 2 +-
src/main/python/docs/source/conf.py | 4 ++--
src/main/python/systemds/project_info.py | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/docs/_config.yml b/docs/_config.yml
index 69da6f054d..6c3ce51e10 100644
--- a/docs/_config.yml
+++ b/docs/_config.yml
@@ -39,5 +39,5 @@ exclude:
- updateAPI.sh
# These allow the documentation to be updated with newer releases
-SYSTEMDS_VERSION: 3.2.0-SNAPSHOT
+SYSTEMDS_VERSION: 3.2.0
diff --git a/src/main/python/docs/source/conf.py b/src/main/python/docs/source/conf.py
index cc8359254d..938f5dce43 100644
--- a/src/main/python/docs/source/conf.py
+++ b/src/main/python/docs/source/conf.py
@@ -34,11 +34,11 @@ sys.path.insert(0, os.path.abspath('../..'))
# -- Project information -----------------------------------------------------
project = 'SystemDS'
-copyright = '2023, Apache SystemDS'
+copyright = '2024, Apache SystemDS'
author = 'Apache SystemDS'
# The full version, including alpha/beta/rc tags
-release = '3.2.0-dev'
+release = '3.2.0'
# -- General configuration ---------------------------------------------------
# Add any Sphinx extension module names here, as strings.
diff --git a/src/main/python/systemds/project_info.py b/src/main/python/systemds/project_info.py
index ff2721e2a4..981b3f9cbf 100644
--- a/src/main/python/systemds/project_info.py
+++ b/src/main/python/systemds/project_info.py
@@ -23,4 +23,4 @@
# via string substitutions using the maven-resources-plugin
__project_group_id__ = 'org.apache.systemds'
__project_artifact_id__ = 'systemds'
-__project_version__ = '3.2.0-dev'
+__project_version__ = '3.2.0'