You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2021/02/26 13:35:52 UTC
[systemds] branch master updated: [SYSTEMDS-2871] Python API
Autogenerator
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 47cd041 [SYSTEMDS-2871] Python API Autogenerator
47cd041 is described below
commit 47cd04191f16e582811a5637afede020b4b15111
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Fri Feb 26 14:28:44 2021 +0100
[SYSTEMDS-2871] Python API Autogenerator
This commit makes the generator work if called from the python folder,
as well as before from the root folder of the project.
---
src/main/python/generator/generator.py | 28 ++++++----
src/main/python/generator/parser.py | 60 +++++++++++++---------
.../python/systemds/operator/algorithm/__init__.py | 2 +-
.../operator/algorithm/builtin/alsTopkPredict.py | 2 +-
.../systemds/operator/algorithm/builtin/kmeans.py | 2 +-
.../operator/algorithm/builtin/kmeansPredict.py | 2 +-
.../systemds/operator/algorithm/builtin/l2svm.py | 2 +-
.../systemds/operator/algorithm/builtin/lm.py | 2 +-
.../operator/algorithm/builtin/multiLogReg.py | 2 +-
.../algorithm/builtin/multiLogRegPredict.py | 2 +-
.../systemds/operator/algorithm/builtin/pca.py | 2 +-
.../operator/algorithm/builtin/toOneHot.py | 2 +-
12 files changed, 64 insertions(+), 44 deletions(-)
diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py
index 4c299d3..c1a9ce2 100644
--- a/src/main/python/generator/generator.py
+++ b/src/main/python/generator/generator.py
@@ -29,21 +29,23 @@ from parser import FunctionParser
class PythonAPIFileGenerator(object):
target_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'systemds', 'operator', 'algorithm', 'builtin')
- source_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))),'scripts', 'builtin')
licence_path = os.path.join('resources', 'template_python_script_license')
template_path = os.path.join('resources', 'template_python_script_imports')
- licence = ""
- imports = ""
- generated_by = ""
- generated_from = ""
+ source_path :str
+ licence : str
+ imports: str
+ generated_by : str
+ generated_from : str
init_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'systemds', 'operator', 'algorithm', '__init__.py')
init_import = u"from .builtin.{function} import {function} \n"
init_all = u"__all__ = {functions} \n"
- def __init__(self, extension: str='py'):
+ def __init__(self, source_path:str, extension: str='py'):
super(PythonAPIFileGenerator, self).__init__()
+ self.source_path = source_path
+
self.extension = '.{extension}'.format(extension=extension)
os.makedirs(self.__class__.target_path, exist_ok = True)
self.function_names = list()
@@ -54,7 +56,7 @@ class PythonAPIFileGenerator(object):
with open(os.path.join(path, self.__class__.licence_path), 'r') as f:
self.licence = f.read()
- self.generated_by = "# Autogenerated By : " + path + ".py\n"
+ self.generated_by = "# Autogenerated By : src/main/python/generator/generator.py\n"
self.generated_from = "# Autogenerated From : "
def generate_file(self, filename: str, file_content: str, dml_file: str):
@@ -68,7 +70,7 @@ class PythonAPIFileGenerator(object):
with open(target_file, "w") as new_script:
new_script.write(self.licence)
new_script.write(self.generated_by)
- new_script.write(self.generated_from + dml_file + "\n")
+ new_script.write((self.generated_from + dml_file + "\n").replace("../","").replace("src/main/python/generator/",""))
new_script.write(self.imports)
new_script.write(file_content)
@@ -311,10 +313,14 @@ class PythonAPIDocumentationGenerator(object):
if __name__ == "__main__":
- f_parser = FunctionParser(PythonAPIFileGenerator.source_path)
- doc_generator = PythonAPIDocumentationGenerator()
+ if "python" in os.getcwd():
+ source_path = os.path.join("../../../",'scripts', 'builtin' )
+ else:
+ source_path = os.path.join(os.path.dirname(__file__),"../../../../",'scripts', 'builtin')
+ file_generator = PythonAPIFileGenerator(source_path)
fun_generator = PythonAPIFunctionGenerator()
- file_generator = PythonAPIFileGenerator()
+ f_parser = FunctionParser(source_path )
+ doc_generator = PythonAPIDocumentationGenerator()
for dml_file in f_parser.files():
try:
diff --git a/src/main/python/generator/parser.py b/src/main/python/generator/parser.py
index dff5228..14883f4 100644
--- a/src/main/python/generator/parser.py
+++ b/src/main/python/generator/parser.py
@@ -35,7 +35,7 @@ class FunctionParser(object):
type_mapping_file = os.path.join('resources', 'type_mapping.json')
- def __init__(self, path: str, extension: str='dml'):
+ def __init__(self, path: str, extension: str = 'dml'):
"""
@param path: path where to look for python scripts
"""
@@ -44,7 +44,7 @@ class FunctionParser(object):
self.extension = '.{extension}'.format(extension=extension)
self.files()
- def parse_function(self, path:str):
+ def parse_function(self, path: str):
"""
@param path: path of file to parse
parses function
@@ -61,20 +61,22 @@ class FunctionParser(object):
function_definition = self.find_function_definition(path)
except AttributeError as e:
print("[ERROR] Could not find function_definition for file \'{file_name}\'.".format(
- file_name = file_name
+ file_name=file_name
))
raise e
# print(function_definition)
- pattern = re.compile(self.__class__.parameter_pattern, flags=re.I|re.M)
+ pattern = re.compile(
+ self.__class__.parameter_pattern, flags=re.I | re.M)
match = pattern.match(function_definition)
- param_str,retval_str = match.group(1,2)
+ param_str, retval_str = match.group(1, 2)
parameters = self.get_parameters(param_str)
return_values = self.get_parameters(retval_str)
- data = {'function_name': function_name, 'parameters': parameters, 'return_values':return_values}
+ data = {'function_name': function_name,
+ 'parameters': parameters, 'return_values': return_values}
return data
-
- def get_parameters(self, param_str:str):
+
+ def get_parameters(self, param_str: str):
# pattern = re.compile(r"[\r\v\n\t]")
# param_str = pattern.sub(" ", param_str)
# print(param_str)
@@ -98,12 +100,14 @@ class FunctionParser(object):
def get_header_parameters(self, param_str: str):
parameters = list()
- pattern = re.compile(self.__class__.header_parameter_pattern, flags=re.I)
+ pattern = re.compile(
+ self.__class__.header_parameter_pattern, flags=re.I)
for param_line in [s for s in param_str.split("\n") if s]:
match = pattern.match(param_line)
try:
- parameters.append((match.group(1), match.group(2), match.group(3), match.group(4)))
+ parameters.append((match.group(1), match.group(
+ 2), match.group(3), match.group(4)))
except Exception as e:
if re.search(pattern=self.__class__.divider_pattern, string=param_line, flags=re.I | re.M) is not None:
continue
@@ -112,7 +116,7 @@ class FunctionParser(object):
return parameters
- def parse_header(self, path:str):
+ def parse_header(self, path: str):
"""
@param path: path of file to parse
parses function
@@ -131,32 +135,39 @@ class FunctionParser(object):
output_parameters = self.get_header_parameters(h_output)
except AttributeError as e:
file_name = os.path.basename(path)
- print("[WARNING] Could not parse header in file \'{file_name}\'.".format(file_name = file_name))
+ print("[WARNING] Could not parse header in file \'{file_name}\'.".format(
+ file_name=file_name))
input_parameters = []
output_parameters = []
- data = {'function_name': None, 'parameters': input_parameters, 'return_values':output_parameters}
+ data = {'function_name': None, 'parameters': input_parameters,
+ 'return_values': output_parameters}
return data
def find_header_input_params(self, path: str):
with open(path, 'r') as f:
content = f.read()
- start = re.search(pattern=self.__class__.header_input_pattern, string=content, flags=re.I | re.M).end()
- end = re.search(pattern=self.__class__.header_output_pattern, string=content, flags=re.I | re.M).start()
+ start = re.search(pattern=self.__class__.header_input_pattern,
+ string=content, flags=re.I | re.M).end()
+ end = re.search(pattern=self.__class__.header_output_pattern,
+ string=content, flags=re.I | re.M).start()
header = content[start:end]
return header
def find_header_output_params(self, path: str):
with open(path, 'r') as f:
content = f.read()
- start = re.search(pattern=self.__class__.header_output_pattern, string=content, flags=re.I | re.M).end()
- end = re.search(pattern=self.__class__.function_pattern, string=content, flags=re.I | re.M).start()
+ start = re.search(pattern=self.__class__.header_output_pattern,
+ string=content, flags=re.I | re.M).end()
+ end = re.search(pattern=self.__class__.function_pattern,
+ string=content, flags=re.I | re.M).start()
header = content[start:end]
return header
def find_function_definition(self, path: str):
with open(path, 'r') as f:
content = f.read()
- match = re.search(pattern=self.__class__.function_pattern, string=content, flags=re.I | re.M)
+ match = re.search(pattern=self.__class__.function_pattern,
+ string=content, flags=re.I | re.M)
start = match.start()
end = match.end()
return content[start:end]
@@ -174,7 +185,8 @@ class FunctionParser(object):
type_mapping_pattern = r"^([^\[\s]+)"
path = os.path.dirname(__file__)
- type_mapping_path = os.path.join(path, self.__class__.type_mapping_file)
+ type_mapping_path = os.path.join(
+ path, self.__class__.type_mapping_file)
# print(type_mapping_path)
with open(type_mapping_path, 'r') as mapping:
type_mapping = json.load(mapping)
@@ -184,10 +196,12 @@ class FunctionParser(object):
if header_param_names != data_param_names:
print("[ERROR] The parameter names of the function does not match with the documentation "
"for file \'{file_name}\'.".format(file_name=data["function_name"]))
- raise ValueError("The parameter names of the function does not match with the documentation")
+ raise ValueError(
+ "The parameter names of the function does not match with the documentation")
header_param_type = [p[1].lower() for p in header["parameters"]]
- header_param_type = [type_mapping["type"].get(item, item) for item in header_param_type]
+ header_param_type = [type_mapping["type"].get(
+ item, item) for item in header_param_type]
data_param_type = [p[1].lower() for p in data["parameters"]]
data_param_type = [type_mapping["type"].get(
@@ -197,7 +211,8 @@ class FunctionParser(object):
if header_param_type != data_param_type:
print("[ERROR] The parameter type of the function does not match with the documentation "
"for file \'{file_name}\'.".format(file_name=data["function_name"]))
- raise ValueError("The parameter type of the function does not match with the documentation")
+ raise ValueError(
+ "The parameter type of the function does not match with the documentation")
# header_param_default = [p[2].lower() for p in header["parameters"]]
# header_param_default = [type_mapping["default"].get(item, item).lower() for item in header_param_default]
@@ -207,4 +222,3 @@ class FunctionParser(object):
# print("[ERROR] The parameter default of the function does not match with the documentation "
# "for file \'{file_name}\'.".format(file_name=data["function_name"]))
# raise ValueError("The parameter default of the function does not match with the documentation")
-
diff --git a/src/main/python/systemds/operator/algorithm/__init__.py b/src/main/python/systemds/operator/algorithm/__init__.py
index ca61bd6..80e5b67 100644
--- a/src/main/python/systemds/operator/algorithm/__init__.py
+++ b/src/main/python/systemds/operator/algorithm/__init__.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
from .builtin.l2svm import l2svm
from .builtin.multiLogRegPredict import multiLogRegPredict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
index aaba420..520e9f8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/alsTopkPredict.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
index b500d56..8994a2b 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/kmeans.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
index 01007a5..0df2267 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/kmeansPredict.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
index c93ed7a..9065f58 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/l2svm.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lm.py b/src/main/python/systemds/operator/algorithm/builtin/lm.py
index 2f25b99..3c19d18 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lm.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/lm.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
index ff0f3ca..d03b25e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/multiLogReg.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
index 4a24d63..00745f9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/multiLogRegPredict.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/pca.py b/src/main/python/systemds/operator/algorithm/builtin/pca.py
index 4523af5..c908636 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/pca.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/pca.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/pca.dml
from typing import Dict
diff --git a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
index c624c15..ab360d7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
@@ -19,7 +19,7 @@
#
# -------------------------------------------------------------
-# Autogenerated By : src/main/python/generator.py
+# Autogenerated By : src/main/python/generator/generator.py
# Autogenerated From : scripts/builtin/toOneHot.dml
from typing import Dict