You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/06/10 14:22:31 UTC
[systemds] 04/05: [MINOR] Update and fix minor syntax for Python Docs
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit d1081a48aeede19c3d5343feba19c9758033eeec
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Fri Jun 10 16:20:42 2022 +0200
[MINOR] Update and fix minor syntax for Python Docs
---
src/main/python/generator/generator.py | 16 +--
src/main/python/generator/parser.py | 10 +-
.../systemds/operator/algorithm/builtin/WoE.py | 12 +-
.../operator/algorithm/builtin/WoEApply.py | 7 +-
.../systemds/operator/algorithm/builtin/abstain.py | 16 +--
.../systemds/operator/algorithm/builtin/als.py | 12 +-
.../systemds/operator/algorithm/builtin/alsCG.py | 12 +-
.../systemds/operator/algorithm/builtin/alsDS.py | 14 ++-
.../operator/algorithm/builtin/alsPredict.py | 11 +-
.../operator/algorithm/builtin/alsTopkPredict.py | 12 +-
.../operator/algorithm/builtin/apply_pipeline.py | 11 +-
.../systemds/operator/algorithm/builtin/arima.py | 7 +-
.../algorithm/builtin/autoencoder_2layer.py | 27 ++--
.../systemds/operator/algorithm/builtin/bandit.py | 9 +-
.../systemds/operator/algorithm/builtin/bivar.py | 14 ++-
.../operator/algorithm/builtin/components.py | 13 +-
.../operator/algorithm/builtin/confusionMatrix.py | 31 ++---
.../systemds/operator/algorithm/builtin/cor.py | 7 +-
.../operator/algorithm/builtin/correctTypos.py | 26 ++--
.../algorithm/builtin/correctTyposApply.py | 29 +++--
.../systemds/operator/algorithm/builtin/cox.py | 52 ++++----
.../systemds/operator/algorithm/builtin/cspline.py | 13 +-
.../operator/algorithm/builtin/csplineCG.py | 8 +-
.../operator/algorithm/builtin/csplineDS.py | 8 +-
.../systemds/operator/algorithm/builtin/cvlm.py | 12 +-
.../systemds/operator/algorithm/builtin/dbscan.py | 7 +-
.../operator/algorithm/builtin/dbscanApply.py | 8 +-
.../operator/algorithm/builtin/decisionTree.py | 27 ++--
.../algorithm/builtin/decisionTreePredict.py | 10 +-
.../operator/algorithm/builtin/deepWalk.py | 7 +-
.../algorithm/builtin/denialConstraints.py | 68 +++++-----
.../operator/algorithm/builtin/discoverFD.py | 8 +-
.../systemds/operator/algorithm/builtin/dist.py | 7 +-
.../systemds/operator/algorithm/builtin/dmv.py | 7 +-
.../systemds/operator/algorithm/builtin/ema.py | 7 +-
.../operator/algorithm/builtin/executePipeline.py | 13 +-
.../operator/algorithm/builtin/ffPredict.py | 7 +-
.../systemds/operator/algorithm/builtin/ffTrain.py | 11 +-
.../operator/algorithm/builtin/fit_pipeline.py | 12 +-
.../algorithm/builtin/fixInvalidLengths.py | 8 +-
.../algorithm/builtin/fixInvalidLengthsApply.py | 9 +-
.../operator/algorithm/builtin/frameSort.py | 9 +-
.../operator/algorithm/builtin/frequencyEncode.py | 8 +-
.../algorithm/builtin/frequencyEncodeApply.py | 7 +-
.../systemds/operator/algorithm/builtin/garch.py | 22 ++--
.../algorithm/builtin/gaussianClassifier.py | 20 +--
.../operator/algorithm/builtin/getAccuracy.py | 7 +-
.../systemds/operator/algorithm/builtin/glm.py | 139 +++++++++++----------
.../operator/algorithm/builtin/glmPredict.py | 80 ++++++------
.../systemds/operator/algorithm/builtin/gmm.py | 16 ++-
.../operator/algorithm/builtin/gmmPredict.py | 11 +-
.../systemds/operator/algorithm/builtin/gnmf.py | 21 ++--
.../operator/algorithm/builtin/gridSearch.py | 12 +-
.../algorithm/builtin/hospitalResidencyMatch.py | 78 +++++++-----
.../operator/algorithm/builtin/hyperband.py | 20 +--
.../operator/algorithm/builtin/img_brightness.py | 7 +-
.../operator/algorithm/builtin/img_crop.py | 7 +-
.../operator/algorithm/builtin/img_cutout.py | 7 +-
.../operator/algorithm/builtin/img_invert.py | 7 +-
.../operator/algorithm/builtin/img_mirror.py | 9 +-
.../operator/algorithm/builtin/img_posterize.py | 9 +-
.../operator/algorithm/builtin/img_rotate.py | 9 +-
.../algorithm/builtin/img_sample_pairing.py | 7 +-
.../operator/algorithm/builtin/img_shear.py | 9 +-
.../operator/algorithm/builtin/img_transform.py | 11 +-
.../operator/algorithm/builtin/img_translate.py | 11 +-
.../operator/algorithm/builtin/impurityMeasures.py | 19 +--
.../operator/algorithm/builtin/imputeByFD.py | 8 +-
.../operator/algorithm/builtin/imputeByFDApply.py | 7 +-
.../operator/algorithm/builtin/imputeByMean.py | 9 +-
.../algorithm/builtin/imputeByMeanApply.py | 9 +-
.../operator/algorithm/builtin/imputeByMedian.py | 10 +-
.../algorithm/builtin/imputeByMedianApply.py | 9 +-
.../operator/algorithm/builtin/imputeByMode.py | 9 +-
.../algorithm/builtin/imputeByModeApply.py | 9 +-
.../operator/algorithm/builtin/intersect.py | 7 +-
.../systemds/operator/algorithm/builtin/km.py | 63 +++++-----
.../systemds/operator/algorithm/builtin/kmeans.py | 8 +-
.../operator/algorithm/builtin/kmeansPredict.py | 7 +-
.../systemds/operator/algorithm/builtin/knn.py | 9 +-
.../operator/algorithm/builtin/knnGraph.py | 7 +-
.../systemds/operator/algorithm/builtin/knnbf.py | 7 +-
.../systemds/operator/algorithm/builtin/l2svm.py | 7 +-
.../operator/algorithm/builtin/l2svmPredict.py | 10 +-
.../systemds/operator/algorithm/builtin/lasso.py | 9 +-
.../operator/algorithm/builtin/lenetPredict.py | 7 +-
.../operator/algorithm/builtin/lenetTrain.py | 11 +-
.../systemds/operator/algorithm/builtin/lm.py | 9 +-
.../systemds/operator/algorithm/builtin/lmCG.py | 7 +-
.../systemds/operator/algorithm/builtin/lmDS.py | 7 +-
.../operator/algorithm/builtin/lmPredict.py | 7 +-
.../operator/algorithm/builtin/logSumExp.py | 7 +-
.../operator/algorithm/builtin/matrixProfile.py | 25 ++--
.../systemds/operator/algorithm/builtin/mcc.py | 7 +-
.../systemds/operator/algorithm/builtin/mdedup.py | 10 +-
.../systemds/operator/algorithm/builtin/mice.py | 14 ++-
.../operator/algorithm/builtin/miceApply.py | 14 ++-
.../systemds/operator/algorithm/builtin/msvm.py | 9 +-
.../operator/algorithm/builtin/msvmPredict.py | 10 +-
.../operator/algorithm/builtin/multiLogReg.py | 13 +-
.../algorithm/builtin/multiLogRegPredict.py | 11 +-
.../systemds/operator/algorithm/builtin/na_locf.py | 7 +-
.../operator/algorithm/builtin/naiveBayes.py | 8 +-
.../algorithm/builtin/naiveBayesPredict.py | 8 +-
.../operator/algorithm/builtin/normalize.py | 11 +-
.../operator/algorithm/builtin/normalizeApply.py | 13 +-
.../systemds/operator/algorithm/builtin/outlier.py | 9 +-
.../operator/algorithm/builtin/outlierByArima.py | 9 +-
.../operator/algorithm/builtin/outlierByIQR.py | 7 +-
.../algorithm/builtin/outlierByIQRApply.py | 7 +-
.../operator/algorithm/builtin/outlierBySd.py | 7 +-
.../operator/algorithm/builtin/outlierBySdApply.py | 7 +-
.../systemds/operator/algorithm/builtin/pca.py | 10 +-
.../operator/algorithm/builtin/pcaInverse.py | 11 +-
.../operator/algorithm/builtin/pcaTransform.py | 11 +-
.../systemds/operator/algorithm/builtin/pnmf.py | 19 +--
.../systemds/operator/algorithm/builtin/ppca.py | 12 +-
.../operator/algorithm/builtin/randomForest.py | 28 +++--
.../systemds/operator/algorithm/builtin/scale.py | 9 +-
.../operator/algorithm/builtin/scaleApply.py | 7 +-
.../operator/algorithm/builtin/scaleMinMax.py | 7 +-
.../algorithm/builtin/selectByVarThresh.py | 7 +-
.../systemds/operator/algorithm/builtin/setdiff.py | 7 +-
.../operator/algorithm/builtin/sherlock.py | 28 +++--
.../operator/algorithm/builtin/sherlockPredict.py | 19 +--
.../operator/algorithm/builtin/shortestPath.py | 19 +--
.../systemds/operator/algorithm/builtin/sigmoid.py | 9 +-
.../operator/algorithm/builtin/slicefinder.py | 17 +--
.../systemds/operator/algorithm/builtin/smote.py | 9 +-
.../systemds/operator/algorithm/builtin/softmax.py | 7 +-
.../systemds/operator/algorithm/builtin/split.py | 10 +-
.../operator/algorithm/builtin/splitBalanced.py | 12 +-
.../operator/algorithm/builtin/stableMarriage.py | 79 ++++++------
.../systemds/operator/algorithm/builtin/statsNA.py | 27 ++--
.../systemds/operator/algorithm/builtin/steplm.py | 33 ++---
.../operator/algorithm/builtin/stratstats.py | 91 +++++++-------
.../algorithm/builtin/symmetricDifference.py | 7 +-
.../systemds/operator/algorithm/builtin/tSNE.py | 9 +-
.../operator/algorithm/builtin/toOneHot.py | 7 +-
.../operator/algorithm/builtin/tomeklink.py | 13 +-
.../operator/algorithm/builtin/topk_cleaning.py | 5 +-
.../operator/algorithm/builtin/underSampling.py | 8 +-
.../systemds/operator/algorithm/builtin/union.py | 7 +-
.../systemds/operator/algorithm/builtin/unique.py | 7 +-
.../systemds/operator/algorithm/builtin/univar.py | 7 +-
.../operator/algorithm/builtin/vectorToCsv.py | 9 +-
.../operator/algorithm/builtin/winsorize.py | 11 +-
.../operator/algorithm/builtin/winsorizeApply.py | 9 +-
.../systemds/operator/algorithm/builtin/xdummy1.py | 7 +-
.../systemds/operator/algorithm/builtin/xdummy2.py | 8 +-
.../systemds/operator/algorithm/builtin/xgboost.py | 42 ++++---
.../builtin/xgboostPredictClassification.py | 9 +-
.../algorithm/builtin/xgboostPredictRegression.py | 9 +-
153 files changed, 1294 insertions(+), 1000 deletions(-)
diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py
index f23f31350d..ffc2c2dc8d 100644
--- a/src/main/python/generator/generator.py
+++ b/src/main/python/generator/generator.py
@@ -291,7 +291,6 @@ class PythonAPIFunctionGenerator(object):
class PythonAPIDocumentationGenerator(object):
param_str = "\n :param {pname}: {meaning}"
- return_str = "\n :return: \'OperationNode\' containing {meaning} \n"
def __init__(self):
super(PythonAPIDocumentationGenerator, self).__init__()
@@ -314,12 +313,10 @@ class PythonAPIDocumentationGenerator(object):
if description == "":
data['function_header'] = ""
elif header_data["return_values"] == []:
- data['function_header'] = '"""\n ' + description + '"""'
+ data['function_header'] = '"""\n ' + description + '"""\n'
else:
- res_str = "\n :return: \'OperationNode\' containing {meaning} \n".format(
- meaning=output_param.lower())
data['function_header'] = '"""\n ' + description + \
- input_param + res_str + ' """'
+ input_param + output_param + '\n """\n'
def header_parameter_string(self, parameter: dict) -> str:
parameter_str = "\n "
@@ -330,14 +327,9 @@ class PythonAPIDocumentationGenerator(object):
return parameter_str
def header_return_string(self, parameter: dict) -> str:
- meaning_str = "\n "
- first = True
+ meaning_str = ""
for param in parameter:
- if first:
- meaning_str += param[1]
- else:
- meaning_str += "\n & " + param[1]
-
+ meaning_str += "\n :return: " + param[1]
return meaning_str
diff --git a/src/main/python/generator/parser.py b/src/main/python/generator/parser.py
index a1162a1623..017abf78b3 100644
--- a/src/main/python/generator/parser.py
+++ b/src/main/python/generator/parser.py
@@ -177,15 +177,15 @@ class FunctionParser(object):
# skip two lines
line = f.readline()
line = f.readline()
-
+
if in_output:
if "----------" not in line:
- h_output += line[2:]
+ h_output += line[1:]
elif in_input:
if "----------" not in line:
- h_input += line[2:]
+ h_input += line[1:]
else:
- description += line[2:]
+ description += line[1:]
line = f.readline()
if description == "" or h_input == "" or h_output == "":
@@ -212,7 +212,7 @@ class FunctionParser(object):
ret = []
for line in data.split("\n"):
if line:
- if line[0] == " ":
+ if line[1] == " ":
prev = ret[-1]
n = (prev[0], prev[1] +"\n " + line.strip())
ret[-1] = n
diff --git a/src/main/python/systemds/operator/algorithm/builtin/WoE.py b/src/main/python/systemds/operator/algorithm/builtin/WoE.py
index f16b878cc4..3a22bad6c9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/WoE.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/WoE.py
@@ -33,26 +33,26 @@ def WoE(X: Matrix,
Y: Matrix,
mask: Matrix):
"""
- function Weight of evidence / information gain
+ function Weight of evidence / information gain
+
:param X: ---
:param Y: ---
:param mask: ---
- :return: 'OperationNode' containing
- ---------
+ :return: Weighted X matrix where the entropy mask is applied
+ :return: A entropy matrix to apply to data
"""
+
params_dict = {'X': X, 'Y': Y, 'mask': mask}
vX_0 = Matrix(X.sds_context, '')
vX_1 = Matrix(X.sds_context, '')
- vX_2 = Matrix(X.sds_context, '')
- output_nodes = [vX_0, vX_1, vX_2, ]
+ output_nodes = [vX_0, vX_1, ]
op = MultiReturn(X.sds_context, 'WoE', output_nodes, named_input_nodes=params_dict)
vX_0._unnamed_input_nodes = [op]
vX_1._unnamed_input_nodes = [op]
- vX_2._unnamed_input_nodes = [op]
return op
diff --git a/src/main/python/systemds/operator/algorithm/builtin/WoEApply.py b/src/main/python/systemds/operator/algorithm/builtin/WoEApply.py
index d6299bb9f3..78e76d558b 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/WoEApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/WoEApply.py
@@ -33,15 +33,16 @@ def WoEApply(X: Matrix,
Y: Matrix,
entropyMatrix: Matrix):
"""
- function Weight of evidence / information gain apply on new data
+ function Weight of evidence / information gain apply on new data
+
:param X: ---
:param Y: ---
:param entropyMatrix: ---
- :return: 'OperationNode' containing
- ---
+ :return: Weighted X matrix where the entropy mask is applied
"""
+
params_dict = {'X': X, 'Y': Y, 'entropyMatrix': entropyMatrix}
return Matrix(X.sds_context,
'WoEApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/abstain.py b/src/main/python/systemds/operator/algorithm/builtin/abstain.py
index ee8611a44e..bfd294f966 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/abstain.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/abstain.py
@@ -34,17 +34,19 @@ def abstain(X: Matrix,
threshold: float,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function calls the multiLogReg-function in which solves Multinomial
- Logistic Regression using Trust Region method
+ This function calls the multiLogReg-function in which solves Multinomial
+ Logistic Regression using Trust Region method
- :param X: Location to read the matrix of feature vectors
- :param Y: Location to read the matrix with category labels
- :param threshold: ---
+
+ :param X: matrix of feature vectors
+ :param Y: matrix with category labels
+ :param threshold: threshold to clear otherwise return X and Y unmodified
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- ------
+ :return: abstained output X
+ :return: abstained output Y
"""
+
params_dict = {'X': X, 'Y': Y, 'threshold': threshold}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/als.py b/src/main/python/systemds/operator/algorithm/builtin/als.py
index f067d94953..3fa7f70caf 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/als.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/als.py
@@ -32,9 +32,10 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def als(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
- using different implementations of the Alternating-Least-Squares (ALS) algorithm.
- Matrices U and V are computed by minimizing a loss function (with regularization).
+ This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
+ using different implementations of the Alternating-Least-Squares (ALS) algorithm.
+ Matrices U and V are computed by minimizing a loss function (with regularization).
+
:param X: Location to read the input matrix X to be factorized
@@ -55,9 +56,10 @@ def als(X: Matrix,
if check is FALSE thr is ignored
:param seed: The seed to random parts of the algorithm
:param verbose: If the algorithm should run verbosely
- :return: 'OperationNode' containing
- an m x r matrix where r is the factorization rankan m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsCG.py b/src/main/python/systemds/operator/algorithm/builtin/alsCG.py
index 80095faeeb..14dff61948 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/alsCG.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/alsCG.py
@@ -32,9 +32,10 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def alsCG(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
- using the Alternating-Least-Squares (ALS) algorithm with conjugate gradient.
- Matrices U and V are computed by minimizing a loss function (with regularization).
+ This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
+ using the Alternating-Least-Squares (ALS) algorithm with conjugate gradient.
+ Matrices U and V are computed by minimizing a loss function (with regularization).
+
:param X: Location to read the input matrix X to be factorized
@@ -55,9 +56,10 @@ def alsCG(X: Matrix,
if check is FALSE thr is ignored
:param seed: The seed to random parts of the algorithm
:param verbose: If the algorithm should run verbosely
- :return: 'OperationNode' containing
- an m x r matrix where r is the factorization rankan m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsDS.py b/src/main/python/systemds/operator/algorithm/builtin/alsDS.py
index fba79a30aa..fa2e33d4d3 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/alsDS.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/alsDS.py
@@ -32,10 +32,11 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def alsDS(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Alternating-Least-Squares (ALS) algorithm using a direct solve method for
- individual least squares problems (reg="L2"). This script computes an
- approximate factorization of a low-rank matrix V into two matrices L and R.
- Matrices L and R are computed by minimizing a loss function (with regularization).
+ Alternating-Least-Squares (ALS) algorithm using a direct solve method for
+ individual least squares problems (reg="L2"). This script computes an
+ approximate factorization of a low-rank matrix V into two matrices L and R.
+ Matrices L and R are computed by minimizing a loss function (with regularization).
+
:param X: Location to read the input matrix V to be factorized
@@ -48,9 +49,10 @@ def alsDS(X: Matrix,
if check is FALSE thr is ignored
:param seed: The seed to random parts of the algorithm
:param verbose: If the algorithm should run verbosely
- :return: 'OperationNode' containing
- an m x r matrix where r is the factorization rankan m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
+ :return: An m x r matrix where r is the factorization rank
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py b/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py
index 38ddcbaf64..83e486665a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/alsPredict.py
@@ -34,18 +34,19 @@ def alsPredict(userIDs: Matrix,
L: Matrix,
R: Matrix):
"""
- This script computes the rating/scores for a given list of userIDs
- using 2 factor matrices L and R. We assume that all users have rates
- at least once and all items have been rates at least once.
+ This script computes the rating/scores for a given list of userIDs
+ using 2 factor matrices L and R. We assume that all users have rates
+ at least once and all items have been rates at least once.
+
:param userIDs: Column vector of user-ids (n x 1)
:param I: Indicator matrix user-id x user-id to exclude from scoring
:param L: The factor matrix L: user-id x feature-id
:param R: The factor matrix R: feature-id x item-id
- :return: 'OperationNode' containing
- the output user-id/item-id/score#
+ :return: The output user-id/item-id/score#
"""
+
params_dict = {'userIDs': userIDs, 'I': I, 'L': L, 'R': R}
return Matrix(userIDs.sds_context,
'alsPredict',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
index 86eb1a9483..1c51abe4b9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/alsTopkPredict.py
@@ -35,9 +35,10 @@ def alsTopkPredict(userIDs: Matrix,
R: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script computes the top-K rating/scores for a given list of userIDs
- using 2 factor matrices L and R. We assume that all users have rates
- at least once and all items have been rates at least once.
+ This script computes the top-K rating/scores for a given list of userIDs
+ using 2 factor matrices L and R. We assume that all users have rates
+ at least once and all items have been rates at least once.
+
:param userIDs: Column vector of user-ids (n x 1)
@@ -45,9 +46,10 @@ def alsTopkPredict(userIDs: Matrix,
:param L: The factor matrix L: user-id x feature-id
:param R: The factor matrix R: feature-id x item-id
:param K: The number of top-K items
- :return: 'OperationNode' containing
- a matrix containing the top-k item-ids with highest predicted ratings for the specified users (rows)a matrix containing the top-k predicted ratings for the specified users (rows)
+ :return: A matrix containing the top-K item-ids with highest predicted ratings for the specified users (rows)
+ :return: A matrix containing the top-K predicted ratings for the specified users (rows)
"""
+
params_dict = {'userIDs': userIDs, 'I': I, 'L': L, 'R': R}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
index 7b232ccd6b..be1100b412 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
@@ -37,9 +37,10 @@ def apply_pipeline(testData: Frame,
iState: List,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
- and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
- in terms of classification accuracy
+ This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
+ and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
+ in terms of classification accuracy
+
:param trainData: ---
@@ -52,9 +53,9 @@ def apply_pipeline(testData: Frame,
:param evalFunHp: ---
:param isLastLabel: ---
:param correctTypos: ---
- :return: 'OperationNode' containing
- ---
+ :return: ---
"""
+
params_dict = {'testData': testData, 'pip': pip, 'applyFunc': applyFunc, 'hp': hp, 'exState': exState, 'iState': iState}
params_dict.update(kwargs)
return Matrix(testData.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/arima.py b/src/main/python/systemds/operator/algorithm/builtin/arima.py
index 38669c7b69..d1f4abc64f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/arima.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/arima.py
@@ -32,7 +32,8 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def arima(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function that implements ARIMA
+ Builtin function that implements ARIMA
+
:param X: The input Matrix to apply Arima on.
@@ -46,9 +47,9 @@ def arima(X: Matrix,
:param s: period in terms of number of time-steps
:param include_mean: center to mean 0, and include in result
:param solver: solver, is either "cg" or "jacobi"
- :return: 'OperationNode' containing
- the calculated coefficients
+ :return: The calculated coefficients
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/autoencoder_2layer.py b/src/main/python/systemds/operator/algorithm/builtin/autoencoder_2layer.py
index b6012ef345..b55fa5eb0d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/autoencoder_2layer.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/autoencoder_2layer.py
@@ -35,13 +35,14 @@ def autoencoder_2layer(X: Matrix,
max_epochs: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Trains a 2-layer autoencoder with minibatch SGD and step-size decay.
- If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
- Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
- The script standardizes the input before training (can be turned off).
- Also, it randomly reshuffles rows before training.
- Currently, tanh is set to be the activation function.
- By re-implementing 'func' DML-bodied function, one can change the activation.
+ Trains a 2-layer autoencoder with minibatch SGD and step-size decay.
+ If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
+ Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
+ The script standardizes the input before training (can be turned off).
+ Also, it randomly reshuffles rows before training.
+ Currently, tanh is set to be the activation function.
+ By re-implementing 'func' DML-bodied function, one can change the activation.
+
:param X: Filename where the input is stored
@@ -59,9 +60,17 @@ def autoencoder_2layer(X: Matrix,
:param W2_rand: ---
:param W3_rand: ---
:param W4_rand: ---
- :return: 'OperationNode' containing
- matrix storing weights between input layer and 1st hidden layermatrix storing bias between input layer and 1st hidden layermatrix storing weights between 1st hidden layer and 2nd hidden layermatrix storing bias between 1st hidden layer and 2nd hidden layermatrix storing weights between 2nd hidden layer and 3rd hidden layermatrix storing bias between 2nd hidden layer and 3rd hidden layermatrix storing weights between 3rd hidden layer and output layermatrix storing bias between 3rd [...]
+ :return: Matrix storing weights between input layer and 1st hidden layer
+ :return: Matrix storing bias between input layer and 1st hidden layer
+ :return: Matrix storing weights between 1st hidden layer and 2nd hidden layer
+ :return: Matrix storing bias between 1st hidden layer and 2nd hidden layer
+ :return: Matrix storing weights between 2nd hidden layer and 3rd hidden layer
+ :return: Matrix storing bias between 2nd hidden layer and 3rd hidden layer
+ :return: Matrix storing weights between 3rd hidden layer and output layer
+ :return: Matrix storing bias between 3rd hidden layer and output layer
+ :return: Matrix storing the hidden (2nd) layer representation if needed
"""
+
params_dict = {'X': X, 'num_hidden1': num_hidden1, 'num_hidden2': num_hidden2, 'max_epochs': max_epochs}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/bandit.py b/src/main/python/systemds/operator/algorithm/builtin/bandit.py
index 64ea2a1a3d..fe433c45a2 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/bandit.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/bandit.py
@@ -44,8 +44,9 @@ def bandit(X_train: Matrix,
cv: bool,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- In The bandit function the objective is to find an arm that optimizes
- a known functional of the unknown arm-reward distributions.
+ In The bandit function the objective is to find an arm that optimizes
+ a known functional of the unknown arm-reward distributions.
+
:param X_train: ---
@@ -65,9 +66,9 @@ def bandit(X_train: Matrix,
:param cvk: ---
:param verbose: ---
:param output: ---
- :return: 'OperationNode' containing
- ---
+ :return: ---
"""
+
params_dict = {'X_train': X_train, 'Y_train': Y_train, 'X_test': X_test, 'Y_test': Y_test, 'metaList': metaList, 'evaluationFunc': evaluationFunc, 'evalFunHp': evalFunHp, 'lp': lp, 'lpHp': lpHp, 'primitives': primitives, 'param': param, 'baseLineScore': baseLineScore, 'cv': cv}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/bivar.py b/src/main/python/systemds/operator/algorithm/builtin/bivar.py
index b2e02a5408..6f278220d8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/bivar.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/bivar.py
@@ -36,9 +36,10 @@ def bivar(X: Matrix,
T2: Matrix,
verbose: bool):
"""
- For a given pair of attribute sets, compute bivariate statistics between all attribute pairs.
- Given, index1 = {A_11, A_12, ... A_1m} and index2 = {A_21, A_22, ... A_2n}
- compute bivariate stats for m*n pairs (A_1i, A_2j), (1<= i <=m) and (1<= j <=n).
+ For a given pair of attribute sets, compute bivariate statistics between all attribute pairs.
+ Given, index1 = {A_11, A_12, ... A_1m} and index2 = {A_21, A_22, ... A_2n}
+ compute bivariate stats for m*n pairs (A_1i, A_2j), (1<= i <=m) and (1<= j <=n).
+
:param X: Input matrix
@@ -47,9 +48,12 @@ def bivar(X: Matrix,
:param T1: Kind for attributes in S1
(kind=1 for scale, kind=2 for nominal, kind=3 for ordinal)
:param verbose: Print bivar stats
- :return: 'OperationNode' containing
- basestats_scale_scale as output with bivar statsbasestats_nominal_scale as output with bivar statsbasestats_nominal_nominal as output with bivar statsbasestats_ordinal_ordinal as output with bivar stats
+ :return: basestats_scale_scale as output with bivar stats
+ :return: basestats_nominal_scale as output with bivar stats
+ :return: basestats_nominal_nominal as output with bivar stats
+ :return: basestats_ordinal_ordinal as output with bivar stats
"""
+
params_dict = {'X': X, 'S1': S1, 'S2': S2, 'T1': T1, 'T2': T2, 'verbose': verbose}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/components.py b/src/main/python/systemds/operator/algorithm/builtin/components.py
index 6133a4bf1e..f94288eb3f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/components.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/components.py
@@ -32,10 +32,11 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def components(G: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Computes the connected components of a graph and returns a
- vector indicating the assignment of vertices to components,
- where each component is identified by the maximum vertex ID
- (i.e., row/column position of the input graph)
+ Computes the connected components of a graph and returns a
+ vector indicating the assignment of vertices to components,
+ where each component is identified by the maximum vertex ID
+ (i.e., row/column position of the input graph)
+
:param X: Location to read the matrix of feature vectors
@@ -48,9 +49,9 @@ def components(G: Matrix,
:param maxi: max. number of outer (Newton) iterations
:param maxii: max. number of inner (conjugate gradient) iterations, 0 = no max
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- regression betas as output for prediction
+ :return: regression betas as output for prediction
"""
+
params_dict = {'G': G}
params_dict.update(kwargs)
return Matrix(G.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
index f2183706b8..b230c3a906 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
@@ -32,27 +32,28 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def confusionMatrix(P: Matrix,
Y: Matrix):
"""
- Accepts a vector for prediction and a one-hot-encoded matrix
- Then it computes the max value of each vector and compare them
- After which, it calculates and returns the sum of classifications
- and the average of each true class.
- True Labels
- 1 2
- 1 TP | FP
- Predictions ----+----
- 2 FN | TN
- TP = True Positives
- FP = False Positives
- FN = False Negatives
- TN = True Negatives
+ Accepts a vector for prediction and a one-hot-encoded matrix
+ Then it computes the max value of each vector and compare them
+ After which, it calculates and returns the sum of classifications
+ and the average of each true class.
+
+ .. code-block:: txt
+
+ True Labels
+ 1 2
+ 1 TP | FP
+ Predictions ----+----
+ 2 FN | TN
+
:param P: vector of Predictions
:param Y: vector of Golden standard One Hot Encoded; the one hot
encoded vector of actual labels
- :return: 'OperationNode' containing
- the confusion matrix sums of classificationsthe confusion matrix averages of each true class
+ :return: The Confusion Matrix Sums of classifications
+ :return: The Confusion Matrix averages of each true class
"""
+
params_dict = {'P': P, 'Y': Y}
vX_0 = Matrix(P.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/cor.py b/src/main/python/systemds/operator/algorithm/builtin/cor.py
index bb5a87c29d..a19af9bccd 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/cor.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/cor.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def cor(X: Matrix):
"""
- This Function compute correlation matrix
+ This Function compute correlation matrix
+
:param X: A Matrix Input to compute the correlation on
- :return: 'OperationNode' containing
- correlation matrix of the input matrix
+ :return: Correlation matrix of the input matrix
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'cor',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
index 59ad8e1d5c..1dd4cdf350 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
@@ -32,24 +32,28 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def correctTypos(strings: Frame,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Corrects corrupted frames of strings
- This algorithm operates on the assumption that most strings are correct
- and simply swaps strings that do not occur often with similar strings that
- occur more often
- References:
- Fred J. Damerau. 1964.
- A technique for computer detection and correction of spelling errors.
- Commun. ACM 7, 3 (March 1964), 171–176.
- DOI:https://doi.org/10.1145/363958.363994
+ Corrects corrupted frames of strings
+ This algorithm operates on the assumption that most strings are correct
+ and simply swaps strings that do not occur often with similar strings that
+ occur more often
+
+ .. code-block:: txt
+
+ References:
+ Fred J. Damerau. 1964.
+ A technique for computer detection and correction of spelling errors.
+ Commun. ACM 7, 3 (March 1964), 171–176.
+ DOI:https://doi.org/10.1145/363958.363994
+
:param strings: The nx1 input frame of corrupted strings
:param frequency_threshold: Strings that occur above this frequency level will not be corrected
:param distance_threshold: Max distance at which strings are considered similar
:param is_verbose: Print debug information
- :return: 'OperationNode' containing
- corrected nx1 output frame
+ :return: Corrected nx1 output frame
"""
+
params_dict = {'strings': strings}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
index c93a4cc418..e9e2ddf11f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
@@ -34,16 +34,21 @@ def correctTyposApply(strings: Frame,
dict: Frame,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Corrects corrupted frames of strings
- This algorithm operates on the assumption that most strings are correct
- and simply swaps strings that do not occur often with similar strings that
- occur more often
- References:
- Fred J. Damerau. 1964.
- A technique for computer detection and correction of spelling errors.
- Commun. ACM 7, 3 (March 1964), 171–176.
- DOI:https://doi.org/10.1145/363958.363994
- TODO: future: add parameter for list of words that are sure to be correct
+ Corrects corrupted frames of strings
+ This algorithm operates on the assumption that most strings are correct
+ and simply swaps strings that do not occur often with similar strings that
+ occur more often
+
+ .. code-block:: txt
+
+ References:
+ Fred J. Damerau. 1964.
+ A technique for computer detection and correction of spelling errors.
+ Commun. ACM 7, 3 (March 1964), 171–176.
+ DOI:https://doi.org/10.1145/363958.363994
+
+ TODO: future: add parameter for list of words that are sure to be correct
+
:param strings: The nx1 input frame of corrupted strings
@@ -52,9 +57,9 @@ def correctTyposApply(strings: Frame,
:param distance_threshold: Max distance at which strings are considered similar
:param distance matrix: ---
:param dict: ---
- :return: 'OperationNode' containing
- corrected nx1 output frame
+ :return: Corrected nx1 output frame
"""
+
params_dict = {'strings': strings, 'distance_matrix': distance_matrix, 'dict': dict}
params_dict.update(kwargs)
return Matrix(strings.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/cox.py b/src/main/python/systemds/operator/algorithm/builtin/cox.py
index b00c511a3a..edd4120289 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/cox.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/cox.py
@@ -35,10 +35,10 @@ def cox(X: Matrix,
R: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script fits a cox Proportional hazard regression model.
- The Breslow method is used for handling ties and the regression parameters
- are computed using trust region newton method with conjugate gradient
-
+ This script fits a cox Proportional hazard regression model.
+ The Breslow method is used for handling ties and the regression parameters
+ are computed using trust region newton method with conjugate gradient
+
:param X: Location to read the input matrix X containing the survival data
@@ -63,31 +63,33 @@ def cox(X: Matrix,
:param tol: Tolerance ("epsilon")
:param moi: Max. number of outer (Newton) iterations
:param mii: Max. number of inner (conjugate gradient) iterations, 0 = no max
- :return: 'OperationNode' containing
- a d x 7 matrix m, where d denotes the number of covariates, with the following schema:
- m[,1]: betas
- m[,2]: exp(betas)
- m[,3]: standard error of betas
- m[,4]: z
- m[,5]: p-value
- m[,6]: lower 100*(1-alpha)% confidence interval of betas
- m[,7]: upper 100*(1-alpha)% confidence interval of betastwo matrices containing a summary of some statistics of the fitted model:
- 1 - file s with the following format
+ :return: A D x 7 matrix M, where D denotes the number of covariates, with the following schema:
+ M[,1]: betas
+ M[,2]: exp(betas)
+ M[,3]: standard error of betas
+ M[,4]: Z
+ M[,5]: P-value
+ M[,6]: lower 100*(1-alpha)% confidence interval of betas
+ M[,7]: upper 100*(1-alpha)% confidence interval of betas
+ :return: Two matrices containing a summary of some statistics of the fitted model:
+ 1 - File S with the following format
- row 1: no. of observations
- row 2: no. of events
- row 3: log-likelihood
- - row 4: aic
- - row 5: rsquare (cox & snell)
- - row 6: max possible rsquare
- 2 - file t with the following format
- - row 1: likelihood ratio test statistic, degree of freedom, p-value
- - row 2: wald test statistic, degree of freedom, p-value
- - row 3: score (log-rank) test statistic, degree of freedom, p-valueadditionally, the following matrices are stored (needed for prediction)
- 1- a column matrix rt that contains the order-preserving recoded timestamps from x
- 2- matrix xo which is matrix x with sorted timestamps
- 3- variance-covariance matrix of the betas cov
- 4- a column matrix mf that contains the column indices of x with the baseline factors removed (if available)
+ - row 4: AIC
+ - row 5: Rsquare (Cox & Snell)
+ - row 6: max possible Rsquare
+ 2 - File T with the following format
+ - row 1: Likelihood ratio test statistic, degree of freedom, P-value
+ - row 2: Wald test statistic, degree of freedom, P-value
+ - row 3: Score (log-rank) test statistic, degree of freedom, P-value
+ :return: Additionally, the following matrices are stored (needed for prediction)
+ 1- A column matrix RT that contains the order-preserving recoded timestamps from X
+ 2- Matrix XO which is matrix X with sorted timestamps
+ 3- Variance-covariance matrix of the betas COV
+ 4- A column matrix MF that contains the column indices of X with the baseline factors removed (if available)
"""
+
params_dict = {'X': X, 'TE': TE, 'F': F, 'R': R}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/cspline.py b/src/main/python/systemds/operator/algorithm/builtin/cspline.py
index e84f9b01d5..bf7db602da 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/cspline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/cspline.py
@@ -34,9 +34,11 @@ def cspline(X: Matrix,
inp_x: float,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Solves Cubic Spline Interpolation
- Algorithms: implement https://en.wikipedia.org/wiki/Spline_interpolation#Algorithm_to_find_the_interpolating_cubic_spline
- It use natural spline with q1''(x0) == qn''(xn) == 0.0
+ Solves Cubic Spline Interpolation
+
+ Algorithms: implement https://en.wikipedia.org/wiki/Spline_interpolation#Algorithm_to_find_the_interpolating_cubic_spline
+ It use natural spline with q1''(x0) == qn''(xn) == 0.0
+
:param X: 1-column matrix of x values knots. It is assumed that x values are
@@ -47,9 +49,10 @@ def cspline(X: Matrix,
:param tol: Tolerance (epsilon); conjugate graduent procedure terminates early if
L2 norm of the beta-residual is less than tolerance * its initial norm
:param maxi: Maximum number of conjugate gradient iterations, 0 = no maximum
- :return: 'OperationNode' containing
- predicted valuematrix of k parameters
+ :return: Predicted value
+ :return: Matrix of k parameters
"""
+
params_dict = {'X': X, 'Y': Y, 'inp_x': inp_x}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/csplineCG.py b/src/main/python/systemds/operator/algorithm/builtin/csplineCG.py
index 1e5e52929a..fa6dcb04f9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/csplineCG.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/csplineCG.py
@@ -34,7 +34,8 @@ def csplineCG(X: Matrix,
inp_x: float,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin that solves cubic spline interpolation using conjugate gradient algorithm
+ Builtin that solves cubic spline interpolation using conjugate gradient algorithm
+
:param X: 1-column matrix of x values knots. It is assumed that x values are
@@ -44,9 +45,10 @@ def csplineCG(X: Matrix,
:param tol: Tolerance (epsilon); conjugate graduent procedure terminates early if
L2 norm of the beta-residual is less than tolerance * its initial norm
:param maxi: Maximum number of conjugate gradient iterations, 0 = no maximum
- :return: 'OperationNode' containing
- predicted valuematrix of k parameters
+ :return: Predicted value
+ :return: Matrix of k parameters
"""
+
params_dict = {'X': X, 'Y': Y, 'inp_x': inp_x}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py b/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py
index dfdfa536df..04a1d6f9a5 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/csplineDS.py
@@ -33,16 +33,18 @@ def csplineDS(X: Matrix,
Y: Matrix,
inp_x: float):
"""
- Builtin that solves cubic spline interpolation using a direct solver.
+ Builtin that solves cubic spline interpolation using a direct solver.
+
:param X: 1-column matrix of x values knots. It is assumed that x values are
monotonically increasing and there is no duplicates points in X
:param Y: 1-column matrix of corresponding y values knots
:param inp_x: the given input x, for which the cspline will find predicted y.
- :return: 'OperationNode' containing
- predicted valuematrix of k parameters
+ :return: Predicted value
+ :return: Matrix of k parameters
"""
+
params_dict = {'X': X, 'Y': Y, 'inp_x': inp_x}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/cvlm.py b/src/main/python/systemds/operator/algorithm/builtin/cvlm.py
index 35f8e519e5..ff9aaed29c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/cvlm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/cvlm.py
@@ -34,9 +34,10 @@ def cvlm(X: Matrix,
k: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The cvlm-function is used for cross-validation of the provided data model. This function follows a non-exhaustive cross
- validation method. It uses lm and lmPredict functions to solve the linear regression and to predict the class of a
- feature vector with no intercept, shifting, and rescaling.
+ The cvlm-function is used for cross-validation of the provided data model. This function follows a non-exhaustive cross
+ validation method. It uses lm and lmPredict functions to solve the linear regression and to predict the class of a
+ feature vector with no intercept, shifting, and rescaling.
+
:param X: Recorded Data set into matrix
@@ -45,9 +46,10 @@ def cvlm(X: Matrix,
:param icpt: Intercept presence, shifting and rescaling the columns of X
:param reg: Regularization constant (lambda) for L2-regularization. set to nonzero for
highly dependant/sparse/numerous features
- :return: 'OperationNode' containing
- response valuesvalidated data set
+ :return: Response values
+ :return: Validated data set
"""
+
params_dict = {'X': X, 'y': y, 'k': k}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py b/src/main/python/systemds/operator/algorithm/builtin/dbscan.py
index 541b272a58..35fb0dda50 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/dbscan.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/dbscan.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def dbscan(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Implements the DBSCAN clustering algorithm using Euclidian distance matrix
+ Implements the DBSCAN clustering algorithm using Euclidian distance matrix
+
:param X: The input Matrix to do DBSCAN on.
:param eps: Maximum distance between two points for one to be considered reachable for the other.
:param minPts: Number of points in a neighborhood for a point to be considered as a core point
(includes the point itself).
- :return: 'OperationNode' containing
- clustering matrix
+ :return: clustering Matrix
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/dbscanApply.py b/src/main/python/systemds/operator/algorithm/builtin/dbscanApply.py
index 0d254e79cc..42a29dc323 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/dbscanApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/dbscanApply.py
@@ -33,16 +33,16 @@ def dbscanApply(X: Matrix,
clusterModel: Matrix,
eps: float):
"""
- Implements the outlier detection/prediction algorithm using a DBScan model
+ Implements the outlier detection/prediction algorithm using a DBScan model
+
- :param NAME: MEANING
:param X: The input Matrix to do outlier detection on.
:param clusterModel: Model of clusters to predict outliers against.
:param eps: Maximum distance between two points for one to be considered reachable for the other.
- :return: 'OperationNode' containing
- meaningpredicted outliers
+ :return: Predicted outliers
"""
+
params_dict = {'X': X, 'clusterModel': clusterModel, 'eps': eps}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
index 9b321898e2..4fb2ee5688 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
@@ -34,7 +34,8 @@ def decisionTree(X: Matrix,
R: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin script implementing classification trees with scale and categorical features
+ Builtin script implementing classification trees with scale and categorical features
+
:param X: Feature matrix X; note that X needs to be both recoded and dummy coded
@@ -46,25 +47,25 @@ def decisionTree(X: Matrix,
:param bins: Number of equiheight bins per scale feature to choose thresholds
:param depth: Maximum depth of the learned tree
:param verbose: boolean specifying if the algorithm should print information while executing
- :return: 'OperationNode' containing
- matrix m where each column corresponds to a node in the learned tree and each row
+ :return: Matrix M where each column corresponds to a node in the learned tree and each row
contains the following information:
- m[1,j]: id of node j (in a complete binary tree)
- m[2,j]: offset (no. of columns) to left child of j if j is an internal node, otherwise 0
- m[3,j]: feature index of the feature (scale feature id if the feature is scale or
+ M[1,j]: id of node j (in a complete binary tree)
+ M[2,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
+ M[3,j]: Feature index of the feature (scale feature id if the feature is scale or
categorical feature id if the feature is categorical)
that node j looks at if j is an internal node, otherwise 0
- m[4,j]: type of the feature that node j looks at if j is an internal node: holds
- the same information as r input vector
- m[5,j]: if j is an internal node: 1 if the feature chosen for j is scale,
+ M[4,j]: Type of the feature that node j looks at if j is an internal node: holds
+ the same information as R input vector
+ M[5,j]: If j is an internal node: 1 if the feature chosen for j is scale,
otherwise the size of the subset of values
stored in rows 6,7,... if j is categorical
- if j is a leaf node: number of misclassified samples reaching at node j
- m[6:,j]: if j is an internal node: threshold the example's feature value is compared
- to is stored at m[6,j] if the feature chosen for j is scale,
+ If j is a leaf node: number of misclassified samples reaching at node j
+ M[6:,j]: If j is an internal node: Threshold the example's feature value is compared
+ to is stored at M[6,j] if the feature chosen for j is scale,
otherwise if the feature chosen for j is categorical rows 6,7,... depict the value subset chosen for j
- if j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0
+ If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0
"""
+
params_dict = {'X': X, 'Y': Y, 'R': R}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/decisionTreePredict.py b/src/main/python/systemds/operator/algorithm/builtin/decisionTreePredict.py
index 7c485fa3c8..51a396eef7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/decisionTreePredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/decisionTreePredict.py
@@ -33,8 +33,10 @@ def decisionTreePredict(M: Matrix,
X: Matrix,
strategy: str):
"""
- Builtin script implementing prediction based on classification trees with scale features using prediction methods of the
- Hummingbird paper (https://www.usenix.org/system/files/osdi20-nakandala.pdf).
+
+ Builtin script implementing prediction based on classification trees with scale features using prediction methods of the
+ Hummingbird paper (https://www.usenix.org/system/files/osdi20-nakandala.pdf).
+
:param M: Decision tree matrix M, as generated by scripts/builtin/decisionTree.dml, where each column corresponds
@@ -57,9 +59,9 @@ def decisionTreePredict(M: Matrix,
:param X: Feature matrix X
:param strategy: Prediction strategy, can be one of ["GEMM", "TT", "PTT"], referring to "Generic matrix multiplication",
"Tree traversal", and "Perfect tree traversal", respectively
- :return: 'OperationNode' containing
- matrix containing the predicted labels for x
+ :return: Matrix containing the predicted labels for X
"""
+
params_dict = {'M': M, 'X': X, 'strategy': strategy}
return Matrix(M.sds_context,
'decisionTreePredict',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/deepWalk.py b/src/main/python/systemds/operator/algorithm/builtin/deepWalk.py
index 3295690676..1cecded9ee 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/deepWalk.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/deepWalk.py
@@ -36,7 +36,8 @@ def deepWalk(Graph: Matrix,
t: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script performs DeepWalk on a given graph (https://arxiv.org/pdf/1403.6652.pdf)
+ This script performs DeepWalk on a given graph (https://arxiv.org/pdf/1403.6652.pdf)
+
:param Graph: adjacency matrix of a graph (n x n)
@@ -46,9 +47,9 @@ def deepWalk(Graph: Matrix,
:param t: walk length
:param alpha: learning rate
:param beta: factor for decreasing learning rate
- :return: 'OperationNode' containing
- matrix of vertex/word representation (n x d)
+ :return: matrix of vertex/word representation (n x d)
"""
+
params_dict = {'Graph': Graph, 'w': w, 'd': d, 'gamma': gamma, 't': t}
params_dict.update(kwargs)
return Matrix(Graph.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
index ab46e7fac8..3c7d8724e6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
@@ -32,31 +32,39 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def denialConstraints(dataFrame: Frame,
constraintsFrame: Frame):
"""
- This function considers some constraints indicating statements that can NOT happen in the data (denial constraints).
- EXAMPLE:
- dataFrame:
- rank discipline yrs.since.phd yrs.service sex salary
- 1 Prof B 19 18 Male 139750
- 2 Prof B 20 16 Male 173200
- 3 AsstProf B 3 3 Male 79750.56
- 4 Prof B 45 39 Male 115000
- 5 Prof B 40 40 Male 141500
- 6 AssocProf B 6 6 Male 97000
- 7 Prof B 30 23 Male 175000
- 8 Prof B 45 45 Male 147765
- 9 Prof B 21 20 Male 119250
- 10 Prof B 18 18 Female 129000
- 11 AssocProf B 12 8 Male 119800
- 12 AsstProf B 7 2 Male 79800
- 13 AsstProf B 1 1 Male 77700
- constraintsFrame:
-
- idx constraint.type group.by group.variable group.option variable1 relation variable2
- 1 variableCompare FALSE yrs.since.phd < yrs.service
- 2 instanceCompare TRUE rank Prof yrs.service >< salary
- 3 valueCompare FALSE salary = 78182
- 4 variableCompare TRUE discipline B yrs.service > yrs.since.phd
- Example: explanation of constraint 2 --> it can't happen that one professor of rank Prof has more years of service than other, but lower salary.
+ This function considers some constraints indicating statements that can NOT happen in the data (denial constraints).
+
+ .. code-block:: txt
+
+ EXAMPLE:
+ dataFrame:
+
+ rank discipline yrs.since.phd yrs.service sex salary
+ 1 Prof B 19 18 Male 139750
+ 2 Prof B 20 16 Male 173200
+ 3 AsstProf B 3 3 Male 79750.56
+ 4 Prof B 45 39 Male 115000
+ 5 Prof B 40 40 Male 141500
+ 6 AssocProf B 6 6 Male 97000
+ 7 Prof B 30 23 Male 175000
+ 8 Prof B 45 45 Male 147765
+ 9 Prof B 21 20 Male 119250
+ 10 Prof B 18 18 Female 129000
+ 11 AssocProf B 12 8 Male 119800
+ 12 AsstProf B 7 2 Male 79800
+ 13 AsstProf B 1 1 Male 77700
+
+ constraintsFrame:
+
+ idx constraint.type group.by group.variable group.option variable1 relation variable2
+ 1 variableCompare FALSE yrs.since.phd < yrs.service
+ 2 instanceCompare TRUE rank Prof yrs.service >< salary
+ 3 valueCompare FALSE salary = 78182
+ 4 variableCompare TRUE discipline B yrs.service > yrs.since.phd
+
+
+ Example: explanation of constraint 2 --> it can't happen that one professor of rank Prof has more years of service than other, but lower salary.
+
:param dataFrame: frame which columns represent the variables of the data and the rows correspond
@@ -77,12 +85,12 @@ def denialConstraints(dataFrame: Frame,
7. relation: (string) can be < , > or = in the case of variableCompare and valueCompare, and < >, < < , > < or > >
in the case of instanceCompare
8. variable2: (string) second variable to compare (name of column in dataFrame) or fixed value for the case of valueCompare.
- :return: 'OperationNode' containing
- matrix of 2 columns.
- - first column shows the indexes of dataframe that are wrong.
- - second column shows the index of the denial constraint that is fulfilled
- if there are no wrong instances to show (0 constrains fulfilled) --> wronginstances=matrix(0,1,2)
+ :return: Matrix of 2 columns.
+ - First column shows the indexes of dataFrame that are wrong.
+ - Second column shows the index of the denial constraint that is fulfilled
+ If there are no wrong instances to show (0 constrains fulfilled) --> WrongInstances=matrix(0,1,2)
"""
+
params_dict = {'dataFrame': dataFrame, 'constraintsFrame': constraintsFrame}
return Matrix(dataFrame.sds_context,
'denialConstraints',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py b/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py
index 11d2b7eb4e..fca2e24df0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/discoverFD.py
@@ -33,17 +33,17 @@ def discoverFD(X: Matrix,
Mask: Matrix,
threshold: float):
"""
- Implements builtin for finding functional dependencies
-
+ Implements builtin for finding functional dependencies
+
:param X: Input Matrix X, encoded Matrix if data is categorical
:param Mask: A row vector for interested features i.e. Mask =[1, 0, 1]
will exclude the second column from processing
:param threshold: threshold value in interval [0, 1] for robust FDs
- :return: 'OperationNode' containing
- matrix of functional dependencies
+ :return: matrix of functional dependencies
"""
+
params_dict = {'X': X, 'Mask': Mask, 'threshold': threshold}
return Matrix(X.sds_context,
'discoverFD',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/dist.py b/src/main/python/systemds/operator/algorithm/builtin/dist.py
index 933bfe0de9..4933299703 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/dist.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/dist.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def dist(X: Matrix):
"""
- Returns Euclidean distance matrix (distances between N n-dimensional points)
+ Returns Euclidean distance matrix (distances between N n-dimensional points)
+
:param X: Matrix to calculate the distance inside
- :return: 'OperationNode' containing
- euclidean distance matrix
+ :return: Euclidean distance matrix
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'dist',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/dmv.py b/src/main/python/systemds/operator/algorithm/builtin/dmv.py
index c95b9535cc..deaf3ea8a6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/dmv.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/dmv.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def dmv(X: Frame,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The dmv-function is used to find disguised missing values utilising syntactical pattern recognition.
+ The dmv-function is used to find disguised missing values utilising syntactical pattern recognition.
+
:param X: Input Frame
:param threshold: Threshold value in interval [0, 1] for dominant pattern per column (e.g., 0.8 means
that 80% of the entries per column must adhere this pattern to be dominant)
:param replace: The string disguised missing values are replaced with
- :return: 'OperationNode' containing
- frame x including detected disguised missing values
+ :return: Frame X including detected disguised missing values
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ema.py b/src/main/python/systemds/operator/algorithm/builtin/ema.py
index 485bc60463..4e0ccca6bb 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ema.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ema.py
@@ -37,7 +37,8 @@ def ema(X: Frame,
beta: float,
gamma: float):
"""
- This function imputes values with exponential moving average (single, double or triple).
+ This function imputes values with exponential moving average (single, double or triple).
+
:param X: Frame that contains time series data that needs to be imputed
@@ -48,9 +49,9 @@ def ema(X: Frame,
:param alpha: alpha- value for EMA
:param beta: beta- value for EMA
:param gamma: gamma- value for EMA
- :return: 'OperationNode' containing
- frame with ema results
+ :return: Frame with EMA results
"""
+
params_dict = {'X': X, 'search_iterations': search_iterations, 'mode': mode, 'freq': freq, 'alpha': alpha, 'beta': beta, 'gamma': gamma}
return Matrix(X.sds_context,
'ema',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
index 52a885f182..021925e2b9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
@@ -40,7 +40,8 @@ def executePipeline(pipeline: Frame,
verbose: bool,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function execute pipeline.
+ This function execute pipeline.
+
:param logical: ---
@@ -56,9 +57,15 @@ def executePipeline(pipeline: Frame,
:param flagsCount: ---
:param test: ---
:param verbose: ---
- :return: 'OperationNode' containing
- ---------------------
+ :return: ---
+ :return: ---
+ :return: ---
+ :return: ---
+ :return: ---
+ :return: ---
+ :return: ---
"""
+
params_dict = {'pipeline': pipeline, 'Xtrain': Xtrain, 'Ytrain': Ytrain, 'Xtest': Xtest, 'Ytest': Ytest, 'metaList': metaList, 'hyperParameters': hyperParameters, 'flagsCount': flagsCount, 'verbose': verbose}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ffPredict.py b/src/main/python/systemds/operator/algorithm/builtin/ffPredict.py
index e262b747e6..93531d2236 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ffPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ffPredict.py
@@ -33,15 +33,16 @@ def ffPredict(model: List,
X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function makes prediction given data and trained feedforward neural network model
+ This builtin function makes prediction given data and trained feedforward neural network model
+
:param Model: Trained ff neural network model
:param X: Data used for making predictions
:param batch_size: Batch size
- :return: 'OperationNode' containing
- predicted value
+ :return: Predicted value
"""
+
params_dict = {'model': model, 'X': X}
params_dict.update(kwargs)
return Matrix(model.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ffTrain.py b/src/main/python/systemds/operator/algorithm/builtin/ffTrain.py
index 366493c136..02435a9550 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ffTrain.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ffTrain.py
@@ -35,9 +35,10 @@ def ffTrain(X: Matrix,
loss_fcn: str,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function trains simple feed-forward neural network. The architecture of the
- networks is: affine1 -> relu -> dropout -> affine2 -> configurable output activation function.
- Hidden layer has 128 neurons. Dropout rate is 0.35. Input and output sizes are inferred from X and Y.
+ This builtin function trains simple feed-forward neural network. The architecture of the
+ networks is: affine1 -> relu -> dropout -> affine2 -> configurable output activation function.
+ Hidden layer has 128 neurons. Dropout rate is 0.35. Input and output sizes are inferred from X and Y.
+
:param X: Training data
@@ -53,9 +54,9 @@ def ffTrain(X: Matrix,
:param validation_split: Fraction of training set used as validation set
:param seed: Seed for model initialization
:param verbose: Flag which indicates if function should print to stdout
- :return: 'OperationNode' containing
- trained model which can be used in ffpredict
+ :return: Trained model which can be used in ffPredict
"""
+
params_dict = {'X': X, 'Y': Y, 'out_activation': out_activation, 'loss_fcn': loss_fcn}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
index 9acea1623a..5de40c745f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
@@ -38,12 +38,12 @@ def fit_pipeline(trainData: Frame,
evalFunHp: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
- and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
- in terms of classification accuracy
+ This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
+ and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
+ in terms of classification accuracy
+
- :param NAME: MEANING
:param trainData: ---
:param testData: ---
:param metaData: ---
@@ -54,9 +54,9 @@ def fit_pipeline(trainData: Frame,
:param evalFunHp: ---
:param isLastLabel: ---
:param correctTypos: ---
- :return: 'OperationNode' containing
- meaning---
+ :return: ---
"""
+
params_dict = {'trainData': trainData, 'testData': testData, 'pip': pip, 'applyFunc': applyFunc, 'hp': hp, 'evaluationFunc': evaluationFunc, 'evalFunHp': evalFunHp}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
index a9730be4dc..b635f31b29 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
@@ -33,16 +33,18 @@ def fixInvalidLengths(F1: Frame,
mask: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Fix invalid lengths
+ Fix invalid lengths
+
:param F1: ---
:param mask: ---
:param ql: ---
:param qu: ---
- :return: 'OperationNode' containing
- ------
+ :return: ---
+ :return: ---
"""
+
params_dict = {'F1': F1, 'mask': mask}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
index 31440f9f98..cc8fe68aac 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
@@ -34,17 +34,18 @@ def fixInvalidLengthsApply(X: Frame,
qLow: Matrix,
qUp: Matrix):
"""
- Fix invalid lengths
+ Fix invalid lengths
+
- :param NAME: MEANING
:param X: ---
:param mask: ---
:param ql: ---
:param qu: ---
- :return: 'OperationNode' containing
- meaning------
+ :return: ---
+ :return: ---
"""
+
params_dict = {'X': X, 'mask': mask, 'qLow': qLow, 'qUp': qUp}
return Matrix(X.sds_context,
'fixInvalidLengthsApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
index 692d925a48..0bfc7f3afe 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
@@ -33,15 +33,16 @@ def frameSort(F: Frame,
mask: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
- Built-in for sorting frames
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ Built-in for sorting frames
+
:param F: Data frame of string values
:param mask: matrix for identifying string columns
- :return: 'OperationNode' containing
- sorted dataset by column 1 in decreasing order
+ :return: sorted dataset by column 1 in decreasing order
"""
+
params_dict = {'F': F, 'mask': mask}
params_dict.update(kwargs)
return Matrix(F.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/frequencyEncode.py b/src/main/python/systemds/operator/algorithm/builtin/frequencyEncode.py
index 3ae4a8cfc0..7de760943c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/frequencyEncode.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/frequencyEncode.py
@@ -32,14 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def frequencyEncode(X: Matrix,
mask: Matrix):
"""
- function frequency conversion
+ function frequency conversion
+
:param X: dataset x
:param mask: mask of the columns for frequency conversion
- :return: 'OperationNode' containing
- categorical columns are replaced with their frequenciesthe frequency counts for the different categoricals
+ :return: categorical columns are replaced with their frequencies
+ :return: the frequency counts for the different categoricals
"""
+
params_dict = {'X': X, 'mask': mask}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/frequencyEncodeApply.py b/src/main/python/systemds/operator/algorithm/builtin/frequencyEncodeApply.py
index 17d7e3118b..50997cf642 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/frequencyEncodeApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/frequencyEncodeApply.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def frequencyEncodeApply(X: Matrix,
freqCount: Matrix):
"""
- frequency code apply
+ frequency code apply
+
:param X: dataset x
:param freqCount: the frequency counts for the different categoricals
- :return: 'OperationNode' containing
- categorical columns are replaced with their frequencies given
+ :return: categorical columns are replaced with their frequencies given
"""
+
params_dict = {'X': X, 'freqCount': freqCount}
return Matrix(X.sds_context,
'frequencyEncodeApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/garch.py b/src/main/python/systemds/operator/algorithm/builtin/garch.py
index 9d647c2249..69a0aa1d87 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/garch.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/garch.py
@@ -39,12 +39,14 @@ def garch(X: Matrix,
sim_seed: int,
verbose: bool):
"""
- This is a builtin function that implements GARCH(1,1), a statistical model used in analyzing time-series data where the variance
- error is believed to be serially autocorrelated
- COMMENTS
- This has some drawbacks: slow convergence of optimization (sort of simulated annealing/gradient descent)
- TODO: use BFGS or BHHH if it is available (this are go to methods)
- TODO: (only then) extend to garch(p,q); otherwise the search space is way too big for the current method
+ This is a builtin function that implements GARCH(1,1), a statistical model used in analyzing time-series data where the variance
+ error is believed to be serially autocorrelated
+
+ COMMENTS
+ This has some drawbacks: slow convergence of optimization (sort of simulated annealing/gradient descent)
+ TODO: use BFGS or BHHH if it is available (this are go to methods)
+ TODO: (only then) extend to garch(p,q); otherwise the search space is way too big for the current method
+
:param X: The input Matrix to apply Arima on.
@@ -56,9 +58,13 @@ def garch(X: Matrix,
:param end_vicinity: same at end (linear decay)
:param sim_seed: seed for simulation of process on fitted coefficients
:param verbose: verbosity, comments during fitting
- :return: 'OperationNode' containing
- simulated garch(1,1) process on fitted coefficientsvariances of simulated fitted processonstant term of fitted process1-st arch-coefficient of fitted process1-st garch-coefficient of fitted process
+ :return: simulated garch(1,1) process on fitted coefficients
+ :return: variances of simulated fitted process
+ :return: onstant term of fitted process
+ :return: 1-st arch-coefficient of fitted process
+ :return: 1-st garch-coefficient of fitted process
"""
+
params_dict = {'X': X, 'kmax': kmax, 'momentum': momentum, 'start_stepsize': start_stepsize, 'end_stepsize': end_stepsize, 'start_vicinity': start_vicinity, 'end_vicinity': end_vicinity, 'sim_seed': sim_seed, 'verbose': verbose}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py b/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py
index 976af9e91e..8d6e513a93 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/gaussianClassifier.py
@@ -33,21 +33,25 @@ def gaussianClassifier(D: Matrix,
C: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Computes the parameters needed for Gaussian Classification.
- Thus it computes the following per class: the prior probability,
- the inverse covariance matrix, the mean per feature and the determinant
- of the covariance matrix. Furthermore (if not explicitly defined), it
- adds some small smoothing value along the variances, to prevent
- numerical errors / instabilities.
+ Computes the parameters needed for Gaussian Classification.
+ Thus it computes the following per class: the prior probability,
+ the inverse covariance matrix, the mean per feature and the determinant
+ of the covariance matrix. Furthermore (if not explicitly defined), it
+ adds some small smoothing value along the variances, to prevent
+ numerical errors / instabilities.
+
:param D: Input matrix (training set)
:param C: Target vector
:param varSmoothing: Smoothing factor for variances
:param verbose: Print accuracy of the training set
- :return: 'OperationNode' containing
- vector storing the class prior probabilitiesmatrix storing the means of the classeslist of inverse covariance matricesvector storing the determinants of the classes
+ :return: Vector storing the class prior probabilities
+ :return: Matrix storing the means of the classes
+ :return: List of inverse covariance matrices
+ :return: Vector storing the determinants of the classes
"""
+
params_dict = {'D': D, 'C': C}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py b/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py
index 4af6063a2a..b9906d61f1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/getAccuracy.py
@@ -33,15 +33,16 @@ def getAccuracy(y: Matrix,
yhat: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function compute the weighted and simple accuracy for given predictions
+ This builtin function compute the weighted and simple accuracy for given predictions
+
:param y: Ground truth (Actual Labels)
:param yhat: Predictions (Predicted labels)
:param isWeighted: Flag for weighted or non-weighted accuracy calculation
- :return: 'OperationNode' containing
- accuracy of the predicted labels
+ :return: accuracy of the predicted labels
"""
+
params_dict = {'y': y, 'yhat': yhat}
params_dict.update(kwargs)
return Matrix(y.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/glm.py b/src/main/python/systemds/operator/algorithm/builtin/glm.py
index 24af5b4d54..15d6973b0b 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/glm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/glm.py
@@ -33,67 +33,78 @@ def glm(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script solves GLM regression using NEWTON/FISHER scoring with trust regions. The glm-function is a flexible
- generalization of ordinary linear regression that allows for response variables that have error distribution models.
- In addition, some GLM statistics are provided as console output by setting verbose=TRUE, one comma-separated name-value
- pair per each line, as follows:
- ----------------------------------------------------------------------------------------------------------------------
- TERMINATION_CODE A positive integer indicating success/failure as follows:
- 1 = Converged successfully; 2 = Maximum number of iterations reached;
- 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported
- BETA_MIN Smallest beta value (regression coefficient), excluding the intercept
- BETA_MIN_INDEX Column index for the smallest beta value
- BETA_MAX Largest beta value (regression coefficient), excluding the intercept
- BETA_MAX_INDEX Column index for the largest beta value
- INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0)
- DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter
- or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0
- DISPERSION_EST Dispersion estimated from the dataset
- DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0
- DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value
- ----------------------------------------------------------------------------------------------------------------------
- The Log file, when requested, contains the following per-iteration variables in CSV format,
- each line containing triple (NAME, ITERATION, VALUE) with ITERATION = 0 for initial values:
- ----------------------------------------------------------------------------------------------------------------------
- NUM_CG_ITERS Number of inner (Conj.Gradient) iterations in this outer iteration
- IS_TRUST_REACHED 1 = trust region boundary was reached, 0 = otherwise
- POINT_STEP_NORM L2-norm of iteration step from old point (i.e. "beta") to new point
- OBJECTIVE The loss function we minimize (i.e. negative partial log-likelihood)
- OBJ_DROP_REAL Reduction in the objective during this iteration, actual value
- OBJ_DROP_PRED Reduction in the objective predicted by a quadratic approximation
- OBJ_DROP_RATIO Actual-to-predicted reduction ratio, used to update the trust region
- GRADIENT_NORM L2-norm of the loss function gradient (NOTE: sometimes omitted)
- LINEAR_TERM_MIN The minimum value of X %*% beta, used to check for overflows
- LINEAR_TERM_MAX The maximum value of X %*% beta, used to check for overflows
- IS_POINT_UPDATED 1 = new point accepted; 0 = new point rejected, old point restored
- TRUST_DELTA Updated trust region size, the "delta"
- ----------------------------------------------------------------------------------------------------------------------
- SOME OF THE SUPPORTED GLM DISTRIBUTION FAMILIES
- AND LINK FUNCTIONS:
- dfam vpow link lpow Distribution.link nical?
- ----------------------------------------------------------------------------------------------------------------------
- 1 0.0 1 -1.0 Gaussian.inverse
- 1 0.0 1 0.0 Gaussian.log
- 1 0.0 1 1.0 Gaussian.id Yes
- 1 1.0 1 0.0 Poisson.log Yes
- 1 1.0 1 0.5 Poisson.sqrt
- 1 1.0 1 1.0 Poisson.id
- 1 2.0 1 -1.0 Gamma.inverse Yes
- 1 2.0 1 0.0 Gamma.log
- 1 2.0 1 1.0 Gamma.id
- 1 3.0 1 -2.0 InvGaussian.1/mu^2 Yes
- 1 3.0 1 -1.0 InvGaussian.inverse
- 1 3.0 1 0.0 InvGaussian.log
- 1 3.0 1 1.0 InvGaussian.id
- 1 * 1 * AnyVariance.AnyLink
- ----------------------------------------------------------------------------------------------------------------------
- 2 * 1 0.0 Binomial.log
- 2 * 1 0.5 Binomial.sqrt
- 2 * 2 * Binomial.logit Yes
- 2 * 3 * Binomial.probit
- 2 * 4 * Binomial.cloglog
- 2 * 5 * Binomial.cauchit
- ----------------------------------------------------------------------------------------------------------------------
+ This script solves GLM regression using NEWTON/FISHER scoring with trust regions. The glm-function is a flexible
+ generalization of ordinary linear regression that allows for response variables that have error distribution models.
+
+ In addition, some GLM statistics are provided as console output by setting verbose=TRUE, one comma-separated name-value
+ pair per each line, as follows:
+
+ .. code-block:: txt
+
+ --------------------------------------------------------------------------------------------
+ TERMINATION_CODE A positive integer indicating success/failure as follows:
+ 1 = Converged successfully; 2 = Maximum number of iterations reached;
+ 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported
+ BETA_MIN Smallest beta value (regression coefficient), excluding the intercept
+ BETA_MIN_INDEX Column index for the smallest beta value
+ BETA_MAX Largest beta value (regression coefficient), excluding the intercept
+ BETA_MAX_INDEX Column index for the largest beta value
+ INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0)
+ DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter
+ or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0
+ DISPERSION_EST Dispersion estimated from the dataset
+ DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0
+ DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value
+ --------------------------------------------------------------------------------------------
+
+ The Log file, when requested, contains the following per-iteration variables in CSV format,
+ each line containing triple (NAME, ITERATION, VALUE) with ITERATION = 0 for initial values:
+
+ --------------------------------------------------------------------------------------------
+ NUM_CG_ITERS Number of inner (Conj.Gradient) iterations in this outer iteration
+ IS_TRUST_REACHED 1 = trust region boundary was reached, 0 = otherwise
+ POINT_STEP_NORM L2-norm of iteration step from old point (i.e. "beta") to new point
+ OBJECTIVE The loss function we minimize (i.e. negative partial log-likelihood)
+ OBJ_DROP_REAL Reduction in the objective during this iteration, actual value
+ OBJ_DROP_PRED Reduction in the objective predicted by a quadratic approximation
+ OBJ_DROP_RATIO Actual-to-predicted reduction ratio, used to update the trust region
+ GRADIENT_NORM L2-norm of the loss function gradient (NOTE: sometimes omitted)
+ LINEAR_TERM_MIN The minimum value of X %*% beta, used to check for overflows
+ LINEAR_TERM_MAX The maximum value of X %*% beta, used to check for overflows
+ IS_POINT_UPDATED 1 = new point accepted; 0 = new point rejected, old point restored
+ TRUST_DELTA Updated trust region size, the "delta"
+ --------------------------------------------------------------------------------------------
+
+ SOME OF THE SUPPORTED GLM DISTRIBUTION FAMILIES
+ AND LINK FUNCTIONS:
+
+ .. code-block:: txt
+
+ dfam vpow link lpow Distribution.link nical?
+ ---------------------------------------------------
+ 1 0.0 1 -1.0 Gaussian.inverse
+ 1 0.0 1 0.0 Gaussian.log
+ 1 0.0 1 1.0 Gaussian.id Yes
+ 1 1.0 1 0.0 Poisson.log Yes
+ 1 1.0 1 0.5 Poisson.sqrt
+ 1 1.0 1 1.0 Poisson.id
+ 1 2.0 1 -1.0 Gamma.inverse Yes
+ 1 2.0 1 0.0 Gamma.log
+ 1 2.0 1 1.0 Gamma.id
+ 1 3.0 1 -2.0 InvGaussian.1/mu^2 Yes
+ 1 3.0 1 -1.0 InvGaussian.inverse
+ 1 3.0 1 0.0 InvGaussian.log
+ 1 3.0 1 1.0 InvGaussian.id
+ 1 * 1 * AnyVariance.AnyLink
+ ---------------------------------------------------
+ 2 * 1 0.0 Binomial.log
+ 2 * 1 0.5 Binomial.sqrt
+ 2 * 2 * Binomial.logit Yes
+ 2 * 3 * Binomial.probit
+ 2 * 4 * Binomial.cloglog
+ 2 * 5 * Binomial.cauchit
+ ---------------------------------------------------
+
:param X: matrix X of feature vectors
@@ -117,10 +128,10 @@ def glm(X: Matrix,
:param moi: Maximum number of outer (Newton / Fisher Scoring) iterations
:param mii: Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum
:param verbose: if the Algorithm should be verbose
- :return: 'OperationNode' containing
- matrix beta, whose size depends on icpt:
- icpt=0: ncol(x) x 1; icpt=1: (ncol(x) + 1) x 1; icpt=2: (ncol(x) + 1) x 2
+ :return: Matrix beta, whose size depends on icpt:
+ icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/glmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/glmPredict.py
index 234078c868..78c5dd31e9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/glmPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/glmPredict.py
@@ -33,37 +33,45 @@ def glmPredict(X: Matrix,
B: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Applies the estimated parameters of a GLM type regression to a new dataset
- Additional statistics are printed one per each line, in the following
- CSV format: NAME,[COLUMN],[SCALED],VALUE
- ---
- NAME is the string identifier for the statistic, see the table below.
- COLUMN is an optional integer value that specifies the Y-column for per-column statistics;
- note that a Binomial/Multinomial one-column Y input is converted into multi-column.
- SCALED is an optional Boolean value (TRUE or FALSE) that tells us whether or not the input
- dispersion parameter (disp) scaling has been applied to this statistic.
- VALUE is the value of the statistic.
- ---
- NAME COLUMN SCALED MEANING
- ---------------------------------------------------------------------------------------------
- LOGLHOOD_Z + Log-Likelihood Z-score (in st.dev's from mean)
- LOGLHOOD_Z_PVAL + Log-Likelihood Z-score p-value
- PEARSON_X2 + Pearson residual X^2 statistic
- PEARSON_X2_BY_DF + Pearson X^2 divided by degrees of freedom
- PEARSON_X2_PVAL + Pearson X^2 p-value
- DEVIANCE_G2 + Deviance from saturated model G^2 statistic
- DEVIANCE_G2_BY_DF + Deviance G^2 divided by degrees of freedom
- DEVIANCE_G2_PVAL + Deviance G^2 p-value
- AVG_TOT_Y + Average of Y column for a single response value
- STDEV_TOT_Y + St.Dev. of Y column for a single response value
- AVG_RES_Y + Average of column residual, i.e. of Y - mean(Y|X)
- STDEV_RES_Y + St.Dev. of column residual, i.e. of Y - mean(Y|X)
- PRED_STDEV_RES + + Model-predicted St.Dev. of column residual
- R2 + R^2 of Y column residual with bias included
- ADJUSTED_R2 + Adjusted R^2 of Y column residual with bias included
- R2_NOBIAS + R^2 of Y column residual with bias subtracted
- ADJUSTED_R2_NOBIAS + Adjusted R^2 of Y column residual with bias subtracted
- ---------------------------------------------------------------------------------------------
+ Applies the estimated parameters of a GLM type regression to a new dataset
+
+ Additional statistics are printed one per each line, in the following
+
+ .. code-block:: txt
+
+ CSV format: NAME,[COLUMN],[SCALED],VALUE
+ ---
+ NAME is the string identifier for the statistic, see the table below.
+ COLUMN is an optional integer value that specifies the Y-column for per-column statistics;
+ note that a Binomial/Multinomial one-column Y input is converted into multi-column.
+ SCALED is an optional Boolean value (TRUE or FALSE) that tells us whether or not the input
+ dispersion parameter (disp) scaling has been applied to this statistic.
+ VALUE is the value of the statistic.
+ ---
+
+ .. code-block:: txt
+
+ NAME COLUMN SCALED MEANING
+ ---------------------------------------------------------------------------------------------
+ LOGLHOOD_Z + Log-Likelihood Z-score (in st.dev's from mean)
+ LOGLHOOD_Z_PVAL + Log-Likelihood Z-score p-value
+ PEARSON_X2 + Pearson residual X^2 statistic
+ PEARSON_X2_BY_DF + Pearson X^2 divided by degrees of freedom
+ PEARSON_X2_PVAL + Pearson X^2 p-value
+ DEVIANCE_G2 + Deviance from saturated model G^2 statistic
+ DEVIANCE_G2_BY_DF + Deviance G^2 divided by degrees of freedom
+ DEVIANCE_G2_PVAL + Deviance G^2 p-value
+ AVG_TOT_Y + Average of Y column for a single response value
+ STDEV_TOT_Y + St.Dev. of Y column for a single response value
+ AVG_RES_Y + Average of column residual, i.e. of Y - mean(Y|X)
+ STDEV_RES_Y + St.Dev. of column residual, i.e. of Y - mean(Y|X)
+ PRED_STDEV_RES + + Model-predicted St.Dev. of column residual
+ R2 + R^2 of Y column residual with bias included
+ ADJUSTED_R2 + Adjusted R^2 of Y column residual with bias included
+ R2_NOBIAS + R^2 of Y column residual with bias subtracted
+ ADJUSTED_R2_NOBIAS + Adjusted R^2 of Y column residual with bias subtracted
+ ---------------------------------------------------------------------------------------------
+
:param X: Matrix X of records (feature vectors)
@@ -84,12 +92,12 @@ def glmPredict(X: Matrix,
-2.0 = 1/mu^2, -1.0 = reciprocal, 0.0 = log, 0.5 = sqrt, 1.0 = identity
:param disp: Dispersion value, when available
:param verbose: Print statistics to stdout
- :return: 'OperationNode' containing
- matrix m of predicted means/probabilities:
- nrow(x) x 1 : for power-type distributions (dfam=1)
- nrow(x) x 2 : for binomial distribution (dfam=2), column 2 is "no"
- nrow(x) x k+1: for multinomial logit (dfam=3), col# k+1 is baseline
+ :return: Matrix M of predicted means/probabilities:
+ nrow(X) x 1 : for Power-type distributions (dfam=1)
+ nrow(X) x 2 : for Binomial distribution (dfam=2), column 2 is "No"
+ nrow(X) x k+1: for Multinomial Logit (dfam=3), col# k+1 is baseline
"""
+
params_dict = {'X': X, 'B': B}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/gmm.py b/src/main/python/systemds/operator/algorithm/builtin/gmm.py
index 726938d159..e219e04448 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/gmm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/gmm.py
@@ -32,8 +32,9 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def gmm(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The gmm-function implements builtin Gaussian Mixture Model with four different types of covariance matrices
- i.e., VVV, EEE, VVI, VII and two initialization methods namely "kmeans" and "random".
+ The gmm-function implements builtin Gaussian Mixture Model with four different types of covariance matrices
+ i.e., VVV, EEE, VVI, VII and two initialization methods namely "kmeans" and "random".
+
:param X: Matrix X
@@ -47,10 +48,15 @@ def gmm(X: Matrix,
:param iterations: Number of iterations
:param reg_covar: regularization parameter for covariance matrix
:param tol: tolerance value for convergence
- :return: 'OperationNode' containing
- prediction matrixprobability of the predictionsnumber of estimated parametersbayesian information criterion for best iterationfitted clusters meana matrix whose [i,k]th entry is the probability that observation i in the test data
- belongs to the kth class
+ :return: Prediction matrix
+ :return: Probability of the predictions
+ :return: Number of estimated parameters
+ :return: Bayesian information criterion for best iteration
+ :return: fitted clusters mean
+ :return: A matrix whose [i,k]th entry is the probability that observation i in the test data
+ belongs to the kth class
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py
index 749ed39d67..0f442c6b17 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/gmmPredict.py
@@ -35,8 +35,10 @@ def gmmPredict(X: Matrix,
precisions_cholesky: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function is a Prediction function for a Gaussian Mixture Model (gmm).
- compute posterior probabilities for new instances given the variance and mean of fitted data
+ This function is a Prediction function for a Gaussian Mixture Model (gmm).
+
+ compute posterior probabilities for new instances given the variance and mean of fitted data
+
:param X: Matrix X (instances to be clustered)
@@ -44,9 +46,10 @@ def gmmPredict(X: Matrix,
:param mu: fitted clusters mean
:param precisions_cholesky: fitted precision matrix for each mixture
:param model: fitted model
- :return: 'OperationNode' containing
- predicted cluster labelsprobabilities of belongingness
+ :return: predicted cluster labels
+ :return: probabilities of belongingness
"""
+
params_dict = {'X': X, 'weight': weight, 'mu': mu, 'precisions_cholesky': precisions_cholesky}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/gnmf.py b/src/main/python/systemds/operator/algorithm/builtin/gnmf.py
index 29fc19c3a5..70acf79f4e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/gnmf.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/gnmf.py
@@ -33,22 +33,25 @@ def gnmf(X: Matrix,
rnk: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The gnmf-function does Gaussian Non-Negative Matrix Factorization. In this, a matrix X is factorized into two
- matrices W and H, such that all three matrices have no negative elements. This non-negativity makes the resulting
- matrices easier to inspect.
- References:
- [Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
- Distributed nonnegative matrix factorization for web-scale dyadic
- data analysis on mapreduce. WWW 2010: 681-690]
+ The gnmf-function does Gaussian Non-Negative Matrix Factorization. In this, a matrix X is factorized into two
+ matrices W and H, such that all three matrices have no negative elements. This non-negativity makes the resulting
+ matrices easier to inspect.
+
+ References:
+ [Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
+ Distributed nonnegative matrix factorization for web-scale dyadic
+ data analysis on mapreduce. WWW 2010: 681-690]
+
:param X: Matrix of feature vectors.
:param rnk: Number of components into which matrix X is to be factored
:param eps: Tolerance
:param maxi: Maximum number of conjugate gradient iterations
- :return: 'OperationNode' containing
- list of pattern matrices, one for each repetitionlist of amplitude matrices, one for each repetition
+ :return: List of pattern matrices, one for each repetition
+ :return: List of amplitude matrices, one for each repetition
"""
+
params_dict = {'X': X, 'rnk': rnk}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py b/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py
index 5eb5709654..dd044c76cb 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/gridSearch.py
@@ -37,8 +37,9 @@ def gridSearch(X: Matrix,
paramValues: List,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The gridSearch-function is used to find the optimal hyper-parameters of a model which results in the most
- accurate predictions. This function takes train and eval functions by name.
+ The gridSearch-function is used to find the optimal hyper-parameters of a model which results in the most
+ accurate predictions. This function takes train and eval functions by name.
+
:param X: Input feature matrix
@@ -60,10 +61,11 @@ def gridSearch(X: Matrix,
:param cv: flag enabling k-fold cross validation, otherwise training loss
:param cvk: if cv=TRUE, specifies the the number of folds, otherwise ignored
:param verbose: flag for verbose debug output
- :return: 'OperationNode' containing
- matrix[double]the trained model with minimal loss (by the 'predict' function)
- multi-column models are returned as a column-major linearized column vectorone-row frame w/ optimal hyper-parameters (by 'params' position)
+ :return: Matrix[Double]the trained model with minimal loss (by the 'predict' function)
+ Multi-column models are returned as a column-major linearized column vector
+ :return: one-row frame w/ optimal hyper-parameters (by 'params' position)
"""
+
params_dict = {'X': X, 'y': y, 'train': train, 'predict': predict, 'params': params, 'paramValues': paramValues}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/hospitalResidencyMatch.py b/src/main/python/systemds/operator/algorithm/builtin/hospitalResidencyMatch.py
index 5db64203ac..c76f5f857f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/hospitalResidencyMatch.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/hospitalResidencyMatch.py
@@ -34,34 +34,43 @@ def hospitalResidencyMatch(R: Matrix,
capacity: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script computes a solution for the hospital residency match problem.
- Residents.mtx:
- 2.0,1.0,3.0
- 1.0,2.0,3.0
- 1.0,2.0,0.0
- Since it is an ORDERED matrix, this means that Resident 1 (row 1) likes hospital 2 the most, followed by hospital 1 and hospital 3.
- If it was UNORDERED, this would mean that resident 1 (row 1) likes hospital 3 the most (since the value at [1,3] is the row max),
- followed by hospital 1 (2.0 preference value) and hospital 2 (1.0 preference value).
- Hospitals.mtx:
- 2.0,1.0,0.0
- 0.0,1.0,2.0
- 1.0,2.0,0.0
- Since it is an UNORDERED matrix this means that Hospital 1 (row 1) likes Resident 1 the most (since the value at [1,1] is the row max).
- capacity.mtx
- 1.0
- 1.0
- 1.0
- residencyMatch.mtx
- 2.0,0.0,0.0
- 1.0,0.0,0.0
- 0.0,2.0,0.0
- hospitalMatch.mtx
- 0.0,1.0,0.0
- 0.0,0.0,2.0
- 1.0,0.0,0.0
- Resident 1 has matched with Hospital 3 (since [1,3] is non-zero) at a preference level of 2.0.
- Resident 2 has matched with Hospital 1 (since [2,1] is non-zero) at a preference level of 1.0.
- Resident 3 has matched with Hospital 2 (since [3,2] is non-zero) at a preference level of 2.0.
+ This script computes a solution for the hospital residency match problem.
+
+ Residents.mtx:
+ 2.0,1.0,3.0
+ 1.0,2.0,3.0
+ 1.0,2.0,0.0
+
+ Since it is an ORDERED matrix, this means that Resident 1 (row 1) likes hospital 2 the most, followed by hospital 1 and hospital 3.
+ If it was UNORDERED, this would mean that resident 1 (row 1) likes hospital 3 the most (since the value at [1,3] is the row max),
+ followed by hospital 1 (2.0 preference value) and hospital 2 (1.0 preference value).
+
+ Hospitals.mtx:
+ 2.0,1.0,0.0
+ 0.0,1.0,2.0
+ 1.0,2.0,0.0
+
+ Since it is an UNORDERED matrix this means that Hospital 1 (row 1) likes Resident 1 the most (since the value at [1,1] is the row max).
+
+ capacity.mtx
+ 1.0
+ 1.0
+ 1.0
+
+ residencyMatch.mtx
+ 2.0,0.0,0.0
+ 1.0,0.0,0.0
+ 0.0,2.0,0.0
+
+ hospitalMatch.mtx
+ 0.0,1.0,0.0
+ 0.0,0.0,2.0
+ 1.0,0.0,0.0
+
+ Resident 1 has matched with Hospital 3 (since [1,3] is non-zero) at a preference level of 2.0.
+ Resident 2 has matched with Hospital 1 (since [2,1] is non-zero) at a preference level of 1.0.
+ Resident 3 has matched with Hospital 2 (since [3,2] is non-zero) at a preference level of 2.0.
+
:param R: Residents matrix R.
@@ -74,13 +83,14 @@ def hospitalResidencyMatch(R: Matrix,
i.e. the leftmost value in a row in P is the preference value for the acceptor
with index 1 and vice-versa (higher is better).
:param verbose: If the operation is verbose
- :return: 'OperationNode' containing
- result matrix
- if cell [i,j] is non-zero, it means that resident i has matched with hospital j.
- further, if cell [i,j] is non-zero, it holds the preference value that led to the match.result matrix
- if cell [i,j] is non-zero, it means that resident i has matched with hospital j.
- further, if cell [i,j] is non-zero, it holds the preference value that led to the match.
+ :return: Result Matrix
+ If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j.
+ Further, if cell [i,j] is non-zero, it holds the preference value that led to the match.
+ :return: Result Matrix
+ If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j.
+ Further, if cell [i,j] is non-zero, it holds the preference value that led to the match.
"""
+
params_dict = {'R': R, 'H': H, 'capacity': capacity}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/hyperband.py b/src/main/python/systemds/operator/algorithm/builtin/hyperband.py
index ea3ad26085..0c0e2795e1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/hyperband.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/hyperband.py
@@ -37,13 +37,14 @@ def hyperband(X_train: Matrix,
paramRanges: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The hyperband-function is used for hyper parameter optimization and is based on multi-armed bandits and early
- elimination. Through multiple parallel brackets and consecutive trials it will return the hyper parameter combination
- which performed best on a validation dataset. A set of hyper parameter combinations is drawn from uniform distributions
- with given ranges; Those make up the candidates for hyperband. Notes:
- hyperband is hard-coded for lmCG, and uses lmPredict for validation
- hyperband is hard-coded to use the number of iterations as a resource
- hyperband can only optimize continuous hyperparameters
+ The hyperband-function is used for hyper parameter optimization and is based on multi-armed bandits and early
+ elimination. Through multiple parallel brackets and consecutive trials it will return the hyper parameter combination
+ which performed best on a validation dataset. A set of hyper parameter combinations is drawn from uniform distributions
+ with given ranges; Those make up the candidates for hyperband. Notes:
+ hyperband is hard-coded for lmCG, and uses lmPredict for validation
+ hyperband is hard-coded to use the number of iterations as a resource
+ hyperband can only optimize continuous hyperparameters
+
:param X_train: Input Matrix of training vectors
@@ -56,9 +57,10 @@ def hyperband(X_train: Matrix,
:param R: Controls number of candidates evaluated
:param eta: Determines fraction of candidates to keep after each trial
:param verbose: If TRUE print messages are activated
- :return: 'OperationNode' containing
- 1-column matrix of weights of best performing candidatehyper parameters of best performing candidate
+ :return: 1-column matrix of weights of best performing candidate
+ :return: hyper parameters of best performing candidate
"""
+
params_dict = {'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val, 'params': params, 'paramRanges': paramRanges}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py
index 2aba1a5b16..55c629b805 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py
@@ -33,15 +33,16 @@ def img_brightness(img_in: Matrix,
value: float,
channel_max: int):
"""
- The img_brightness-function is an image data augmentation function. It changes the brightness of the image.
+ The img_brightness-function is an image data augmentation function. It changes the brightness of the image.
+
:param img_in: Input matrix/image
:param value: The amount of brightness to be changed for the image
:param channel_max: Maximum value of the brightness of the image
- :return: 'OperationNode' containing
- type meaningmatrix[double] output matrix/image
+ :return: Output matrix/image
"""
+
params_dict = {'img_in': img_in, 'value': value, 'channel_max': channel_max}
return Matrix(img_in.sds_context,
'img_brightness',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py
index a5affa8ac1..059aae933a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py
@@ -35,7 +35,8 @@ def img_crop(img_in: Matrix,
x_offset: int,
y_offset: int):
"""
- The img_crop-function is an image data augmentation function. It cuts out a subregion of an image.
+ The img_crop-function is an image data augmentation function. It cuts out a subregion of an image.
+
:param img_in: Input matrix/image
@@ -43,9 +44,9 @@ def img_crop(img_in: Matrix,
:param h: The height of the subregion required
:param x_offset: The horizontal coordinate in the image to begin the crop operation
:param y_offset: The vertical coordinate in the image to begin the crop operation
- :return: 'OperationNode' containing
- cropped matrix/image
+ :return: Cropped matrix/image
"""
+
params_dict = {'img_in': img_in, 'w': w, 'h': h, 'x_offset': x_offset, 'y_offset': y_offset}
return Matrix(img_in.sds_context,
'img_crop',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py b/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py
index de2e15026c..8bb4b37e18 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py
@@ -36,7 +36,8 @@ def img_cutout(img_in: Matrix,
height: int,
fill_value: float):
"""
- Image Cutout function replaces a rectangular section of an image with a constant value.
+ Image Cutout function replaces a rectangular section of an image with a constant value.
+
:param img_in: Input image as 2D matrix with top left corner at [1, 1]
@@ -45,9 +46,9 @@ def img_cutout(img_in: Matrix,
:param width: Width of the rectangle (must be positive)
:param height: Height of the rectangle (must be positive)
:param fill_value: The value to set for the rectangle
- :return: 'OperationNode' containing
- output image as 2d matrix with top left corner at [1, 1]
+ :return: Output image as 2D matrix with top left corner at [1, 1]
"""
+
params_dict = {'img_in': img_in, 'x': x, 'y': y, 'width': width, 'height': height, 'fill_value': fill_value}
return Matrix(img_in.sds_context,
'img_cutout',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_invert.py b/src/main/python/systemds/operator/algorithm/builtin/img_invert.py
index a88c0f4339..0fecf7899e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_invert.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_invert.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def img_invert(img_in: Matrix,
max_value: float):
"""
- This is an image data augmentation function. It inverts an image.
+ This is an image data augmentation function. It inverts an image.
+
:param img_in: Input image
:param max_value: The maximum value pixels can have
- :return: 'OperationNode' containing
- output image
+ :return: Output image
"""
+
params_dict = {'img_in': img_in, 'max_value': max_value}
return Matrix(img_in.sds_context,
'img_invert',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py
index 8dc15c5830..4729f8d400 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def img_mirror(img_in: Matrix,
horizontal_axis: bool):
"""
- This function is an image data augmentation function.
- It flips an image on the X (horizontal) or Y (vertical) axis.
+ This function is an image data augmentation function.
+ It flips an image on the X (horizontal) or Y (vertical) axis.
+
:param img_in: Input matrix/image
:param max_value: The maximum value pixels can have
- :return: 'OperationNode' containing
- flipped matrix/image
+ :return: Flipped matrix/image
"""
+
params_dict = {'img_in': img_in, 'horizontal_axis': horizontal_axis}
return Matrix(img_in.sds_context,
'img_mirror',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py b/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py
index 083824e37e..62dda22a13 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def img_posterize(img_in: Matrix,
bits: int):
"""
- The Image Posterize function limits pixel values to 2^bits different values in the range [0, 255].
- Assumes the input image can attain values in the range [0, 255].
+ The Image Posterize function limits pixel values to 2^bits different values in the range [0, 255].
+ Assumes the input image can attain values in the range [0, 255].
+
:param img_in: Input image
:param bits: The number of bits keep for the values.
1 means black and white, 8 means every integer between 0 and 255.
- :return: 'OperationNode' containing
- output image
+ :return: Output image
"""
+
params_dict = {'img_in': img_in, 'bits': bits}
return Matrix(img_in.sds_context,
'img_posterize',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py b/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py
index 2ed1131889..eedc643586 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py
@@ -33,16 +33,17 @@ def img_rotate(img_in: Matrix,
radians: float,
fill_value: float):
"""
- The Image Rotate function rotates the input image counter-clockwise around the center.
- Uses nearest neighbor sampling.
+ The Image Rotate function rotates the input image counter-clockwise around the center.
+ Uses nearest neighbor sampling.
+
:param img_in: Input image as 2D matrix with top left corner at [1, 1]
:param radians: The value by which to rotate in radian.
:param fill_value: The background color revealed by the rotation
- :return: 'OperationNode' containing
- output image as 2d matrix with top left corner at [1, 1]
+ :return: Output image as 2D matrix with top left corner at [1, 1]
"""
+
params_dict = {'img_in': img_in, 'radians': radians, 'fill_value': fill_value}
return Matrix(img_in.sds_context,
'img_rotate',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py
index fbd27ac91c..b5bcddf2ba 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py
@@ -33,16 +33,17 @@ def img_sample_pairing(img_in1: Matrix,
img_in2: Matrix,
weight: float):
"""
- The image sample pairing function blends two images together.
+ The image sample pairing function blends two images together.
+
:param img_in1: First input image
:param img_in2: Second input image
:param weight: The weight given to the second image.
0 means only img_in1, 1 means only img_in2 will be visible
- :return: 'OperationNode' containing
- output image
+ :return: Output image
"""
+
params_dict = {'img_in1': img_in1, 'img_in2': img_in2, 'weight': weight}
return Matrix(img_in1.sds_context,
'img_sample_pairing',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_shear.py b/src/main/python/systemds/operator/algorithm/builtin/img_shear.py
index 3e622e1ddf..41a3daef80 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_shear.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_shear.py
@@ -34,17 +34,18 @@ def img_shear(img_in: Matrix,
shear_y: float,
fill_value: float):
"""
- This function applies a shearing transformation to an image.
- Uses nearest neighbor sampling.
+ This function applies a shearing transformation to an image.
+ Uses nearest neighbor sampling.
+
:param img_in: Input image as 2D matrix with top left corner at [1, 1]
:param shear_x: Shearing factor for horizontal shearing
:param shear_y: Shearing factor for vertical shearing
:param fill_value: The background color revealed by the shearing
- :return: 'OperationNode' containing
- output image as 2d matrix with top left corner at [1, 1]
+ :return: Output image as 2D matrix with top left corner at [1, 1]
"""
+
params_dict = {'img_in': img_in, 'shear_x': shear_x, 'shear_y': shear_y, 'fill_value': fill_value}
return Matrix(img_in.sds_context,
'img_shear',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_transform.py b/src/main/python/systemds/operator/algorithm/builtin/img_transform.py
index 64afe15c05..701433621d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_transform.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_transform.py
@@ -40,9 +40,10 @@ def img_transform(img_in: Matrix,
f: float,
fill_value: float):
"""
- The Image Transform function applies an affine transformation to an image.
- Optionally resizes the image (without scaling).
- Uses nearest neighbor sampling.
+ The Image Transform function applies an affine transformation to an image.
+ Optionally resizes the image (without scaling).
+ Uses nearest neighbor sampling.
+
:param img_in: Input image as 2D matrix with top left corner at [1, 1]
@@ -50,9 +51,9 @@ def img_transform(img_in: Matrix,
:param out_h: Height of the output image
:param a,b,c,d,e,f: The first two rows of the affine matrix in row-major order
:param fill_value: The background of the image
- :return: 'OperationNode' containing
- output image as 2d matrix with top left corner at [1, 1]
+ :return: Output image as 2D matrix with top left corner at [1, 1]
"""
+
params_dict = {'img_in': img_in, 'out_w': out_w, 'out_h': out_h, 'a': a, 'b': b, 'c': c, 'd': d, 'e': e, 'f': f, 'fill_value': fill_value}
return Matrix(img_in.sds_context,
'img_transform',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_translate.py b/src/main/python/systemds/operator/algorithm/builtin/img_translate.py
index 7583748d9f..0ae64a7d06 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/img_translate.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/img_translate.py
@@ -36,9 +36,10 @@ def img_translate(img_in: Matrix,
out_h: int,
fill_value: float):
"""
- The Image Translate function translates the image.
- Optionally resizes the image (without scaling).
- Uses nearest neighbor sampling.
+ The Image Translate function translates the image.
+ Optionally resizes the image (without scaling).
+ Uses nearest neighbor sampling.
+
:param img_in: Input image as 2D matrix with top left corner at [1, 1]
@@ -47,9 +48,9 @@ def img_translate(img_in: Matrix,
:param out_w: Width of the output image
:param out_h: Height of the output image
:param fill_value: The background of the image
- :return: 'OperationNode' containing
- output image as 2d matrix with top left corner at [1, 1]
+ :return: Output image as 2D matrix with top left corner at [1, 1]
"""
+
params_dict = {'img_in': img_in, 'offset_x': offset_x, 'offset_y': offset_y, 'out_w': out_w, 'out_h': out_h, 'fill_value': fill_value}
return Matrix(img_in.sds_context,
'img_translate',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/impurityMeasures.py b/src/main/python/systemds/operator/algorithm/builtin/impurityMeasures.py
index ea71fba72b..ddda0e3eb8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/impurityMeasures.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/impurityMeasures.py
@@ -35,8 +35,9 @@ def impurityMeasures(X: Matrix,
method: str,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function computes the measure of impurity for the given dataset based on the passed method (gini or entropy).
- The current version expects the target vector to contain only 0 or 1 values.
+ This function computes the measure of impurity for the given dataset based on the passed method (gini or entropy).
+ The current version expects the target vector to contain only 0 or 1 values.
+
:param X: Feature matrix.
@@ -45,16 +46,16 @@ def impurityMeasures(X: Matrix,
1 denotes a continuous feature, 2 denotes a categorical feature.
:param n_bins: Number of bins for binning in case of scale features.
:param method: String indicating the method to use; either "entropy" or "gini".
- :return: 'OperationNode' containing
- (1 x ncol(x)) row vector containing information/gini gain for
+ :return: (1 x ncol(X)) row vector containing information/gini gain for
each feature of the dataset.
- in case of gini, the values denote the gini gains, i.e. how much
- impurity was removed with the respective split. the higher the
+ In case of gini, the values denote the gini gains, i.e. how much
+ impurity was removed with the respective split. The higher the
value, the better the split.
- in case of entropy, the values denote the information gain, i.e.
- how much entropy was removed. the higher the information gain,
- the better the split.
+ In case of entropy, the values denote the information gain, i.e.
+ how much entropy was removed. The higher the information gain,
+ the better the split.
"""
+
params_dict = {'X': X, 'Y': Y, 'R': R, 'method': method}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py
index 9d46f434d6..26b42498a7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByFD.py
@@ -34,16 +34,18 @@ def imputeByFD(X: Matrix,
threshold: float,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies
+ Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies
+
:param X: Vector X, source attribute of functional dependency
:param Y: Vector Y, target attribute of functional dependency and imputation
:param threshold: threshold value in interval [0, 1] for robust FDs
:param verbose: flag for printing verbose debug output
- :return: 'OperationNode' containing
- vector y, with missing values mapped to a new max valuevector y, with imputed missing values
+ :return: Vector Y, with missing values mapped to a new max value
+ :return: Vector Y, with imputed missing values
"""
+
params_dict = {'X': X, 'Y': Y, 'threshold': threshold}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByFDApply.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByFDApply.py
index f9d67a2a7c..d9a731155d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByFDApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByFDApply.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByFDApply(X: Matrix,
Y_imp: Matrix):
"""
- Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies
+ Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies
+
:param X: Matrix X
:param source: source attribute to use for imputation and error correction
:param target: attribute to be fixed
:param threshold: threshold value in interval [0, 1] for robust FDs
- :return: 'OperationNode' containing
- matrix with possible imputations
+ :return: Matrix with possible imputations
"""
+
params_dict = {'X': X, 'Y_imp': Y_imp}
return Matrix(X.sds_context,
'imputeByFDApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py
index 70f55890bb..4c8b4ab11c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMean.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByMean(X: Matrix,
mask: Matrix):
"""
- impute the data by mean value and if the feature is categorical then by mode value
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ impute the data by mean value and if the feature is categorical then by mode value
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
:param X: Data Matrix (Recoded Matrix for categorical features)
:param mask: A 0/1 row vector for identifying numeric (0) and categorical features (1)
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'mask': mask}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMeanApply.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMeanApply.py
index dd2bf0673d..587f7da1c4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMeanApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMeanApply.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByMeanApply(X: Matrix,
imputedVec: Matrix):
"""
- impute the data by mean value and if the feature is categorical then by mode value
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ impute the data by mean value and if the feature is categorical then by mode value
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
:param X: Data Matrix (Recoded Matrix for categorical features)
:param imputationVector: column mean vector
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'imputedVec': imputedVec}
return Matrix(X.sds_context,
'imputeByMeanApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py
index b74977e0f9..bcdc52eb87 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedian.py
@@ -32,15 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByMedian(X: Matrix,
mask: Matrix):
"""
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
- impute the data by median value and if the feature is categorical then by mode value
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
+ impute the data by median value and if the feature is categorical then by mode value
+
:param X: Data Matrix (Recoded Matrix for categorical features)
:param mask: A 0/1 row vector for identifying numeric (0) and categorical features (1)
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'mask': mask}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedianApply.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedianApply.py
index 4cb487a0e5..f8b84d19a3 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMedianApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMedianApply.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByMedianApply(X: Matrix,
imputedVec: Matrix):
"""
- impute the data by median value and if the feature is categorical then by mode value
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ impute the data by median value and if the feature is categorical then by mode value
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
:param X: Data Matrix (Recoded Matrix for categorical features)
:param imputationVector: column median vector
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'imputedVec': imputedVec}
return Matrix(X.sds_context,
'imputeByMedianApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py
index 8787aacdfb..35bfe7c7ae 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByMode.py
@@ -31,14 +31,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByMode(X: Matrix):
"""
- This function impute the data by mode value
- Related to [SYSTEMDS-2902] dependency function for cleaning pipelines
+ This function impute the data by mode value
+ Related to [SYSTEMDS-2902] dependency function for cleaning pipelines
+
:param X: Data Matrix (Recoded Matrix for categorical features)
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByModeApply.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByModeApply.py
index ca85c28d3f..1a7987e0a0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByModeApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByModeApply.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByModeApply(X: Matrix,
imputedVec: Matrix):
"""
- impute the data by most frequent value (recoded data only)
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ impute the data by most frequent value (recoded data only)
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
:param X: Data Matrix (Recoded Matrix for categorical features)
:param imputationVector: column mean vector
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'imputedVec': imputedVec}
return Matrix(X.sds_context,
'imputeByModeApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/intersect.py b/src/main/python/systemds/operator/algorithm/builtin/intersect.py
index 7e86ecf50f..b23f4c5295 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/intersect.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/intersect.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def intersect(X: Matrix,
Y: Matrix):
"""
- Implements set intersection for numeric data
+ Implements set intersection for numeric data
+
:param X: matrix X, set A
:param Y: matrix Y, set B
- :return: 'OperationNode' containing
- intersection matrix, set of intersecting items
+ :return: intersection matrix, set of intersecting items
"""
+
params_dict = {'X': X, 'Y': Y}
return Matrix(X.sds_context,
'intersect',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/km.py b/src/main/python/systemds/operator/algorithm/builtin/km.py
index ff30b4aa35..a053f08be1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/km.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/km.py
@@ -35,7 +35,8 @@ def km(X: Matrix,
SI: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function that implements the analysis of survival data with KAPLAN-MEIER estimates
+ Builtin function that implements the analysis of survival data with KAPLAN-MEIER estimates
+
:param X: Input matrix X containing the survival data:
@@ -55,43 +56,45 @@ def km(X: Matrix,
:param test_type: If survival data for multiple groups is available specifies which test to
perform for comparing survival data across multiple groups: "none" (the default)
"log-rank" or "wilcoxon" test
- :return: 'OperationNode' containing
- matrix km whose dimension depends on the number of groups (denoted by g) and
+ :return: Matrix KM whose dimension depends on the number of groups (denoted by g) and
strata (denoted by s) in the data:
- each collection of 7 consecutive columns in km corresponds to a unique
+ each collection of 7 consecutive columns in KM corresponds to a unique
combination of groups and strata in the data with the following schema
1. col: timestamp
2. col: no. at risk
3. col: no. of events
- 4. col: kaplan-meier estimate of survivor function surv
+ 4. col: Kaplan-Meier estimate of survivor function surv
5. col: standard error of surv
6. col: lower 100*(1-alpha)% confidence interval for surv
- 7. col: upper 100*(1-alpha)% confidence interval for survmatrix m whose dimension depends on the number of groups (g) and strata (s) in
- the data (k denotes the number of factors used for grouping ,i.e., ncol(gi) and
- l denotes the number of factors used for stratifying, i.e., ncol(si))
- m[,1:k]: unique combination of values in the k factors used for grouping
- m[,(k+1):(k+l)]: unique combination of values in the l factors used for stratifying
- m[,k+l+1]: total number of records
- m[,k+l+2]: total number of events
- m[,k+l+3]: median of surv
- m[,k+l+4]: lower 100*(1-alpha)% confidence interval of the median of surv
- m[,k+l+5]: upper 100*(1-alpha)% confidence interval of the median of surv
- if the number of groups and strata is equal to 1, m will have 4 columns with
- m[,1]: total number of events
- m[,2]: median of surv
- m[,3]: lower 100*(1-alpha)% confidence interval of the median of surv
- m[,4]: upper 100*(1-alpha)% confidence interval of the median of survif survival data from multiple groups available and ttype=log-rank or wilcoxon,
- a 1 x 4 matrix t and an g x 5 matrix t_groups_oe with
- t_groups_oe[,1] = no. of events
- t_groups_oe[,2] = observed value (o)
- t_groups_oe[,3] = expected value (e)
- t_groups_oe[,4] = (o-e)^2/e
- t_groups_oe[,5] = (o-e)^2/v
- t[1,1] = no. of groups
- t[1,2] = degree of freedom for chi-squared distributed test statistic
- t[1,3] = test statistic
- t[1,4] = p-value
+ 7. col: upper 100*(1-alpha)% confidence interval for surv
+ :return: Matrix M whose dimension depends on the number of groups (g) and strata (s) in
+ the data (k denotes the number of factors used for grouping ,i.e., ncol(GI) and
+ l denotes the number of factors used for stratifying, i.e., ncol(SI))
+ M[,1:k]: unique combination of values in the k factors used for grouping
+ M[,(k+1):(k+l)]: unique combination of values in the l factors used for stratifying
+ M[,k+l+1]: total number of records
+ M[,k+l+2]: total number of events
+ M[,k+l+3]: median of surv
+ M[,k+l+4]: lower 100*(1-alpha)% confidence interval of the median of surv
+ M[,k+l+5]: upper 100*(1-alpha)% confidence interval of the median of surv
+ If the number of groups and strata is equal to 1, M will have 4 columns with
+ M[,1]: total number of events
+ M[,2]: median of surv
+ M[,3]: lower 100*(1-alpha)% confidence interval of the median of surv
+ M[,4]: upper 100*(1-alpha)% confidence interval of the median of surv
+ :return: If survival data from multiple groups available and ttype=log-rank or wilcoxon,
+ a 1 x 4 matrix T and an g x 5 matrix T_GROUPS_OE with
+ T_GROUPS_OE[,1] = no. of events
+ T_GROUPS_OE[,2] = observed value (O)
+ T_GROUPS_OE[,3] = expected value (E)
+ T_GROUPS_OE[,4] = (O-E)^2/E
+ T_GROUPS_OE[,5] = (O-E)^2/V
+ T[1,1] = no. of groups
+ T[1,2] = degree of freedom for Chi-squared distributed test statistic
+ T[1,3] = test statistic
+ T[1,4] = P-value
"""
+
params_dict = {'X': X, 'TE': TE, 'GI': GI, 'SI': SI}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
index 0b53dfb397..4c1f69ac31 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/kmeans.py
@@ -32,7 +32,8 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def kmeans(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function that implements the k-Means clustering algorithm
+ Builtin function that implements the k-Means clustering algorithm
+
:param X: The input Matrix to do KMeans on.
@@ -44,9 +45,10 @@ def kmeans(X: Matrix,
:param avg_sample_size_per_centroid: Average number of records per centroid in data samples
:param seed: The seed used for initial sampling. If set to -1
random seeds are selected.
- :return: 'OperationNode' containing
- the mapping of records to centroidsthe output matrix with the centroids
+ :return: The mapping of records to centroids
+ :return: The output matrix with the centroids
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
index e7aafe6307..5d99ea7bb0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/kmeansPredict.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def kmeansPredict(X: Matrix,
C: Matrix):
"""
- Builtin function that does predictions based on a set of centroids provided.
+ Builtin function that does predictions based on a set of centroids provided.
+
:param X: The input Matrix to do KMeans on.
:param C: The input Centroids to map X onto.
- :return: 'OperationNode' containing
- the mapping of records to centroids
+ :return: The mapping of records to centroids
"""
+
params_dict = {'X': X, 'C': C}
return Matrix(X.sds_context,
'kmeansPredict',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/knn.py b/src/main/python/systemds/operator/algorithm/builtin/knn.py
index 4d70089315..1239b72c98 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/knn.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/knn.py
@@ -35,7 +35,8 @@ def knn(Train: Matrix,
START_SELECTED: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script implements KNN (K Nearest Neighbor) algorithm.
+ This script implements KNN (K Nearest Neighbor) algorithm.
+
:param Train: The input matrix as features
@@ -58,9 +59,11 @@ def knn(Train: Matrix,
(TRUE means yes)
:param predict_con_tg: Continuous target predict function: mean(=0) or median(=1)
:param START_SELECTED: feature selection initial value
- :return: 'OperationNode' containing
- ------feature importance value
+ :return: Applied clusters to X
+ :return: Cluster matrix
+ :return: Feature importance value
"""
+
params_dict = {'Train': Train, 'Test': Test, 'CL': CL, 'START_SELECTED': START_SELECTED}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/knnGraph.py b/src/main/python/systemds/operator/algorithm/builtin/knnGraph.py
index 5b5086f1a7..4c45ef12ee 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/knnGraph.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/knnGraph.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def knnGraph(X: Matrix,
k: int):
"""
- Builtin for k nearest neighbor graph construction
+ Builtin for k nearest neighbor graph construction
+
:param X: ---
:param k: ---
- :return: 'OperationNode' containing
- ---
+ :return: ---
"""
+
params_dict = {'X': X, 'k': k}
return Matrix(X.sds_context,
'knnGraph',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/knnbf.py b/src/main/python/systemds/operator/algorithm/builtin/knnbf.py
index e16c412a02..d0fe8e3c59 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/knnbf.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/knnbf.py
@@ -33,15 +33,16 @@ def knnbf(X: Matrix,
T: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script implements KNN (K Nearest Neighbor) algorithm.
+ This script implements KNN (K Nearest Neighbor) algorithm.
+
:param X: ---
:param T: ---
:param k_value: ---
- :return: 'OperationNode' containing
- ---
+ :return: ---
"""
+
params_dict = {'X': X, 'T': T}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
index e605188cdf..5ae30f44b0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/l2svm.py
@@ -33,7 +33,8 @@ def l2svm(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function Implements binary-class SVM with squared slack variables
+ Builtin function Implements binary-class SVM with squared slack variables
+
:param X: matrix X of feature vectors
@@ -47,9 +48,9 @@ def l2svm(X: Matrix,
:param verbose: Set to true if one wants print statements updating on loss.
:param columnId: The column Id used if one wants to add a ID to the print statement,
eg. used when L2SVM is used in MSVM.
- :return: 'OperationNode' containing
- the trained model
+ :return: the trained model
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py
index f5abc4fccd..7dbedd0d43 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/l2svmPredict.py
@@ -33,16 +33,18 @@ def l2svmPredict(X: Matrix,
W: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function Implements binary-class SVM with squared slack variables.
+ Builtin function Implements binary-class SVM with squared slack variables.
+
:param X: matrix X of feature vectors to classify
:param W: matrix of the trained variables
:param verbose: Set to true if one wants print statements.
- :return: 'OperationNode' containing
- classification labels raw, meaning not modified to clean
- labels of 1's and -1'sclassification labels maxed to ones and zeros.
+ :return: Classification Labels Raw, meaning not modified to clean
+ labels of 1's and -1's
+ :return: Classification Labels Maxed to ones and zeros.
"""
+
params_dict = {'X': X, 'W': W}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lasso.py b/src/main/python/systemds/operator/algorithm/builtin/lasso.py
index c5b3060980..3f21d97855 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lasso.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lasso.py
@@ -33,8 +33,9 @@ def lasso(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function for the SpaRSA algorithm to perform lasso regression
- (SpaRSA .. Sparse Reconstruction by Separable Approximation)
+ Builtin function for the SpaRSA algorithm to perform lasso regression
+ (SpaRSA .. Sparse Reconstruction by Separable Approximation)
+
:param X: input feature matrix
@@ -44,9 +45,9 @@ def lasso(X: Matrix,
:param tau: regularization component
:param maxi: maximum number of iterations until convergence
:param verbose: if the builtin should be verbose
- :return: 'OperationNode' containing
- model matrix
+ :return: model matrix
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lenetPredict.py b/src/main/python/systemds/operator/algorithm/builtin/lenetPredict.py
index 32eaddc75d..cc83bc3174 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lenetPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lenetPredict.py
@@ -36,7 +36,8 @@ def lenetPredict(model: List,
Win: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function makes prediction given data and trained LeNet model
+ This builtin function makes prediction given data and trained LeNet model
+
:param model: Trained LeNet model
@@ -45,9 +46,9 @@ def lenetPredict(model: List,
:param Hin: Input height
:param Win: Input width
:param batch_size: Batch size
- :return: 'OperationNode' containing
- predicted values
+ :return: Predicted values
"""
+
params_dict = {'model': model, 'X': X, 'C': C, 'Hin': Hin, 'Win': Win}
params_dict.update(kwargs)
return Matrix(model.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lenetTrain.py b/src/main/python/systemds/operator/algorithm/builtin/lenetTrain.py
index d181ccb3dd..597fd7966e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lenetTrain.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lenetTrain.py
@@ -38,9 +38,10 @@ def lenetTrain(X: Matrix,
Win: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function trains LeNet CNN. The architecture of the
- networks is:conv1 -> relu1 -> pool1 -> conv2 -> relu2 -> pool2 ->
- affine3 -> relu3 -> affine4 -> softmax
+ This builtin function trains LeNet CNN. The architecture of the
+ networks is:conv1 -> relu1 -> pool1 -> conv2 -> relu2 -> pool2 ->
+ affine3 -> relu3 -> affine4 -> softmax
+
:param X: Input data matrix, of shape (N, C*Hin*Win)
@@ -58,9 +59,9 @@ def lenetTrain(X: Matrix,
:param reg: Regularization strength
:param seed: Seed for model initialization
:param verbose: Flag indicates if function should print to stdout
- :return: 'OperationNode' containing
- trained model which can be used in lenetpredict
+ :return: Trained model which can be used in lenetPredict
"""
+
params_dict = {'X': X, 'Y': Y, 'X_val': X_val, 'Y_val': Y_val, 'C': C, 'Hin': Hin, 'Win': Win}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lm.py b/src/main/python/systemds/operator/algorithm/builtin/lm.py
index a063bfb172..06f5f6e7c5 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lm.py
@@ -33,8 +33,9 @@ def lm(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The lm-function solves linear regression using either the direct solve method or the conjugate gradient
- algorithm depending on the input size of the matrices (See lmDS-function and lmCG-function respectively).
+ The lm-function solves linear regression using either the direct solve method or the conjugate gradient
+ algorithm depending on the input size of the matrices (See lmDS-function and lmCG-function respectively).
+
:param X: Matrix of feature vectors.
@@ -46,9 +47,9 @@ def lm(X: Matrix,
norm of the beta-residual is less than tolerance * its initial norm
:param maxi: Maximum number of conjugate gradient iterations. 0 = no maximum
:param verbose: If TRUE print messages are activated
- :return: 'OperationNode' containing
- the model fit
+ :return: The model fit
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lmCG.py b/src/main/python/systemds/operator/algorithm/builtin/lmCG.py
index c8d66db6b2..607579f869 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lmCG.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lmCG.py
@@ -33,7 +33,8 @@ def lmCG(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The lmCG function solves linear regression using the conjugate gradient algorithm
+ The lmCG function solves linear regression using the conjugate gradient algorithm
+
:param X: Matrix of feature vectors.
@@ -45,9 +46,9 @@ def lmCG(X: Matrix,
norm of the beta-residual is less than tolerance * its initial norm
:param maxi: Maximum number of conjugate gradient iterations. 0 = no maximum
:param verbose: If TRUE print messages are activated
- :return: 'OperationNode' containing
- the model fit
+ :return: The model fit
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lmDS.py b/src/main/python/systemds/operator/algorithm/builtin/lmDS.py
index e130a53642..8fcdbd5b46 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lmDS.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lmDS.py
@@ -33,7 +33,8 @@ def lmDS(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The lmDC function solves linear regression using the direct solve method
+ The lmDC function solves linear regression using the direct solve method
+
:param X: Matrix of feature vectors.
@@ -45,9 +46,9 @@ def lmDS(X: Matrix,
norm of the beta-residual is less than tolerance * its initial norm
:param maxi: Maximum number of conjugate gradient iterations. 0 = no maximum
:param verbose: If TRUE print messages are activated
- :return: 'OperationNode' containing
- the model fit
+ :return: The model fit
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py
index 0bc9ff1aa7..24836882f1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/lmPredict.py
@@ -33,7 +33,8 @@ def lmPredict(X: Matrix,
B: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The lmPredict-function predicts the class of a feature vector
+ The lmPredict-function predicts the class of a feature vector
+
:param X: Matrix of feature vectors
@@ -42,9 +43,9 @@ def lmPredict(X: Matrix,
if verbose output is not wanted
:param icpt: Intercept presence, shifting and rescaling the columns of X
:param verbose: If TRUE print messages are activated
- :return: 'OperationNode' containing
- 1-column matrix of classes
+ :return: 1-column matrix of classes
"""
+
params_dict = {'X': X, 'B': B}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py b/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py
index 3200e6a141..48b675f7a6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/logSumExp.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def logSumExp(M: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Built-in LOGSUMEXP
+ Built-in LOGSUMEXP
+
:param M: matrix to perform Log sum exp on.
:param margin: if the logsumexp of rows is required set margin = "row"
if the logsumexp of columns is required set margin = "col"
if set to "none" then a single scalar is returned computing logsumexp of matrix
- :return: 'OperationNode' containing
- a 1*1 matrix, row vector or column vector depends on margin value
+ :return: a 1*1 matrix, row vector or column vector depends on margin value
"""
+
params_dict = {'M': M}
params_dict.update(kwargs)
return Matrix(M.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/matrixProfile.py b/src/main/python/systemds/operator/algorithm/builtin/matrixProfile.py
index b7ccdd361b..2f8ddf0737 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/matrixProfile.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/matrixProfile.py
@@ -32,14 +32,18 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def matrixProfile(ts: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function that computes the MatrixProfile of a time series efficiently
- using the SCRIMP++ algorithm.
- References:
- Yan Zhu et al.. 2018.
- Matrix Profile XI: SCRIMP++: Time Series Motif Discovery at Interactive Speeds.
- 2018 IEEE International Conference on Data Mining (ICDM), 2018, pp. 837-846.
- DOI: 10.1109/ICDM.2018.00099.
- https://www.cs.ucr.edu/~eamonn/SCRIMP_ICDM_camera_ready_updated.pdf
+ Builtin function that computes the MatrixProfile of a time series efficiently
+ using the SCRIMP++ algorithm.
+
+ .. code-block:: txt
+
+ References:
+ Yan Zhu et al.. 2018.
+ Matrix Profile XI: SCRIMP++: Time Series Motif Discovery at Interactive Speeds.
+ 2018 IEEE International Conference on Data Mining (ICDM), 2018, pp. 837-846.
+ DOI: 10.1109/ICDM.2018.00099.
+ https://www.cs.ucr.edu/~eamonn/SCRIMP_ICDM_camera_ready_updated.pdf
+
:param ts: Time series to profile
@@ -48,9 +52,10 @@ def matrixProfile(ts: Matrix,
between zero and one (1
computes the exact solution)
:param is_verbose: Print debug information
- :return: 'OperationNode' containing
- the computed matrix profileindices of least distances
+ :return: The computed matrix profile
+ :return: Indices of least distances
"""
+
params_dict = {'ts': ts}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/mcc.py b/src/main/python/systemds/operator/algorithm/builtin/mcc.py
index 8cf399ab67..33f051aca8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/mcc.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/mcc.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def mcc(predictions: Matrix,
labels: Matrix):
"""
- Built-in function mcc: Matthews' Correlation Coefficient for binary classification evaluation
+ Built-in function mcc: Matthews' Correlation Coefficient for binary classification evaluation
+
:param predictions: Vector of predicted 0/1 values.
(requires setting 'labels' parameter)
:param labels: Vector of 0/1 labels.
- :return: 'OperationNode' containing
- matthews' correlation coefficient
+ :return: Matthews' Correlation Coefficient
"""
+
params_dict = {'predictions': predictions, 'labels': labels}
return Matrix(predictions.sds_context,
'mcc',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/mdedup.py b/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
index 1c9c55126b..85d93d5c2c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
@@ -36,9 +36,9 @@ def mdedup(X: Frame,
RHSthreshold: Matrix,
verbose: bool):
"""
- Implements builtin for deduplication using matching dependencies (e.g. Street 0.95, City 0.90 -> ZIP 1.0)
- and Jaccard distance.
-
+ Implements builtin for deduplication using matching dependencies (e.g. Street 0.95, City 0.90 -> ZIP 1.0)
+ and Jaccard distance.
+
:param X: Input Frame X
@@ -48,9 +48,9 @@ def mdedup(X: Frame,
:param RHSfeatures: A matrix 1xd with numbers of columns for MDs
:param RHSthreshold: A matrix 1xd with threshold values in interval [0, 1] for MDs
:param verbose: To print the output
- :return: 'OperationNode' containing
- matrix nx1 of duplicates
+ :return: Matrix nx1 of duplicates
"""
+
params_dict = {'X': X, 'LHSfeatures': LHSfeatures, 'LHSthreshold': LHSthreshold, 'RHSfeatures': RHSfeatures, 'RHSthreshold': RHSthreshold, 'verbose': verbose}
return Matrix(X.sds_context,
'mdedup',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/mice.py b/src/main/python/systemds/operator/algorithm/builtin/mice.py
index 8f5654f5a7..7a08907a4a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/mice.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/mice.py
@@ -33,10 +33,12 @@ def mice(X: Matrix,
cMask: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This Builtin function implements multiple imputation using Chained Equations (MICE)
- Assumption missing value are represented with empty string i.e ",," in CSV file
- variables with suffix n are storing continuos/numeric data and variables with
- suffix c are storing categorical data
+ This Builtin function implements multiple imputation using Chained Equations (MICE)
+
+ Assumption missing value are represented with empty string i.e ",," in CSV file
+ variables with suffix n are storing continuos/numeric data and variables with
+ suffix c are storing categorical data
+
:param X: Data Matrix (Recoded Matrix for categorical features)
@@ -46,9 +48,9 @@ def mice(X: Matrix,
if the predicted value has probability greater than threshold,
only applicable for categorical data
:param verbose: Boolean value.
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'cMask': cMask}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/miceApply.py b/src/main/python/systemds/operator/algorithm/builtin/miceApply.py
index 1e3f387f71..070bf6c01a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/miceApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/miceApply.py
@@ -35,10 +35,12 @@ def miceApply(X: Matrix,
dM: Frame,
betaList: List):
"""
- This Builtin function implements multiple imputation using Chained Equations (MICE)
- Assumption missing value are represented with empty string i.e ",," in CSV file
- variables with suffix n are storing continuos/numeric data and variables with
- suffix c are storing categorical data
+ This Builtin function implements multiple imputation using Chained Equations (MICE)
+
+ Assumption missing value are represented with empty string i.e ",," in CSV file
+ variables with suffix n are storing continuos/numeric data and variables with
+ suffix c are storing categorical data
+
:param X: Data Matrix (Recoded Matrix for categorical features)
@@ -51,9 +53,9 @@ def miceApply(X: Matrix,
:param dM: meta frame from OHE on original data
:param betaList: List of machine learning models trained for each column imputation
:param verbose: Boolean value.
- :return: 'OperationNode' containing
- imputed dataset
+ :return: imputed dataset
"""
+
params_dict = {'X': X, 'meta': meta, 'threshold': threshold, 'dM': dM, 'betaList': betaList}
return Matrix(X.sds_context,
'miceApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/msvm.py b/src/main/python/systemds/operator/algorithm/builtin/msvm.py
index cb9dc4f27a..988d873518 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/msvm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/msvm.py
@@ -33,8 +33,9 @@ def msvm(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Implements builtin multi-class SVM with squared slack variables,
- learns one-against-the-rest binary-class classifiers by making a function call to l2SVM
+ Implements builtin multi-class SVM with squared slack variables,
+ learns one-against-the-rest binary-class classifiers by making a function call to l2SVM
+
:param X: matrix X of feature vectors
@@ -46,9 +47,9 @@ def msvm(X: Matrix,
:param reg: Regularization parameter (lambda) for L2 regularization
:param maxIterations: Maximum number of conjugate gradient iterations
:param verbose: Set to true to print while training.
- :return: 'OperationNode' containing
- model matrix
+ :return: model matrix
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py b/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py
index 7f8d39b9cf..930428043e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/msvmPredict.py
@@ -32,15 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def msvmPredict(X: Matrix,
W: Matrix):
"""
- This Scripts helps in applying an trained MSVM
+ This Scripts helps in applying an trained MSVM
+
:param X: matrix X of feature vectors to classify
:param W: matrix of the trained variables
- :return: 'OperationNode' containing
- classification labels raw, meaning not modified to clean
- labeles of 1's and -1'sclassification labels maxed to ones and zeros.
+ :return: Classification Labels Raw, meaning not modified to clean
+ Labeles of 1's and -1's
+ :return: Classification Labels Maxed to ones and zeros.
"""
+
params_dict = {'X': X, 'W': W}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
index 01466158bc..624ae8da1e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogReg.py
@@ -33,10 +33,11 @@ def multiLogReg(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Solves Multinomial Logistic Regression using Trust Region method.
- (See: Trust Region Newton Method for Logistic Regression, Lin, Weng and Keerthi, JMLR 9 (2008) 627-650)
- The largest label represents the baseline category; if label -1 or 0 is present, then it is
- the baseline label (and it is converted to the largest label).
+ Solves Multinomial Logistic Regression using Trust Region method.
+ (See: Trust Region Newton Method for Logistic Regression, Lin, Weng and Keerthi, JMLR 9 (2008) 627-650)
+ The largest label represents the baseline category; if label -1 or 0 is present, then it is
+ the baseline label (and it is converted to the largest label).
+
:param X: Location to read the matrix of feature vectors
@@ -49,9 +50,9 @@ def multiLogReg(X: Matrix,
:param maxi: max. number of outer (Newton) iterations
:param maxii: max. number of inner (conjugate gradient) iterations, 0 = no max
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- regression betas as output for prediction
+ :return: regression betas as output for prediction
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
index 12970d0177..3fcd41c25c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/multiLogRegPredict.py
@@ -34,17 +34,20 @@ def multiLogRegPredict(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- THIS SCRIPT APPLIES THE ESTIMATED PARAMETERS OF MULTINOMIAL LOGISTIC REGRESSION TO A NEW (TEST) DATASET
- Matrix M of predicted means/probabilities, some statistics in CSV format (see below)
+ THIS SCRIPT APPLIES THE ESTIMATED PARAMETERS OF MULTINOMIAL LOGISTIC REGRESSION TO A NEW (TEST) DATASET
+ Matrix M of predicted means/probabilities, some statistics in CSV format (see below)
+
:param X: Data Matrix X
:param B: Regression parameters betas
:param Y: Response vector Y
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- matrix m of predicted means/probabilitiespredicted response vectorscalar value of accuracy
+ :return: Matrix M of predicted means/probabilities
+ :return: Predicted response vector
+ :return: scalar value of accuracy
"""
+
params_dict = {'X': X, 'B': B, 'Y': Y}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/na_locf.py b/src/main/python/systemds/operator/algorithm/builtin/na_locf.py
index b098b1f4ce..5aebab6240 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/na_locf.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/na_locf.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def na_locf(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function for imputing missing values using forward fill and backward fill techniques
+ Builtin function for imputing missing values using forward fill and backward fill techniques
+
:param X: Matrix X
:param option: String "locf" (last observation moved forward) to do forward fill
"nocb" (next observation carried backward) to do backward fill
:param verbose: to print output on screen
- :return: 'OperationNode' containing
- matrix with no missing values
+ :return: Matrix with no missing values
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/naiveBayes.py b/src/main/python/systemds/operator/algorithm/builtin/naiveBayes.py
index 54e9283ff5..e51fb1dff3 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/naiveBayes.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/naiveBayes.py
@@ -33,16 +33,18 @@ def naiveBayes(D: Matrix,
C: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The naiveBayes-function computes the class conditional probabilities and class priors.
+ The naiveBayes-function computes the class conditional probabilities and class priors.
+
:param D: One dimensional column matrix with N rows.
:param C: One dimensional column matrix with N rows.
:param laplace: Any Double value.
:param verbose: Boolean value.
- :return: 'OperationNode' containing
- class priors, one dimensional column matrix with n rows.class conditional probabilities, one dimensional column matrix with n rows.
+ :return: Class priors, One dimensional column matrix with N rows.
+ :return: Class conditional probabilities, One dimensional column matrix with N rows.
"""
+
params_dict = {'D': D, 'C': C}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/naiveBayesPredict.py b/src/main/python/systemds/operator/algorithm/builtin/naiveBayesPredict.py
index e73462d3f1..5e0fe122b7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/naiveBayesPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/naiveBayesPredict.py
@@ -33,15 +33,17 @@ def naiveBayesPredict(X: Matrix,
P: Matrix,
C: Matrix):
"""
- The naiveBaysePredict-function predicts the scoring with a naive Bayes model.
+ The naiveBaysePredict-function predicts the scoring with a naive Bayes model.
+
:param X: Matrix of test data with N rows.
:param P: Class priors, One dimensional column matrix with N rows.
:param C: Class conditional probabilities, matrix with N rows
- :return: 'OperationNode' containing
- a matrix containing the top-k item-ids with highest predicted ratings.a matrix containing predicted ratings.
+ :return: A matrix containing the top-K item-ids with highest predicted ratings.
+ :return: A matrix containing predicted ratings.
"""
+
params_dict = {'X': X, 'P': P, 'C': C}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/normalize.py b/src/main/python/systemds/operator/algorithm/builtin/normalize.py
index 19c7f1d68f..d09b5b2f93 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/normalize.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/normalize.py
@@ -31,14 +31,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def normalize(X: Matrix):
"""
- Min-max normalization (a.k.a. min-max scaling) to range [0,1]. For matrices
- of positive values, this normalization preserves the input sparsity.
+ Min-max normalization (a.k.a. min-max scaling) to range [0,1]. For matrices
+ of positive values, this normalization preserves the input sparsity.
+
:param X: Input feature matrix of shape n-by-m
- :return: 'OperationNode' containing
- modified output feature matrix of shape n-by-mcolumn minima of shape 1-by-mcolumn maxima of shape 1-by-m
+ :return: Modified output feature matrix of shape n-by-m
+ :return: Column minima of shape 1-by-m
+ :return: Column maxima of shape 1-by-m
"""
+
params_dict = {'X': X}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/normalizeApply.py b/src/main/python/systemds/operator/algorithm/builtin/normalizeApply.py
index abcd6c54ed..063e457f25 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/normalizeApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/normalizeApply.py
@@ -33,18 +33,19 @@ def normalizeApply(X: Matrix,
cmin: Matrix,
cmax: Matrix):
"""
- Min-max normalization (a.k.a. min-max scaling) to range [0,1], given
- existing min-max ranges. For matrices of positive values, this normalization
- preserves the input sparsity. The validity of the provided min-max range
- and post-processing is under control of the caller.
+ Min-max normalization (a.k.a. min-max scaling) to range [0,1], given
+ existing min-max ranges. For matrices of positive values, this normalization
+ preserves the input sparsity. The validity of the provided min-max range
+ and post-processing is under control of the caller.
+
:param X: Input feature matrix of shape n-by-m
:param cmin: Colunm minima of shape 1-by-m
:param cmax: Column maxima of shape 1-by-m
- :return: 'OperationNode' containing
- modified output feature matrix of shape n-by-m
+ :return: Modified output feature matrix of shape n-by-m
"""
+
params_dict = {'X': X, 'cmin': cmin, 'cmax': cmax}
return Matrix(X.sds_context,
'normalizeApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlier.py b/src/main/python/systemds/operator/algorithm/builtin/outlier.py
index 8664b1310e..dd0d6ff7c8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlier.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlier.py
@@ -32,16 +32,17 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def outlier(X: Matrix,
opposite: bool):
"""
- This outlier-function takes a matrix data set as input from where it determines
- which point(s) have the largest difference from mean.
+ This outlier-function takes a matrix data set as input from where it determines
+ which point(s) have the largest difference from mean.
+
:param X: Matrix of Recoded dataset for outlier evaluation
:param opposite: (1)TRUE for evaluating outlier from upper quartile range,
(0)FALSE for evaluating outlier from lower quartile range
- :return: 'OperationNode' containing
- matrix indicating outlier values
+ :return: matrix indicating outlier values
"""
+
params_dict = {'X': X, 'opposite': opposite}
return Matrix(X.sds_context,
'outlier',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py b/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py
index e865f66932..5f5b048665 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlierByArima.py
@@ -32,8 +32,9 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def outlierByArima(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Built-in function for detecting and repairing outliers in time series, by training an ARIMA model
- and classifying values that are more than k standard-deviations away from the predicated values as outliers.
+ Built-in function for detecting and repairing outliers in time series, by training an ARIMA model
+ and classifying values that are more than k standard-deviations away from the predicated values as outliers.
+
:param X: Matrix X
@@ -49,9 +50,9 @@ def outlierByArima(X: Matrix,
:param s: period in terms of number of time-steps
:param include_mean: If the mean should be included
:param solver: solver, is either "cg" or "jacobi"
- :return: 'OperationNode' containing
- matrix x with no outliers
+ :return: Matrix X with no outliers
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py
index 5ac90a6755..2c0dd71aa6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQR.py
@@ -34,7 +34,8 @@ def outlierByIQR(X: Matrix,
max_iterations: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function for detecting and repairing outliers using standard deviation
+ Builtin function for detecting and repairing outliers using standard deviation
+
:param X: Matrix X
@@ -46,9 +47,9 @@ def outlierByIQR(X: Matrix,
:param max_iterations: values: 0 = arbitrary number of iteraition until all outliers are removed,
n = any constant defined by user
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- matrix x with no outliers
+ :return: Matrix X with no outliers
"""
+
params_dict = {'X': X, 'k': k, 'max_iterations': max_iterations}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQRApply.py b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQRApply.py
index 2f3a8f0881..9794d7ae0e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlierByIQRApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlierByIQRApply.py
@@ -36,7 +36,8 @@ def outlierByIQRApply(X: Matrix,
k: float,
repairMethod: int):
"""
- Builtin function for repairing outliers by IQR
+ Builtin function for repairing outliers by IQR
+
:param X: Matrix X
@@ -47,9 +48,9 @@ def outlierByIQRApply(X: Matrix,
:param repairMethod: values: 0 = delete rows having outliers,
1 = replace outliers with zeros
2 = replace outliers as missing values
- :return: 'OperationNode' containing
- matrix x with no outliers
+ :return: Matrix X with no outliers
"""
+
params_dict = {'X': X, 'Q1': Q1, 'Q3': Q3, 'IQR': IQR, 'k': k, 'repairMethod': repairMethod}
return Matrix(X.sds_context,
'outlierByIQRApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py b/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py
index 0030ecdc8a..b173eabf00 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlierBySd.py
@@ -33,7 +33,8 @@ def outlierBySd(X: Matrix,
max_iterations: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function for detecting and repairing outliers using standard deviation
+ Builtin function for detecting and repairing outliers using standard deviation
+
:param X: Matrix X
@@ -42,9 +43,9 @@ def outlierBySd(X: Matrix,
2 = replace outliers as missing values
:param max_iterations: values: 0 = arbitrary number of iteration until all outliers are removed,
n = any constant defined by user
- :return: 'OperationNode' containing
- matrix x with no outliers
+ :return: Matrix X with no outliers
"""
+
params_dict = {'X': X, 'max_iterations': max_iterations}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/outlierBySdApply.py b/src/main/python/systemds/operator/algorithm/builtin/outlierBySdApply.py
index d51f89130a..4343113d17 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/outlierBySdApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/outlierBySdApply.py
@@ -35,7 +35,8 @@ def outlierBySdApply(X: Matrix,
k: float,
repairMethod: int):
"""
- Builtin function for detecting and repairing outliers using standard deviation
+ Builtin function for detecting and repairing outliers using standard deviation
+
:param X: Matrix X
@@ -48,9 +49,9 @@ def outlierBySdApply(X: Matrix,
:param max_iterations: values: 0 = arbitrary number of iteraition until all outliers are removed,
n = any constant defined by user
:param verbose: flag specifying if logging information should be printed
- :return: 'OperationNode' containing
- matrix x with no outliers
+ :return: Matrix X with no outliers
"""
+
params_dict = {'X': X, 'colMean': colMean, 'colSD': colSD, 'k': k, 'repairMethod': repairMethod}
return Matrix(X.sds_context,
'outlierBySdApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/pca.py b/src/main/python/systemds/operator/algorithm/builtin/pca.py
index 56be35704c..403f9cfca1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/pca.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/pca.py
@@ -32,16 +32,20 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def pca(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The function Principal Component Analysis (PCA) is used for dimensionality reduction
+ The function Principal Component Analysis (PCA) is used for dimensionality reduction
+
:param X: Input feature matrix
:param K: Number of reduced dimensions (i.e., columns)
:param Center: Indicates whether or not to center the feature matrix
:param Scale: Indicates whether or not to scale the feature matrix
- :return: 'OperationNode' containing
- output feature matrix with k columnsoutput dominant eigen vectors (can be used for projections)the column means of the input, subtracted to construct the pcathe scaling of the values, to make each dimension same size.
+ :return: Output feature matrix with K columns
+ :return: Output dominant eigen vectors (can be used for projections)
+ :return: The column means of the input, subtracted to construct the PCA
+ :return: The Scaling of the values, to make each dimension same size.
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/pcaInverse.py b/src/main/python/systemds/operator/algorithm/builtin/pcaInverse.py
index f721c5ea66..f59891f6d2 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/pcaInverse.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/pcaInverse.py
@@ -34,18 +34,19 @@ def pcaInverse(Y: Matrix,
Centering: Matrix,
ScaleFactor: Matrix):
"""
- Principal Component Analysis (PCA) for reconstruction of approximation of the original data.
- This methods allows to reconstruct an approximation of the original matrix, and is useful for
- calculating how much information is lost in the PCA.
+ Principal Component Analysis (PCA) for reconstruction of approximation of the original data.
+ This methods allows to reconstruct an approximation of the original matrix, and is useful for
+ calculating how much information is lost in the PCA.
+
:param Y: Input features that have PCA applied to them
:param Clusters: The previous PCA components computed
:param Centering: The column means of the PCA model, subtracted to construct the PCA
:param ScaleFactor: The scaling of each dimension in the PCA model
- :return: 'OperationNode' containing
- output feature matrix reconstructing and approximation of the original matrix
+ :return: Output feature matrix reconstructing and approximation of the original matrix
"""
+
params_dict = {'Y': Y, 'Clusters': Clusters, 'Centering': Centering, 'ScaleFactor': ScaleFactor}
return Matrix(Y.sds_context,
'pcaInverse',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/pcaTransform.py b/src/main/python/systemds/operator/algorithm/builtin/pcaTransform.py
index de4b307f7a..9b815f8f21 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/pcaTransform.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/pcaTransform.py
@@ -34,18 +34,19 @@ def pcaTransform(X: Matrix,
Centering: Matrix,
ScaleFactor: Matrix):
"""
- Principal Component Analysis (PCA) for dimensionality reduction prediction
- This method is used to transpose data, which the PCA model was not trained on. To validate how good
- The PCA is, and to apply in production.
+ Principal Component Analysis (PCA) for dimensionality reduction prediction
+ This method is used to transpose data, which the PCA model was not trained on. To validate how good
+ The PCA is, and to apply in production.
+
:param X: Input feature matrix
:param Clusters: The previously computed principal components
:param Centering: The column means of the PCA model, subtracted to construct the PCA
:param ScaleFactor: The scaling of each dimension in the PCA model
- :return: 'OperationNode' containing
- output feature matrix dimensionally reduced by pca
+ :return: Output feature matrix dimensionally reduced by PCA
"""
+
params_dict = {'X': X, 'Clusters': Clusters, 'Centering': Centering, 'ScaleFactor': ScaleFactor}
return Matrix(X.sds_context,
'pcaTransform',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/pnmf.py b/src/main/python/systemds/operator/algorithm/builtin/pnmf.py
index 014be52627..e0011af45c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/pnmf.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/pnmf.py
@@ -33,12 +33,14 @@ def pnmf(X: Matrix,
rnk: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The pnmf-function implements Poisson Non-negative Matrix Factorization (PNMF). Matrix X is factorized into two
- non-negative matrices, W and H based on Poisson probabilistic assumption. This non-negativity makes the resulting
- matrices easier to inspect.
- [Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
- Distributed nonnegative matrix factorization for web-scale dyadic
- data analysis on mapreduce. WWW 2010: 681-690]
+ The pnmf-function implements Poisson Non-negative Matrix Factorization (PNMF). Matrix X is factorized into two
+ non-negative matrices, W and H based on Poisson probabilistic assumption. This non-negativity makes the resulting
+ matrices easier to inspect.
+
+ [Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
+ Distributed nonnegative matrix factorization for web-scale dyadic
+ data analysis on mapreduce. WWW 2010: 681-690]
+
:param X: Matrix of feature vectors.
@@ -46,9 +48,10 @@ def pnmf(X: Matrix,
:param eps: Tolerance
:param maxi: Maximum number of conjugate gradient iterations.
:param verbose: If TRUE, 'iter' and 'obj' are printed.
- :return: 'OperationNode' containing
- list of pattern matrices, one for each repetition.list of amplitude matrices, one for each repetition.
+ :return: List of pattern matrices, one for each repetition.
+ :return: List of amplitude matrices, one for each repetition.
"""
+
params_dict = {'X': X, 'rnk': rnk}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ppca.py b/src/main/python/systemds/operator/algorithm/builtin/ppca.py
index 97721f957b..07bec55b19 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ppca.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ppca.py
@@ -32,9 +32,10 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def ppca(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script performs Probabilistic Principal Component Analysis (PCA) on the given input data.
- It is based on paper: sPCA: Scalable Principal Component Analysis for Big Data on Distributed
- Platforms. Tarek Elgamal et.al.
+ This script performs Probabilistic Principal Component Analysis (PCA) on the given input data.
+ It is based on paper: sPCA: Scalable Principal Component Analysis for Big Data on Distributed
+ Platforms. Tarek Elgamal et.al.
+
:param X: n x m input feature matrix
@@ -43,9 +44,10 @@ def ppca(X: Matrix,
:param tolobj: objective function tolerance value to stop ppca algorithm
:param tolrecerr: reconstruction error tolerance value to stop the algorithm
:param verbose: verbose debug output
- :return: 'OperationNode' containing
- output feature matrix with k columnsoutput dominant eigen vectors (can be used for projections)
+ :return: Output feature matrix with K columns
+ :return: Output dominant eigen vectors (can be used for projections)
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
index 061565acb5..b2b4424ff6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
@@ -34,7 +34,8 @@ def randomForest(X: Matrix,
R: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script implement classification random forest with both scale and categorical features.
+ This script implement classification random forest with both scale and categorical features.
+
:param X: Feature matrix X; note that X needs to be both recoded and dummy coded
@@ -55,23 +56,26 @@ def randomForest(X: Matrix,
as a power of number of features in the dataset;
by default square root of features (i.e., feature_subset = 0.5) are used at each tree node
:param impurity: Impurity measure: entropy or Gini (the default)
- :return: 'OperationNode' containing
- matrix m containing the learned tree, where each column corresponds to a node
+ :return: Matrix M containing the learned tree, where each column corresponds to a node
in the learned tree and each row contains the following information:
- m[1,j]: id of node j (in a complete binary tree)
- m[2,j]: tree id to which node j belongs
- m[3,j]: offset (no. of columns) to left child of j
- m[4,j]: feature index of the feature that node j looks at if j is an internal node, otherwise 0
- m[5,j]: type of the feature that node j looks at if j is an internal node: 1 for scale and 2
+ M[1,j]: id of node j (in a complete binary tree)
+ M[2,j]: tree id to which node j belongs
+ M[3,j]: Offset (no. of columns) to left child of j
+ M[4,j]: Feature index of the feature that node j looks at if j is an internal node, otherwise 0
+ M[5,j]: Type of the feature that node j looks at if j is an internal node: 1 for scale and 2
for categorical features,
otherwise the label that leaf node j is supposed to predict
- m[6,j]: 1 if j is an internal node and the feature chosen for j is scale, otherwise the
+ M[6,j]: 1 if j is an internal node and the feature chosen for j is scale, otherwise the
size of the subset of values
stored in rows 7,8,... if j is categorical
- m[7:,j]: only applicable for internal nodes. threshold the example's feature value is
- compared to is stored at m[7,j] if the feature chosen for j is scale;
- if the feature chosen for j is categorical rows 7,8,... depict the value subset chosen for jmatrix c containing the number of times samples are chosen in each tree of the random forestmappings from scale feature ids to global feature idsmappings from categorical feature ids to global feature ids
+ M[7:,j]: Only applicable for internal nodes. Threshold the example's feature value is
+ compared to is stored at M[7,j] if the feature chosen for j is scale;
+ If the feature chosen for j is categorical rows 7,8,... depict the value subset chosen for j
+ :return: Matrix C containing the number of times samples are chosen in each tree of the random forest
+ :return: Mappings from scale feature ids to global feature ids
+ :return: Mappings from categorical feature ids to global feature ids
"""
+
params_dict = {'X': X, 'Y': Y, 'R': R}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/scale.py b/src/main/python/systemds/operator/algorithm/builtin/scale.py
index 11ebdc722a..015709d8c6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/scale.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/scale.py
@@ -32,15 +32,18 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def scale(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function scales and center individual features in the input matrix (column wise.) using z-score to scale the values.
+ This function scales and center individual features in the input matrix (column wise.) using z-score to scale the values.
+
:param X: Input feature matrix
:param center: Indicates whether or not to center the feature matrix
:param scale: Indicates whether or not to scale the feature matrix
- :return: 'OperationNode' containing
- output feature matrix with k columnsthe column means of the input, subtracted if center was truethe scaling of the values, to make each dimension have similar value ranges
+ :return: Output feature matrix with K columns
+ :return: The column means of the input, subtracted if Center was TRUE
+ :return: The Scaling of the values, to make each dimension have similar value ranges
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py b/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py
index b2f86aab83..88520460e5 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/scaleApply.py
@@ -33,15 +33,16 @@ def scaleApply(X: Matrix,
Centering: Matrix,
ScaleFactor: Matrix):
"""
- This function scales and center individual features in the input matrix (column wise.) using the input matrices.
+ This function scales and center individual features in the input matrix (column wise.) using the input matrices.
+
:param X: Input feature matrix
:param Centering: The column means to subtract from X (not done if empty)
:param ScaleFactor: The column scaling to multiply with X (not done if empty)
- :return: 'OperationNode' containing
- output feature matrix with k columns
+ :return: Output feature matrix with K columns
"""
+
params_dict = {'X': X, 'Centering': Centering, 'ScaleFactor': ScaleFactor}
return Matrix(X.sds_context,
'scaleApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/scaleMinMax.py b/src/main/python/systemds/operator/algorithm/builtin/scaleMinMax.py
index 05a0fccc68..b2a59859f6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/scaleMinMax.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/scaleMinMax.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def scaleMinMax(X: Matrix):
"""
- This function performs min-max normalization (rescaling to [0,1]).
+ This function performs min-max normalization (rescaling to [0,1]).
+
:param X: Input feature matrix
- :return: 'OperationNode' containing
- scaled output matrix
+ :return: Scaled output matrix
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'scaleMinMax',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/selectByVarThresh.py b/src/main/python/systemds/operator/algorithm/builtin/selectByVarThresh.py
index 4a00e3f3c2..12a50b2a28 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/selectByVarThresh.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/selectByVarThresh.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def selectByVarThresh(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function drops feature with <= thresh variance (by default drop constants).
+ This function drops feature with <= thresh variance (by default drop constants).
+
:param X: Matrix of feature vectors.
:param thresh: The threshold for to drop
- :return: 'OperationNode' containing
- matrix of feature vectors with <= thresh variance.
+ :return: Matrix of feature vectors with <= thresh variance.
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/setdiff.py b/src/main/python/systemds/operator/algorithm/builtin/setdiff.py
index 7f1b32756a..d1a28f2ac0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/setdiff.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/setdiff.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def setdiff(X: Matrix,
Y: Matrix):
"""
- Builtin function that implements difference operation on vectors
+ Builtin function that implements difference operation on vectors
+
:param X: input vector
:param Y: input vector
- :return: 'OperationNode' containing
- vector with all elements that are present in x but not in y
+ :return: vector with all elements that are present in X but not in Y
"""
+
params_dict = {'X': X, 'Y': Y}
return Matrix(X.sds_context,
'setdiff',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/sherlock.py b/src/main/python/systemds/operator/algorithm/builtin/sherlock.py
index 6e510f0c78..7dce268807 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/sherlock.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/sherlock.py
@@ -32,19 +32,31 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def sherlock(X_train: Matrix,
y_train: Matrix):
"""
- This function implements training phase of Sherlock: A Deep Learning Approach to Semantic Data Type Detection
- [Hulsebos, Madelon, et al. "Sherlock: A deep learning approach to semantic data type detection."
- Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
- 2019.]
- Split feature matrix into four different feature categories and train neural networks on the
- respective single features. Then combine all trained features to train final neural network.
+ This function implements training phase of Sherlock: A Deep Learning Approach to Semantic Data Type Detection
+
+ [Hulsebos, Madelon, et al. "Sherlock: A deep learning approach to semantic data type detection."
+ Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
+ 2019.]
+
+ Split feature matrix into four different feature categories and train neural networks on the
+ respective single features. Then combine all trained features to train final neural network.
+
:param X_train: matrix of feature vectors
:param y_train: matrix Y of class labels of semantic data type
- :return: 'OperationNode' containing
- weights (parameters) matrices for character distributionsbiases vectors for character distributionsweights (parameters) matrices for word embeddingsbiases vectors for word embeddingsweights (parameters) matrices for paragraph vectorsbiases vectors for paragraph vectorsweights (parameters) matrices for global statisticsbiases vectors for global statisticsweights (parameters) matrices for combining all trained features (final)biases vectors for combining all trained features (final)
+ :return: weights (parameters) matrices for character distributions
+ :return: biases vectors for character distributions
+ :return: weights (parameters) matrices for word embeddings
+ :return: biases vectors for word embeddings
+ :return: weights (parameters) matrices for paragraph vectors
+ :return: biases vectors for paragraph vectors
+ :return: weights (parameters) matrices for global statistics
+ :return: biases vectors for global statistics
+ :return: weights (parameters) matrices for combining all trained features (final)
+ :return: biases vectors for combining all trained features (final)
"""
+
params_dict = {'X_train': X_train, 'y_train': y_train}
vX_0 = Matrix(X_train.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py b/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py
index 2556f5d5e9..c187872b9e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/sherlockPredict.py
@@ -61,13 +61,14 @@ def sherlockPredict(X: Matrix,
fW3: Matrix,
fb3: Matrix):
"""
- This function implements prediction and evaluation phase of Sherlock:
- Split feature matrix into four different feature categories and predicting the class probability
- on the respective features. Then combine all predictions for final predicted probabilities.
- A Deep Learning Approach to Semantic Data Type Detection.
- [Hulsebos, Madelon, et al. "Sherlock: A deep learning approach to semantic data type detection."
- Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
- 2019.]
+ This function implements prediction and evaluation phase of Sherlock:
+ Split feature matrix into four different feature categories and predicting the class probability
+ on the respective features. Then combine all predictions for final predicted probabilities.
+ A Deep Learning Approach to Semantic Data Type Detection.
+ [Hulsebos, Madelon, et al. "Sherlock: A deep learning approach to semantic data type detection."
+ Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
+ 2019.]
+
:param X: matrix of values which are to be classified
@@ -81,9 +82,9 @@ def sherlockPredict(X: Matrix,
:param sb: biases vectors for global statistics
:param fW: weights (parameters) matrices for combining all trained features (final)
:param fb: biases vectors for combining all trained features (final)
- :return: 'OperationNode' containing
- class probabilities of shape (n, k)
+ :return: class probabilities of shape (N, K)
"""
+
params_dict = {'X': X, 'cW1': cW1, 'cb1': cb1, 'cW2': cW2, 'cb2': cb2, 'cW3': cW3, 'cb3': cb3, 'wW1': wW1, 'wb1': wb1, 'wW2': wW2, 'wb2': wb2, 'wW3': wW3, 'wb3': wb3, 'pW1': pW1, 'pb1': pb1, 'pW2': pW2, 'pb2': pb2, 'pW3': pW3, 'pb3': pb3, 'sW1': sW1, 'sb1': sb1, 'sW2': sW2, 'sb2': sb2, 'sW3': sW3, 'sb3': sb3, 'fW1': fW1, 'fb1': fb1, 'fW2': fW2, 'fb2': fb2, 'fW3': fW3, 'fb3': fb3}
return Matrix(X.sds_context,
'sherlockPredict',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/shortestPath.py b/src/main/python/systemds/operator/algorithm/builtin/shortestPath.py
index 737553cb32..521fdbc462 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/shortestPath.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/shortestPath.py
@@ -33,11 +33,12 @@ def shortestPath(G: Matrix,
sourceNode: int,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Computes the minimum distances (shortest-path) between a single source vertex and every other vertex in the graph.
+ Computes the minimum distances (shortest-path) between a single source vertex and every other vertex in the graph.
+
+ Grzegorz Malewicz, Matthew H. Austern, Aart J. C. Bilk,
+ James C. Dehnert, Ikkan Horn, Naty Leiser and Grzegorz Czajkowski:
+ Pregel: A System for Large-Scale Graph Processing, SIGMOD 2010
- Grzegorz Malewicz, Matthew H. Austern, Aart J. C. Bilk,
- James C. Dehnert, Ikkan Horn, Naty Leiser and Grzegorz Czajkowski:
- Pregel: A System for Large-Scale Graph Processing, SIGMOD 2010
:param G: adjacency matrix of the labeled graph: Such graph can be directed
@@ -49,13 +50,13 @@ def shortestPath(G: Matrix,
max number of iterations not defined)
:param sourceNode: node index to calculate the shortest paths to all other nodes.
:param verbose: flag for verbose debug output
- :return: 'OperationNode' containing
- output matrix (double) of minimum distances (shortest-path) between
- vertices: the value of the ith row and the jth column of the output
+ :return: Output matrix (double) of minimum distances (shortest-path) between
+ vertices: The value of the ith row and the jth column of the output
matrix is the minimum distance shortest-path from vertex i to vertex j.
- when the value of the minimum distance is infinity, the two nodes are
- not connected.
+ When the value of the minimum distance is infinity, the two nodes are
+ not connected.
"""
+
params_dict = {'G': G, 'sourceNode': sourceNode}
params_dict.update(kwargs)
return Matrix(G.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py b/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py
index 127d0f6fb1..1c6dac5539 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/sigmoid.py
@@ -31,14 +31,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def sigmoid(X: Matrix):
"""
- The Sigmoid function is a type of activation function, and also defined as a squashing function which limit the
- output to a range between 0 and 1, which will make these functions useful in the prediction of probabilities.
+ The Sigmoid function is a type of activation function, and also defined as a squashing function which limit the
+ output to a range between 0 and 1, which will make these functions useful in the prediction of probabilities.
+
:param X: Matrix of feature vectors.
- :return: 'OperationNode' containing
- 1-column matrix of weights.
+ :return: 1-column matrix of weights.
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'sigmoid',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py b/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py
index efe8dc80f6..a8c34cc0b9 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/slicefinder.py
@@ -33,11 +33,12 @@ def slicefinder(X: Matrix,
e: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This builtin function implements SliceLine, a linear-algebra-based
- ML model debugging technique for finding the top-k data slices where
- a trained models performs significantly worse than on the overall
- dataset. For a detailed description and experimental results, see:
- Svetlana Sagadeeva, Matthias Boehm: SliceLine: Fast, Linear-Algebra-based Slice Finding for ML Model Debugging.(SIGMOD 2021)
+ This builtin function implements SliceLine, a linear-algebra-based
+ ML model debugging technique for finding the top-k data slices where
+ a trained models performs significantly worse than on the overall
+ dataset. For a detailed description and experimental results, see:
+ Svetlana Sagadeeva, Matthias Boehm: SliceLine: Fast, Linear-Algebra-based Slice Finding for ML Model Debugging.(SIGMOD 2021)
+
:param X: Recoded dataset into Matrix
@@ -52,9 +53,11 @@ def slicefinder(X: Matrix,
:param selFeat: flag for removing one-hot-encoded features that don't satisfy
the initial minimum-support constraint and/or have zero error
:param verbose: flag for verbose debug output
- :return: 'OperationNode' containing
- top-k slices (k x ncol(x) if successful)score, size, error of slices (k x 3)debug matrix, populated with enumeration stats if verbose
+ :return: top-k slices (k x ncol(X) if successful)
+ :return: score, size, error of slices (k x 3)
+ :return: debug matrix, populated with enumeration stats if verbose
"""
+
params_dict = {'X': X, 'e': e}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/smote.py b/src/main/python/systemds/operator/algorithm/builtin/smote.py
index 70b74985fc..6389ddfb5c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/smote.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/smote.py
@@ -33,8 +33,9 @@ def smote(X: Matrix,
mask: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- Builtin function for handing class imbalance using Synthetic Minority Over-sampling Technique (SMOTE)
- by Nitesh V. Chawla et. al. In Journal of Artificial Intelligence Research 16 (2002). 321–357
+ Builtin function for handing class imbalance using Synthetic Minority Over-sampling Technique (SMOTE)
+ by Nitesh V. Chawla et. al. In Journal of Artificial Intelligence Research 16 (2002). 321–357
+
:param X: Matrix of minority class samples
@@ -42,9 +43,9 @@ def smote(X: Matrix,
:param s: Amount of SMOTE (percentage of oversampling), integral multiple of 100
:param k: Number of nearest neighbor
:param verbose: if the algorithm should be verbose
- :return: 'OperationNode' containing
- matrix of (n/100)-1 * nrow(x) synthetic minority class samples
+ :return: Matrix of (N/100)-1 * nrow(X) synthetic minority class samples
"""
+
params_dict = {'X': X, 'mask': mask}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/softmax.py b/src/main/python/systemds/operator/algorithm/builtin/softmax.py
index 88d56e3745..63fff167c0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/softmax.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/softmax.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def softmax(S: Matrix):
"""
- Performs softmax on the given input matrix.
+ Performs softmax on the given input matrix.
+
:param S: Inputs of shape (N, D).
- :return: 'OperationNode' containing
- outputs of shape (n, d).
+ :return: Outputs of shape (N, D).
"""
+
params_dict = {'S': S}
return Matrix(S.sds_context,
'softmax',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/split.py b/src/main/python/systemds/operator/algorithm/builtin/split.py
index 0882baf382..311eb9d1a7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/split.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/split.py
@@ -33,7 +33,8 @@ def split(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function split input data X and Y into contiguous or samples train/test sets
+ This function split input data X and Y into contiguous or samples train/test sets
+
:param X: Input feature matrix
@@ -41,9 +42,12 @@ def split(X: Matrix,
:param f: Train set fraction [0,1]
:param cont: contiguous splits, otherwise sampled
:param seed: The seed to randomly select rows in sampled mode
- :return: 'OperationNode' containing
- train split of feature matrixtest split of feature matrixtrain split of label matrixtest split of label matrix
+ :return: Train split of feature matrix
+ :return: Test split of feature matrix
+ :return: Train split of label matrix
+ :return: Test split of label matrix
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py b/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py
index 47ed8085cd..e678e343d4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/splitBalanced.py
@@ -33,17 +33,21 @@ def splitBalanced(X: Matrix,
Y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This functions split input data X and Y into contiguous balanced ratio
- Related to [SYSTEMDS-2902] dependency function for cleaning pipelines
+ This functions split input data X and Y into contiguous balanced ratio
+ Related to [SYSTEMDS-2902] dependency function for cleaning pipelines
+
:param X: Input feature matrix
:param Y: Input Labels
:param f: Train set fraction [0,1]
:param verbose: print available
- :return: 'OperationNode' containing
- train split of feature matrixtest split of feature matrixtrain split of label matrixtest split of label matrix
+ :return: Train split of feature matrix
+ :return: Test split of feature matrix
+ :return: Train split of label matrix
+ :return: Test split of label matrix
"""
+
params_dict = {'X': X, 'Y': Y}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/stableMarriage.py b/src/main/python/systemds/operator/algorithm/builtin/stableMarriage.py
index 4aeb055663..94024a4f7f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/stableMarriage.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/stableMarriage.py
@@ -33,45 +33,48 @@ def stableMarriage(P: Matrix,
A: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This script computes a solution for the stable marriage problem.
- result description:
- If cell [i,j] is non-zero, it means that acceptor i has matched with
- proposer j. Further, if cell [i,j] is non-zero, it holds the preference
- value that led to the match.
- Proposers.mtx:
- 2.0,1.0,3.0
- 1.0,2.0,3.0
- 1.0,3.0,2.0
+ This script computes a solution for the stable marriage problem.
- Since ordered=TRUE, this means that proposer 1 (row 1) likes acceptor 2
- the most, followed by acceptor 1 and acceptor 3.
- If ordered=FALSE, this would mean that proposer 1 (row 1) likes acceptor 3
- the most (since the value at [1,3] is the row max),
- followed by acceptor 1 (2.0 preference value) and acceptor 2 (1.0 preference value).
+ result description:
- Acceptors.mtx:
- 3.0,1.0,2.0
- 2.0,1.0,3.0
- 3.0,2.0,1.0
+ If cell [i,j] is non-zero, it means that acceptor i has matched with
+ proposer j. Further, if cell [i,j] is non-zero, it holds the preference
+ value that led to the match.
+ Proposers.mtx:
+ 2.0,1.0,3.0
+ 1.0,2.0,3.0
+ 1.0,3.0,2.0
+
+ Since ordered=TRUE, this means that proposer 1 (row 1) likes acceptor 2
+ the most, followed by acceptor 1 and acceptor 3.
+ If ordered=FALSE, this would mean that proposer 1 (row 1) likes acceptor 3
+ the most (since the value at [1,3] is the row max),
+ followed by acceptor 1 (2.0 preference value) and acceptor 2 (1.0 preference value).
+
+ Acceptors.mtx:
+ 3.0,1.0,2.0
+ 2.0,1.0,3.0
+ 3.0,2.0,1.0
+
+ Since ordered=TRUE, this means that acceptor 1 (row 1) likes proposer 3
+ the most, followed by proposer 1 and proposer 2.
+ If ordered=FALSE, this would mean that acceptor 1 (row 1) likes proposer 1
+ the most (since the value at [1,1] is the row max),
+ followed by proposer 3 (2.0 preference value) and proposer 2
+ (1.0 preference value).
+
+ Output.mtx (assuming ordered=TRUE):
+ 0.0,0.0,3.0
+ 0.0,3.0,0.0
+ 1.0,0.0,0.0
+
+ Acceptor 1 has matched with proposer 3 (since [1,3] is non-zero) at a
+ preference level of 3.0.
+ Acceptor 2 has matched with proposer 2 (since [2,2] is non-zero) at a
+ preference level of 3.0.
+ Acceptor 3 has matched with proposer 1 (since [3,1] is non-zero) at a
+ preference level of 1.0.
- Since ordered=TRUE, this means that acceptor 1 (row 1) likes proposer 3
- the most, followed by proposer 1 and proposer 2.
- If ordered=FALSE, this would mean that acceptor 1 (row 1) likes proposer 1
- the most (since the value at [1,1] is the row max),
- followed by proposer 3 (2.0 preference value) and proposer 2
- (1.0 preference value).
-
- Output.mtx (assuming ordered=TRUE):
- 0.0,0.0,3.0
- 0.0,3.0,0.0
- 1.0,0.0,0.0
-
- Acceptor 1 has matched with proposer 3 (since [1,3] is non-zero) at a
- preference level of 3.0.
- Acceptor 2 has matched with proposer 2 (since [2,2] is non-zero) at a
- preference level of 3.0.
- Acceptor 3 has matched with proposer 1 (since [3,1] is non-zero) at a
- preference level of 1.0.
:param P: proposer matrix P.
@@ -83,9 +86,9 @@ def stableMarriage(P: Matrix,
i.e. the leftmost value in a row in P is the preference value for the acceptor with
index 1 and vice-versa (higher is better).
:param verbose: if the algorithm should print verbosely
- :return: 'OperationNode' containing
- result matrix
+ :return: Result Matrix
"""
+
params_dict = {'P': P, 'A': A}
params_dict.update(kwargs)
return Matrix(P.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/statsNA.py b/src/main/python/systemds/operator/algorithm/builtin/statsNA.py
index 57d602f986..0f0800e2c7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/statsNA.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/statsNA.py
@@ -32,16 +32,19 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def statsNA(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The statsNA-function Print summary stats about the distribution of missing values in a univariate time series.
- result matrix contains the following:
- 1. Length of time series (including NAs)
- 2. Number of Missing Values (NAs)
- 3. Percentage of Missing Values (#2/#1)
- 4. Number of Gaps (consisting of one or more consecutive NAs)
- 5. Average Gap Size - Average size of consecutive NAs for the NA gaps
- 6. Longest NA gap - Longest series of consecutive missing values
- 7. Most frequent gap size - Most frequently occurring gap size
- 8. Gap size accounting for most NAs
+ The statsNA-function Print summary stats about the distribution of missing values in a univariate time series.
+
+ result matrix contains the following:
+ 1. Length of time series (including NAs)
+ 2. Number of Missing Values (NAs)
+ 3. Percentage of Missing Values (#2/#1)
+ 4. Number of Gaps (consisting of one or more consecutive NAs)
+ 5. Average Gap Size - Average size of consecutive NAs for the NA gaps
+ 6. Longest NA gap - Longest series of consecutive missing values
+ 7. Most frequent gap size - Most frequently occurring gap size
+ 8. Gap size accounting for most NAs
+
+
:param X: Numeric Vector ('vector') object containing NAs
@@ -51,9 +54,9 @@ def statsNA(X: Matrix,
:param verbose: Print detailed information.
For print_only = TRUE, the missing value stats are printed with
more information ("Stats for Bins" and "overview NA series").
- :return: 'OperationNode' containing
- column vector where each row correspond to described values
+ :return: Column vector where each row correspond to described values
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/steplm.py b/src/main/python/systemds/operator/algorithm/builtin/steplm.py
index f7f14edbaf..6f8bca7594 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/steplm.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/steplm.py
@@ -33,19 +33,23 @@ def steplm(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The steplm-function (stepwise linear regression) implements a classical forward feature selection method.
- This method iteratively runs what-if scenarios and greedily selects the next best feature
- until the Akaike information criterion (AIC) does not improve anymore. Each configuration trains a regression model
- via lm, which in turn calls either the closed form lmDS or iterative lmGC.
+ The steplm-function (stepwise linear regression) implements a classical forward feature selection method.
+ This method iteratively runs what-if scenarios and greedily selects the next best feature
+ until the Akaike information criterion (AIC) does not improve anymore. Each configuration trains a regression model
+ via lm, which in turn calls either the closed form lmDS or iterative lmGC.
+
+ .. code-block:: txt
+
+ return: Matrix of regression parameters (the betas) and its size depend on icpt input value:
+ OUTPUT SIZE: OUTPUT CONTENTS: HOW TO PREDICT Y FROM X AND B:
+ icpt=0: ncol(X) x 1 Betas for X only Y ~ X %*% B[1:ncol(X), 1], or just X %*% B
+ icpt=1: ncol(X)+1 x 1 Betas for X and intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
+ icpt=2: ncol(X)+1 x 2 Col.1: betas for X & intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
+ Col.2: betas for shifted/rescaled X and intercept
+
+ In addition, in the last run of linear regression some statistics are provided in CSV format, one comma-separated
+ name-value pair per each line, as follows:
- return: Matrix of regression parameters (the betas) and its size depend on icpt input value:
- OUTPUT SIZE: OUTPUT CONTENTS: HOW TO PREDICT Y FROM X AND B:
- icpt=0: ncol(X) x 1 Betas for X only Y ~ X %*% B[1:ncol(X), 1], or just X %*% B
- icpt=1: ncol(X)+1 x 1 Betas for X and intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
- icpt=2: ncol(X)+1 x 2 Col.1: betas for X & intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
- Col.2: betas for shifted/rescaled X and intercept
- In addition, in the last run of linear regression some statistics are provided in CSV format, one comma-separated
- name-value pair per each line, as follows:
:param X: Location (on HDFS) to read the matrix X of feature vectors
@@ -58,9 +62,10 @@ def steplm(X: Matrix,
:param tol: Tolerance threshold to train until achieved
:param maxi: maximum iterations 0 means until tolerance is reached
:param verbose: If the algorithm should be verbose
- :return: 'OperationNode' containing
- matrix of regression parameters (the betas) and its size depend on icpt input value.matrix of selected features ordered as computed by the algorithm.
+ :return: Matrix of regression parameters (the betas) and its size depend on icpt input value.
+ :return: Matrix of selected features ordered as computed by the algorithm.
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/stratstats.py b/src/main/python/systemds/operator/algorithm/builtin/stratstats.py
index 4128936ef2..ee3d1b2cc6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/stratstats.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/stratstats.py
@@ -32,48 +32,51 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def stratstats(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The stratstats.dml script computes common bivariate statistics, such as correlation, slope, and their p-value,
- in parallel for many pairs of input variables in the presence of a confounding categorical variable.
- Output contains:
- (1st covariante, 2nd covariante)
- 40 columns containing the following information:
- Col 01: 1st covariate X-column number
- Col 02: 1st covariate global presence count
- Col 03: 1st covariate global mean
- Col 04: 1st covariate global standard deviation
- Col 05: 1st covariate stratified standard deviation
- Col 06: R-squared, 1st covariate vs. strata
- Col 07: adjusted R-squared, 1st covariate vs. strata
- Col 08: P-value, 1st covariate vs. strata
- Col 09-10: Reserved
- Col 11: 2nd covariate Y-column number
- Col 12: 2nd covariate global presence count
- Col 13: 2nd covariate global mean
- Col 14: 2nd covariate global standard deviation
- Col 15: 2nd covariate stratified standard deviation
- Col 16: R-squared, 2nd covariate vs. strata
- Col 17: adjusted R-squared, 2nd covariate vs. strata
- Col 18: P-value, 2nd covariate vs. strata
- Col 19-20: Reserved
- Col 21: Global 1st & 2nd covariate presence count
- Col 22: Global regression slope (2nd vs. 1st covariate)
- Col 23: Global regression slope standard deviation
- Col 24: Global correlation = +/- sqrt(R-squared)
- Col 25: Global residual standard deviation
- Col 26: Global R-squared
- Col 27: Global adjusted R-squared
- Col 28: Global P-value for hypothesis "slope = 0"
- Col 29-30: Reserved
- Col 31: Stratified 1st & 2nd covariate presence count
- Col 32: Stratified regression slope (2nd vs. 1st covariate)
- Col 33: Stratified regression slope standard deviation
- Col 34: Stratified correlation = +/- sqrt(R-squared)
- Col 35: Stratified residual standard deviation
- Col 36: Stratified R-squared
- Col 37: Stratified adjusted R-squared
- Col 38: Stratified P-value for hypothesis "slope = 0"
- Col 39: Number of strata with at least two counted points
- Col 40: Reserved
+ The stratstats.dml script computes common bivariate statistics, such as correlation, slope, and their p-value,
+ in parallel for many pairs of input variables in the presence of a confounding categorical variable.
+
+ Output contains:
+ (1st covariante, 2nd covariante)
+ 40 columns containing the following information:
+ Col 01: 1st covariate X-column number
+ Col 02: 1st covariate global presence count
+ Col 03: 1st covariate global mean
+ Col 04: 1st covariate global standard deviation
+ Col 05: 1st covariate stratified standard deviation
+ Col 06: R-squared, 1st covariate vs. strata
+ Col 07: adjusted R-squared, 1st covariate vs. strata
+ Col 08: P-value, 1st covariate vs. strata
+ Col 09-10: Reserved
+ Col 11: 2nd covariate Y-column number
+ Col 12: 2nd covariate global presence count
+ Col 13: 2nd covariate global mean
+ Col 14: 2nd covariate global standard deviation
+ Col 15: 2nd covariate stratified standard deviation
+ Col 16: R-squared, 2nd covariate vs. strata
+ Col 17: adjusted R-squared, 2nd covariate vs. strata
+ Col 18: P-value, 2nd covariate vs. strata
+ Col 19-20: Reserved
+ Col 21: Global 1st & 2nd covariate presence count
+ Col 22: Global regression slope (2nd vs. 1st covariate)
+ Col 23: Global regression slope standard deviation
+ Col 24: Global correlation = +/- sqrt(R-squared)
+ Col 25: Global residual standard deviation
+ Col 26: Global R-squared
+ Col 27: Global adjusted R-squared
+ Col 28: Global P-value for hypothesis "slope = 0"
+ Col 29-30: Reserved
+ Col 31: Stratified 1st & 2nd covariate presence count
+ Col 32: Stratified regression slope (2nd vs. 1st covariate)
+ Col 33: Stratified regression slope standard deviation
+ Col 34: Stratified correlation = +/- sqrt(R-squared)
+ Col 35: Stratified residual standard deviation
+ Col 36: Stratified R-squared
+ Col 37: Stratified adjusted R-squared
+ Col 38: Stratified P-value for hypothesis "slope = 0"
+ Col 39: Number of strata with at least two counted points
+ Col 40: Reserved
+
+
:param X: Matrix X that has all 1-st covariates
@@ -86,9 +89,9 @@ def stratstats(X: Matrix,
:param Ycid: 2-nd covariate Y-column indices
the default value empty means "use columns 1 : ncol(Y)"
:param Scid: Column index of the stratum column in S
- :return: 'OperationNode' containing
- output matrix, one row per each distinct pair
+ :return: Output matrix, one row per each distinct pair
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/symmetricDifference.py b/src/main/python/systemds/operator/algorithm/builtin/symmetricDifference.py
index 2a72c99fe5..c9f81a738c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/symmetricDifference.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/symmetricDifference.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def symmetricDifference(X: Matrix,
Y: Matrix):
"""
- Builtin function that implements symmetric difference set-operation on vectors
+ Builtin function that implements symmetric difference set-operation on vectors
+
:param X: input vector
:param Y: input vector
- :return: 'OperationNode' containing
- vector with all elements in x and y but not in both
+ :return: vector with all elements in X and Y but not in both
"""
+
params_dict = {'X': X, 'Y': Y}
return Matrix(X.sds_context,
'symmetricDifference',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/tSNE.py b/src/main/python/systemds/operator/algorithm/builtin/tSNE.py
index ea3e3eaf60..3c659160c6 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/tSNE.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/tSNE.py
@@ -32,8 +32,9 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def tSNE(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function performs dimensionality reduction using tSNE algorithm based on
- the paper: Visualizing Data using t-SNE, Maaten et. al.
+ This function performs dimensionality reduction using tSNE algorithm based on
+ the paper: Visualizing Data using t-SNE, Maaten et. al.
+
:param X: Data Matrix of shape
@@ -46,9 +47,9 @@ def tSNE(X: Matrix,
:param seed: The seed used for initial values.
If set to -1 random seeds are selected.
:param is_verbose: Print debug information
- :return: 'OperationNode' containing
- data matrix of shape (number of data points, reduced_dims)
+ :return: Data Matrix of shape (number of data points, reduced_dims)
"""
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
index b2912347f5..f8201c72f4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def toOneHot(X: Matrix,
numClasses: int):
"""
- The toOneHot-function encodes unordered categorical vector to multiple binary vectors.
+ The toOneHot-function encodes unordered categorical vector to multiple binary vectors.
+
:param X: Vector with N integer entries between 1 and numClasses
:param numclasses: Number of columns, must be be greater than or equal to largest value in X
- :return: 'OperationNode' containing
- one-hot-encoded matrix with shape (n, numclasses)
+ :return: One-hot-encoded matrix with shape (N, numClasses)
"""
+
params_dict = {'X': X, 'numClasses': numClasses}
return Matrix(X.sds_context,
'toOneHot',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py b/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py
index 14414c46cd..9a05193d3c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/tomeklink.py
@@ -32,16 +32,19 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def tomeklink(X: Matrix,
y: Matrix):
"""
- The tomekLink-function performs under sampling by removing Tomek's links for imbalanced multi-class problems
- Computes TOMEK links and drops them from data matrix and label vector.
- Drops only the majority label and corresponding point of TOMEK links.
+ The tomekLink-function performs under sampling by removing Tomek's links for imbalanced multi-class problems
+ Computes TOMEK links and drops them from data matrix and label vector.
+ Drops only the majority label and corresponding point of TOMEK links.
+
:param X: Data Matrix (nxm)
:param y: Label Matrix (nx1), greater than zero
- :return: 'OperationNode' containing
- data matrix without tomek linkslabels corresponding to under sampled dataindices of dropped rows/labels wrt input
+ :return: Data Matrix without Tomek links
+ :return: Labels corresponding to under sampled data
+ :return: Indices of dropped rows/labels wrt input
"""
+
params_dict = {'X': X, 'y': y}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
index 7463465bc0..16a20d20e0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
@@ -36,9 +36,10 @@ def topk_cleaning(dataTrain: Frame,
evalFunHp: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function cleans top-K item (where K is given as input)for a given list of users.
- metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask
+ This function cleans top-K item (where K is given as input)for a given list of users.
+ metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask
"""
+
params_dict = {'dataTrain': dataTrain, 'primitives': primitives, 'parameters': parameters, 'evaluationFunc': evaluationFunc, 'evalFunHp': evalFunHp}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/underSampling.py b/src/main/python/systemds/operator/algorithm/builtin/underSampling.py
index b161133ac5..aecf7bab11 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/underSampling.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/underSampling.py
@@ -33,15 +33,17 @@ def underSampling(X: Matrix,
Y: Matrix,
ratio: float):
"""
- Builtin to perform random under sampling on data.
+ Builtin to perform random under sampling on data.
+
:param X: X data to sample from
:param Y: Y data to sample from it will sample the same rows from x.
:param ratio: The ratio to sample
- :return: 'OperationNode' containing
- the under sample data xthe under sample data y
+ :return: The under sample data X
+ :return: The under sample data Y
"""
+
params_dict = {'X': X, 'Y': Y, 'ratio': ratio}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/union.py b/src/main/python/systemds/operator/algorithm/builtin/union.py
index 6e1e92c806..45871cca31 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/union.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/union.py
@@ -32,14 +32,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def union(X: Matrix,
Y: Matrix):
"""
- Builtin function that implements union operation on vectors
+ Builtin function that implements union operation on vectors
+
:param X: input vector
:param Y: input vector
- :return: 'OperationNode' containing
- matrix with all unique rows existing in x and y
+ :return: matrix with all unique rows existing in X and Y
"""
+
params_dict = {'X': X, 'Y': Y}
return Matrix(X.sds_context,
'union',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/unique.py b/src/main/python/systemds/operator/algorithm/builtin/unique.py
index 825dc51a25..fd77b1fd55 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/unique.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/unique.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def unique(X: Matrix):
"""
- Builtin function that implements unique operation on vectors
+ Builtin function that implements unique operation on vectors
+
:param X: input vector
- :return: 'OperationNode' containing
- matrix with only unique rows
+ :return: matrix with only unique rows
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'unique',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/univar.py b/src/main/python/systemds/operator/algorithm/builtin/univar.py
index 6ab83730f3..7830b81bd7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/univar.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/univar.py
@@ -32,15 +32,16 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def univar(X: Matrix,
types: Matrix):
"""
- Computes univariate statistics for all attributes in a given data set
+ Computes univariate statistics for all attributes in a given data set
+
:param X: Input matrix of the shape (N, D)
:param TYPES: Matrix of the shape (1, D) with features types:
1 for scale, 2 for nominal, 3 for ordinal
- :return: 'OperationNode' containing
- univariate statistics for all attributes
+ :return: univariate statistics for all attributes
"""
+
params_dict = {'X': X, 'types': types}
return Matrix(X.sds_context,
'univar',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py b/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py
index 2cf97b07b5..aacb24831a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/vectorToCsv.py
@@ -31,14 +31,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def vectorToCsv(mask: Matrix):
"""
- This builtin function convert vector into csv string such as [1 0 0 1 1 0 1] = "1,4,5,7"
- Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+ This builtin function convert vector into csv string such as [1 0 0 1 1 0 1] = "1,4,5,7"
+ Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+
:param mask: Data vector (having 0 for excluded indexes)
- :return: 'OperationNode' containing
- indexes
+ :return: indexes
"""
+
params_dict = {'mask': mask}
return Matrix(mask.sds_context,
'vectorToCsv',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/winsorize.py b/src/main/python/systemds/operator/algorithm/builtin/winsorize.py
index f672ce1522..d22660f55f 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/winsorize.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/winsorize.py
@@ -33,16 +33,17 @@ def winsorize(X: Matrix,
verbose: bool,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- The winsorize-function removes outliers from the data. It does so by computing upper and
- lower quartile range of the given data then it replaces any value that falls outside this range
- (less than lower quartile range or more than upper quartile range).
+ The winsorize-function removes outliers from the data. It does so by computing upper and
+ lower quartile range of the given data then it replaces any value that falls outside this range
+ (less than lower quartile range or more than upper quartile range).
+
:param X: Input feature matrix
:param verbose: To print output on screen
- :return: 'OperationNode' containing
- matrix without outlier values
+ :return: Matrix without outlier values
"""
+
params_dict = {'X': X, 'verbose': verbose}
params_dict.update(kwargs)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/winsorizeApply.py b/src/main/python/systemds/operator/algorithm/builtin/winsorizeApply.py
index 35a9856dd3..b6a55b39fb 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/winsorizeApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/winsorizeApply.py
@@ -33,16 +33,17 @@ def winsorizeApply(X: Matrix,
qLower: Matrix,
qUpper: Matrix):
"""
- winsorizeApply takes the upper and lower quantile values per column, and
- remove outliers by replacing them with these upper and lower bound values.
+ winsorizeApply takes the upper and lower quantile values per column, and
+ remove outliers by replacing them with these upper and lower bound values.
+
:param X: Input feature matrix
:param qLower: row vector of upper bounds per column
:param qUpper: row vector of lower bounds per column
- :return: 'OperationNode' containing
- matrix without outlier values
+ :return: Matrix without outlier values
"""
+
params_dict = {'X': X, 'qLower': qLower, 'qUpper': qUpper}
return Matrix(X.sds_context,
'winsorizeApply',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py b/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py
index 80ff91f288..ed444f1a75 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/xdummy1.py
@@ -31,13 +31,14 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def xdummy1(X: Matrix):
"""
- This builtin function is here for debugging purposes
+ This builtin function is here for debugging purposes
+
:param X: test input
- :return: 'OperationNode' containing
- test result
+ :return: test result
"""
+
params_dict = {'X': X}
return Matrix(X.sds_context,
'xdummy1',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py b/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py
index 1aaf63b0cb..5acc5c36fb 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/xdummy2.py
@@ -31,13 +31,15 @@ from systemds.utils.consts import VALID_INPUT_TYPES
def xdummy2(X: Matrix):
"""
- This builtin function is here for debugging purposes
+ This builtin function is here for debugging purposes
+
:param X: Debug input
- :return: 'OperationNode' containing
- ------
+ :return: ---
+ :return: ---
"""
+
params_dict = {'X': X}
vX_0 = Matrix(X.sds_context, '')
diff --git a/src/main/python/systemds/operator/algorithm/builtin/xgboost.py b/src/main/python/systemds/operator/algorithm/builtin/xgboost.py
index eb459f5722..8876332f0d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/xgboost.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/xgboost.py
@@ -33,24 +33,26 @@ def xgboost(X: Matrix,
y: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
- implementation supports classification and regression and is capable of working with categorical and scalar features.
- Output explained:
- (the first node is the init prediction) and each row contains
- the following information:
- M[1,j]: id of node j (in a complete binary tree)
- M[2,j]: tree id to which node j belongs
- M[3,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
- M[4,j]: Feature index of the feature (scale feature id if the feature is
- scale or categorical feature id if the feature is categorical)
- that node j looks at if j is an internal node, otherwise 0
- M[5,j]: Type of the feature that node j looks at if j is an internal node.
- if leaf = 0, if scalar = 1, if categorical = 2
- M[6:,j]: If j is an internal node: Threshold the example's feature value is
- compared to is stored at M[6,j] if the feature chosen for j is scale,
- otherwise if the feature chosen for j is categorical rows 6,7,... depict
- the value subset chosen for j
- If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0
+ XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+ implementation supports classification and regression and is capable of working with categorical and scalar features.
+
+ Output explained:
+ (the first node is the init prediction) and each row contains
+ the following information:
+ M[1,j]: id of node j (in a complete binary tree)
+ M[2,j]: tree id to which node j belongs
+ M[3,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
+ M[4,j]: Feature index of the feature (scale feature id if the feature is
+ scale or categorical feature id if the feature is categorical)
+ that node j looks at if j is an internal node, otherwise 0
+ M[5,j]: Type of the feature that node j looks at if j is an internal node.
+ if leaf = 0, if scalar = 1, if categorical = 2
+ M[6:,j]: If j is an internal node: Threshold the example's feature value is
+ compared to is stored at M[6,j] if the feature chosen for j is scale,
+ otherwise if the feature chosen for j is categorical rows 6,7,... depict
+ the value subset chosen for j
+ If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0
+
:param X: Feature matrix X; note that X needs to be both recoded and dummy coded
@@ -65,9 +67,9 @@ def xgboost(X: Matrix,
:param learning_rate: Alias: eta. After each boosting step the learning rate controls the weights of the new predictions
:param max_depth: Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit
:param lambda: L2 regularization term on weights. Increasing this value will make model more conservative and reduce amount of leaves of a tree
- :return: 'OperationNode' containing
- matrix m where each column corresponds to a node in the learned tree
+ :return: Matrix M where each column corresponds to a node in the learned tree
"""
+
params_dict = {'X': X, 'y': y}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictClassification.py b/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictClassification.py
index bb93895f06..571025e33a 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictClassification.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictClassification.py
@@ -33,16 +33,17 @@ def xgboostPredictClassification(X: Matrix,
M: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
- implementation supports classification and is capable of working with categorical features.
+ XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+ implementation supports classification and is capable of working with categorical features.
+
:param X: Matrix of feature vectors we want to predict (X_test)
:param M: The model created at xgboost
:param learning_rate: The learning rate used in the model
- :return: 'OperationNode' containing
- the predictions of the samples using the given xgboost model. (y_prediction)
+ :return: The predictions of the samples using the given xgboost model. (y_prediction)
"""
+
params_dict = {'X': X, 'M': M}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictRegression.py b/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictRegression.py
index 2efd5d2565..f7921bcfb4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictRegression.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/xgboostPredictRegression.py
@@ -33,16 +33,17 @@ def xgboostPredictRegression(X: Matrix,
M: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
- implementation supports regression.
+ XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+ implementation supports regression.
+
:param X: Matrix of feature vectors we want to predict (X_test)
:param M: The model created at xgboost
:param learning_rate: The learning rate used in the model
- :return: 'OperationNode' containing
- the predictions of the samples using the given xgboost model. (y_prediction)
+ :return: The predictions of the samples using the given xgboost model. (y_prediction)
"""
+
params_dict = {'X': X, 'M': M}
params_dict.update(kwargs)
return Matrix(X.sds_context,