You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2025/09/24 13:26:26 UTC
(systemds) 01/03: [SYSTEMDS-3330] Fixes in generator.py, builtin function fixes
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit 9a7c3c7227403c993f7c7b56c920b05b8ea44228
Author: anuunchin <88...@users.noreply.github.com>
AuthorDate: Wed Sep 24 15:01:06 2025 +0200
[SYSTEMDS-3330] Fixes in generator.py, builtin function fixes
This commit resolves some inconsistent behavior in the generation
of builtin functions in python generated files.
Closes #2291
---
scripts/builtin/ampute.dml | 1 -
scripts/builtin/confusionMatrix.dml | 2 +-
scripts/builtin/cooccurrenceMatrix.dml | 13 ++--
scripts/builtin/decisionTree.dml | 4 +-
scripts/builtin/differenceStatistics.dml | 5 ++
scripts/builtin/glove.dml | 87 +++++++++++-----------
scripts/builtin/imputeByKNN.dml | 25 +++----
scripts/builtin/quantizeByCluster.dml | 5 +-
scripts/builtin/randomForest.dml | 7 +-
scripts/builtin/shapExplainer.dml | 1 +
scripts/builtin/topk_cleaning.dml | 38 +++++++++-
src/main/python/docs/README.md | 2 +-
src/main/python/docs/requires-docs.txt | 3 +-
src/main/python/generator/dml_parser.py | 16 ++--
.../python/systemds/operator/algorithm/__init__.py | 36 +++++++++
.../systemds/operator/algorithm/builtin/ampute.py | 10 +++
.../operator/algorithm/builtin/apply_pipeline.py | 1 -
.../operator/algorithm/builtin/confusionMatrix.py | 2 +-
.../{confusionMatrix.py => cooccurrenceMatrix.py} | 37 ++++-----
.../operator/algorithm/builtin/correctTypos.py | 1 -
.../algorithm/builtin/correctTyposApply.py | 1 -
.../operator/algorithm/builtin/decisionTree.py | 4 +-
.../algorithm/builtin/denialConstraints.py | 1 -
.../algorithm/builtin/differenceStatistics.py | 5 ++
.../systemds/operator/algorithm/builtin/dmv.py | 1 -
.../systemds/operator/algorithm/builtin/ema.py | 1 -
.../operator/algorithm/builtin/executePipeline.py | 38 ++++++++--
.../operator/algorithm/builtin/fit_pipeline.py | 1 -
.../algorithm/builtin/fixInvalidLengths.py | 1 -
.../algorithm/builtin/fixInvalidLengthsApply.py | 1 -
.../operator/algorithm/builtin/frameSort.py | 1 -
.../systemds/operator/algorithm/builtin/glove.py | 67 +++++++++++++++++
.../operator/algorithm/builtin/imputeByKNN.py | 19 ++++-
.../systemds/operator/algorithm/builtin/mdedup.py | 1 -
.../algorithm/builtin/quantizeByCluster.py | 83 +++++++++++++++++++++
.../operator/algorithm/builtin/randomForest.py | 7 +-
.../operator/algorithm/builtin/shapExplainer.py | 78 +++++++++++++++++++
.../operator/algorithm/builtin/topk_cleaning.py | 34 ++++++++-
.../builtin/{fixInvalidLengthsApply.py => wer.py} | 28 +++----
39 files changed, 516 insertions(+), 152 deletions(-)
diff --git a/scripts/builtin/ampute.dml b/scripts/builtin/ampute.dml
index 0da6af20fd..ffc645b5b3 100644
--- a/scripts/builtin/ampute.dml
+++ b/scripts/builtin/ampute.dml
@@ -30,7 +30,6 @@
# mech a string [either "MAR", "MNAR", or "MCAR"] specifying the missingness mechanism. Chosen "MAR" and "MNAR" settings will be overridden if a non-default weight matrix is specified
# weights a weight matrix [shape: k-by-m], containing weights that will be used to calculate the weighted sum scores. Will be overridden if mech == "MCAR"
# seed a manually defined seed for reproducible RNG
-
# -------------------------------------------------------------------------------------
#
# OUTPUT:
diff --git a/scripts/builtin/confusionMatrix.dml b/scripts/builtin/confusionMatrix.dml
index 3ac70fb3f8..b21088f2cf 100644
--- a/scripts/builtin/confusionMatrix.dml
+++ b/scripts/builtin/confusionMatrix.dml
@@ -23,7 +23,7 @@
# and actual labels. We return both the counts and relative frequency
# (normalized by sum of true labels)
#
-# .. code-block::
+# .. code-block:: text
#
# True Labels
# 1 2
diff --git a/scripts/builtin/cooccurrenceMatrix.dml b/scripts/builtin/cooccurrenceMatrix.dml
index 86b8b9ca16..590f4ba1e0 100644
--- a/scripts/builtin/cooccurrenceMatrix.dml
+++ b/scripts/builtin/cooccurrenceMatrix.dml
@@ -18,22 +18,21 @@
# under the License.
#
#-------------------------------------------------------------
-#
-# The implementation is based on
+
+# Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting.
+# Adds an index column to the result. The implementation is based on
# https://github.com/stanfordnlp/GloVe/blob/master/src/cooccur.c
#
-#-------------------------------------------------------------
-
-## Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting.
-## Adds an index column to the result.
# INPUT:
# ------------------------------------------------------------------------------
# S (Frame[Unknown]): 1D input data frame containing text data.
# ------------------------------------------------------------------------------
+#
# OUTPUT:
# ------------------------------------------------------------------------------
# result (Frame[Unknown]): Processed text data with an index column.
# ------------------------------------------------------------------------------
+
processText = function(Frame[Unknown] S) return (Frame[Unknown] result){
print("processText");
tmpStr = map(S[,1], "x -> x.replaceAll(\"[.]\", \"\")");
@@ -172,4 +171,4 @@ f_cooccurrenceMatrix = function(
[wordPosition, docID] = getWordPosition(processedResult, maxTokens);
[recodedWordPosition, tableSize, column] = getRecodedMatrix(wordPosition);
coocMatrix = createCoocMatrix(cbind(docID, recodedWordPosition), tableSize, distanceWeighting, symmetric, windowSize);
-}
+}
\ No newline at end of file
diff --git a/scripts/builtin/decisionTree.dml b/scripts/builtin/decisionTree.dml
index 69bf12af90..94c292d855 100644
--- a/scripts/builtin/decisionTree.dml
+++ b/scripts/builtin/decisionTree.dml
@@ -30,9 +30,9 @@
# and the following trees, M would look as follows:
#
# (L1) |d<5|
-# / \
+# / \\
# (L2) P1:2 |a<7|
-# / \
+# / \\
# (L3) P2:2 P3:1
#
# --> M :=
diff --git a/scripts/builtin/differenceStatistics.dml b/scripts/builtin/differenceStatistics.dml
index 0e9019f096..30f207091e 100644
--- a/scripts/builtin/differenceStatistics.dml
+++ b/scripts/builtin/differenceStatistics.dml
@@ -28,6 +28,11 @@
# X First Matrix to compare
# Y Second Matrix to compare
# --------------------------------------------------------------------------------
+#
+# OUTPUT:
+# -------------------------------------------------------------------------------------
+# stats. Difference statistics
+# -------------------------------------------------------------------------------------
m_differenceStatistics = function(Matrix[Double] X, Matrix[Double] Y) {
diff --git a/scripts/builtin/glove.dml b/scripts/builtin/glove.dml
index fc5ee9bafb..9acf52975c 100644
--- a/scripts/builtin/glove.dml
+++ b/scripts/builtin/glove.dml
@@ -18,6 +18,51 @@
# under the License.
#-------------------------------------------------------------
+
+# Computes the vector embeddings for words in a large text corpus.
+#
+# INPUT:
+# --------------------------------------------------------------------------------
+# input 1DInput corpus in CSV format.
+# seed Random seed for reproducibility.
+# vector_size Dimensionality of word vectors, V.
+# eta Learning rate for optimization, recommended value: 0.05.
+# alpha Weighting function parameter, recommended value: 0.75.
+# x_max Maximum co-occurrence value as per the GloVe paper: 100.
+# tol Tolerance value to avoid overfitting, recommended value: 1e-4.
+# iterations Total number of training iterations.
+# print_loss_it Interval (in iterations) for printing the loss.
+# maxTokens Maximum number of tokens per text entry.
+# windowSize Context window size.
+# distanceWeighting Whether to apply distance-based weighting.
+# symmetric Determines if the matrix is symmetric (TRUE) or asymmetric (FALSE).
+# ------------------------------------------------------------------------------
+#
+# OUTPUT:
+# ------------------------------------------------------------------------------
+# G The word indices and their word vectors, of shape (N, V). Each represented as a vector, of shape (1,V)
+# ------------------------------------------------------------------------------
+
+
+f_glove = function(
+ Frame[Unknown] input,
+ int seed, int vector_size,
+ double alpha, double eta,
+ double x_max,
+ double tol,
+ int iterations,
+ int print_loss_it,
+ Int maxTokens,
+ Int windowSize,
+ Boolean distanceWeighting,
+ Boolean symmetric)
+ return (frame[Unknown] G){
+
+ [cooc_matrix, cooc_index] = cooccurrenceMatrix(input, maxTokens, windowSize, distanceWeighting, symmetric);
+ G = gloveWithCoocMatrix(cooc_matrix, cooc_index, seed, vector_size, alpha, eta, x_max, tol, iterations, print_loss_it);
+}
+
+
init = function(matrix[double] cooc_matrix, double x_max, double alpha)
return(matrix[double] weights, matrix[double] log_cooc_matrix){
E = 2.718281828;
@@ -118,45 +163,3 @@ gloveWithCoocMatrix = function(matrix[double] cooc_matrix, frame[Unknown] cooc_i
print("Given " + iterations + " iterations, " + "stopped (or converged) at the " + final_iter + " iteration / error: " + error);
G = cbind(cooc_index[,2], as.frame(G));
}
-
-glove = function(
- Frame[Unknown] input,
- int seed, int vector_size,
- double alpha, double eta,
- double x_max,
- double tol,
- int iterations,
- int print_loss_it,
- Int maxTokens,
- Int windowSize,
- Boolean distanceWeighting,
- Boolean symmetric)
- return (frame[Unknown] G){
-
- /*
- * Main function to Computes the vector embeddings for words in a large text corpus.
- * INPUT:
- * ------------------------------------------------------------------------------
- * - input (Frame[Unknown]): 1DInput corpus in CSV format.
- * - seed: Random seed for reproducibility.
- * - vector_size: Dimensionality of word vectors, V.
- * - eta: Learning rate for optimization, recommended value: 0.05.
- * - alpha: Weighting function parameter, recommended value: 0.75.
- * - x_max: Maximum co-occurrence value as per the GloVe paper: 100.
- * - tol: Tolerance value to avoid overfitting, recommended value: 1e-4.
- * - iterations: Total number of training iterations.
- * - print_loss_it: Interval (in iterations) for printing the loss.
- * - maxTokens (Int): Maximum number of tokens per text entry.
- * - windowSize (Int): Context window size.
- * - distanceWeighting (Boolean): Whether to apply distance-based weighting.
- * - symmetric (Boolean): Determines if the matrix is symmetric (TRUE) or asymmetric (FALSE).
- * ------------------------------------------------------------------------------
- * OUTPUT:
- * ------------------------------------------------------------------------------
- * G (Frame[Unknown]): The word indices and their word vectors, of shape (N, V). Each represented as a vector, of shape (1,V)
- * ------------------------------------------------------------------------------
- */
-
- [cooc_matrix, cooc_index] = cooccurrenceMatrix(input, maxTokens, windowSize, distanceWeighting, symmetric);
- G = gloveWithCoocMatrix(cooc_matrix, cooc_index, seed, vector_size, alpha, eta, x_max, tol, iterations, print_loss_it);
-}
diff --git a/scripts/builtin/imputeByKNN.dml b/scripts/builtin/imputeByKNN.dml
index 13136ff2c9..edd8e7727d 100644
--- a/scripts/builtin/imputeByKNN.dml
+++ b/scripts/builtin/imputeByKNN.dml
@@ -25,23 +25,16 @@
# the missing values by column means. Currently, only the column with the most
# missing values is actually imputed.
#
-# ------------------------------------------------------------------------------
# INPUT:
# ------------------------------------------------------------------------------
-# X Matrix with missing values, which are represented as NaNs
-# method Method used for imputing missing values with different performance
-# and accuracy tradeoffs:
-# 'dist' (default): Compute all-pairs distances and impute the
-# missing values by closest. O(N^2 * #features)
-# 'dist_missing': Compute distances between data and records with
-# missing values. O(N*M * #features), assuming
-# that the number of records with MV is M<<N.
-# 'dist_sample': Compute distances between sample of data and
-# records with missing values. O(S*M * #features)
-# with M<<N and S<<N, but suboptimal imputation.
-# seed Root seed value for random/sample calls for deterministic behavior
-# -1 for true randomization
-# sample_frac Sample fraction for 'dist_sample' (value between 0 and 1)
+# X Matrix with missing values, which are represented as NaNs
+# method Method used for imputing missing values with different performance and accuracy tradeoffs:\n
+# - 'dist' (default): Compute all-pairs distances and impute the missing values by closest. O(N^2 * #features)
+# - 'dist_missing': Compute distances between data and records with missing values. O(N*M * #features), assuming that the number of records with MV is M<<N.
+# - 'dist_sample': Compute distances between sample of data and records with missing values. O(S*M * #features) with M<<N and S<<N, but suboptimal imputation.
+#
+# seed Root seed value for random/sample calls for deterministic behavior. -1 for true randomization
+# sample_frac Sample fraction for 'dist_sample' (value between 0 and 1)
# ------------------------------------------------------------------------------
#
# OUTPUT:
@@ -136,4 +129,4 @@ compute_missing_values = function (Matrix[Double] X, Matrix[Double] filled_matri
#Get the subset records that need to be imputed
imputedValue = t(reshaped) %*% aligned
imputedValue = t(imputedValue)
-}
+}
\ No newline at end of file
diff --git a/scripts/builtin/quantizeByCluster.dml b/scripts/builtin/quantizeByCluster.dml
index 824ac35053..20b0b0c89d 100644
--- a/scripts/builtin/quantizeByCluster.dml
+++ b/scripts/builtin/quantizeByCluster.dml
@@ -58,7 +58,7 @@
# the product quantization. Only relevant when space_decomp = TRUE.
# ------------------------------------------------------------------------------------------
-m_quantizeByCluster = function(Matrix[Double]X, Integer M = 4, Integer k = 10, Integer runs = 10,
+m_quantizeByCluster = function(Matrix[Double] X, Integer M = 4, Integer k = 10, Integer runs = 10,
Integer max_iter = 1000, Double eps = 1e-6, Integer avg_sample_size_per_centroid = 50, Boolean separate=TRUE, Boolean space_decomp=FALSE, Integer seed = -1)
return(Matrix[Double] codebook, Matrix[Double] codes, Matrix[Double] R)
{
@@ -118,5 +118,4 @@ m_quantizeByCluster = function(Matrix[Double]X, Integer M = 4, Integer k = 10, I
codes[,i] = tmp_c + offset
}
}
-}
-
+}
\ No newline at end of file
diff --git a/scripts/builtin/randomForest.dml b/scripts/builtin/randomForest.dml
index 8daeb5bc7f..53529afb9a 100644
--- a/scripts/builtin/randomForest.dml
+++ b/scripts/builtin/randomForest.dml
@@ -26,16 +26,17 @@
# and optionally subset of features (columns). During tree construction, split
# candidates are additionally chosen on a sample of remaining features.
#
-# .. code-block::
+# .. code-block:: text
#
# For example, given a feature matrix with features [a,b,c,d]
# and the following two trees, M (the output) would look as follows:
#
# (L1) |a<7| |d<5|
-# / \ / \
+# / \\ / \\
# (L2) |c<3| |b<4| |a<7| P3:2
-# / \ / \ / \
+# / \\ / \\ / \\
# (L3) P1:2 P2:1 P3:1 P4:2 P1:2 P2:1
+#
# --> M :=
# [[1, 7, 3, 3, 2, 4, 0, 2, 0, 1, 0, 1, 0, 2], (1st tree)
# [4, 5, 1, 7, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0]] (2nd tree)
diff --git a/scripts/builtin/shapExplainer.dml b/scripts/builtin/shapExplainer.dml
index b78a5dbcef..39d365bf01 100644
--- a/scripts/builtin/shapExplainer.dml
+++ b/scripts/builtin/shapExplainer.dml
@@ -51,6 +51,7 @@
# S Matrix holding the shapley values along the cols, one row per instance.
# expected Double holding the average prediction of all instances.
# -----------------------------------------------------------------------------
+
s_shapExplainer = function(String model_function, list[unknown] model_args, Matrix[Double] x_instances,
Matrix[Double] X_bg, Integer n_permutations = 10, Integer n_samples = 100, Integer remove_non_var=0,
Matrix[Double] partitions=as.matrix(-1), Integer seed = -1, Integer verbose = 0)
diff --git a/scripts/builtin/topk_cleaning.dml b/scripts/builtin/topk_cleaning.dml
index 6f946c7729..c998732092 100644
--- a/scripts/builtin/topk_cleaning.dml
+++ b/scripts/builtin/topk_cleaning.dml
@@ -19,8 +19,44 @@
#
#-------------------------------------------------------------
-# This function cleans top-K item (where K is given as input)for a given list of users.
+# This function cleans top-K item (where K is given as input) for a given list of users.
# metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask
+#
+# INPUT:
+# ------------------------------------------------------------------------------
+# dataTrain Training set
+# dataTest Test set ignored when cv is set to True
+# metaData 3×n frame with schema, categorical mask, and FD mask for dataTrain
+# primitives Library of primitive cleaning operators
+# parameters Hyperparameter search space that matches the primitives
+# refSol Reference solution
+# evaluationFunc Name of a SystemDS DML function that scores a pipeline
+# evalFunHp Hyperparameter matrix for the above evaluation function
+# topK Number of best pipelines to return
+# resource_val Maximum resource R for the Bandit search
+# max_iter Maximum iterations while enumerating logical pipelines
+# lq Lower quantile used by utils::doErrorSample when triggered
+# uq Upper quantile used by utils::doErrorSample when triggered
+# sample Fraction of rows to subsample from dataTrain
+# expectedIncrease Minimum improvement over dirtyScore that a candidate must deliver
+# seed Seed number
+# cv TRUE means k-fold CV, FALSE means hold-out split
+# cvk Number of folds if cv = TRUE
+# isLastLabel TRUE if the last column is the label
+# rowCount Row-count threshold above which doErrorSample may replace uniform sampling
+# correctTypos Run spelling correction in the string preprocessing step
+# enablePruning Enable pruning inside the Bandit phase
+# ------------------------------------------------------------------------------
+#
+# OUTPUT:
+#-------------------------------------------------------------------------------
+# topKPipelines K cleaned-data pipelines
+# topKHyperParams Hyperparameter matrix with rows aligning with topKPipelines
+# topKScores Evaluation scores with rows aligning with topKPipelines
+# dirtyScore Baseline score on the unclean data
+# evalFunHp Updated evaluation function hyperparameters
+# applyFunc Frame of “apply” functions for deploying each of the top-K pipelines
+#-------------------------------------------------------------------------------
source("scripts/pipelines/scripts/utils.dml") as utils;
source("scripts/pipelines/scripts/enumerateLogical.dml") as lg;
diff --git a/src/main/python/docs/README.md b/src/main/python/docs/README.md
index 61bdd24a3e..e5bc5c5958 100644
--- a/src/main/python/docs/README.md
+++ b/src/main/python/docs/README.md
@@ -39,4 +39,4 @@ and then run `make html`:
make html
```
-The docs will then be created at: `/src/main/python/build`in HTML will be placed in the `./_build` directory.
+The docs will then be created at: `/src/main/python/docs/build/html/`.
\ No newline at end of file
diff --git a/src/main/python/docs/requires-docs.txt b/src/main/python/docs/requires-docs.txt
index 9305d9320f..1022b65240 100644
--- a/src/main/python/docs/requires-docs.txt
+++ b/src/main/python/docs/requires-docs.txt
@@ -24,4 +24,5 @@ sphinx_rtd_theme
numpy
py4j
scipy
-requests
\ No newline at end of file
+requests
+pandas
\ No newline at end of file
diff --git a/src/main/python/generator/dml_parser.py b/src/main/python/generator/dml_parser.py
index 2abffb021f..8e835e96a1 100644
--- a/src/main/python/generator/dml_parser.py
+++ b/src/main/python/generator/dml_parser.py
@@ -28,7 +28,7 @@ import re
class FunctionParser(object):
header_input_pattern = r"^[ \t\n]*[#]+[ \t\n]*input[ \t\n\w:;.,#]*[\s#\-]*[#]+[\w\s\d:,.()\" \t\n\-]*[\s#\-]*$"
header_output_pattern = r"[\s#\-]*[#]+[ \t]*(return|output)[ \t\w:;.,#]*[\s#\-]*[#]+[\w\s\d:,.()\" \t\-]*[\s#\-]*$"
- function_pattern = r"^[ms]_[\w]+[ \t\n]*=[ \t\n]+function[^#{]*"
+ function_pattern = r"^[fms]_[\w]+[ \t\n]*=[ \t\n]+function[^#{]*"
# parameter_pattern = r"^m_[\w]+[\s]+=[\s]+function[\s]*\([\s]*(?=return)[\s]*\)[\s]*return[\s]*\([\s]*([\w\[\]\s,\d=.\-_]*)[\s]*\)[\s]*"
header_parameter_pattern = r"[\s#\-]*[#]+[ \t]*([\w|-]+)[\s]+([\w]+)[\s]+([\w,\d.\"\-]+)[\s]+([\w|\W]+)"
divider_pattern = r"[\s#\-]*"
@@ -57,15 +57,13 @@ class FunctionParser(object):
"""
file_name = os.path.basename(path)
function_name, extension = os.path.splitext(file_name)
- # try:
- function_definition = self.find_function_definition(path)
- # pattern = re.compile(
- # self.__class__.parameter_pattern, flags=re.I | re.M)
- # match = pattern.match(function_definition)
-
- # if match:
+ try:
+ function_definition = self.find_function_definition(path)
+ except AttributeError:
+ print(f"[INFO] Skipping '{function_name}': does not match function name pattern. It is likely an internal function.")
+ return
- func_split = function_definition.split("function")[1].split("return")
+ func_split = function_definition.split("function", 1)[1].split("return")
param_str = self.extract_param_str(func_split[0])
retval_str = None
diff --git a/src/main/python/systemds/operator/algorithm/__init__.py b/src/main/python/systemds/operator/algorithm/__init__.py
index bd611ee6cc..c198a7138e 100644
--- a/src/main/python/systemds/operator/algorithm/__init__.py
+++ b/src/main/python/systemds/operator/algorithm/__init__.py
@@ -31,6 +31,7 @@ from .builtin.alsDS import alsDS
from .builtin.alsPredict import alsPredict
from .builtin.alsTopkPredict import alsTopkPredict
from .builtin.ampute import ampute
+from .builtin.apply_pipeline import apply_pipeline
from .builtin.arima import arima
from .builtin.auc import auc
from .builtin.autoencoder_2layer import autoencoder_2layer
@@ -38,7 +39,10 @@ from .builtin.bandit import bandit
from .builtin.bivar import bivar
from .builtin.components import components
from .builtin.confusionMatrix import confusionMatrix
+from .builtin.cooccurrenceMatrix import cooccurrenceMatrix
from .builtin.cor import cor
+from .builtin.correctTypos import correctTypos
+from .builtin.correctTyposApply import correctTyposApply
from .builtin.cov import cov
from .builtin.cox import cox
from .builtin.cspline import cspline
@@ -50,15 +54,22 @@ from .builtin.dbscanApply import dbscanApply
from .builtin.decisionTree import decisionTree
from .builtin.decisionTreePredict import decisionTreePredict
from .builtin.deepWalk import deepWalk
+from .builtin.denialConstraints import denialConstraints
from .builtin.differenceStatistics import differenceStatistics
from .builtin.discoverFD import discoverFD
from .builtin.dist import dist
+from .builtin.dmv import dmv
+from .builtin.ema import ema
from .builtin.executePipeline import executePipeline
from .builtin.f1Score import f1Score
from .builtin.fdr import fdr
from .builtin.ffPredict import ffPredict
from .builtin.ffTrain import ffTrain
+from .builtin.fit_pipeline import fit_pipeline
+from .builtin.fixInvalidLengths import fixInvalidLengths
+from .builtin.fixInvalidLengthsApply import fixInvalidLengthsApply
from .builtin.flattenQuantile import flattenQuantile
+from .builtin.frameSort import frameSort
from .builtin.frequencyEncode import frequencyEncode
from .builtin.frequencyEncodeApply import frequencyEncodeApply
from .builtin.garch import garch
@@ -66,6 +77,7 @@ from .builtin.gaussianClassifier import gaussianClassifier
from .builtin.getAccuracy import getAccuracy
from .builtin.glm import glm
from .builtin.glmPredict import glmPredict
+from .builtin.glove import glove
from .builtin.gmm import gmm
from .builtin.gmmPredict import gmmPredict
from .builtin.gnmf import gnmf
@@ -97,6 +109,7 @@ from .builtin.img_translate_linearized import img_translate_linearized
from .builtin.impurityMeasures import impurityMeasures
from .builtin.imputeByFD import imputeByFD
from .builtin.imputeByFDApply import imputeByFDApply
+from .builtin.imputeByKNN import imputeByKNN
from .builtin.imputeByMean import imputeByMean
from .builtin.imputeByMeanApply import imputeByMeanApply
from .builtin.imputeByMedian import imputeByMedian
@@ -126,6 +139,7 @@ from .builtin.mae import mae
from .builtin.mape import mape
from .builtin.matrixProfile import matrixProfile
from .builtin.mcc import mcc
+from .builtin.mdedup import mdedup
from .builtin.mice import mice
from .builtin.miceApply import miceApply
from .builtin.mse import mse
@@ -153,6 +167,7 @@ from .builtin.pcaTransform import pcaTransform
from .builtin.pnmf import pnmf
from .builtin.ppca import ppca
from .builtin.psnr import psnr
+from .builtin.quantizeByCluster import quantizeByCluster
from .builtin.raGroupby import raGroupby
from .builtin.raJoin import raJoin
from .builtin.raSelection import raSelection
@@ -165,6 +180,7 @@ from .builtin.scaleMinMax import scaleMinMax
from .builtin.selectByVarThresh import selectByVarThresh
from .builtin.ses import ses
from .builtin.setdiff import setdiff
+from .builtin.shapExplainer import shapExplainer
from .builtin.sherlock import sherlock
from .builtin.sherlockPredict import sherlockPredict
from .builtin.shortestPath import shortestPath
@@ -189,10 +205,12 @@ from .builtin.symmetricDifference import symmetricDifference
from .builtin.tSNE import tSNE
from .builtin.toOneHot import toOneHot
from .builtin.tomeklink import tomeklink
+from .builtin.topk_cleaning import topk_cleaning
from .builtin.underSampling import underSampling
from .builtin.union import union
from .builtin.univar import univar
from .builtin.vectorToCsv import vectorToCsv
+from .builtin.wer import wer
from .builtin.winsorize import winsorize
from .builtin.winsorizeApply import winsorizeApply
from .builtin.xdummy1 import xdummy1
@@ -211,6 +229,7 @@ __all__ = ['WoE',
'alsPredict',
'alsTopkPredict',
'ampute',
+ 'apply_pipeline',
'arima',
'auc',
'autoencoder_2layer',
@@ -218,7 +237,10 @@ __all__ = ['WoE',
'bivar',
'components',
'confusionMatrix',
+ 'cooccurrenceMatrix',
'cor',
+ 'correctTypos',
+ 'correctTyposApply',
'cov',
'cox',
'cspline',
@@ -230,15 +252,22 @@ __all__ = ['WoE',
'decisionTree',
'decisionTreePredict',
'deepWalk',
+ 'denialConstraints',
'differenceStatistics',
'discoverFD',
'dist',
+ 'dmv',
+ 'ema',
'executePipeline',
'f1Score',
'fdr',
'ffPredict',
'ffTrain',
+ 'fit_pipeline',
+ 'fixInvalidLengths',
+ 'fixInvalidLengthsApply',
'flattenQuantile',
+ 'frameSort',
'frequencyEncode',
'frequencyEncodeApply',
'garch',
@@ -246,6 +275,7 @@ __all__ = ['WoE',
'getAccuracy',
'glm',
'glmPredict',
+ 'glove',
'gmm',
'gmmPredict',
'gnmf',
@@ -277,6 +307,7 @@ __all__ = ['WoE',
'impurityMeasures',
'imputeByFD',
'imputeByFDApply',
+ 'imputeByKNN',
'imputeByMean',
'imputeByMeanApply',
'imputeByMedian',
@@ -306,6 +337,7 @@ __all__ = ['WoE',
'mape',
'matrixProfile',
'mcc',
+ 'mdedup',
'mice',
'miceApply',
'mse',
@@ -333,6 +365,7 @@ __all__ = ['WoE',
'pnmf',
'ppca',
'psnr',
+ 'quantizeByCluster',
'raGroupby',
'raJoin',
'raSelection',
@@ -345,6 +378,7 @@ __all__ = ['WoE',
'selectByVarThresh',
'ses',
'setdiff',
+ 'shapExplainer',
'sherlock',
'sherlockPredict',
'shortestPath',
@@ -369,10 +403,12 @@ __all__ = ['WoE',
'tSNE',
'toOneHot',
'tomeklink',
+ 'topk_cleaning',
'underSampling',
'union',
'univar',
'vectorToCsv',
+ 'wer',
'winsorize',
'winsorizeApply',
'xdummy1',
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ampute.py b/src/main/python/systemds/operator/algorithm/builtin/ampute.py
index d323000710..fb3a82a380 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ampute.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ampute.py
@@ -33,6 +33,16 @@ def ampute(X: Matrix,
"""
This function injects missing values into a multivariate a given dataset, similarly to the ampute() method in R's MICE package.
+
+
+ :param X: a multivariate numeric dataset [shape: n-by-m]
+ :param prop: a number in the (0, 1] range specifying the proportion of amputed rows across the entire dataset
+ :param patterns: a pattern matrix of 0's and 1's [shape: k-by-m] where each row corresponds to a pattern. 0 indicates that a variable should have missing values and 1 indicating that a variable should remain complete
+ :param freq: a vector [length: k] containing the relative frequency with which each pattern in the patterns matrix should occur
+ :param mech: a string [either "MAR", "MNAR", or "MCAR"] specifying the missingness mechanism. Chosen "MAR" and "MNAR" settings will be overridden if a non-default weight matrix is specified
+ :param weights: a weight matrix [shape: k-by-m], containing weights that will be used to calculate the weighted sum scores. Will be overridden if mech == "MCAR"
+ :param seed: a manually defined seed for reproducible RNG
+ :return: amputed output dataset
"""
params_dict = {'X': X}
diff --git a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
index be1100b412..63ffc3f66b 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
index 81c549b598..66a01780b0 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
@@ -35,7 +35,7 @@ def confusionMatrix(P: Matrix,
and actual labels. We return both the counts and relative frequency
(normalized by sum of true labels)
- .. code-block::
+ .. code-block:: text
True Labels
1 2
diff --git a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py
similarity index 56%
copy from src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
copy to src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py
index 81c549b598..6df77d3e7d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py
@@ -20,7 +20,7 @@
# -------------------------------------------------------------
# Autogenerated By : src/main/python/generator/generator.py
-# Autogenerated From : scripts/builtin/confusionMatrix.dml
+# Autogenerated From : scripts/builtin/cooccurrenceMatrix.dml
from typing import Dict, Iterable
@@ -28,36 +28,29 @@ from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, S
from systemds.utils.consts import VALID_INPUT_TYPES
-def confusionMatrix(P: Matrix,
- Y: Matrix):
+def cooccurrenceMatrix(input: Frame,
+ maxTokens: int,
+ windowSize: int,
+ distanceWeighting: bool,
+ symmetric: bool):
"""
- Computes the confusion matrix for input vectors of predictions
- and actual labels. We return both the counts and relative frequency
- (normalized by sum of true labels)
+ Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting.
+ Adds an index column to the result. The implementation is based on
+ https://github.com/stanfordnlp/GloVe/blob/master/src/cooccur.c
- .. code-block::
- True Labels
- 1 2
- 1 TP | FP
- Predictions ----+----
- 2 FN | TN
-
-
- :param P: vector of predictions (1-based, recoded)
- :param Y: vector of actual labels (1-based, recoded)
- :return: the confusion matrix as absolute counts
- :return: the confusion matrix as relative frequencies
+ :param S: (Frame[Unknown]): 1D input data frame containing text data.
+ :return: (Frame[Unknown]): Processed text data with an index column.
"""
- params_dict = {'P': P, 'Y': Y}
+ params_dict = {'input': input, 'maxTokens': maxTokens, 'windowSize': windowSize, 'distanceWeighting': distanceWeighting, 'symmetric': symmetric}
- vX_0 = Matrix(P.sds_context, '')
- vX_1 = Matrix(P.sds_context, '')
+ vX_0 = Matrix(input.sds_context, '')
+ vX_1 = Frame(input.sds_context, '')
output_nodes = [vX_0, vX_1, ]
- op = MultiReturn(P.sds_context, 'confusionMatrix', output_nodes, named_input_nodes=params_dict)
+ op = MultiReturn(input.sds_context, 'cooccurrenceMatrix', output_nodes, named_input_nodes=params_dict)
vX_0._unnamed_input_nodes = [op]
vX_1._unnamed_input_nodes = [op]
diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
index 321a1949f5..64354a9bc3 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
index 0a2c61a6f4..5da8769509 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
index a1a751d0aa..3fe565b8c7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py
@@ -44,9 +44,9 @@ def decisionTree(X: Matrix,
and the following trees, M would look as follows:
(L1) |d<5|
- / \
+ / \\
(L2) P1:2 |a<7|
- / \
+ / \\
(L3) P2:2 P3:1
--> M :=
diff --git a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
index 347502b848..5cdec21296 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py b/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py
index dfe2218a42..b6597bb6e4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py
@@ -35,6 +35,11 @@ def differenceStatistics(X: Matrix,
they are different. This can be used for instance in comparison of lossy
compression techniques, that reduce the fidelity of the data.
+
+
+ :param X: First Matrix to compare
+ :param Y: Second Matrix to compare
+ :return: Difference statistics
"""
params_dict = {'X': X, 'Y': Y}
diff --git a/src/main/python/systemds/operator/algorithm/builtin/dmv.py b/src/main/python/systemds/operator/algorithm/builtin/dmv.py
index deaf3ea8a6..2955e505e1 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/dmv.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/dmv.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/ema.py b/src/main/python/systemds/operator/algorithm/builtin/ema.py
index 4e0ccca6bb..90f9a852d7 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/ema.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/ema.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
index 1fffb46f10..66750fc071 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py
@@ -28,7 +28,18 @@ from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, S
from systemds.utils.consts import VALID_INPUT_TYPES
-def executePipeline(X: Matrix):
+def executePipeline(pipeline: Frame,
+ Xtrain: Matrix,
+ Ytrain: Matrix,
+ Xtest: Matrix,
+ Ytest: Matrix,
+ metaList: List,
+ hyperParameters: Matrix,
+ flagsCount: int,
+ verbose: bool,
+ startInd: int,
+ endInd: int,
+ **kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
This function execute pipeline.
@@ -56,17 +67,30 @@ def executePipeline(X: Matrix):
:return: ---
"""
- params_dict = {'X': X}
+ params_dict = {'pipeline': pipeline, 'Xtrain': Xtrain, 'Ytrain': Ytrain, 'Xtest': Xtest, 'Ytest': Ytest, 'metaList': metaList, 'hyperParameters': hyperParameters, 'flagsCount': flagsCount, 'verbose': verbose, 'startInd': startInd, 'endInd': endInd}
+ params_dict.update(kwargs)
- vX_0 = Matrix(X.sds_context, '')
- vX_1 = Matrix(X.sds_context, '')
- vX_2 = Matrix(X.sds_context, '')
- output_nodes = [vX_0, vX_1, vX_2, ]
+ vX_0 = Matrix(pipeline.sds_context, '')
+ vX_1 = Matrix(pipeline.sds_context, '')
+ vX_2 = Matrix(pipeline.sds_context, '')
+ vX_3 = Matrix(pipeline.sds_context, '')
+ vX_4 = Scalar(pipeline.sds_context, '')
+ vX_5 = Matrix(pipeline.sds_context, '')
+ vX_6 = Matrix(pipeline.sds_context, '')
+ vX_7 = Scalar(pipeline.sds_context, '')
+ vX_8 = List(pipeline.sds_context, '')
+ output_nodes = [vX_0, vX_1, vX_2, vX_3, vX_4, vX_5, vX_6, vX_7, vX_8, ]
- op = MultiReturn(X.sds_context, 'executePipeline', output_nodes, named_input_nodes=params_dict)
+ op = MultiReturn(pipeline.sds_context, 'executePipeline', output_nodes, named_input_nodes=params_dict)
vX_0._unnamed_input_nodes = [op]
vX_1._unnamed_input_nodes = [op]
vX_2._unnamed_input_nodes = [op]
+ vX_3._unnamed_input_nodes = [op]
+ vX_4._unnamed_input_nodes = [op]
+ vX_5._unnamed_input_nodes = [op]
+ vX_6._unnamed_input_nodes = [op]
+ vX_7._unnamed_input_nodes = [op]
+ vX_8._unnamed_input_nodes = [op]
return op
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
index 5de40c745f..48363035d8 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
index b635f31b29..cc0e83a51e 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
index cc8fe68aac..ed2572368d 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
index 0bfc7f3afe..2575baefe4 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/glove.py b/src/main/python/systemds/operator/algorithm/builtin/glove.py
new file mode 100644
index 0000000000..cbf9c421c4
--- /dev/null
+++ b/src/main/python/systemds/operator/algorithm/builtin/glove.py
@@ -0,0 +1,67 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+
+# Autogenerated By : src/main/python/generator/generator.py
+# Autogenerated From : scripts/builtin/glove.dml
+
+from typing import Dict, Iterable
+
+from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
+from systemds.utils.consts import VALID_INPUT_TYPES
+
+
+def glove(input: Frame,
+ seed: int,
+ vector_size: int,
+ alpha: float,
+ eta: float,
+ x_max: float,
+ tol: float,
+ iterations: int,
+ print_loss_it: int,
+ maxTokens: int,
+ windowSize: int,
+ distanceWeighting: bool,
+ symmetric: bool):
+ """
+
+
+
+ :param input: 1DInput corpus in CSV format.
+ :param seed: Random seed for reproducibility.
+ :param vector_size: Dimensionality of word vectors, V.
+ :param eta: Learning rate for optimization, recommended value: 0.05.
+ :param alpha: Weighting function parameter, recommended value: 0.75.
+ :param x_max: Maximum co-occurrence value as per the GloVe paper: 100.
+ :param tol: Tolerance value to avoid overfitting, recommended value: 1e-4.
+ :param iterations: Total number of training iterations.
+ :param print_loss_it: Interval (in iterations) for printing the loss.
+ :param maxTokens: Maximum number of tokens per text entry.
+ :param windowSize: Context window size.
+ :param distanceWeighting: Whether to apply distance-based weighting.
+ :param symmetric: Determines if the matrix is symmetric (TRUE) or asymmetric (FALSE).
+ :return: The word indices and their word vectors, of shape (N, V). Each represented as a vector, of shape (1,V)
+ """
+
+ params_dict = {'input': input, 'seed': seed, 'vector_size': vector_size, 'alpha': alpha, 'eta': eta, 'x_max': x_max, 'tol': tol, 'iterations': iterations, 'print_loss_it': print_loss_it, 'maxTokens': maxTokens, 'windowSize': windowSize, 'distanceWeighting': distanceWeighting, 'symmetric': symmetric}
+ return Matrix(input.sds_context,
+ 'glove',
+ named_input_nodes=params_dict)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py
index fcc096180b..f04aa09851 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py
@@ -25,13 +25,30 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
def imputeByKNN(X: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
+ """
+ Imputes missing values, indicated by NaNs, using KNN-based methods
+ (k-nearest neighbors by euclidean distance). In order to avoid NaNs in
+ distance computation and meaningful nearest neighbor search, we initialize
+ the missing values by column means. Currently, only the column with the most
+ missing values is actually imputed.
+
+
+ :param X: Matrix with missing values, which are represented as NaNs
+ :param method: Method used for imputing missing values with different performance and accuracy tradeoffs:\n
+ - 'dist' (default): Compute all-pairs distances and impute the missing values by closest. O(N^2 * #features)
+ - 'dist_missing': Compute distances between data and records with missing values. O(N*M * #features), assuming that the number of records with MV is M<<N.
+ - 'dist_sample': Compute distances between sample of data and records with missing values. O(S*M * #features) with M<<N and S<<N, but suboptimal imputation.
+ :param seed: Root seed value for random/sample calls for deterministic behavior. -1 for true randomization
+ :param sample_frac: Sample fraction for 'dist_sample' (value between 0 and 1)
+ :return: Imputed dataset
+ """
+
params_dict = {'X': X}
params_dict.update(kwargs)
return Matrix(X.sds_context,
diff --git a/src/main/python/systemds/operator/algorithm/builtin/mdedup.py b/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
index 85d93d5c2c..cbcc15d43b 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/mdedup.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
diff --git a/src/main/python/systemds/operator/algorithm/builtin/quantizeByCluster.py b/src/main/python/systemds/operator/algorithm/builtin/quantizeByCluster.py
new file mode 100644
index 0000000000..5afb96412b
--- /dev/null
+++ b/src/main/python/systemds/operator/algorithm/builtin/quantizeByCluster.py
@@ -0,0 +1,83 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+
+# Autogenerated By : src/main/python/generator/generator.py
+# Autogenerated From : scripts/builtin/quantizeByCluster.dml
+
+from typing import Dict, Iterable
+
+from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
+from systemds.utils.consts import VALID_INPUT_TYPES
+
+
+def quantizeByCluster(X: Matrix,
+ **kwargs: Dict[str, VALID_INPUT_TYPES]):
+ """
+ The quantizeByCluster-function implements product quantization. Initially, it
+ divides the original vector space into M subspaces. The resulting lower dimensional
+ subvectors are then quantized. If the column count is not divisible by the number of
+ subspaces M, the data is padded with zeros. Optimal space decomposition can be
+ computed, when the data follows a Gaussian distribution. The function uses kmeans for
+ quantizing and svd to compute the space decomposition.
+
+
+
+ :param X: The input matrix to perform product quantization on
+ :param M: Number of subspaces
+ :param k: Number of vectors in the subcodebooks
+ :param runs: Number of runs (with different initial centroids)
+ :param max_iter: Maximum number of iterations per run
+ :param eps: Tolerance (epsilon) for WCSS change ratio
+ :param avg_sample_size_per_centroid: Average number of records per centroid in data samples
+ :param separate: Cluster subspaces separately. If value is set to true,
+ kmeans is run M times, once for each subspace. Otherwise
+ kmeans is run only once.
+ :param space_decomp: Decompose the vector space by multiplying the input
+ matrix X with an orthogonal matrix R. Assumes the data
+ follows a parametric Gaussian distribution.
+ Time complexity in O(nrow(X)^2 * min(nrow(X), ncol(X))).
+ :param seed: The seed used for initial sampling. If set to -1 random
+ seeds are selected.
+ :return: The matrix containing the centroids. If clustered separately, the ith
+ subcodebook is the ith chunk of size k. The codebook matrix has the dimensions
+ [k*M x ncol(X)/M].
+ :return: The mapping of vectors to centroids. Each vector of the input matrix X is mapped
+ onto a vector of codes. The entries in the codes matrix are the indices of
+ the vectors in the codebook. The codes matrix has the dimensions [nrow(X) x M].
+ :return: The orthogonal matrix R which is applied to the input matrix X before performing
+ the product quantization. Only relevant when space_decomp = TRUE.
+ """
+
+ params_dict = {'X': X}
+ params_dict.update(kwargs)
+
+ vX_0 = Matrix(X.sds_context, '')
+ vX_1 = Matrix(X.sds_context, '')
+ vX_2 = Matrix(X.sds_context, '')
+ output_nodes = [vX_0, vX_1, vX_2, ]
+
+ op = MultiReturn(X.sds_context, 'quantizeByCluster', output_nodes, named_input_nodes=params_dict)
+
+ vX_0._unnamed_input_nodes = [op]
+ vX_1._unnamed_input_nodes = [op]
+ vX_2._unnamed_input_nodes = [op]
+
+ return op
diff --git a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
index 88b1c9145b..177ebd3fd3 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py
@@ -40,16 +40,17 @@ def randomForest(X: Matrix,
and optionally subset of features (columns). During tree construction, split
candidates are additionally chosen on a sample of remaining features.
- .. code-block::
+ .. code-block:: text
For example, given a feature matrix with features [a,b,c,d]
and the following two trees, M (the output) would look as follows:
(L1) |a<7| |d<5|
- / \ / \
+ / \\ / \\
(L2) |c<3| |b<4| |a<7| P3:2
- / \ / \ / \
+ / \\ / \\ / \\
(L3) P1:2 P2:1 P3:1 P4:2 P1:2 P2:1
+
--> M :=
[[1, 7, 3, 3, 2, 4, 0, 2, 0, 1, 0, 1, 0, 2], (1st tree)
[4, 5, 1, 7, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0]] (2nd tree)
diff --git a/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py b/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py
new file mode 100644
index 0000000000..42a0afb6e6
--- /dev/null
+++ b/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py
@@ -0,0 +1,78 @@
+# -------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# -------------------------------------------------------------
+
+# Autogenerated By : src/main/python/generator/generator.py
+# Autogenerated From : scripts/builtin/shapExplainer.dml
+
+from typing import Dict, Iterable
+
+from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
+from systemds.utils.consts import VALID_INPUT_TYPES
+
+
+def shapExplainer(model_function: str,
+ model_args: List,
+ x_instances: Matrix,
+ X_bg: Matrix,
+ **kwargs: Dict[str, VALID_INPUT_TYPES]):
+ """
+ Computes shapley values for multiple instances in parallel using antithetic permutation sampling.
+ The resulting matrix phis holds the shapley values for each feature in the column given by the index of the feature in the sample.
+
+ This method first creates two large matrices for masks and masked background data for all permutations and
+ then runs in paralell on all instances in x.
+ While the prepared matrices can become very large (2 * #features * #permuations * #n_samples * #features),
+ the preparation of a row for the model call breaks down to a single element-wise multiplication of this mask with the row and
+ an addition to the masked background data, since masks can be reused for each instance.
+
+
+
+ :param model_function: The function of the model to be evaluated as a String. This function has to take a matrix of samples
+ and return a vector of predictions.
+ It might be usefull to wrap the model into a function the takes and returns the desired shapes and
+ use this wrapper here.
+ :param model_args: Arguments in order for the model, if desired. This will be prepended by the created instances-matrix.
+ :param x_instances: Multiple instances as rows for which to compute the shapley values.
+ :param X_bg: The background dataset from which to pull the random samples to perform Monte Carlo integration.
+ :param n_permutations: The number of permutaions. Defaults to 10. Theoretical 1 should already be enough for models with up
+ to second order interaction effects.
+ :param n_samples: Number of samples from X_bg used for marginalization.
+ :param remove_non_var: EXPERIMENTAL: If set, for every instance the varaince of each feature is checked against this feature in the
+ background data. If it does not change, we do not run any model cals for it.
+ :param seed: A seed, in case the sampling has to be deterministic.
+ :param verbose: A boolean to enable logging of each step of the function.
+ :return: Matrix holding the shapley values along the cols, one row per instance.
+ :return: Double holding the average prediction of all instances.
+ """
+
+ params_dict = {'model_function': model_function, 'model_args': model_args, 'x_instances': x_instances, 'X_bg': X_bg}
+ params_dict.update(kwargs)
+
+ vX_0 = Matrix(model_function.sds_context, '')
+ vX_1 = Scalar(model_function.sds_context, '')
+ output_nodes = [vX_0, vX_1, ]
+
+ op = MultiReturn(model_function.sds_context, 'shapExplainer', output_nodes, named_input_nodes=params_dict)
+
+ vX_0._unnamed_input_nodes = [op]
+ vX_1._unnamed_input_nodes = [op]
+
+ return op
diff --git a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
index 16a20d20e0..270a6d7b16 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py
@@ -25,7 +25,6 @@
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
@@ -36,8 +35,39 @@ def topk_cleaning(dataTrain: Frame,
evalFunHp: Matrix,
**kwargs: Dict[str, VALID_INPUT_TYPES]):
"""
- This function cleans top-K item (where K is given as input)for a given list of users.
+ This function cleans top-K item (where K is given as input) for a given list of users.
metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask
+
+
+
+ :param dataTrain: Training set
+ :param dataTest: Test set ignored when cv is set to True
+ :param metaData: 3×n frame with schema, categorical mask, and FD mask for dataTrain
+ :param primitives: Library of primitive cleaning operators
+ :param parameters: Hyperparameter search space that matches the primitives
+ :param refSol: Reference solution
+ :param evaluationFunc: Name of a SystemDS DML function that scores a pipeline
+ :param evalFunHp: Hyperparameter matrix for the above evaluation function
+ :param topK: Number of best pipelines to return
+ :param resource_val: Maximum resource R for the Bandit search
+ :param max_iter: Maximum iterations while enumerating logical pipelines
+ :param lq: Lower quantile used by utils::doErrorSample when triggered
+ :param uq: Upper quantile used by utils::doErrorSample when triggered
+ :param sample: Fraction of rows to subsample from dataTrain
+ :param expectedIncrease: Minimum improvement over dirtyScore that a candidate must deliver
+ :param seed: Seed number
+ :param cv: TRUE means k-fold CV, FALSE means hold-out split
+ :param cvk: Number of folds if cv = TRUE
+ :param isLastLabel: TRUE if the last column is the label
+ :param rowCount: Row-count threshold above which doErrorSample may replace uniform sampling
+ :param correctTypos: Run spelling correction in the string preprocessing step
+ :param enablePruning: Enable pruning inside the Bandit phase
+ :return: K cleaned-data pipelines
+ :return: Hyperparameter matrix with rows aligning with topKPipelines
+ :return: Evaluation scores with rows aligning with topKPipelines
+ :return: Baseline score on the unclean data
+ :return: Updated evaluation function hyperparameters
+ :return: Frame of “apply” functions for deploying each of the top-K pipelines
"""
params_dict = {'dataTrain': dataTrain, 'primitives': primitives, 'parameters': parameters, 'evaluationFunc': evaluationFunc, 'evalFunHp': evalFunHp}
diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py b/src/main/python/systemds/operator/algorithm/builtin/wer.py
similarity index 69%
copy from src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
copy to src/main/python/systemds/operator/algorithm/builtin/wer.py
index cc8fe68aac..99d278461c 100644
--- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py
+++ b/src/main/python/systemds/operator/algorithm/builtin/wer.py
@@ -20,33 +20,29 @@
# -------------------------------------------------------------
# Autogenerated By : src/main/python/generator/generator.py
-# Autogenerated From : scripts/builtin/fixInvalidLengthsApply.dml
+# Autogenerated From : scripts/builtin/wer.dml
from typing import Dict, Iterable
from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar
-from systemds.script_building.dag import OutputType
from systemds.utils.consts import VALID_INPUT_TYPES
-def fixInvalidLengthsApply(X: Frame,
- mask: Matrix,
- qLow: Matrix,
- qUp: Matrix):
+def wer(R: Frame,
+ H: Frame):
"""
- Fix invalid lengths
+ This built-in function computes the word error rate (WER)
+ defined as wer = (numSubst + numDel + numIns) / length(r)
- :param X: ---
- :param mask: ---
- :param ql: ---
- :param qu: ---
- :return: ---
- :return: ---
+ :param R: Input frame of reference strings, shape: [N x 1]
+ :param H: Input frame of hypothesis strings, shape: [N x 1]
+ :return: Output matrix of word error rate per pair of strings,
+ shape: [N x 1], where W[i,1] = wer(R[i,1], H[i,1])
"""
- params_dict = {'X': X, 'mask': mask, 'qLow': qLow, 'qUp': qUp}
- return Matrix(X.sds_context,
- 'fixInvalidLengthsApply',
+ params_dict = {'R': R, 'H': H}
+ return Matrix(R.sds_context,
+ 'wer',
named_input_nodes=params_dict)