You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by ba...@apache.org on 2022/06/10 14:22:32 UTC
[systemds] 05/05: [DOCS] update python docs
This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
commit ce0a30781fade0c1c1dd1e507c99fa2d9d749bcf
Author: baunsgaard <ba...@tugraz.at>
AuthorDate: Fri Jun 10 16:21:01 2022 +0200
[DOCS] update python docs
---
docs/api/python/api/operator/algorithms.html | 3972 +++++++++++++++++++++----
docs/api/python/api/operator/node/scalar.html | 4 +-
docs/api/python/genindex.html | 181 +-
docs/api/python/guide/federated.html | 27 +
docs/api/python/objects.inv | Bin 1867 -> 2262 bytes
docs/api/python/searchindex.js | 2 +-
6 files changed, 3622 insertions(+), 564 deletions(-)
diff --git a/docs/api/python/api/operator/algorithms.html b/docs/api/python/api/operator/algorithms.html
index 5368ab2a5e..0655bc44d3 100644
--- a/docs/api/python/api/operator/algorithms.html
+++ b/docs/api/python/api/operator/algorithms.html
@@ -216,18 +216,75 @@
</pre></div>
</div>
<span class="target" id="module-systemds.operator.algorithm"></span><dl class="py function">
+<dt id="systemds.operator.algorithm.WoE">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">WoE</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:< [...]
+<dd><blockquote>
+<div><p>function Weight of evidence / information gain</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Y</strong> – <p>—</p>
+</p></li>
+<li><p><strong>mask</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Weighted X matrix where the entropy mask is applied</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>A entropy matrix to apply to data</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.WoEApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">WoEApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">entropyMatrix</span><spa [...]
+<dd><blockquote>
+<div><p>function Weight of evidence / information gain apply on new data</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Y</strong> – <p>—</p>
+</p></li>
+<li><p><strong>entropyMatrix</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Weighted X matrix where the entropy mask is applied</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
<dt id="systemds.operator.algorithm.abstain">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">abstain</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><span cla [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This function calls the multiLogReg-function in which solves Multinomial
+Logistic Regression using Trust Region method</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>threshold</strong> – <p>—</p>
-</p></li>
+<li><p><strong>X</strong> – matrix of feature vectors</p></li>
+<li><p><strong>Y</strong> – matrix with category labels</p></li>
+<li><p><strong>threshold</strong> – threshold to clear otherwise return X and Y unmodified</p></li>
<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>abstained output X</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>abstained output Y</p>
</dd>
</dl>
</dd></dl>
@@ -235,21 +292,39 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.als">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">als</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DAG [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
+using different implementations of the Alternating-Least-Squares (ALS) algorithm.
+Matrices U and V are computed by minimizing a loss function (with regularization).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Location to read the input matrix X to be factorized</p></li>
<li><p><strong>rank</strong> – Rank of the factorization</p></li>
-<li><p><strong>reg</strong> – Regularization:</p></li>
-<li><p><strong>lambda</strong> – Regularization parameter, no regularization if 0.0</p></li>
+<li><p><strong>regType</strong> – Regularization:
+“L2” = L2 regularization;
+f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
++ 0.5 * reg * (sum (U ^ 2) + sum (V ^ 2))
+“wL2” = weighted L2 regularization
+f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
++ 0.5 * reg * (sum (U ^ 2 * row_nonzeros)
++ sum (V ^ 2 * col_nonzeros))</p></li>
+<li><p><strong>reg</strong> – Regularization parameter, no regularization if 0.0</p></li>
<li><p><strong>maxi</strong> – Maximum number of iterations</p></li>
<li><p><strong>check</strong> – Check for convergence after every iteration, i.e., updating U and V once</p></li>
-<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared</p></li>
-<li><p><strong>if</strong> – in loss in any two consecutive iterations falls below this threshold;</p></li>
-<li><p><strong>if</strong> – FALSE thr is ignored</p></li>
+<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared
+if the decrease in loss in any two consecutive iterations falls below this threshold;
+if check is FALSE thr is ignored</p></li>
+<li><p><strong>seed</strong> – The seed to random parts of the algorithm</p></li>
+<li><p><strong>verbose</strong> – If the algorithm should run verbosely</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing m x r matrix where r is the factorization rank & m x r matrix where r is the factorization rank</p>
+<dd class="field-even"><p>An m x r matrix where r is the factorization rank</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>An m x r matrix where r is the factorization rank</p>
</dd>
</dl>
</dd></dl>
@@ -257,21 +332,39 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.alsCG">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">alsCG</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>D [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script computes an approximate factorization of a low-rank matrix X into two matrices U and V
+using the Alternating-Least-Squares (ALS) algorithm with conjugate gradient.
+Matrices U and V are computed by minimizing a loss function (with regularization).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Location to read the input matrix X to be factorized</p></li>
<li><p><strong>rank</strong> – Rank of the factorization</p></li>
-<li><p><strong>reg</strong> – Regularization:</p></li>
-<li><p><strong>lambda</strong> – Regularization parameter, no regularization if 0.0</p></li>
+<li><p><strong>regType</strong> – Regularization:
+“L2” = L2 regularization;
+f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
++ 0.5 * reg * (sum (U ^ 2) + sum (V ^ 2))
+“wL2” = weighted L2 regularization
+f (U, V) = 0.5 * sum (W * (U %*% V - X) ^ 2)
++ 0.5 * reg * (sum (U ^ 2 * row_nonzeros)
++ sum (V ^ 2 * col_nonzeros))</p></li>
+<li><p><strong>reg</strong> – Regularization parameter, no regularization if 0.0</p></li>
<li><p><strong>maxi</strong> – Maximum number of iterations</p></li>
<li><p><strong>check</strong> – Check for convergence after every iteration, i.e., updating U and V once</p></li>
-<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared</p></li>
-<li><p><strong>if</strong> – in loss in any two consecutive iterations falls below this threshold;</p></li>
-<li><p><strong>if</strong> – FALSE thr is ignored</p></li>
+<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared
+if the decrease in loss in any two consecutive iterations falls below this threshold;
+if check is FALSE thr is ignored</p></li>
+<li><p><strong>seed</strong> – The seed to random parts of the algorithm</p></li>
+<li><p><strong>verbose</strong> – If the algorithm should run verbosely</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>An m x r matrix where r is the factorization rank</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>An m x r matrix where r is the factorization rank</p>
</dd>
</dl>
</dd></dl>
@@ -279,20 +372,55 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.alsDS">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">alsDS</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>D [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Alternating-Least-Squares (ALS) algorithm using a direct solve method for
+individual least squares problems (reg=”L2”). This script computes an
+approximate factorization of a low-rank matrix V into two matrices L and R.
+Matrices L and R are computed by minimizing a loss function (with regularization).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Location to read the input matrix V to be factorized</p></li>
<li><p><strong>rank</strong> – Rank of the factorization</p></li>
-<li><p><strong>lambda</strong> – Regularization parameter, no regularization if 0.0</p></li>
+<li><p><strong>reg</strong> – Regularization parameter, no regularization if 0.0</p></li>
<li><p><strong>maxi</strong> – Maximum number of iterations</p></li>
<li><p><strong>check</strong> – Check for convergence after every iteration, i.e., updating L and R once</p></li>
-<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared</p></li>
-<li><p><strong>if</strong> – in loss in any two consecutive iterations falls below this threshold;</p></li>
-<li><p><strong>if</strong> – FALSE thr is ignored</p></li>
+<li><p><strong>thr</strong> – Assuming check is set to TRUE, the algorithm stops and convergence is declared
+if the decrease in loss in any two consecutive iterations falls below this threshold;
+if check is FALSE thr is ignored</p></li>
+<li><p><strong>seed</strong> – The seed to random parts of the algorithm</p></li>
+<li><p><strong>verbose</strong> – If the algorithm should run verbosely</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>An m x r matrix where r is the factorization rank</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>An m x r matrix where r is the factorization rank</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.alsPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">alsPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">userIDs</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">I</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">L</span><span cl [...]
+<dd><blockquote>
+<div><p>This script computes the rating/scores for a given list of userIDs
+using 2 factor matrices L and R. We assume that all users have rates
+at least once and all items have been rates at least once.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>userIDs</strong> – Column vector of user-ids (n x 1)</p></li>
+<li><p><strong>I</strong> – Indicator matrix user-id x user-id to exclude from scoring</p></li>
+<li><p><strong>L</strong> – The factor matrix L: user-id x feature-id</p></li>
+<li><p><strong>R</strong> – The factor matrix R: feature-id x item-id</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The output user-id/item-id/score#</p>
</dd>
</dl>
</dd></dl>
@@ -300,12 +428,66 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.alsTopkPredict">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">alsTopkPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">userIDs</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">I</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">L</span><spa [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script computes the top-K rating/scores for a given list of userIDs
+using 2 factor matrices L and R. We assume that all users have rates
+at least once and all items have been rates at least once.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>userIDs</strong> – Column vector of user-ids (n x 1)</p></li>
+<li><p><strong>I</strong> – Indicator matrix user-id x user-id to exclude from scoring</p></li>
+<li><p><strong>L</strong> – The factor matrix L: user-id x feature-id</p></li>
+<li><p><strong>R</strong> – The factor matrix R: feature-id x item-id</p></li>
+<li><p><strong>K</strong> – The number of top-K items</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>A matrix containing the top-K item-ids with highest predicted ratings for the specified users (rows)</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>A matrix containing the top-K predicted ratings for the specified users (rows)</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.apply_pipeline">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">apply_pipeline</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">testData</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">pip</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">applyFunc</sp [...]
+<dd><blockquote>
+<div><p>This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
+and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
+in terms of classification accuracy</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>K</strong> – The number of top-K items</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>trainData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>testData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>metaData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>lp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>pip</strong> – <p>—</p>
+</p></li>
+<li><p><strong>hp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evaluationFunc</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evalFunHp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>isLastLabel</strong> – <p>—</p>
+</p></li>
+<li><p><strong>correctTypos</strong> – <p>—</p>
+</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p><p>—</p>
+</p>
</dd>
</dl>
</dd></dl>
@@ -313,9 +495,15 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.arima">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">arima</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>D [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function that implements ARIMA</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to apply Arima on.</p></li>
+<li><p><strong>max_func_invoc</strong> – <p>—</p>
+</p></li>
<li><p><strong>p</strong> – non-seasonal AR order</p></li>
<li><p><strong>d</strong> – non-seasonal differencing order</p></li>
<li><p><strong>q</strong> – non-seasonal MA order</p></li>
@@ -328,7 +516,125 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The calculated coefficients</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.autoencoder_2layer">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">autoencoder_2layer</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">num_hidden1</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">num_hidden2</span><span class="p">:</s [...]
+<dd><blockquote>
+<div><p>Trains a 2-layer autoencoder with minibatch SGD and step-size decay.
+If invoked with H1 > H2 then it becomes a ‘bowtie’ structured autoencoder
+Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
+The script standardizes the input before training (can be turned off).
+Also, it randomly reshuffles rows before training.
+Currently, tanh is set to be the activation function.
+By re-implementing ‘func’ DML-bodied function, one can change the activation.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Filename where the input is stored</p></li>
+<li><p><strong>num_hidden1</strong> – Number of neurons in the 1st hidden layer</p></li>
+<li><p><strong>num_hidden2</strong> – Number of neurons in the 2nd hidden layer</p></li>
+<li><p><strong>max_epochs</strong> – Number of epochs to train for</p></li>
+<li><p><strong>full_obj</strong> – If TRUE, Computes objective function value (squared-loss)
+at the end of each epoch. Note that, computing the full
+objective can take a lot of time.</p></li>
+<li><p><strong>batch_size</strong> – Mini-batch size (training parameter)</p></li>
+<li><p><strong>step</strong> – Initial step size (training parameter)</p></li>
+<li><p><strong>decay</strong> – Decays step size after each epoch (training parameter)</p></li>
+<li><p><strong>mu</strong> – Momentum parameter (training parameter)</p></li>
+<li><p><strong>W1_rand</strong> – Weights might be initialized via input matrices</p></li>
+<li><p><strong>W2_rand</strong> – <p>—</p>
+</p></li>
+<li><p><strong>W3_rand</strong> – <p>—</p>
+</p></li>
+<li><p><strong>W4_rand</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix storing weights between input layer and 1st hidden layer</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix storing bias between input layer and 1st hidden layer</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix storing weights between 1st hidden layer and 2nd hidden layer</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix storing bias between 1st hidden layer and 2nd hidden layer</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix storing weights between 2nd hidden layer and 3rd hidden layer</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix storing bias between 2nd hidden layer and 3rd hidden layer</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix storing weights between 3rd hidden layer and output layer</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix storing bias between 3rd hidden layer and output layer</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix storing the hidden (2nd) layer representation if needed</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.bandit">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">bandit</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">X_test</span>< [...]
+<dd><blockquote>
+<div><p>In The bandit function the objective is to find an arm that optimizes
+a known functional of the unknown arm-reward distributions.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X_train</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Y_train</strong> – <p>—</p>
+</p></li>
+<li><p><strong>X_test</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Y_test</strong> – <p>—</p>
+</p></li>
+<li><p><strong>metaList</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evaluationFunc</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evalFunHp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>lp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>primitives</strong> – <p>—</p>
+</p></li>
+<li><p><strong>params</strong> – <p>—</p>
+</p></li>
+<li><p><strong>K</strong> – <p>—</p>
+</p></li>
+<li><p><strong>R</strong> – <p>—</p>
+</p></li>
+<li><p><strong>baseLineScore</strong> – <p>—</p>
+</p></li>
+<li><p><strong>cv</strong> – <p>—</p>
+</p></li>
+<li><p><strong>cvk</strong> – <p>—</p>
+</p></li>
+<li><p><strong>verbose</strong> – <p>—</p>
+</p></li>
+<li><p><strong>output</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
</dd>
</dl>
</dd></dl>
@@ -336,12 +642,33 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.bivar">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">bivar</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">S1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">S2</span><span class="p">: [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>For a given pair of attribute sets, compute bivariate statistics between all attribute pairs.
+Given, index1 = {A_11, A_12, … A_1m} and index2 = {A_21, A_22, … A_2n}
+compute bivariate stats for m*n pairs (A_1i, A_2j), (1<= i <=m) and (1<= j <=n).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>verbose</strong> – Print bivar stats</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input matrix</p></li>
+<li><p><strong>S1</strong> – First attribute set {A_11, A_12, … A_1m}</p></li>
+<li><p><strong>S2</strong> – Second attribute set {A_21, A_22, … A_2n}</p></li>
+<li><p><strong>T1</strong> – Kind for attributes in S1
+(kind=1 for scale, kind=2 for nominal, kind=3 for ordinal)</p></li>
+<li><p><strong>verbose</strong> – Print bivar stats</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>basestats_scale_scale as output with bivar stats</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>basestats_nominal_scale as output with bivar stats</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing as output with bivar stats & as output with bivar stats & as output with bivar stats & as output with bivar stats</p>
+<dd class="field-even"><p>basestats_nominal_nominal as output with bivar stats</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>basestats_ordinal_ordinal as output with bivar stats</p>
</dd>
</dl>
</dd></dl>
@@ -349,12 +676,20 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.components">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">components</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">G</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</s [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Computes the connected components of a graph and returns a
+vector indicating the assignment of vertices to components,
+where each component is identified by the maximum vertex ID
+(i.e., row/column position of the input graph)</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>X</strong> – Location to read the matrix of feature vectors</p></li>
<li><p><strong>Y</strong> – Location to read the matrix with category labels</p></li>
-<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling X columns: 0 = no intercept,</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling X columns: 0 = no intercept,
+no shifting, no rescaling; 1 = add intercept, but neither shift nor rescale X;
+2 = add intercept, shift & rescale X columns to mean = 0, variance = 1</p></li>
<li><p><strong>tol</strong> – tolerance (“epsilon”)</p></li>
<li><p><strong>reg</strong> – regularization parameter (lambda = 1/C); intercept is not regularized</p></li>
<li><p><strong>maxi</strong> – max. number of outer (Newton) iterations</p></li>
@@ -363,7 +698,7 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>regression betas as output for prediction</p>
</dd>
</dl>
</dd></dl>
@@ -371,12 +706,116 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.confusionMatrix">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">confusionMatrix</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">P</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" hre [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Accepts a vector for prediction and a one-hot-encoded matrix
+Then it computes the max value of each vector and compare them
+After which, it calculates and returns the sum of classifications
+and the average of each true class.</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span> True Labels
+ 1 2
+ 1 TP | FP
+Predictions ----+----
+ 2 FN | TN
+</pre></div>
+</div>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>P</strong> – vector of Predictions</p></li>
+<li><p><strong>Y</strong> – vector of Golden standard One Hot Encoded; the one hot
+encoded vector of actual labels</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The Confusion Matrix Sums of classifications</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>The Confusion Matrix averages of each true class</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.cor">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">cor</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.cor" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This Function compute correlation matrix</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>X</strong> – A Matrix Input to compute the correlation on</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Correlation matrix of the input matrix</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.correctTypos">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">correctTypos</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">strings</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p [...]
+<dd><blockquote>
+<div><p>Corrects corrupted frames of strings
+This algorithm operates on the assumption that most strings are correct
+and simply swaps strings that do not occur often with similar strings that
+occur more often</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>References:
+Fred J. Damerau. 1964.
+ A technique for computer detection and correction of spelling errors.
+ Commun. ACM 7, 3 (March 1964), 171–176.
+ DOI:https://doi.org/10.1145/363958.363994
+</pre></div>
+</div>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>encoded</strong> – actual labels</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>strings</strong> – The nx1 input frame of corrupted strings</p></li>
+<li><p><strong>frequency_threshold</strong> – Strings that occur above this frequency level will not be corrected</p></li>
+<li><p><strong>distance_threshold</strong> – Max distance at which strings are considered similar</p></li>
+<li><p><strong>is_verbose</strong> – Print debug information</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Corrected nx1 output frame</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.correctTyposApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">correctTyposApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">strings</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">distance_matrix</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class=" [...]
+<dd><blockquote>
+<div><p>Corrects corrupted frames of strings
+This algorithm operates on the assumption that most strings are correct
+and simply swaps strings that do not occur often with similar strings that
+occur more often</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>References:
+Fred J. Damerau. 1964.
+ A technique for computer detection and correction of spelling errors.
+ Commun. ACM 7, 3 (March 1964), 171–176.
+ DOI:https://doi.org/10.1145/363958.363994
+</pre></div>
+</div>
+<p>TODO: future: add parameter for list of words that are sure to be correct</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>strings</strong> – The nx1 input frame of corrupted strings</p></li>
+<li><p><strong>nullMask</strong> – <p>—</p>
+</p></li>
+<li><p><strong>frequency_threshold</strong> – Strings that occur above this frequency level will not be corrected</p></li>
+<li><p><strong>distance_threshold</strong> – Max distance at which strings are considered similar</p></li>
+<li><p><strong>matrix</strong> (<em>distance</em>) – <p>—</p>
+</p></li>
+<li><p><strong>dict</strong> – <p>—</p>
+</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Corrected nx1 output frame</p>
</dd>
</dl>
</dd></dl>
@@ -384,19 +823,32 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.cox">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">cox</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">TE</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">F</span><span class="p">:</s [...]
-<dd><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>X</strong> – Location to read the input matrix X containing the survival data</p></li>
-<li><p><strong>containing</strong> – information</p></li>
-<li><p><strong>TE</strong> – Column indices of X as a column vector which contain timestamp</p></li>
-<li><p><strong>F</strong> – Column indices of X as a column vector which are to be used for</p></li>
-<li><p><strong>fitting</strong> – model</p></li>
-<li><p><strong>R</strong> – If factors (categorical variables) are available in the input matrix</p></li>
-<li><p><strong>the</strong> – X</p></li>
-<li><p><strong>each</strong> – needs to be removed from X; in this case the start</p></li>
-<li><p><strong>and</strong> – corresponding to the baseline level need to be the same;</p></li>
-<li><p><strong>if</strong> – not provided by default all variables are considered to be continuous</p></li>
+<dd><blockquote>
+<div><p>This script fits a cox Proportional hazard regression model.
+The Breslow method is used for handling ties and the regression parameters
+are computed using trust region newton method with conjugate gradient</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Location to read the input matrix X containing the survival data
+containing the following information
+1: timestamps
+2: whether an event occurred (1) or data is censored (0)
+3: feature vectors</p></li>
+<li><p><strong>TE</strong> – Column indices of X as a column vector which contain timestamp
+(first row) and event information (second row)</p></li>
+<li><p><strong>F</strong> – Column indices of X as a column vector which are to be used for
+fitting the Cox model</p></li>
+<li><p><strong>R</strong> – If factors (categorical variables) are available in the input matrix
+X, location to read matrix R containing the start and end indices of
+the factors in X
+R[,1]: start indices
+R[,2]: end indices
+Alternatively, user can specify the indices of the baseline level of
+each factor which needs to be removed from X; in this case the start
+and end indices corresponding to the baseline level need to be the same;
+if R is not provided by default all variables are considered to be continuous</p></li>
<li><p><strong>alpha</strong> – Parameter to compute a 100*(1-alpha)% confidence interval for the betas</p></li>
<li><p><strong>tol</strong> – Tolerance (“epsilon”)</p></li>
<li><p><strong>moi</strong> – Max. number of outer (Newton) iterations</p></li>
@@ -404,7 +856,35 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing matrix rt that contains the order-preserving recoded timestamps from x & which is matrix x with sorted timestamps & matrix mf that contains the column indices of x with the baseline factors removed (if available)</p>
+<dd class="field-even"><p>A D x 7 matrix M, where D denotes the number of covariates, with the following schema:
+M[,1]: betas
+M[,2]: exp(betas)
+M[,3]: standard error of betas
+M[,4]: Z
+M[,5]: P-value
+M[,6]: lower 100*(1-alpha)% confidence interval of betas
+M[,7]: upper 100*(1-alpha)% confidence interval of betas</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Two matrices containing a summary of some statistics of the fitted model:
+1 - File S with the following format
+- row 1: no. of observations
+- row 2: no. of events
+- row 3: log-likelihood
+- row 4: AIC
+- row 5: Rsquare (Cox & Snell)
+- row 6: max possible Rsquare
+2 - File T with the following format
+- row 1: Likelihood ratio test statistic, degree of freedom, P-value
+- row 2: Wald test statistic, degree of freedom, P-value
+- row 3: Score (log-rank) test statistic, degree of freedom, P-value</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Additionally, the following matrices are stored (needed for prediction)
+1- A column matrix RT that contains the order-preserving recoded timestamps from X
+2- Matrix XO which is matrix X with sorted timestamps
+3- Variance-covariance matrix of the betas COV
+4- A column matrix MF that contains the column indices of X with the baseline factors removed (if available)</p>
</dd>
</dl>
</dd></dl>
@@ -412,19 +892,29 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.cspline">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">cspline</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">inp_x</span><span class=" [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Solves Cubic Spline Interpolation</p>
+<p>Algorithms: implement <a class="reference external" href="https://en.wikipedia.org/wiki/Spline_interpolation#Algorithm_to_find_the_interpolating_cubic_spline">https://en.wikipedia.org/wiki/Spline_interpolation#Algorithm_to_find_the_interpolating_cubic_spline</a>
+It use natural spline with q1’’(x0) == qn’’(xn) == 0.0</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>monotonically</strong> – there is no duplicates points in X</p></li>
+<li><p><strong>X</strong> – 1-column matrix of x values knots. It is assumed that x values are
+monotonically increasing and there is no duplicates points in X</p></li>
+<li><p><strong>Y</strong> – 1-column matrix of corresponding y values knots</p></li>
<li><p><strong>inp_x</strong> – the given input x, for which the cspline will find predicted y</p></li>
<li><p><strong>mode</strong> – Specifies the method for cspline (DS - Direct Solve, CG - Conjugate Gradient)</p></li>
-<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate graduent procedure terminates early if</p></li>
-<li><p><strong>L2</strong> – the beta-residual is less than tolerance * its initial norm</p></li>
+<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate graduent procedure terminates early if
+L2 norm of the beta-residual is less than tolerance * its initial norm</p></li>
<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations, 0 = no maximum</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Predicted value</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix of k parameters</p>
</dd>
</dl>
</dd></dl>
@@ -432,18 +922,26 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.csplineCG">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">csplineCG</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">inp_x</span><span class [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin that solves cubic spline interpolation using conjugate gradient algorithm</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>monotonically</strong> – there is no duplicates points in X</p></li>
+<li><p><strong>X</strong> – 1-column matrix of x values knots. It is assumed that x values are
+monotonically increasing and there is no duplicates points in X</p></li>
+<li><p><strong>Y</strong> – 1-column matrix of corresponding y values knots</p></li>
<li><p><strong>inp_x</strong> – the given input x, for which the cspline will find predicted y.</p></li>
-<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate graduent procedure terminates early if</p></li>
-<li><p><strong>L2</strong> – the beta-residual is less than tolerance * its initial norm</p></li>
+<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate graduent procedure terminates early if
+L2 norm of the beta-residual is less than tolerance * its initial norm</p></li>
<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations, 0 = no maximum</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Predicted value</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix of k parameters</p>
</dd>
</dl>
</dd></dl>
@@ -451,15 +949,23 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.csplineDS">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">csplineDS</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">inp_x</span><span class [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin that solves cubic spline interpolation using a direct solver.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>monotonically</strong> – there is no duplicates points in X</p></li>
+<li><p><strong>X</strong> – 1-column matrix of x values knots. It is assumed that x values are
+monotonically increasing and there is no duplicates points in X</p></li>
+<li><p><strong>Y</strong> – 1-column matrix of corresponding y values knots</p></li>
<li><p><strong>inp_x</strong> – the given input x, for which the cspline will find predicted y.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Predicted value</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix of k parameters</p>
</dd>
</dl>
</dd></dl>
@@ -467,16 +973,27 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.cvlm">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">cvlm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">k</span><span class="p">:</s [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The cvlm-function is used for cross-validation of the provided data model. This function follows a non-exhaustive cross
+validation method. It uses lm and lmPredict functions to solve the linear regression and to predict the class of a
+feature vector with no intercept, shifting, and rescaling.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Recorded Data set into matrix</p></li>
+<li><p><strong>y</strong> – 1-column matrix of response values.</p></li>
<li><p><strong>k</strong> – Number of subsets needed, It should always be more than 1 and less than nrow(X)</p></li>
<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
-<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero for</p></li>
+<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero for
+highly dependant/sparse/numerous features</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Response values</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Validated data set</p>
</dd>
</dl>
</dd></dl>
@@ -484,15 +1001,20 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.dbscan">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">dbscan</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span> [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Implements the DBSCAN clustering algorithm using Euclidian distance matrix</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to do DBSCAN on.</p></li>
<li><p><strong>eps</strong> – Maximum distance between two points for one to be considered reachable for the other.</p></li>
-<li><p><strong>minPts</strong> – Number of points in a neighborhood for a point to be considered as a core point</p></li>
+<li><p><strong>minPts</strong> – Number of points in a neighborhood for a point to be considered as a core point
+(includes the point itself).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>clustering Matrix</p>
</dd>
</dl>
</dd></dl>
@@ -500,31 +1022,61 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.dbscanApply">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">dbscanApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">clusterModel</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">eps</span> [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Implements the outlier detection/prediction algorithm using a DBScan model</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>eps</strong> – Maximum distance between two points for one to be considered reachable for the other.</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to do outlier detection on.</p></li>
+<li><p><strong>clusterModel</strong> – Model of clusters to predict outliers against.</p></li>
+<li><p><strong>eps</strong> – Maximum distance between two points for one to be considered reachable for the other.</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Predicted outliers</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
<dt id="systemds.operator.algorithm.decisionTree">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">decisionTree</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">R</span><span class= [...]
-<dd><dl class="field-list simple">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">decisionTree</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">R</span><span class= [...]
+<dd><blockquote>
+<div><p>Builtin script implementing classification trees with scale and categorical features</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>a</strong> – vector, other positive Integers indicate the number of categories</p></li>
-<li><p><strong>If</strong> – not provided by default all variables are assumed to be scale</p></li>
+<li><p><strong>X</strong> – Feature matrix X; note that X needs to be both recoded and dummy coded</p></li>
+<li><p><strong>Y</strong> – Label matrix Y; note that Y needs to be both recoded and dummy coded</p></li>
+<li><p><strong>R</strong> – Matrix R which for each feature in X contains the following information
+- R[1,]: Row Vector which indicates if feature vector is scalar or categorical. 1 indicates
+a scalar feature vector, other positive Integers indicate the number of categories
+If R is not provided by default all variables are assumed to be scale</p></li>
<li><p><strong>bins</strong> – Number of equiheight bins per scale feature to choose thresholds</p></li>
<li><p><strong>depth</strong> – Maximum depth of the learned tree</p></li>
<li><p><strong>verbose</strong> – boolean specifying if the algorithm should print information while executing</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing information: & if the feature is categorical) & looks at if j is an internal node, otherwise 0 & as r input vector & of the subset of values & 6,7,… if j is categorical & a leaf node: number of misclassified samples reaching at node j & at m[6,j] if the feature chosen for j is scale, & feature chosen for j is categorical rows 6,7,… depict the value subset chosen for j & a leaf node 1 if j is impure a [...]
+<dd class="field-even"><p>Matrix M where each column corresponds to a node in the learned tree and each row
+contains the following information:
+M[1,j]: id of node j (in a complete binary tree)
+M[2,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
+M[3,j]: Feature index of the feature (scale feature id if the feature is scale or
+categorical feature id if the feature is categorical)
+that node j looks at if j is an internal node, otherwise 0
+M[4,j]: Type of the feature that node j looks at if j is an internal node: holds
+the same information as R input vector
+M[5,j]: If j is an internal node: 1 if the feature chosen for j is scale,
+otherwise the size of the subset of values
+stored in rows 6,7,… if j is categorical
+If j is a leaf node: number of misclassified samples reaching at node j
+M[6:,j]: If j is an internal node: Threshold the example’s feature value is compared
+to is stored at M[6,j] if the feature chosen for j is scale,
+otherwise if the feature chosen for j is categorical rows 6,7,… depict the value subset chosen for j
+If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0</p>
</dd>
</dl>
</dd></dl>
@@ -532,24 +1084,37 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.decisionTreePredict">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">decisionTreePredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">M</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">strategy</spa [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin script implementing prediction based on classification trees with scale features using prediction methods of the
+Hummingbird paper (<a class="reference external" href="https://www.usenix.org/system/files/osdi20-nakandala.pdf">https://www.usenix.org/system/files/osdi20-nakandala.pdf</a>).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>to</strong> – in the learned tree and each row contains the following information:</p></li>
-<li><p><strong>categorical</strong> – if the feature is categorical)</p></li>
-<li><p><strong>that</strong> – looks at if j is an internal node, otherwise 0</p></li>
-<li><p><strong>the</strong> – as R input vector</p></li>
-<li><p><strong>otherwise</strong> – of the subset of values</p></li>
-<li><p><strong>stored</strong> – 6,7,… if j is categorical</p></li>
-<li><p><strong>If</strong> – a leaf node: number of misclassified samples reaching at node j</p></li>
-<li><p><strong>to</strong> – at M[6,j] if the feature chosen for j is scale,</p></li>
-<li><p><strong>otherwise</strong> – feature chosen for j is categorical rows 6,7,… depict the value subset chosen for j</p></li>
-<li><p><strong>If</strong> – a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0</p></li>
-<li><p><strong>strategy</strong> – strategy, can be one of [“GEMM”, “TT”, “PTT”], referring to “Generic matrix multiplication”,</p></li>
+<li><p><strong>M</strong> – Decision tree matrix M, as generated by scripts/builtin/decisionTree.dml, where each column corresponds
+to a node in the learned tree and each row contains the following information:
+M[1,j]: id of node j (in a complete binary tree)
+M[2,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
+M[3,j]: Feature index of the feature (scale feature id if the feature is scale or
+categorical feature id if the feature is categorical)
+that node j looks at if j is an internal node, otherwise 0
+M[4,j]: Type of the feature that node j looks at if j is an internal node: holds
+the same information as R input vector
+M[5,j]: If j is an internal node: 1 if the feature chosen for j is scale,
+otherwise the size of the subset of values
+stored in rows 6,7,… if j is categorical
+If j is a leaf node: number of misclassified samples reaching at node j
+M[6:,j]: If j is an internal node: Threshold the example’s feature value is compared
+to is stored at M[6,j] if the feature chosen for j is scale,
+otherwise if the feature chosen for j is categorical rows 6,7,… depict the value subset chosen for j
+If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0</p></li>
+<li><p><strong>X</strong> – Feature matrix X</p></li>
+<li><p><strong>strategy</strong> – Prediction strategy, can be one of [“GEMM”, “TT”, “PTT”], referring to “Generic matrix multiplication”,
+“Tree traversal”, and “Perfect tree traversal”, respectively</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix containing the predicted labels for X</p>
</dd>
</dl>
</dd></dl>
@@ -557,9 +1122,13 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.deepWalk">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">deepWalk</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">Graph</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">w</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">d</span><span class="p">:</span> <span class="n">int</ [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script performs DeepWalk on a given graph (<a class="reference external" href="https://arxiv.org/pdf/1403.6652.pdf">https://arxiv.org/pdf/1403.6652.pdf</a>)</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>Graph</strong> – adjacency matrix of a graph (n x n)</p></li>
<li><p><strong>w</strong> – window size</p></li>
<li><p><strong>d</strong> – embedding size</p></li>
<li><p><strong>gamma</strong> – walks per vertex</p></li>
@@ -569,149 +1138,733 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>matrix of vertex/word representation (n x d)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.discoverFD">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">discoverFD</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><sp [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.denialConstraints">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">denialConstraints</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">dataFrame</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">constraintsFrame</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em><span class="sig-paren">)</span><a c [...]
+<dd><blockquote>
+<div><p>This function considers some constraints indicating statements that can NOT happen in the data (denial constraints).</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>EXAMPLE:
+dataFrame:
+
+ rank discipline yrs.since.phd yrs.service sex salary
+1 Prof B 19 18 Male 139750
+2 Prof B 20 16 Male 173200
+3 AsstProf B 3 3 Male 79750.56
+4 Prof B 45 39 Male 115000
+5 Prof B 40 40 Male 141500
+6 AssocProf B 6 6 Male 97000
+7 Prof B 30 23 Male 175000
+8 Prof B 45 45 Male 147765
+9 Prof B 21 20 Male 119250
+10 Prof B 18 18 Female 129000
+11 AssocProf B 12 8 Male 119800
+12 AsstProf B 7 2 Male 79800
+13 AsstProf B 1 1 Male 77700
+
+constraintsFrame:
+
+idx constraint.type group.by group.variable group.option variable1 relation variable2
+1 variableCompare FALSE yrs.since.phd < yrs.service
+2 instanceCompare TRUE rank Prof yrs.service >< salary
+3 valueCompare FALSE salary = 78182
+4 variableCompare TRUE discipline B yrs.service > yrs.since.phd
+</pre></div>
+</div>
+<p>Example: explanation of constraint 2 –> it can’t happen that one professor of rank Prof has more years of service than other, but lower salary.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>will</strong> – second column from processing</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>dataFrame</strong> – frame which columns represent the variables of the data and the rows correspond
+to different tuples or instances.
+Recommended to have a column indexing the instances from 1 to N (N=number of instances).</p></li>
+<li><p><strong>constraintsFrame</strong> – frame with fixed columns and each row representing one constraint.
+1. idx: (double) index of the constraint, from 1 to M (number of constraints)
+2. constraint.type: (string) The constraints can be of 3 different kinds:
+- variableCompare: for each instance, it will compare the values of two variables (with a relation <, > or =).
+- valueCompare: for each instance, it will compare a fixed value and a variable value (with a relation <, > or =).
+- instanceCompare: for every couple of instances, it will compare the relation between two variables,
+ie if the value of the variable 1 in instance 1 is lower/higher than the value of variable 1 in instance 2,
+then the value of of variable 2 in instance 2 can’t be lower/higher than the value of variable 2 in instance 2.
+3. group.by: (boolean) if TRUE only one group of data (defined by a variable option) will be considered for the constraint.
+4. group.variable: (string, only if group.by TRUE) name of the variable (column in dataFrame) that will divide our data in groups.
+5. group.option: (only if group.by TRUE) option of the group.variable that defines the group to consider.
+6. variable1: (string) first variable to compare (name of column in dataFrame).
+7. relation: (string) can be < , > or = in the case of variableCompare and valueCompare, and < >, < < , > < or > >
+in the case of instanceCompare
+8. variable2: (string) second variable to compare (name of column in dataFrame) or fixed value for the case of valueCompare.</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix of 2 columns.
+- First column shows the indexes of dataFrame that are wrong.
+- Second column shows the index of the denial constraint that is fulfilled
+If there are no wrong instances to show (0 constrains fulfilled) –> WrongInstances=matrix(0,1,2)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.executePipeline">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">executePipeline</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.executePipeline" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.discoverFD">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">discoverFD</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><sp [...]
+<dd><blockquote>
+<div><p>Implements builtin for finding functional dependencies</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>flagsCount</strong> – <p>—</p>
-</p></li>
-<li><p><strong>test</strong> – <p>—</p>
-</p></li>
+<li><p><strong>X</strong> – Input Matrix X, encoded Matrix if data is categorical</p></li>
+<li><p><strong>Mask</strong> – A row vector for interested features i.e. Mask =[1, 0, 1]
+will exclude the second column from processing</p></li>
+<li><p><strong>threshold</strong> – threshold value in interval [0, 1] for robust FDs</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing validation check & convert the matrix row-vector into list & flag & append flag & append flag & append flag & of hyper-parameters and loop till that & flag & and remove categorical & and remove numerics & + 1 for nan replacement & matrix & matrix & ohe call, to call inside eval as a function & encoding of categorical features & features & ohe call, to call inside eval as [...]
+<dd class="field-even"><p>matrix of functional dependencies</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.ffTrain">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ffTrain</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">out_activation</span><spa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.dist">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">dist</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.dist" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>Returns Euclidean distance matrix (distances between N n-dimensional points)</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>batch_size</strong> – Batch size</p></li>
-<li><p><strong>epochs</strong> – Number of epochs</p></li>
-<li><p><strong>learning_rate</strong> – Learning rate</p></li>
-<li><p><strong>out_activation</strong> – User specified ouptut activation function. Possible values:</p></li>
-<li><p><strong>loss_fcn</strong> – User specified loss function. Possible values:</p></li>
-<li><p><strong>shuffle</strong> – Flag which indicates if dataset should be shuffled or not</p></li>
-<li><p><strong>validation_split</strong> – Fraction of training set used as validation set</p></li>
-<li><p><strong>seed</strong> – Seed for model initialization</p></li>
-<li><p><strong>verbose</strong> – Flag which indicates if function should print to stdout</p></li>
-</ul>
+<dd class="field-odd"><p><strong>X</strong> – Matrix to calculate the distance inside</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing by the model & by the model</p>
+<dd class="field-even"><p>Euclidean distance matrix</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.garch">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">garch</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">kmax</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">momentum</span><span class="p">:</span> <span class="n">fl [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.dmv">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">dmv</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DAGNo [...]
+<dd><blockquote>
+<div><p>The dmv-function is used to find disguised missing values utilising syntactical pattern recognition.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>kmax</strong> – Number of iterations</p></li>
-<li><p><strong>momentum</strong> – Momentum for momentum-gradient descent (set to 0 to deactivate)</p></li>
-<li><p><strong>start_stepsize</strong> – Initial gradient-descent stepsize</p></li>
-<li><p><strong>end_stepsize</strong> – gradient-descent stepsize at end (linear descent)</p></li>
-<li><p><strong>start_vicinity</strong> – proportion of randomness of restart-location for gradient descent at beginning</p></li>
-<li><p><strong>end_vicinity</strong> – same at end (linear decay)</p></li>
-<li><p><strong>sim_seed</strong> – seed for simulation of process on fitted coefficients</p></li>
-<li><p><strong>verbose</strong> – verbosity, comments during fitting</p></li>
+<li><p><strong>X</strong> – Input Frame</p></li>
+<li><p><strong>threshold</strong> – Threshold value in interval [0, 1] for dominant pattern per column (e.g., 0.8 means
+that 80% of the entries per column must adhere this pattern to be dominant)</p></li>
+<li><p><strong>replace</strong> – The string disguised missing values are replaced with</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing term of fitted process & arch-coefficient of fitted process & garch-coefficient of fitted process & drawbacks: slow convergence of optimization (sort of simulated annealing/gradient descent)</p>
+<dd class="field-even"><p>Frame X including detected disguised missing values</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.gaussianClassifier">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gaussianClassifier</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">D</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.ema">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ema</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">search_iterations</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">mode</span><span class="p">:</span> <span class=" [...]
+<dd><blockquote>
+<div><p>This function imputes values with exponential moving average (single, double or triple).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>varSmoothing</strong> – Smoothing factor for variances</p></li>
-<li><p><strong>verbose</strong> – Print accuracy of the training set</p></li>
+<li><p><strong>X</strong> – Frame that contains time series data that needs to be imputed
+search_iterations Integer – Budget iterations for parameter optimization,
+used if parameters weren’t set</p></li>
+<li><p><strong>mode</strong> – Type of EMA method. Either “single”, “double” or “triple”</p></li>
+<li><p><strong>freq</strong> – Seasonality when using triple EMA.</p></li>
+<li><p><strong>alpha</strong> – alpha- value for EMA</p></li>
+<li><p><strong>beta</strong> – beta- value for EMA</p></li>
+<li><p><strong>gamma</strong> – gamma- value for EMA</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Frame with EMA results</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.getAccuracy">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">getAccuracy</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">yhat</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span cla [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.executePipeline">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">executePipeline</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">pipeline</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">Xtrain</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Ytrain< [...]
+<dd><blockquote>
+<div><p>This function execute pipeline.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>isWeighted</strong> – Flag for weighted or non-weighted accuracy calculation</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>logical</strong> – <p>—</p>
+</p></li>
+<li><p><strong>pipeline</strong> – <p>—</p>
+</p></li>
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Y</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Xtest</strong> – <p>—</p>
+</p></li>
+<li><p><strong>Ytest</strong> – <p>—</p>
+</p></li>
+<li><p><strong>metaList</strong> – <p>—</p>
+</p></li>
+<li><p><strong>hyperParameters</strong> – <p>—</p>
+</p></li>
+<li><p><strong>hpForPruning</strong> – <p>—</p>
+</p></li>
+<li><p><strong>changesByOp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>flagsCount</strong> – <p>—</p>
+</p></li>
+<li><p><strong>test</strong> – <p>—</p>
+</p></li>
+<li><p><strong>verbose</strong> – <p>—</p>
+</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing of the predicted labels</p>
+<dd class="field-even"><p><p>—</p>
+</p>
</dd>
-</dl>
-</dd></dl>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.ffPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ffPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">model</span><span class="p">:</span> <span class="n">systemds.operator.nodes.list.List</span></em>, <em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n [...]
+<dd><blockquote>
+<div><p>This builtin function makes prediction given data and trained feedforward neural network model</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>Model</strong> – Trained ff neural network model</p></li>
+<li><p><strong>X</strong> – Data used for making predictions</p></li>
+<li><p><strong>batch_size</strong> – Batch size</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Predicted value</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.ffTrain">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ffTrain</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">out_activation</span><spa [...]
+<dd><blockquote>
+<div><p>This builtin function trains simple feed-forward neural network. The architecture of the
+networks is: affine1 -> relu -> dropout -> affine2 -> configurable output activation function.
+Hidden layer has 128 neurons. Dropout rate is 0.35. Input and output sizes are inferred from X and Y.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Training data</p></li>
+<li><p><strong>Y</strong> – Labels/Target values</p></li>
+<li><p><strong>batch_size</strong> – Batch size</p></li>
+<li><p><strong>epochs</strong> – Number of epochs</p></li>
+<li><p><strong>learning_rate</strong> – Learning rate</p></li>
+<li><p><strong>out_activation</strong> – User specified output activation function. Possible values:
+“sigmoid”, “relu”, “lrelu”, “tanh”, “softmax”, “logits” (no activation).</p></li>
+<li><p><strong>loss_fcn</strong> – User specified loss function. Possible values:
+“l1”, “l2”, “log_loss”, “logcosh_loss”, “cel” (cross-entropy loss).</p></li>
+<li><p><strong>shuffle</strong> – Flag which indicates if dataset should be shuffled or not</p></li>
+<li><p><strong>validation_split</strong> – Fraction of training set used as validation set</p></li>
+<li><p><strong>seed</strong> – Seed for model initialization</p></li>
+<li><p><strong>verbose</strong> – Flag which indicates if function should print to stdout</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Trained model which can be used in ffPredict</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.fit_pipeline">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">fit_pipeline</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">trainData</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">testData</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">pip</span [...]
+<dd><blockquote>
+<div><p>This script will read the dirty and clean data, then it will apply the best pipeline on dirty data
+and then will classify both cleaned dataset and check if the cleaned dataset is performing same as original dataset
+in terms of classification accuracy</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>trainData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>testData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>metaData</strong> – <p>—</p>
+</p></li>
+<li><p><strong>lp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>pip</strong> – <p>—</p>
+</p></li>
+<li><p><strong>hp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evaluationFunc</strong> – <p>—</p>
+</p></li>
+<li><p><strong>evalFunHp</strong> – <p>—</p>
+</p></li>
+<li><p><strong>isLastLabel</strong> – <p>—</p>
+</p></li>
+<li><p><strong>correctTypos</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.fixInvalidLengths">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">fixInvalidLengths</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">F1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><spa [...]
+<dd><blockquote>
+<div><p>Fix invalid lengths</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>F1</strong> – <p>—</p>
+</p></li>
+<li><p><strong>mask</strong> – <p>—</p>
+</p></li>
+<li><p><strong>ql</strong> – <p>—</p>
+</p></li>
+<li><p><strong>qu</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.fixInvalidLengthsApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">fixInvalidLengthsApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">qLow</spa [...]
+<dd><blockquote>
+<div><p>Fix invalid lengths</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>mask</strong> – <p>—</p>
+</p></li>
+<li><p><strong>ql</strong> – <p>—</p>
+</p></li>
+<li><p><strong>qu</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.frameSort">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">frameSort</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">F</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class=" [...]
+<dd><blockquote>
+<div><p>Related to [SYSTEMDS-2662] dependency function for cleaning pipelines
+Built-in for sorting frames</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>F</strong> – Data frame of string values</p></li>
+<li><p><strong>mask</strong> – matrix for identifying string columns</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>sorted dataset by column 1 in decreasing order</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.frequencyEncode">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">frequencyEncode</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" [...]
+<dd><blockquote>
+<div><p>function frequency conversion</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – dataset x</p></li>
+<li><p><strong>mask</strong> – mask of the columns for frequency conversion</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>categorical columns are replaced with their frequencies</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>the frequency counts for the different categoricals</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.frequencyEncodeApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">frequencyEncodeApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">freqCount</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="he [...]
+<dd><blockquote>
+<div><p>frequency code apply</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – dataset x</p></li>
+<li><p><strong>freqCount</strong> – the frequency counts for the different categoricals</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>categorical columns are replaced with their frequencies given</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.garch">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">garch</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">kmax</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">momentum</span><span class="p">:</span> <span class="n">fl [...]
+<dd><blockquote>
+<div><p>This is a builtin function that implements GARCH(1,1), a statistical model used in analyzing time-series data where the variance
+error is believed to be serially autocorrelated</p>
+<p>COMMENTS
+This has some drawbacks: slow convergence of optimization (sort of simulated annealing/gradient descent)
+TODO: use BFGS or BHHH if it is available (this are go to methods)
+TODO: (only then) extend to garch(p,q); otherwise the search space is way too big for the current method</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to apply Arima on.</p></li>
+<li><p><strong>kmax</strong> – Number of iterations</p></li>
+<li><p><strong>momentum</strong> – Momentum for momentum-gradient descent (set to 0 to deactivate)</p></li>
+<li><p><strong>start_stepsize</strong> – Initial gradient-descent stepsize</p></li>
+<li><p><strong>end_stepsize</strong> – gradient-descent stepsize at end (linear descent)</p></li>
+<li><p><strong>start_vicinity</strong> – proportion of randomness of restart-location for gradient descent at beginning</p></li>
+<li><p><strong>end_vicinity</strong> – same at end (linear decay)</p></li>
+<li><p><strong>sim_seed</strong> – seed for simulation of process on fitted coefficients</p></li>
+<li><p><strong>verbose</strong> – verbosity, comments during fitting</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>simulated garch(1,1) process on fitted coefficients</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>variances of simulated fitted process</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>onstant term of fitted process</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>1-st arch-coefficient of fitted process</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>1-st garch-coefficient of fitted process</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.gaussianClassifier">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gaussianClassifier</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">D</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span [...]
+<dd><blockquote>
+<div><p>Computes the parameters needed for Gaussian Classification.
+Thus it computes the following per class: the prior probability,
+the inverse covariance matrix, the mean per feature and the determinant
+of the covariance matrix. Furthermore (if not explicitly defined), it
+adds some small smoothing value along the variances, to prevent
+numerical errors / instabilities.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>D</strong> – Input matrix (training set)</p></li>
+<li><p><strong>C</strong> – Target vector</p></li>
+<li><p><strong>varSmoothing</strong> – Smoothing factor for variances</p></li>
+<li><p><strong>verbose</strong> – Print accuracy of the training set</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Vector storing the class prior probabilities</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix storing the means of the classes</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>List of inverse covariance matrices</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Vector storing the determinants of the classes</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.getAccuracy">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">getAccuracy</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">yhat</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span cla [...]
+<dd><blockquote>
+<div><p>This builtin function compute the weighted and simple accuracy for given predictions</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>y</strong> – Ground truth (Actual Labels)</p></li>
+<li><p><strong>yhat</strong> – Predictions (Predicted labels)</p></li>
+<li><p><strong>isWeighted</strong> – Flag for weighted or non-weighted accuracy calculation</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>accuracy of the predicted labels</p>
+</dd>
+</dl>
+</dd></dl>
<dl class="py function">
<dt id="systemds.operator.algorithm.glm">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">glm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwar [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script solves GLM regression using NEWTON/FISHER scoring with trust regions. The glm-function is a flexible
+generalization of ordinary linear regression that allows for response variables that have error distribution models.</p>
+<p>In addition, some GLM statistics are provided as console output by setting verbose=TRUE, one comma-separated name-value
+pair per each line, as follows:</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>--------------------------------------------------------------------------------------------
+TERMINATION_CODE A positive integer indicating success/failure as follows:
+ 1 = Converged successfully; 2 = Maximum number of iterations reached;
+ 3 = Input (X, Y) out of range; 4 = Distribution/link is not supported
+BETA_MIN Smallest beta value (regression coefficient), excluding the intercept
+BETA_MIN_INDEX Column index for the smallest beta value
+BETA_MAX Largest beta value (regression coefficient), excluding the intercept
+BETA_MAX_INDEX Column index for the largest beta value
+INTERCEPT Intercept value, or NaN if there is no intercept (if icpt=0)
+DISPERSION Dispersion used to scale deviance, provided as "disp" input parameter
+ or estimated (same as DISPERSION_EST) if the "disp" parameter is <= 0
+DISPERSION_EST Dispersion estimated from the dataset
+DEVIANCE_UNSCALED Deviance from the saturated model, assuming dispersion == 1.0
+DEVIANCE_SCALED Deviance from the saturated model, scaled by the DISPERSION value
+--------------------------------------------------------------------------------------------
+
+The Log file, when requested, contains the following per-iteration variables in CSV format,
+each line containing triple (NAME, ITERATION, VALUE) with ITERATION = 0 for initial values:
+
+--------------------------------------------------------------------------------------------
+NUM_CG_ITERS Number of inner (Conj.Gradient) iterations in this outer iteration
+IS_TRUST_REACHED 1 = trust region boundary was reached, 0 = otherwise
+POINT_STEP_NORM L2-norm of iteration step from old point (i.e. "beta") to new point
+OBJECTIVE The loss function we minimize (i.e. negative partial log-likelihood)
+OBJ_DROP_REAL Reduction in the objective during this iteration, actual value
+OBJ_DROP_PRED Reduction in the objective predicted by a quadratic approximation
+OBJ_DROP_RATIO Actual-to-predicted reduction ratio, used to update the trust region
+GRADIENT_NORM L2-norm of the loss function gradient (NOTE: sometimes omitted)
+LINEAR_TERM_MIN The minimum value of X %*% beta, used to check for overflows
+LINEAR_TERM_MAX The maximum value of X %*% beta, used to check for overflows
+IS_POINT_UPDATED 1 = new point accepted; 0 = new point rejected, old point restored
+TRUST_DELTA Updated trust region size, the "delta"
+--------------------------------------------------------------------------------------------
+</pre></div>
+</div>
+<p>SOME OF THE SUPPORTED GLM DISTRIBUTION FAMILIES
+AND LINK FUNCTIONS:</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>dfam vpow link lpow Distribution.link nical?
+---------------------------------------------------
+ 1 0.0 1 -1.0 Gaussian.inverse
+ 1 0.0 1 0.0 Gaussian.log
+ 1 0.0 1 1.0 Gaussian.id Yes
+ 1 1.0 1 0.0 Poisson.log Yes
+ 1 1.0 1 0.5 Poisson.sqrt
+ 1 1.0 1 1.0 Poisson.id
+ 1 2.0 1 -1.0 Gamma.inverse Yes
+ 1 2.0 1 0.0 Gamma.log
+ 1 2.0 1 1.0 Gamma.id
+ 1 3.0 1 -2.0 InvGaussian.1/mu^2 Yes
+ 1 3.0 1 -1.0 InvGaussian.inverse
+ 1 3.0 1 0.0 InvGaussian.log
+ 1 3.0 1 1.0 InvGaussian.id
+ 1 * 1 * AnyVariance.AnyLink
+---------------------------------------------------
+ 2 * 1 0.0 Binomial.log
+ 2 * 1 0.5 Binomial.sqrt
+ 2 * 2 * Binomial.logit Yes
+ 2 * 3 * Binomial.probit
+ 2 * 4 * Binomial.cloglog
+ 2 * 5 * Binomial.cauchit
+---------------------------------------------------
+</pre></div>
+</div>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X of feature vectors</p></li>
+<li><p><strong>Y</strong> – matrix Y with either 1 or 2 columns:
+if dfam = 2, Y is 1-column Bernoulli or 2-column Binomial (#pos, #neg)</p></li>
<li><p><strong>dfam</strong> – Distribution family code: 1 = Power, 2 = Binomial</p></li>
-<li><p><strong>vpow</strong> – Power for Variance defined as (mean)^power (ignored if dfam != 1):</p></li>
-<li><p><strong>link</strong> – Link function code: 0 = canonical (depends on distribution),</p></li>
-<li><p><strong>lpow</strong> – Power for Link function defined as (mean)^power (ignored if link != 1):</p></li>
+<li><p><strong>vpow</strong> – Power for Variance defined as (mean)^power (ignored if dfam != 1):
+0.0 = Gaussian, 1.0 = Poisson, 2.0 = Gamma, 3.0 = Inverse Gaussian</p></li>
+<li><p><strong>link</strong> – Link function code: 0 = canonical (depends on distribution),
+1 = Power, 2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit</p></li>
+<li><p><strong>lpow</strong> – Power for Link function defined as (mean)^power (ignored if link != 1):
+-2.0 = 1/mu^2, -1.0 = reciprocal, 0.0 = log, 0.5 = sqrt, 1.0 = identity</p></li>
<li><p><strong>yneg</strong> – Response value for Bernoulli “No” label, usually 0.0 or -1.0</p></li>
-<li><p><strong>icpt</strong> – Intercept presence, X columns shifting and rescaling:</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, X columns shifting and rescaling:
+0 = no intercept, no shifting, no rescaling;
+1 = add intercept, but neither shift nor rescale X;
+2 = add intercept, shift & rescale X columns to mean = 0, variance = 1</p></li>
<li><p><strong>reg</strong> – Regularization parameter (lambda) for L2 regularization</p></li>
<li><p><strong>tol</strong> – Tolerance (epsilon)</p></li>
<li><p><strong>disp</strong> – (Over-)dispersion value, or 0.0 to estimate it from data</p></li>
<li><p><strong>moi</strong> – Maximum number of outer (Newton / Fisher Scoring) iterations</p></li>
<li><p><strong>mii</strong> – Maximum number of inner (Conjugate Gradient) iterations, 0 = no maximum</p></li>
+<li><p><strong>verbose</strong> – if the Algorithm should be verbose</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing line, as follows: & integer indicating success/failure as follows: & value (regression coefficient), excluding the intercept & for the smallest beta value & value (regression coefficient), excluding the intercept & for the largest beta value & or nan if there is no intercept (if icpt=0) & to scale deviance, provided as “disp” input parameter & from the dataset & the saturated model, assuming dispersi [...]
+<dd class="field-even"><p>Matrix beta, whose size depends on icpt:
+icpt=0: ncol(X) x 1; icpt=1: (ncol(X) + 1) x 1; icpt=2: (ncol(X) + 1) x 2</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.glmPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">glmPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">B</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class=" [...]
+<dd><blockquote>
+<div><p>Applies the estimated parameters of a GLM type regression to a new dataset</p>
+<p>Additional statistics are printed one per each line, in the following</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>CSV format: NAME,[COLUMN],[SCALED],VALUE
+---
+NAME is the string identifier for the statistic, see the table below.
+COLUMN is an optional integer value that specifies the Y-column for per-column statistics;
+ note that a Binomial/Multinomial one-column Y input is converted into multi-column.
+SCALED is an optional Boolean value (TRUE or FALSE) that tells us whether or not the input
+ dispersion parameter (disp) scaling has been applied to this statistic.
+VALUE is the value of the statistic.
+---
+</pre></div>
+</div>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>NAME COLUMN SCALED MEANING
+---------------------------------------------------------------------------------------------
+LOGLHOOD_Z + Log-Likelihood Z-score (in st.dev's from mean)
+LOGLHOOD_Z_PVAL + Log-Likelihood Z-score p-value
+PEARSON_X2 + Pearson residual X^2 statistic
+PEARSON_X2_BY_DF + Pearson X^2 divided by degrees of freedom
+PEARSON_X2_PVAL + Pearson X^2 p-value
+DEVIANCE_G2 + Deviance from saturated model G^2 statistic
+DEVIANCE_G2_BY_DF + Deviance G^2 divided by degrees of freedom
+DEVIANCE_G2_PVAL + Deviance G^2 p-value
+AVG_TOT_Y + Average of Y column for a single response value
+STDEV_TOT_Y + St.Dev. of Y column for a single response value
+AVG_RES_Y + Average of column residual, i.e. of Y - mean(Y|X)
+STDEV_RES_Y + St.Dev. of column residual, i.e. of Y - mean(Y|X)
+PRED_STDEV_RES + + Model-predicted St.Dev. of column residual
+R2 + R^2 of Y column residual with bias included
+ADJUSTED_R2 + Adjusted R^2 of Y column residual with bias included
+R2_NOBIAS + R^2 of Y column residual with bias subtracted
+ADJUSTED_R2_NOBIAS + Adjusted R^2 of Y column residual with bias subtracted
+---------------------------------------------------------------------------------------------
+</pre></div>
+</div>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X of records (feature vectors)</p></li>
+<li><p><strong>B</strong> – GLM regression parameters (the betas), with dimensions
+ncol(X) x k: do not add intercept
+ncol(X)+1 x k: add intercept as given by the last B-row
+if k > 1, use only B[, 1] unless it is Multinomial Logit (dfam=3)</p></li>
+<li><p><strong>ytest</strong> – Response matrix Y, with the following dimensions:
+nrow(X) x 1 : for all distributions (dfam=1 or 2 or 3)
+nrow(X) x 2 : for Binomial (dfam=2) given by (#pos, #neg) counts
+nrow(X) x k+1: for Multinomial (dfam=3) given by category counts</p></li>
+<li><p><strong>dfam</strong> – GLM distribution family: 1 = Power, 2 = Binomial, 3 = Multinomial Logit</p></li>
+<li><p><strong>vpow</strong> – Power for Variance defined as (mean)^power (ignored if dfam != 1):
+0.0 = Gaussian, 1.0 = Poisson, 2.0 = Gamma, 3.0 = Inverse Gaussian</p></li>
+<li><p><strong>link</strong> – Link function code: 0 = canonical (depends on distribution), 1 = Power,
+2 = Logit, 3 = Probit, 4 = Cloglog, 5 = Cauchit; ignored if Multinomial</p></li>
+<li><p><strong>lpow</strong> – Power for Link function defined as (mean)^power (ignored if link != 1):
+-2.0 = 1/mu^2, -1.0 = reciprocal, 0.0 = log, 0.5 = sqrt, 1.0 = identity</p></li>
+<li><p><strong>disp</strong> – Dispersion value, when available</p></li>
+<li><p><strong>verbose</strong> – Print statistics to stdout</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix M of predicted means/probabilities:
+nrow(X) x 1 : for Power-type distributions (dfam=1)
+nrow(X) x 2 : for Binomial distribution (dfam=2), column 2 is “No”
+nrow(X) x k+1: for Multinomial Logit (dfam=3), col# k+1 is baseline</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
<dt id="systemds.operator.algorithm.gmm">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gmm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">verbose</span><span class="p">:</span> <span class="n">bool</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:< [...]
-<dd><dl class="field-list simple">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gmm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DAG [...]
+<dd><blockquote>
+<div><p>The gmm-function implements builtin Gaussian Mixture Model with four different types of covariance matrices
+i.e., VVV, EEE, VVI, VII and two initialization methods namely “kmeans” and “random”.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
<li><p><strong>n_components</strong> – Number of n_components in the Gaussian mixture model</p></li>
-<li><p><strong>model</strong> – “VVV”: unequal variance (full),each component has its own general covariance matrix</p></li>
+<li><p><strong>model</strong> – “VVV”: unequal variance (full),each component has its own general covariance matrix
+“EEE”: equal variance (tied), all components share the same general covariance matrix
+“VVI”: spherical, unequal volume (diag), each component has its own diagonal
+covariance matrix
+“VII”: spherical, equal volume (spherical), each component has its own single variance</p></li>
<li><p><strong>init_param</strong> – initialize weights with “kmeans” or “random”</p></li>
<li><p><strong>iterations</strong> – Number of iterations</p></li>
<li><p><strong>reg_covar</strong> – regularization parameter for covariance matrix</p></li>
@@ -719,7 +1872,23 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing of estimated parameters & information criterion for best iteration & kth class</p>
+<dd class="field-even"><p>Prediction matrix</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Probability of the predictions</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Number of estimated parameters</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Bayesian information criterion for best iteration</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>fitted clusters mean</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>A matrix whose [i,k]th entry is the probability that observation i in the test data
+belongs to the kth class</p>
</dd>
</dl>
</dd></dl>
@@ -727,12 +1896,25 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.gmmPredict">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gmmPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">weight</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mu</span><span cl [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This function is a Prediction function for a Gaussian Mixture Model (gmm).</p>
+<p>compute posterior probabilities for new instances given the variance and mean of fitted data</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>model</strong> – fitted model</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X (instances to be clustered)</p></li>
+<li><p><strong>weight</strong> – Weight of learned model</p></li>
+<li><p><strong>mu</strong> – fitted clusters mean</p></li>
+<li><p><strong>precisions_cholesky</strong> – fitted precision matrix for each mixture</p></li>
+<li><p><strong>model</strong> – fitted model</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing cluster labels & of belongingness & for new instances given the variance and mean of fitted data</p>
+<dd class="field-even"><p>predicted cluster labels</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>probabilities of belongingness</p>
</dd>
</dl>
</dd></dl>
@@ -740,16 +1922,70 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.gnmf">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gnmf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">rnk</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</spa [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The gnmf-function does Gaussian Non-Negative Matrix Factorization. In this, a matrix X is factorized into two
+matrices W and H, such that all three matrices have no negative elements. This non-negativity makes the resulting
+matrices easier to inspect.</p>
+<p>References:
+[Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
+Distributed nonnegative matrix factorization for web-scale dyadic
+data analysis on mapreduce. WWW 2010: 681-690]</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
<li><p><strong>rnk</strong> – Number of components into which matrix X is to be factored</p></li>
<li><p><strong>eps</strong> – Tolerance</p></li>
<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>List of pattern matrices, one for each repetition</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>List of amplitude matrices, one for each repetition</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.gridSearch">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">gridSearch</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">train</span><span clas [...]
+<dd><blockquote>
+<div><p>The gridSearch-function is used to find the optimal hyper-parameters of a model which results in the most
+accurate predictions. This function takes train and eval functions by name.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>y</strong> – Input Matrix of vectors.</p></li>
+<li><p><strong>train</strong> – Name ft of the train function to call via ft(trainArgs)</p></li>
+<li><p><strong>predict</strong> – Name fp of the loss function to call via fp((predictArgs,B))</p></li>
+<li><p><strong>numB</strong> – Maximum number of parameters in model B (pass the max because the size
+may vary with parameters like icpt or multi-class classification)</p></li>
+<li><p><strong>params</strong> – List of varied hyper-parameter names</p></li>
+<li><p><strong>dataArgs</strong> – List of data parameters (to identify data parameters by name i.e. list(“X”, “Y”))</p></li>
+<li><p><strong>paramValues</strong> – List of matrices providing the parameter values as
+columnvectors for position-aligned hyper-parameters in ‘params’</p></li>
+<li><p><strong>trainArgs</strong> – named List of arguments to pass to the ‘train’ function, where
+gridSearch replaces enumerated hyper-parameter by name, if
+not provided or an empty list, the lm parameters are used</p></li>
+<li><p><strong>predictArgs</strong> – List of arguments to pass to the ‘predict’ function, where
+gridSearch appends the trained models at the end, if
+not provided or an empty list, list(X, y) is used instead</p></li>
+<li><p><strong>cv</strong> – flag enabling k-fold cross validation, otherwise training loss</p></li>
+<li><p><strong>cvk</strong> – if cv=TRUE, specifies the the number of folds, otherwise ignored</p></li>
+<li><p><strong>verbose</strong> – flag for verbose debug output</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix[Double]the trained model with minimal loss (by the ‘predict’ function)
+Multi-column models are returned as a column-major linearized column vector</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>one-row frame w/ optimal hyper-parameters (by ‘params’ position)</p>
</dd>
</dl>
</dd></dl>
@@ -757,34 +1993,96 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.hospitalResidencyMatch">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">hospitalResidencyMatch</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">R</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">H</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">capacity</ [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This script computes a solution for the hospital residency match problem.</p>
+<p>Residents.mtx:
+2.0,1.0,3.0
+1.0,2.0,3.0
+1.0,2.0,0.0</p>
+<p>Since it is an ORDERED matrix, this means that Resident 1 (row 1) likes hospital 2 the most, followed by hospital 1 and hospital 3.
+If it was UNORDERED, this would mean that resident 1 (row 1) likes hospital 3 the most (since the value at [1,3] is the row max),
+followed by hospital 1 (2.0 preference value) and hospital 2 (1.0 preference value).</p>
+<p>Hospitals.mtx:
+2.0,1.0,0.0
+0.0,1.0,2.0
+1.0,2.0,0.0</p>
+<p>Since it is an UNORDERED matrix this means that Hospital 1 (row 1) likes Resident 1 the most (since the value at [1,1] is the row max).</p>
+<p>capacity.mtx
+1.0
+1.0
+1.0</p>
+<p>residencyMatch.mtx
+2.0,0.0,0.0
+1.0,0.0,0.0
+0.0,2.0,0.0</p>
+<p>hospitalMatch.mtx
+0.0,1.0,0.0
+0.0,0.0,2.0
+1.0,0.0,0.0</p>
+<p>Resident 1 has matched with Hospital 3 (since [1,3] is non-zero) at a preference level of 2.0.
+Resident 2 has matched with Hospital 1 (since [2,1] is non-zero) at a preference level of 1.0.
+Resident 3 has matched with Hospital 2 (since [3,2] is non-zero) at a preference level of 2.0.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>It</strong> – an ORDERED matrix.</p></li>
-<li><p><strong>It</strong> – an UNORDRED matrix.</p></li>
-<li><p><strong>It</strong> – a [n*1] matrix with non zero values.</p></li>
-<li><p><strong>with</strong> – and vice-versa (higher is better).</p></li>
+<li><p><strong>R</strong> – Residents matrix R.
+It must be an ORDERED matrix.</p></li>
+<li><p><strong>H</strong> – Hospitals matrix H.
+It must be an UNORDRED matrix.</p></li>
+<li><p><strong>capacity</strong> – capacity of Hospitals matrix C.
+It must be a [n*1] matrix with non zero values.
+i.e. the leftmost value in a row is the most preferred partner’s index.
+i.e. the leftmost value in a row in P is the preference value for the acceptor
+with index 1 and vice-versa (higher is better).</p></li>
<li><p><strong>verbose</strong> – If the operation is verbose</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing an ordered matrix, this means that resident 1 (row 1) likes hospital 2 the most, followed by hospital 1 and hospital 3. & unordered, this would mean that resident 1 (row 1) likes hospital 3 the most (since the value at [1,3] is the row max), & 1 (2.0 preference value) and hospital 2 (1.0 preference value). & an unordered matrix this means that hospital 1 (row 1) likes resident 1 the most (since the value at [1,1] is the ro [...]
+<dd class="field-even"><p>Result Matrix
+If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j.
+Further, if cell [i,j] is non-zero, it holds the preference value that led to the match.</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Result Matrix
+If cell [i,j] is non-zero, it means that Resident i has matched with Hospital j.
+Further, if cell [i,j] is non-zero, it holds the preference value that led to the match.</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
<dt id="systemds.operator.algorithm.hyperband">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">hyperband</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">X_val</span [...]
-<dd><dl class="field-list simple">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">hyperband</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">X_val</span [...]
+<dd><blockquote>
+<div><p>The hyperband-function is used for hyper parameter optimization and is based on multi-armed bandits and early
+elimination. Through multiple parallel brackets and consecutive trials it will return the hyper parameter combination
+which performed best on a validation dataset. A set of hyper parameter combinations is drawn from uniform distributions
+with given ranges; Those make up the candidates for hyperband. Notes:
+hyperband is hard-coded for lmCG, and uses lmPredict for validation
+hyperband is hard-coded to use the number of iterations as a resource
+hyperband can only optimize continuous hyperparameters</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>One</strong> – hyper parameter, first column specifies min, second column max value.</p></li>
+<li><p><strong>X_train</strong> – Input Matrix of training vectors</p></li>
+<li><p><strong>y_train</strong> – Labels for training vectors</p></li>
+<li><p><strong>X_val</strong> – Input Matrix of validation vectors</p></li>
+<li><p><strong>y_val</strong> – Labels for validation vectors</p></li>
+<li><p><strong>params</strong> – List of parameters to optimize</p></li>
+<li><p><strong>paramRanges</strong> – The min and max values for the uniform distributions to draw from.
+One row per hyper parameter, first column specifies min, second column max value.</p></li>
+<li><p><strong>R</strong> – Controls number of candidates evaluated</p></li>
+<li><p><strong>eta</strong> – Determines fraction of candidates to keep after each trial</p></li>
<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>1-column matrix of weights of best performing candidate</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>hyper parameters of best performing candidate</p>
</dd>
</dl>
</dd></dl>
@@ -792,15 +2090,19 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_brightness">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_brightness</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">value</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">channel_max</span><span class="p">:</span [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The img_brightness-function is an image data augmentation function. It changes the brightness of the image.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input matrix/image</p></li>
<li><p><strong>value</strong> – The amount of brightness to be changed for the image</p></li>
<li><p><strong>channel_max</strong> – Maximum value of the brightness of the image</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output matrix/image</p>
</dd>
</dl>
</dd></dl>
@@ -808,9 +2110,13 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_crop">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_crop</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">w</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">h</span><span class="p">:</span> <span class="n">int< [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The img_crop-function is an image data augmentation function. It cuts out a subregion of an image.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input matrix/image</p></li>
<li><p><strong>w</strong> – The width of the subregion required</p></li>
<li><p><strong>h</strong> – The height of the subregion required</p></li>
<li><p><strong>x_offset</strong> – The horizontal coordinate in the image to begin the crop operation</p></li>
@@ -818,7 +2124,7 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Cropped matrix/image</p>
</dd>
</dl>
</dd></dl>
@@ -826,9 +2132,13 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_cutout">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_cutout</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">x</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">in [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Image Cutout function replaces a rectangular section of an image with a constant value.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image as 2D matrix with top left corner at [1, 1]</p></li>
<li><p><strong>x</strong> – Column index of the top left corner of the rectangle (starting at 1)</p></li>
<li><p><strong>y</strong> – Row index of the top left corner of the rectangle (starting at 1)</p></li>
<li><p><strong>width</strong> – Width of the rectangle (must be positive)</p></li>
@@ -837,7 +2147,7 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image as 2D matrix with top left corner at [1, 1]</p>
</dd>
</dl>
</dd></dl>
@@ -845,12 +2155,18 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_invert">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_invert</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">max_value</span><span class="p">:</span> <span class="n">float</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.al [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This is an image data augmentation function. It inverts an image.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>max_value</strong> – The maximum value pixels can have</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image</p></li>
+<li><p><strong>max_value</strong> – The maximum value pixels can have</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image</p>
</dd>
</dl>
</dd></dl>
@@ -858,12 +2174,19 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_mirror">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_mirror</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">horizontal_axis</span><span class="p">:</span> <span class="n">bool</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operat [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This function is an image data augmentation function.
+It flips an image on the X (horizontal) or Y (vertical) axis.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>max_value</strong> – The maximum value pixels can have</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input matrix/image</p></li>
+<li><p><strong>max_value</strong> – The maximum value pixels can have</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Flipped matrix/image</p>
</dd>
</dl>
</dd></dl>
@@ -871,15 +2194,20 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_posterize">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_posterize</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">bits</span><span class="p">:</span> <span class="n">int</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algori [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The Image Posterize function limits pixel values to 2^bits different values in the range [0, 255].
+Assumes the input image can attain values in the range [0, 255].</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>bits</strong> – The number of bits keep for the values.</p></li>
-<li><p><strong>1</strong> – and white, 8 means every integer between 0 and 255.</p></li>
+<li><p><strong>img_in</strong> – Input image</p></li>
+<li><p><strong>bits</strong> – The number of bits keep for the values.
+1 means black and white, 8 means every integer between 0 and 255.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image</p>
</dd>
</dl>
</dd></dl>
@@ -887,15 +2215,20 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_rotate">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_rotate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">radians</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">fill_value</span><span class="p">:</span> < [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The Image Rotate function rotates the input image counter-clockwise around the center.
+Uses nearest neighbor sampling.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image as 2D matrix with top left corner at [1, 1]</p></li>
<li><p><strong>radians</strong> – The value by which to rotate in radian.</p></li>
<li><p><strong>fill_value</strong> – The background color revealed by the rotation</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image as 2D matrix with top left corner at [1, 1]</p>
</dd>
</dl>
</dd></dl>
@@ -903,15 +2236,20 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_sample_pairing">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_sample_pairing</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">img_in2</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">we [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The image sample pairing function blends two images together.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>weight</strong> – The weight given to the second image.</p></li>
-<li><p><strong>0</strong> – img_in1, 1 means only img_in2 will be visible</p></li>
+<li><p><strong>img_in1</strong> – First input image</p></li>
+<li><p><strong>img_in2</strong> – Second input image</p></li>
+<li><p><strong>weight</strong> – The weight given to the second image.
+0 means only img_in1, 1 means only img_in2 will be visible</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image</p>
</dd>
</dl>
</dd></dl>
@@ -919,16 +2257,21 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_shear">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_shear</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">shear_x</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">shear_y</span><span class="p">:</span> <span [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This function applies a shearing transformation to an image.
+Uses nearest neighbor sampling.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image as 2D matrix with top left corner at [1, 1]</p></li>
<li><p><strong>shear_x</strong> – Shearing factor for horizontal shearing</p></li>
<li><p><strong>shear_y</strong> – Shearing factor for vertical shearing</p></li>
<li><p><strong>fill_value</strong> – The background color revealed by the shearing</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image as 2D matrix with top left corner at [1, 1]</p>
</dd>
</dl>
</dd></dl>
@@ -936,16 +2279,23 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_transform">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_transform</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">out_w</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="n">out_h</span><span class="p">:</span> <span c [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The Image Transform function applies an affine transformation to an image.
+Optionally resizes the image (without scaling).
+Uses nearest neighbor sampling.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image as 2D matrix with top left corner at [1, 1]</p></li>
<li><p><strong>out_w</strong> – Width of the output image</p></li>
<li><p><strong>out_h</strong> – Height of the output image</p></li>
+<li><p><strong>a</strong><strong>,</strong><strong>b</strong><strong>,</strong><strong>c</strong><strong>,</strong><strong>d</strong><strong>,</strong><strong>e</strong><strong>,</strong><strong>f</strong> – The first two rows of the affine matrix in row-major order</p></li>
<li><p><strong>fill_value</strong> – The background of the image</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing image as 2d matrix with top left corner at [1, 1]</p>
+<dd class="field-even"><p>Output image as 2D matrix with top left corner at [1, 1]</p>
</dd>
</dl>
</dd></dl>
@@ -953,9 +2303,15 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.img_translate">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">img_translate</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">img_in</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">offset_x</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">offset_y</span><span class="p">:</span> [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The Image Translate function translates the image.
+Optionally resizes the image (without scaling).
+Uses nearest neighbor sampling.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>img_in</strong> – Input image as 2D matrix with top left corner at [1, 1]</p></li>
<li><p><strong>offset_x</strong> – The distance to move the image in x direction</p></li>
<li><p><strong>offset_y</strong> – The distance to move the image in y direction</p></li>
<li><p><strong>out_w</strong> – Width of the output image</p></li>
@@ -964,7 +2320,38 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output image as 2D matrix with top left corner at [1, 1]</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.impurityMeasures">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">impurityMeasures</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">R</span><span cl [...]
+<dd><blockquote>
+<div><p>This function computes the measure of impurity for the given dataset based on the passed method (gini or entropy).
+The current version expects the target vector to contain only 0 or 1 values.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Feature matrix.</p></li>
+<li><p><strong>Y</strong> – Target vector containing 0 and 1 values.</p></li>
+<li><p><strong>R</strong> – Vector indicating whether a feature is categorical or continuous.
+1 denotes a continuous feature, 2 denotes a categorical feature.</p></li>
+<li><p><strong>n_bins</strong> – Number of bins for binning in case of scale features.</p></li>
+<li><p><strong>method</strong> – String indicating the method to use; either “entropy” or “gini”.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>(1 x ncol(X)) row vector containing information/gini gain for
+each feature of the dataset.
+In case of gini, the values denote the gini gains, i.e. how much
+impurity was removed with the respective split. The higher the
+value, the better the split.
+In case of entropy, the values denote the information gain, i.e.
+how much entropy was removed. The higher the information gain,
+the better the split.</p>
</dd>
</dl>
</dd></dl>
@@ -972,15 +2359,23 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.imputeByFD">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByFD</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><span [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Vector X, source attribute of functional dependency</p></li>
+<li><p><strong>Y</strong> – Vector Y, target attribute of functional dependency and imputation</p></li>
<li><p><strong>threshold</strong> – threshold value in interval [0, 1] for robust FDs</p></li>
<li><p><strong>verbose</strong> – flag for printing verbose debug output</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Vector Y, with missing values mapped to a new max value</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Vector Y, with imputed missing values</p>
</dd>
</dl>
</dd></dl>
@@ -988,39 +2383,230 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.imputeByFDApply">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByFDApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y_imp</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Implements builtin for imputing missing values from observed values (if exist) using robust functional dependencies</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
<li><p><strong>source</strong> – source attribute to use for imputation and error correction</p></li>
<li><p><strong>target</strong> – attribute to be fixed</p></li>
<li><p><strong>threshold</strong> – threshold value in interval [0, 1] for robust FDs</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix with possible imputations</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.km">
+<dt id="systemds.operator.algorithm.imputeByMean">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByMean</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" hre [...]
+<dd><blockquote>
+<div><p>impute the data by mean value and if the feature is categorical then by mode value
+Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>mask</strong> – A 0/1 row vector for identifying numeric (0) and categorical features (1)</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.imputeByMeanApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByMeanApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">imputedVec</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="head [...]
+<dd><blockquote>
+<div><p>impute the data by mean value and if the feature is categorical then by mode value
+Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>imputationVector</strong> – column mean vector</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.imputeByMedian">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByMedian</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" h [...]
+<dd><blockquote>
+<div><p>Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+<p>impute the data by median value and if the feature is categorical then by mode value</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>mask</strong> – A 0/1 row vector for identifying numeric (0) and categorical features (1)</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.imputeByMedianApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByMedianApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">imputedVec</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="he [...]
+<dd><blockquote>
+<div><p>impute the data by median value and if the feature is categorical then by mode value
+Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>imputationVector</strong> – column median vector</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.imputeByMode">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByMode</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.imputeByMode" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This function impute the data by mode value
+Related to [SYSTEMDS-2902] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.imputeByModeApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">imputeByModeApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">imputedVec</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="head [...]
+<dd><blockquote>
+<div><p>impute the data by most frequent value (recoded data only)
+Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>imputationVector</strong> – column mean vector</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.intersect">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">intersect</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#sy [...]
+<dd><blockquote>
+<div><p>Implements set intersection for numeric data</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X, set A</p></li>
+<li><p><strong>Y</strong> – matrix Y, set B</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>intersection matrix, set of intersecting items</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.km">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">km</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">TE</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">GI</span><span class="p">:</s [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function that implements the analysis of survival data with KAPLAN-MEIER estimates</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>number</strong> – (categorical features) for grouping and/or stratifying</p></li>
-<li><p><strong>alpha</strong> – Parameter to compute 100*(1-alpha)% confidence intervals for the survivor</p></li>
-<li><p><strong>function</strong> – median</p></li>
-<li><p><strong>err_type</strong> – Parameter to specify the error type according to “greenwood” (the default) or “peto”</p></li>
-<li><p><strong>conf_type</strong> – Parameter to modify the confidence interval; “plain” keeps the lower and</p></li>
-<li><p><strong>upper</strong> – the confidence interval unmodified, “log” (the default)</p></li>
-<li><p><strong>corresponds</strong> – transformation and “log-log” corresponds to the</p></li>
-<li><p><strong>test_type</strong> – If survival data for multiple groups is available specifies which test to</p></li>
-<li><p><strong>perform</strong> – survival data across multiple groups: “none” (the default)</p></li>
+<li><p><strong>X</strong> – Input matrix X containing the survival data:
+timestamps, whether event occurred (1) or data is censored (0), and a
+number of factors (categorical features) for grouping and/or stratifying</p></li>
+<li><p><strong>TE</strong> – Column indices of X which contain timestamps (first entry) and event
+information (second entry)</p></li>
+<li><p><strong>GI</strong> – Column indices of X corresponding to the factors to be used for grouping</p></li>
+<li><p><strong>SI</strong> – Column indices of X corresponding to the factors to be used for stratifying</p></li>
+<li><p><strong>alpha</strong> – Parameter to compute 100*(1-alpha)% confidence intervals for the survivor
+function and its median</p></li>
+<li><p><strong>err_type</strong> – “greenwood” Parameter to specify the error type according to “greenwood” (the default) or “peto”</p></li>
+<li><p><strong>conf_type</strong> – Parameter to modify the confidence interval; “plain” keeps the lower and
+upper bound of the confidence interval unmodified, “log” (the default)
+corresponds to logistic transformation and “log-log” corresponds to the
+complementary log-log transformation</p></li>
+<li><p><strong>test_type</strong> – If survival data for multiple groups is available specifies which test to
+perform for comparing survival data across multiple groups: “none” (the default)
+“log-rank” or “wilcoxon” test</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing 7 consecutive columns in km corresponds to a unique & and strata in the data with the following schema & number of factors used for stratifying, i.e., ncol(si)) & of groups and strata is equal to 1, m will have 4 columns with & 4 matrix t and an g x 5 matrix t_groups_oe with</p>
+<dd class="field-even"><p>Matrix KM whose dimension depends on the number of groups (denoted by g) and
+strata (denoted by s) in the data:
+each collection of 7 consecutive columns in KM corresponds to a unique
+combination of groups and strata in the data with the following schema
+1. col: timestamp
+2. col: no. at risk
+3. col: no. of events
+4. col: Kaplan-Meier estimate of survivor function surv
+5. col: standard error of surv
+6. col: lower 100*(1-alpha)% confidence interval for surv
+7. col: upper 100*(1-alpha)% confidence interval for surv</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix M whose dimension depends on the number of groups (g) and strata (s) in
+the data (k denotes the number of factors used for grouping ,i.e., ncol(GI) and
+l denotes the number of factors used for stratifying, i.e., ncol(SI))
+M[,1:k]: unique combination of values in the k factors used for grouping
+M[,(k+1):(k+l)]: unique combination of values in the l factors used for stratifying
+M[,k+l+1]: total number of records
+M[,k+l+2]: total number of events
+M[,k+l+3]: median of surv
+M[,k+l+4]: lower 100*(1-alpha)% confidence interval of the median of surv
+M[,k+l+5]: upper 100*(1-alpha)% confidence interval of the median of surv
+If the number of groups and strata is equal to 1, M will have 4 columns with
+M[,1]: total number of events
+M[,2]: median of surv
+M[,3]: lower 100*(1-alpha)% confidence interval of the median of surv
+M[,4]: upper 100*(1-alpha)% confidence interval of the median of surv</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>If survival data from multiple groups available and ttype=log-rank or wilcoxon,
+a 1 x 4 matrix T and an g x 5 matrix T_GROUPS_OE with
+T_GROUPS_OE[,1] = no. of events
+T_GROUPS_OE[,2] = observed value (O)
+T_GROUPS_OE[,3] = expected value (E)
+T_GROUPS_OE[,4] = (O-E)^2/E
+T_GROUPS_OE[,5] = (O-E)^2/V
+T[1,1] = no. of groups
+T[1,2] = degree of freedom for Chi-squared distributed test statistic
+T[1,3] = test statistic
+T[1,4] = P-value</p>
</dd>
</dl>
</dd></dl>
@@ -1028,21 +2614,136 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.kmeans">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">kmeans</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span> [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function that implements the k-Means clustering algorithm</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to do KMeans on.</p></li>
<li><p><strong>k</strong> – Number of centroids</p></li>
<li><p><strong>runs</strong> – Number of runs (with different initial centroids)</p></li>
<li><p><strong>max_iter</strong> – Maximum number of iterations per run</p></li>
<li><p><strong>eps</strong> – Tolerance (epsilon) for WCSS change ratio</p></li>
<li><p><strong>is_verbose</strong> – do not print per-iteration stats</p></li>
<li><p><strong>avg_sample_size_per_centroid</strong> – Average number of records per centroid in data samples</p></li>
-<li><p><strong>seed</strong> – The seed used for initial sampling. If set to -1</p></li>
-<li><p><strong>random</strong> – selected.</p></li>
+<li><p><strong>seed</strong> – The seed used for initial sampling. If set to -1
+random seeds are selected.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The mapping of records to centroids</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>The output matrix with the centroids</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.kmeansPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">kmeansPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href= [...]
+<dd><blockquote>
+<div><p>Builtin function that does predictions based on a set of centroids provided.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – The input Matrix to do KMeans on.</p></li>
+<li><p><strong>C</strong> – The input Centroids to map X onto.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The mapping of records to centroids</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.knn">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">knn</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">Train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Test</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">CL</span><span class=" [...]
+<dd><blockquote>
+<div><p>This script implements KNN (K Nearest Neighbor) algorithm.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>Train</strong> – The input matrix as features</p></li>
+<li><p><strong>Test</strong> – The input matrix for nearest neighbor search</p></li>
+<li><p><strong>CL</strong> – The input matrix as target</p></li>
+<li><p><strong>CL_T</strong> – The target type of matrix CL whether
+columns in CL are continuous ( =1 ) or
+categorical ( =2 ) or not specified ( =0 )</p></li>
+<li><p><strong>trans_continuous</strong> – Option flag for continuous feature transformed to [-1,1]:
+FALSE = do not transform continuous variable;
+TRUE = transform continuous variable;</p></li>
+<li><p><strong>k_value</strong> – k value for KNN, ignore if select_k enable</p></li>
+<li><p><strong>select_k</strong> – Use k selection algorithm to estimate k (TRUE means yes)</p></li>
+<li><p><strong>k_min</strong> – Min k value( available if select_k = 1 )</p></li>
+<li><p><strong>k_max</strong> – Max k value( available if select_k = 1 )</p></li>
+<li><p><strong>select_feature</strong> – Use feature selection algorithm to select feature (TRUE means yes)</p></li>
+<li><p><strong>feature_max</strong> – Max feature selection</p></li>
+<li><p><strong>interval</strong> – Interval value for K selecting ( available if select_k = 1 )</p></li>
+<li><p><strong>feature_importance</strong> – Use feature importance algorithm to estimate each feature
+(TRUE means yes)</p></li>
+<li><p><strong>predict_con_tg</strong> – Continuous target predict function: mean(=0) or median(=1)</p></li>
+<li><p><strong>START_SELECTED</strong> – feature selection initial value</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Applied clusters to X</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Cluster matrix</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Feature importance value</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.knnGraph">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">knnGraph</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">k</span><span class="p">:</span> <span class="n">int</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.knnGraph" [...]
+<dd><blockquote>
+<div><p>Builtin for k nearest neighbor graph construction</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>k</strong> – <p>—</p>
+</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.knnbf">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">knnbf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">T</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kw [...]
+<dd><blockquote>
+<div><p>This script implements KNN (K Nearest Neighbor) algorithm.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – <p>—</p>
+</p></li>
+<li><p><strong>T</strong> – <p>—</p>
+</p></li>
+<li><p><strong>k_value</strong> – <p>—</p>
+</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p><p>—</p>
+</p>
</dd>
</dl>
</dd></dl>
@@ -1050,24 +2751,27 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.l2svm">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">l2svm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kw [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function Implements binary-class SVM with squared slack variables</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X of feature vectors</p></li>
+<li><p><strong>Y</strong> – matrix Y of class labels have to be a single column</p></li>
<li><p><strong>intercept</strong> – No Intercept ( If set to TRUE then a constant bias column is added to X)</p></li>
-<li><p><strong>epsilon</strong> – Procedure terminates early if the reduction in objective function value is less</p></li>
-<li><p><strong>lambda</strong> – Regularization parameter (lambda) for L2 regularization</p></li>
+<li><p><strong>epsilon</strong> – Procedure terminates early if the reduction in objective function value is less
+than epsilon (tolerance) times the initial objective function value.</p></li>
+<li><p><strong>reg</strong> – Regularization parameter (reg) for L2 regularization</p></li>
<li><p><strong>maxIterations</strong> – Maximum number of conjugate gradient iterations</p></li>
-<li><p><strong>maxii</strong> – <ul>
-<li></li>
-</ul>
-</p></li>
+<li><p><strong>maxii</strong> – max inner for loop iterations</p></li>
<li><p><strong>verbose</strong> – Set to true if one wants print statements updating on loss.</p></li>
-<li><p><strong>columnId</strong> – The column Id used if one wants to add a ID to the print statement, Specificly</p></li>
-<li><p><strong>usefull</strong> – is used in MSVM.</p></li>
+<li><p><strong>columnId</strong> – The column Id used if one wants to add a ID to the print statement,
+eg. used when L2SVM is used in MSVM.</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>the trained model</p>
</dd>
</dl>
</dd></dl>
@@ -1075,12 +2779,23 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.l2svmPredict">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">l2svmPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">W</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function Implements binary-class SVM with squared slack variables.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>verbose</strong> – Set to true if one wants print statements.</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X of feature vectors to classify</p></li>
+<li><p><strong>W</strong> – matrix of the trained variables</p></li>
+<li><p><strong>verbose</strong> – Set to true if one wants print statements.</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Classification Labels Raw, meaning not modified to clean
+labels of 1’s and -1’s</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Classification Labels Maxed to ones and zeros.</p>
</dd>
</dl>
</dd></dl>
@@ -1088,9 +2803,15 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.lasso">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lasso</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kw [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>Builtin function for the SpaRSA algorithm to perform lasso regression
+(SpaRSA .. Sparse Reconstruction by Separable Approximation)</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – input feature matrix</p></li>
+<li><p><strong>y</strong> – matrix Y columns of the design matrix</p></li>
<li><p><strong>tol</strong> – target convergence tolerance</p></li>
<li><p><strong>M</strong> – history length</p></li>
<li><p><strong>tau</strong> – regularization component</p></li>
@@ -1099,7 +2820,30 @@
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>model matrix</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.lenetPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lenetPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">model</span><span class="p">:</span> <span class="n">systemds.operator.nodes.list.List</span></em>, <em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class= [...]
+<dd><blockquote>
+<div><p>This builtin function makes prediction given data and trained LeNet model</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>model</strong> – Trained LeNet model</p></li>
+<li><p><strong>X</strong> – Input data matrix, of shape (N, C*Hin*Win)</p></li>
+<li><p><strong>C</strong> – Number of input channels</p></li>
+<li><p><strong>Hin</strong> – Input height</p></li>
+<li><p><strong>Win</strong> – Input width</p></li>
+<li><p><strong>batch_size</strong> – Batch size</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Predicted values</p>
</dd>
</dl>
</dd></dl>
@@ -1107,9 +2851,18 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.lenetTrain">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lenetTrain</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">X_val</span><span clas [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>This builtin function trains LeNet CNN. The architecture of the
+networks is:conv1 -> relu1 -> pool1 -> conv2 -> relu2 -> pool2 ->
+affine3 -> relu3 -> affine4 -> softmax</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input data matrix, of shape (N, C*Hin*Win)</p></li>
+<li><p><strong>Y</strong> – Target matrix, of shape (N, K)</p></li>
+<li><p><strong>X_val</strong> – Validation data matrix, of shape (N, C*Hin*Win)</p></li>
+<li><p><strong>Y_val</strong> – Validation target matrix, of shape (N, K)</p></li>
<li><p><strong>C</strong> – Number of input channels (dimensionality of input depth)</p></li>
<li><p><strong>Hin</strong> – Input width</p></li>
<li><p><strong>Win</strong> – Input height</p></li>
@@ -1118,13 +2871,13 @@
<li><p><strong>lr</strong> – Learning rate</p></li>
<li><p><strong>mu</strong> – Momentum value</p></li>
<li><p><strong>decay</strong> – Learning rate decay</p></li>
-<li><p><strong>lambda</strong> – Regularization strength</p></li>
+<li><p><strong>reg</strong> – Regularization strength</p></li>
<li><p><strong>seed</strong> – Seed for model initialization</p></li>
<li><p><strong>verbose</strong> – Flag indicates if function should print to stdout</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Trained model which can be used in lenetPredict</p>
</dd>
</dl>
</dd></dl>
@@ -1132,633 +2885,1770 @@
<dl class="py function">
<dt id="systemds.operator.algorithm.lm">
<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwarg [...]
-<dd><dl class="field-list simple">
+<dd><blockquote>
+<div><p>The lm-function solves linear regression using either the direct solve method or the conjugate gradient
+algorithm depending on the input size of the matrices (See lmDS-function and lmCG-function respectively).</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
+<li><p><strong>y</strong> – 1-column matrix of response values.</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
+<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero
+for highly dependant/sparse/numerous features</p></li>
+<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2
+norm of the beta-residual is less than tolerance * its initial norm</p></li>
+<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations. 0 = no maximum</p></li>
+<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The model fit</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.lmCG">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lmCG</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
+<dd><blockquote>
+<div><p>The lmCG function solves linear regression using the conjugate gradient algorithm</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
+<li><p><strong>y</strong> – 1-column matrix of response values.</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
+<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero
+for highly dependant/sparse/numerous features</p></li>
+<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2
+norm of the beta-residual is less than tolerance * its initial norm</p></li>
+<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations. 0 = no maximum</p></li>
+<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The model fit</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.lmDS">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lmDS</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
+<dd><blockquote>
+<div><p>The lmDC function solves linear regression using the direct solve method</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
+<li><p><strong>y</strong> – 1-column matrix of response values.</p></li>
<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
-<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero</p></li>
-<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2</p></li>
-<li><p><strong>norm</strong> – beta-residual is less than tolerance * its initial norm</p></li>
+<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero
+for highly dependant/sparse/numerous features</p></li>
+<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2
+norm of the beta-residual is less than tolerance * its initial norm</p></li>
<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations. 0 = no maximum</p></li>
<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The model fit</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.lmPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lmPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">B</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n [...]
+<dd><blockquote>
+<div><p>The lmPredict-function predicts the class of a feature vector</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors</p></li>
+<li><p><strong>B</strong> – 1-column matrix of weights.</p></li>
+<li><p><strong>ytest</strong> – test labels, used only for verbose output. can be set to matrix(0,1,1)
+if verbose output is not wanted</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
+<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>1-column matrix of classes</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.logSumExp">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">logSumExp</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">M</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</sp [...]
+<dd><blockquote>
+<div><p>Built-in LOGSUMEXP</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>M</strong> – matrix to perform Log sum exp on.</p></li>
+<li><p><strong>margin</strong> – if the logsumexp of rows is required set margin = “row”
+if the logsumexp of columns is required set margin = “col”
+if set to “none” then a single scalar is returned computing logsumexp of matrix</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>a 1*1 matrix, row vector or column vector depends on margin value</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.matrixProfile">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">matrixProfile</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">ts</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p"> [...]
+<dd><blockquote>
+<div><p>Builtin function that computes the MatrixProfile of a time series efficiently
+using the SCRIMP++ algorithm.</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>References:
+Yan Zhu et al.. 2018.
+ Matrix Profile XI: SCRIMP++: Time Series Motif Discovery at Interactive Speeds.
+ 2018 IEEE International Conference on Data Mining (ICDM), 2018, pp. 837-846.
+ DOI: 10.1109/ICDM.2018.00099.
+ https://www.cs.ucr.edu/~eamonn/SCRIMP_ICDM_camera_ready_updated.pdf
+</pre></div>
+</div>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>ts</strong> – Time series to profile</p></li>
+<li><p><strong>window_size</strong> – Sliding window size</p></li>
+<li><p><strong>sample_percent</strong> – Degree of approximation
+between zero and one (1
+computes the exact solution)</p></li>
+<li><p><strong>is_verbose</strong> – Print debug information</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The computed matrix profile</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Indices of least distances</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.mcc">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">mcc</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">predictions</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">labels</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" [...]
+<dd><blockquote>
+<div><p>Built-in function mcc: Matthews’ Correlation Coefficient for binary classification evaluation</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>predictions</strong> – Vector of predicted 0/1 values.
+(requires setting ‘labels’ parameter)</p></li>
+<li><p><strong>labels</strong> – Vector of 0/1 labels.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matthews’ Correlation Coefficient</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.mdedup">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">mdedup</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">LHSfeatures</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">LHSthreshold</span [...]
+<dd><blockquote>
+<div><p>Implements builtin for deduplication using matching dependencies (e.g. Street 0.95, City 0.90 -> ZIP 1.0)
+and Jaccard distance.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input Frame X</p></li>
+<li><p><strong>LHSfeatures</strong> – A matrix 1xd with numbers of columns for MDs
+(e.g. Street 0.95, City 0.90 -> ZIP 1.0)</p></li>
+<li><p><strong>LHSthreshold</strong> – A matrix 1xd with threshold values in interval [0, 1] for MDs</p></li>
+<li><p><strong>RHSfeatures</strong> – A matrix 1xd with numbers of columns for MDs</p></li>
+<li><p><strong>RHSthreshold</strong> – A matrix 1xd with threshold values in interval [0, 1] for MDs</p></li>
+<li><p><strong>verbose</strong> – To print the output</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix nx1 of duplicates</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.mice">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">mice</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">cMask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n" [...]
+<dd><blockquote>
+<div><p>This Builtin function implements multiple imputation using Chained Equations (MICE)</p>
+<p>Assumption missing value are represented with empty string i.e “,,” in CSV file
+variables with suffix n are storing continuos/numeric data and variables with
+suffix c are storing categorical data</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>cMask</strong> – A 0/1 row vector for identifying numeric (0) and categorical features (1)</p></li>
+<li><p><strong>iter</strong> – Number of iteration for multiple imputations</p></li>
+<li><p><strong>threshold</strong> – confidence value [0, 1] for robust imputation, values will only be imputed
+if the predicted value has probability greater than threshold,
+only applicable for categorical data</p></li>
+<li><p><strong>verbose</strong> – Boolean value.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.miceApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">miceApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">meta</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><spa [...]
+<dd><blockquote>
+<div><p>This Builtin function implements multiple imputation using Chained Equations (MICE)</p>
+<p>Assumption missing value are represented with empty string i.e “,,” in CSV file
+variables with suffix n are storing continuos/numeric data and variables with
+suffix c are storing categorical data</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix (Recoded Matrix for categorical features)</p></li>
+<li><p><strong>mtea</strong> – A meta matrix with each rows storing values 1) mask of original matrix,
+2) information of columns with missing values on original data 0 for no missing value in column and 1 otherwise
+3) dist values in each columns in original data 1 for continuous columns and colMax for categorical</p></li>
+<li><p><strong>threshold</strong> – confidence value [0, 1] for robust imputation, values will only be imputed
+if the predicted value has probability greater than threshold,
+only applicable for categorical data</p></li>
+<li><p><strong>dM</strong> – meta frame from OHE on original data</p></li>
+<li><p><strong>betaList</strong> – List of machine learning models trained for each column imputation</p></li>
+<li><p><strong>verbose</strong> – Boolean value.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>imputed dataset</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.msvm">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">msvm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
+<dd><blockquote>
+<div><p>Implements builtin multi-class SVM with squared slack variables,
+learns one-against-the-rest binary-class classifiers by making a function call to l2SVM</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X of feature vectors</p></li>
+<li><p><strong>Y</strong> – matrix Y of class labels</p></li>
+<li><p><strong>intercept</strong> – No Intercept ( If set to TRUE then a constant bias column is added to X)</p></li>
+<li><p><strong>num_classes</strong> – Number of classes</p></li>
+<li><p><strong>epsilon</strong> – Procedure terminates early if the reduction in objective function
+value is less than epsilon (tolerance) times the initial objective function value.</p></li>
+<li><p><strong>reg</strong> – Regularization parameter (lambda) for L2 regularization</p></li>
+<li><p><strong>maxIterations</strong> – Maximum number of conjugate gradient iterations</p></li>
+<li><p><strong>verbose</strong> – Set to true to print while training.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>model matrix</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.msvmPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">msvmPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">W</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="# [...]
+<dd><blockquote>
+<div><p>This Scripts helps in applying an trained MSVM</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – matrix X of feature vectors to classify</p></li>
+<li><p><strong>W</strong> – matrix of the trained variables</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Classification Labels Raw, meaning not modified to clean
+Labeles of 1’s and -1’s</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Classification Labels Maxed to ones and zeros.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.multiLogReg">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">multiLogReg</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class= [...]
+<dd><blockquote>
+<div><p>Solves Multinomial Logistic Regression using Trust Region method.
+(See: Trust Region Newton Method for Logistic Regression, Lin, Weng and Keerthi, JMLR 9 (2008) 627-650)
+The largest label represents the baseline category; if label -1 or 0 is present, then it is
+the baseline label (and it is converted to the largest label).</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Location to read the matrix of feature vectors</p></li>
+<li><p><strong>Y</strong> – Location to read the matrix with category labels</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling X columns: 0 = no intercept,
+no shifting, no rescaling; 1 = add intercept, but neither shift nor
+rescale X; 2 = add intercept, shift & rescale X columns to mean = 0, variance = 1</p></li>
+<li><p><strong>tol</strong> – tolerance (“epsilon”)</p></li>
+<li><p><strong>reg</strong> – regularization parameter (lambda = 1/C); intercept is not regularized</p></li>
+<li><p><strong>maxi</strong> – max. number of outer (Newton) iterations</p></li>
+<li><p><strong>maxii</strong> – max. number of inner (conjugate gradient) iterations, 0 = no max</p></li>
+<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>regression betas as output for prediction</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.multiLogRegPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">multiLogRegPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">B</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span [...]
+<dd><blockquote>
+<div><p>THIS SCRIPT APPLIES THE ESTIMATED PARAMETERS OF MULTINOMIAL LOGISTIC REGRESSION TO A NEW (TEST) DATASET
+Matrix M of predicted means/probabilities, some statistics in CSV format (see below)</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Data Matrix X</p></li>
+<li><p><strong>B</strong> – Regression parameters betas</p></li>
+<li><p><strong>Y</strong> – Response vector Y</p></li>
+<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix M of predicted means/probabilities</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Predicted response vector</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>scalar value of accuracy</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.na_locf">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">na_locf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span [...]
+<dd><blockquote>
+<div><p>Builtin function for imputing missing values using forward fill and backward fill techniques</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>option</strong> – String “locf” (last observation moved forward) to do forward fill
+“nocb” (next observation carried backward) to do backward fill</p></li>
+<li><p><strong>verbose</strong> – to print output on screen</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix with no missing values</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.naiveBayes">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">naiveBayes</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">D</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class=" [...]
+<dd><blockquote>
+<div><p>The naiveBayes-function computes the class conditional probabilities and class priors.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>D</strong> – One dimensional column matrix with N rows.</p></li>
+<li><p><strong>C</strong> – One dimensional column matrix with N rows.</p></li>
+<li><p><strong>laplace</strong> – Any Double value.</p></li>
+<li><p><strong>verbose</strong> – Boolean value.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Class priors, One dimensional column matrix with N rows.</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Class conditional probabilities, One dimensional column matrix with N rows.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.naiveBayesPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">naiveBayesPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">P</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span c [...]
+<dd><blockquote>
+<div><p>The naiveBaysePredict-function predicts the scoring with a naive Bayes model.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of test data with N rows.</p></li>
+<li><p><strong>P</strong> – Class priors, One dimensional column matrix with N rows.</p></li>
+<li><p><strong>C</strong> – Class conditional probabilities, matrix with N rows</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>A matrix containing the top-K item-ids with highest predicted ratings.</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>A matrix containing predicted ratings.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.normalize">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">normalize</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.normalize" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>Min-max normalization (a.k.a. min-max scaling) to range [0,1]. For matrices
+of positive values, this normalization preserves the input sparsity.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><p><strong>X</strong> – Input feature matrix of shape n-by-m</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Modified output feature matrix of shape n-by-m</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Column minima of shape 1-by-m</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Column maxima of shape 1-by-m</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.normalizeApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">normalizeApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">cmin</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">cmax</span><spa [...]
+<dd><blockquote>
+<div><p>Min-max normalization (a.k.a. min-max scaling) to range [0,1], given
+existing min-max ranges. For matrices of positive values, this normalization
+preserves the input sparsity. The validity of the provided min-max range
+and post-processing is under control of the caller.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input feature matrix of shape n-by-m</p></li>
+<li><p><strong>cmin</strong> – Colunm minima of shape 1-by-m</p></li>
+<li><p><strong>cmax</strong> – Column maxima of shape 1-by-m</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Modified output feature matrix of shape n-by-m</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlier">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlier</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">opposite</span><span class="p">:</span> <span class="n">bool</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.ou [...]
+<dd><blockquote>
+<div><p>This outlier-function takes a matrix data set as input from where it determines
+which point(s) have the largest difference from mean.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of Recoded dataset for outlier evaluation</p></li>
+<li><p><strong>opposite</strong> – (1)TRUE for evaluating outlier from upper quartile range,
+(0)FALSE for evaluating outlier from lower quartile range</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>matrix indicating outlier values</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlierByArima">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByArima</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p"> [...]
+<dd><blockquote>
+<div><p>Built-in function for detecting and repairing outliers in time series, by training an ARIMA model
+and classifying values that are more than k standard-deviations away from the predicated values as outliers.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>k</strong> – threshold values 1, 2, 3 for 68%, 95%, 99.7% respectively (3-sigma rule)</p></li>
+<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers, 1 = replace outliers as zeros
+2 = replace outliers as missing values</p></li>
+<li><p><strong>p</strong> – non-seasonal AR order</p></li>
+<li><p><strong>d</strong> – non-seasonal differencing order</p></li>
+<li><p><strong>q</strong> – non-seasonal MA order</p></li>
+<li><p><strong>P</strong> – seasonal AR order</p></li>
+<li><p><strong>D</strong> – seasonal differencing order</p></li>
+<li><p><strong>Q</strong> – seasonal MA order</p></li>
+<li><p><strong>s</strong> – period in terms of number of time-steps</p></li>
+<li><p><strong>include_mean</strong> – If the mean should be included</p></li>
+<li><p><strong>solver</strong> – solver, is either “cg” or “jacobi”</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix X with no outliers</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlierByIQR">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByIQR</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">k</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">max_iterations</span><span class="p">:</span> <span [...]
+<dd><blockquote>
+<div><p>Builtin function for detecting and repairing outliers using standard deviation</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
+<li><p><strong>isIterative</strong> – iterative repair or single repair</p></li>
+<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,
+1 = replace outliers with zeros
+2 = replace outliers as missing values</p></li>
+<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteraition until all outliers are removed,
+n = any constant defined by user</p></li>
+<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix X with no outliers</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlierByIQRApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByIQRApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Q1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Q3</span><span [...]
+<dd><blockquote>
+<div><p>Builtin function for repairing outliers by IQR</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>Q1</strong> – first quartile</p></li>
+<li><p><strong>Q3</strong> – third quartile</p></li>
+<li><p><strong>IQR</strong> – Inter-quartile range</p></li>
+<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
+<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,
+1 = replace outliers with zeros
+2 = replace outliers as missing values</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix X with no outliers</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlierBySd">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierBySd</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">max_iterations</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><spa [...]
+<dd><blockquote>
+<div><p>Builtin function for detecting and repairing outliers using standard deviation</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>k</strong> – threshold values 1, 2, 3 for 68%, 95%, 99.7% respectively (3-sigma rule)</p></li>
+<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers, 1 = replace outliers as zeros
+2 = replace outliers as missing values</p></li>
+<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteration until all outliers are removed,
+n = any constant defined by user</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix X with no outliers</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.outlierBySdApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierBySdApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">colMean</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">colSD</spa [...]
+<dd><blockquote>
+<div><p>Builtin function for detecting and repairing outliers using standard deviation</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix X</p></li>
+<li><p><strong>colMean</strong> – Matrix X</p></li>
+<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
+<li><p><strong>isIterative</strong> – iterative repair or single repair</p></li>
+<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,
+1 = replace outliers with zeros
+2 = replace outliers as missing values</p></li>
+<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteraition until all outliers are removed,
+n = any constant defined by user</p></li>
+<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix X with no outliers</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.pca">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pca</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DAG [...]
+<dd><blockquote>
+<div><p>The function Principal Component Analysis (PCA) is used for dimensionality reduction</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>K</strong> – Number of reduced dimensions (i.e., columns)</p></li>
+<li><p><strong>Center</strong> – Indicates whether or not to center the feature matrix</p></li>
+<li><p><strong>Scale</strong> – Indicates whether or not to scale the feature matrix</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Output feature matrix with K columns</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Output dominant eigen vectors (can be used for projections)</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The column means of the input, subtracted to construct the PCA</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>The Scaling of the values, to make each dimension same size.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.pcaInverse">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pcaInverse</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Clusters</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Centering</span [...]
+<dd><blockquote>
+<div><p>Principal Component Analysis (PCA) for reconstruction of approximation of the original data.
+This methods allows to reconstruct an approximation of the original matrix, and is useful for
+calculating how much information is lost in the PCA.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>Y</strong> – Input features that have PCA applied to them</p></li>
+<li><p><strong>Clusters</strong> – The previous PCA components computed</p></li>
+<li><p><strong>Centering</strong> – The column means of the PCA model, subtracted to construct the PCA</p></li>
+<li><p><strong>ScaleFactor</strong> – The scaling of each dimension in the PCA model</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Output feature matrix reconstructing and approximation of the original matrix</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.pcaTransform">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pcaTransform</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Clusters</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Centering</sp [...]
+<dd><blockquote>
+<div><p>Principal Component Analysis (PCA) for dimensionality reduction prediction
+This method is used to transpose data, which the PCA model was not trained on. To validate how good
+The PCA is, and to apply in production.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>Clusters</strong> – The previously computed principal components</p></li>
+<li><p><strong>Centering</strong> – The column means of the PCA model, subtracted to construct the PCA</p></li>
+<li><p><strong>ScaleFactor</strong> – The scaling of each dimension in the PCA model</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Output feature matrix dimensionally reduced by PCA</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.pnmf">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pnmf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">rnk</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</spa [...]
+<dd><blockquote>
+<div><p>The pnmf-function implements Poisson Non-negative Matrix Factorization (PNMF). Matrix X is factorized into two
+non-negative matrices, W and H based on Poisson probabilistic assumption. This non-negativity makes the resulting
+matrices easier to inspect.</p>
+<p>[Chao Liu, Hung-chih Yang, Jinliang Fan, Li-Wei He, Yi-Min Wang:
+Distributed nonnegative matrix factorization for web-scale dyadic
+data analysis on mapreduce. WWW 2010: 681-690]</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
+<li><p><strong>rnk</strong> – Number of components into which matrix X is to be factored.</p></li>
+<li><p><strong>eps</strong> – Tolerance</p></li>
+<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations.</p></li>
+<li><p><strong>verbose</strong> – If TRUE, ‘iter’ and ‘obj’ are printed.</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>List of pattern matrices, one for each repetition.</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>List of amplitude matrices, one for each repetition.</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.ppca">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ppca</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DA [...]
+<dd><blockquote>
+<div><p>This script performs Probabilistic Principal Component Analysis (PCA) on the given input data.
+It is based on paper: sPCA: Scalable Principal Component Analysis for Big Data on Distributed
+Platforms. Tarek Elgamal et.al.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – n x m input feature matrix</p></li>
+<li><p><strong>k</strong> – indicates dimension of the new vector space constructed from eigen vectors</p></li>
+<li><p><strong>maxi</strong> – maximum number of iterations until convergence</p></li>
+<li><p><strong>tolobj</strong> – objective function tolerance value to stop ppca algorithm</p></li>
+<li><p><strong>tolrecerr</strong> – reconstruction error tolerance value to stop the algorithm</p></li>
+<li><p><strong>verbose</strong> – verbose debug output</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Output feature matrix with K columns</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Output dominant eigen vectors (can be used for projections)</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.randomForest">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">randomForest</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">R</span><span class= [...]
+<dd><blockquote>
+<div><p>This script implement classification random forest with both scale and categorical features.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Feature matrix X; note that X needs to be both recoded and dummy coded</p></li>
+<li><p><strong>Y</strong> – Label matrix Y; note that Y needs to be both recoded and dummy coded</p></li>
+<li><p><strong>R</strong> – Matrix which for each feature in X contains the following information
+- R[,1]: column ids TODO pass recorded and binned
+- R[,2]: start indices
+- R[,3]: end indices
+If R is not provided by default all variables are assumed to be scale</p></li>
+<li><p><strong>bins</strong> – Number of equiheight bins per scale feature to choose thresholds</p></li>
+<li><p><strong>depth</strong> – Maximum depth of the learned tree</p></li>
+<li><p><strong>num_leaf</strong> – Number of samples when splitting stops and a leaf node is added</p></li>
+<li><p><strong>num_samples</strong> – Number of samples at which point we switch to in-memory subtree building</p></li>
+<li><p><strong>num_trees</strong> – Number of trees to be learned in the random forest model</p></li>
+<li><p><strong>subsamp_rate</strong> – Parameter controlling the size of each tree in the forest; samples are selected from a
+Poisson distribution with parameter subsamp_rate (the default value is 1.0)</p></li>
+<li><p><strong>feature_subset</strong> – Parameter that controls the number of feature used as candidates for splitting at each tree node
+as a power of number of features in the dataset;
+by default square root of features (i.e., feature_subset = 0.5) are used at each tree node</p></li>
+<li><p><strong>impurity</strong> – Impurity measure: entropy or Gini (the default)</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Matrix M containing the learned tree, where each column corresponds to a node
+in the learned tree and each row contains the following information:
+M[1,j]: id of node j (in a complete binary tree)
+M[2,j]: tree id to which node j belongs
+M[3,j]: Offset (no. of columns) to left child of j
+M[4,j]: Feature index of the feature that node j looks at if j is an internal node, otherwise 0
+M[5,j]: Type of the feature that node j looks at if j is an internal node: 1 for scale and 2
+for categorical features,
+otherwise the label that leaf node j is supposed to predict
+M[6,j]: 1 if j is an internal node and the feature chosen for j is scale, otherwise the
+size of the subset of values
+stored in rows 7,8,… if j is categorical
+M[7:,j]: Only applicable for internal nodes. Threshold the example’s feature value is
+compared to is stored at M[7,j] if the feature chosen for j is scale;
+If the feature chosen for j is categorical rows 7,8,… depict the value subset chosen for j</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix C containing the number of times samples are chosen in each tree of the random forest</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Mappings from scale feature ids to global feature ids</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Mappings from categorical feature ids to global feature ids</p>
+</dd>
+</dl>
+</dd></dl>
+
+<dl class="py function">
+<dt id="systemds.operator.algorithm.scale">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">scale</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>D [...]
+<dd><blockquote>
+<div><p>This function scales and center individual features in the input matrix (column wise.) using z-score to scale the values.</p>
+</div></blockquote>
+<dl class="field-list simple">
+<dt class="field-odd">Parameters</dt>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>center</strong> – Indicates whether or not to center the feature matrix</p></li>
+<li><p><strong>scale</strong> – Indicates whether or not to scale the feature matrix</p></li>
+</ul>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Output feature matrix with K columns</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>The column means of the input, subtracted if Center was TRUE</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>The Scaling of the values, to make each dimension have similar value ranges</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.lmCG">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lmCG</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.scaleApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">scaleApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Centering</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">ScaleFactor</s [...]
+<dd><blockquote>
+<div><p>This function scales and center individual features in the input matrix (column wise.) using the input matrices.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
-<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero</p></li>
-<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2</p></li>
-<li><p><strong>norm</strong> – beta-residual is less than tolerance * its initial norm</p></li>
-<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations. 0 = no maximum</p></li>
-<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>Centering</strong> – The column means to subtract from X (not done if empty)</p></li>
+<li><p><strong>ScaleFactor</strong> – The column scaling to multiply with X (not done if empty)</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output feature matrix with K columns</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.lmDS">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">lmDS</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.scaleMinMax">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">scaleMinMax</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.scaleMinMax" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This function performs min-max normalization (rescaling to [0,1]).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X</p></li>
-<li><p><strong>reg</strong> – Regularization constant (lambda) for L2-regularization. set to nonzero</p></li>
-<li><p><strong>tol</strong> – Tolerance (epsilon); conjugate gradient procedure terminates early if L2</p></li>
-<li><p><strong>norm</strong> – beta-residual is less than tolerance * its initial norm</p></li>
-<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations. 0 = no maximum</p></li>
-<li><p><strong>verbose</strong> – If TRUE print messages are activated</p></li>
-</ul>
+<dd class="field-odd"><p><strong>X</strong> – Input feature matrix</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Scaled output matrix</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.logSumExp">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">logSumExp</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">M</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</sp [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.selectByVarThresh">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">selectByVarThresh</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class=" [...]
+<dd><blockquote>
+<div><p>This function drops feature with <= thresh variance (by default drop constants).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>margin</strong> – if the logsumexp of rows is required set margin = “row”</p></li>
-<li><p><strong>if</strong> – of columns is required set margin = “col”</p></li>
-<li><p><strong>if</strong> – “none” then a single scalar is returned computing logsumexp of matrix</p></li>
+<li><p><strong>X</strong> – Matrix of feature vectors.</p></li>
+<li><p><strong>thresh</strong> – The threshold for to drop</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix of feature vectors with <= thresh variance.</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.matrixProfile">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">matrixProfile</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">ts</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p"> [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.setdiff">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">setdiff</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#syst [...]
+<dd><blockquote>
+<div><p>Builtin function that implements difference operation on vectors</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>window_size</strong> – Sliding window size</p></li>
-<li><p><strong>sample_percent</strong> – Degree of approximation</p></li>
-<li><p><strong>between</strong> – one (1</p></li>
-<li><p><strong>computes</strong> – solution)</p></li>
-<li><p><strong>is_verbose</strong> – Print debug information</p></li>
+<li><p><strong>X</strong> – input vector</p></li>
+<li><p><strong>Y</strong> – input vector</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>vector with all elements that are present in X but not in Y</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.miceApply">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">miceApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">meta</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">threshold</span><spa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.sherlock">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">sherlock</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y_train</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink [...]
+<dd><blockquote>
+<div><p>This function implements training phase of Sherlock: A Deep Learning Approach to Semantic Data Type Detection</p>
+<p>[Hulsebos, Madelon, et al. “Sherlock: A deep learning approach to semantic data type detection.”
+Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
+2019.]</p>
+<p>Split feature matrix into four different feature categories and train neural networks on the
+respective single features. Then combine all trained features to train final neural network.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>threshold</strong> – confidence value [0, 1] for robust imputation, values will only be imputed</p></li>
-<li><p><strong>if</strong> – value has probability greater than threshold,</p></li>
-<li><p><strong>only</strong> – categorical data</p></li>
-<li><p><strong>verbose</strong> – Boolean value.</p></li>
+<li><p><strong>X_train</strong> – matrix of feature vectors</p></li>
+<li><p><strong>y_train</strong> – matrix Y of class labels of semantic data type</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing are represented with empty string i.e “,,” in csv file & n are storing continuos/numeric data and variables with & storing categorical data</p>
+<dd class="field-even"><p>weights (parameters) matrices for character distributions</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>biases vectors for character distributions</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>weights (parameters) matrices for word embeddings</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>biases vectors for word embeddings</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>weights (parameters) matrices for paragraph vectors</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>biases vectors for paragraph vectors</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>weights (parameters) matrices for global statistics</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>biases vectors for global statistics</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>weights (parameters) matrices for combining all trained features (final)</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>biases vectors for combining all trained features (final)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.msvm">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">msvm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.sherlockPredict">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">sherlockPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">cW1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">cb1</span><span [...]
+<dd><blockquote>
+<div><p>This function implements prediction and evaluation phase of Sherlock:
+Split feature matrix into four different feature categories and predicting the class probability
+on the respective features. Then combine all predictions for final predicted probabilities.
+A Deep Learning Approach to Semantic Data Type Detection.
+[Hulsebos, Madelon, et al. “Sherlock: A deep learning approach to semantic data type detection.”
+Proceedings of the 25th ACM SIGKDD International Conference on Knowledge Discovery & Data Mining.
+2019.]</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>intercept</strong> – No Intercept ( If set to TRUE then a constant bias column is added to X)</p></li>
-<li><p><strong>num_classes</strong> – Number of classes</p></li>
-<li><p><strong>epsilon</strong> – Procedure terminates early if the reduction in objective function</p></li>
-<li><p><strong>value</strong> – than epsilon (tolerance) times the initial objective function value.</p></li>
-<li><p><strong>lambda</strong> – Regularization parameter (lambda) for L2 regularization</p></li>
-<li><p><strong>maxIterations</strong> – Maximum number of conjugate gradient iterations</p></li>
-<li><p><strong>verbose</strong> – Set to true to print while training.</p></li>
+<li><p><strong>X</strong> – matrix of values which are to be classified</p></li>
+<li><p><strong>cW</strong> – weights (parameters) matrices for character distribtions</p></li>
+<li><p><strong>cb</strong> – biases vectors for character distribtions</p></li>
+<li><p><strong>wW</strong> – weights (parameters) matrices for word embeddings</p></li>
+<li><p><strong>wb</strong> – biases vectors for word embeddings</p></li>
+<li><p><strong>pW</strong> – weights (parameters) matrices for paragraph vectors</p></li>
+<li><p><strong>pb</strong> – biases vectors for paragraph vectors</p></li>
+<li><p><strong>sW</strong> – weights (parameters) matrices for global statistics</p></li>
+<li><p><strong>sb</strong> – biases vectors for global statistics</p></li>
+<li><p><strong>fW</strong> – weights (parameters) matrices for combining all trained features (final)</p></li>
+<li><p><strong>fb</strong> – biases vectors for combining all trained features (final)</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>class probabilities of shape (N, K)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.multiLogReg">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">multiLogReg</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class= [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.shortestPath">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">shortestPath</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">G</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">sourceNode</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span c [...]
+<dd><blockquote>
+<div><p>Computes the minimum distances (shortest-path) between a single source vertex and every other vertex in the graph.</p>
+<p>Grzegorz Malewicz, Matthew H. Austern, Aart J. C. Bilk,
+James C. Dehnert, Ikkan Horn, Naty Leiser and Grzegorz Czajkowski:
+Pregel: A System for Large-Scale Graph Processing, SIGMOD 2010</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling X columns: 0 = no intercept,</p></li>
-<li><p><strong>tol</strong> – tolerance (“epsilon”)</p></li>
-<li><p><strong>reg</strong> – regularization parameter (lambda = 1/C); intercept is not regularized</p></li>
-<li><p><strong>maxi</strong> – max. number of outer (Newton) iterations</p></li>
-<li><p><strong>maxii</strong> – max. number of inner (conjugate gradient) iterations, 0 = no max</p></li>
-<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+<li><p><strong>G</strong> – adjacency matrix of the labeled graph: Such graph can be directed
+(G is symmetric) or undirected (G is not symmetric).
+The values of G can be 0/1 (just specifying whether the nodes
+are connected or not) or integer values (representing the weight
+of the edges or the distances between nodes, 0 if not connected).</p></li>
+<li><p><strong>maxi</strong> – Integer max number of iterations accepted (0 for FALSE, i.e.
+max number of iterations not defined)</p></li>
+<li><p><strong>sourceNode</strong> – node index to calculate the shortest paths to all other nodes.</p></li>
+<li><p><strong>verbose</strong> – flag for verbose debug output</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output matrix (double) of minimum distances (shortest-path) between
+vertices: The value of the ith row and the jth column of the output
+matrix is the minimum distance shortest-path from vertex i to vertex j.
+When the value of the minimum distance is infinity, the two nodes are
+not connected.</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.multiLogRegPredict">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">multiLogRegPredict</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">B</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.sigmoid">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">sigmoid</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.sigmoid" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>The Sigmoid function is a type of activation function, and also defined as a squashing function which limit the
+output to a range between 0 and 1, which will make these functions useful in the prediction of probabilities.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>verbose</strong> – flag specifying if logging information should be printed</p>
+<dd class="field-odd"><p><strong>X</strong> – Matrix of feature vectors.</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing value of accuracy</p>
+<dd class="field-even"><p>1-column matrix of weights.</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.na_locf">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">na_locf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.slicefinder">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">slicefinder</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">e</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class= [...]
+<dd><blockquote>
+<div><p>This builtin function implements SliceLine, a linear-algebra-based
+ML model debugging technique for finding the top-k data slices where
+a trained models performs significantly worse than on the overall
+dataset. For a detailed description and experimental results, see:
+Svetlana Sagadeeva, Matthias Boehm: SliceLine: Fast, Linear-Algebra-based Slice Finding for ML Model Debugging.(SIGMOD 2021)</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>option</strong> – String “locf” (last observation moved forward) to do forward fill</p></li>
-<li><p><strong>verbose</strong> – to print output on screen</p></li>
+<li><p><strong>X</strong> – Recoded dataset into Matrix</p></li>
+<li><p><strong>e</strong> – Trained model</p></li>
+<li><p><strong>k</strong> – Number of subsets required</p></li>
+<li><p><strong>maxL</strong> – maximum level L (conjunctions of L predicates), 0 unlimited</p></li>
+<li><p><strong>minSup</strong> – minimum support (min number of rows per slice)</p></li>
+<li><p><strong>alpha</strong> – weight [0,1]: 0 only size, 1 only error</p></li>
+<li><p><strong>tpEval</strong> – flag for task-parallel slice evaluation,
+otherwise data-parallel</p></li>
+<li><p><strong>tpBlksz</strong> – block size for task-parallel execution (num slices)</p></li>
+<li><p><strong>selFeat</strong> – flag for removing one-hot-encoded features that don’t satisfy
+the initial minimum-support constraint and/or have zero error</p></li>
+<li><p><strong>verbose</strong> – flag for verbose debug output</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>top-k slices (k x ncol(X) if successful)</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>score, size, error of slices (k x 3)</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>debug matrix, populated with enumeration stats if verbose</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.naiveBayes">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">naiveBayes</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">D</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">C</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class=" [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.smote">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">smote</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n" [...]
+<dd><blockquote>
+<div><p>Builtin function for handing class imbalance using Synthetic Minority Over-sampling Technique (SMOTE)
+by Nitesh V. Chawla et. al. In Journal of Artificial Intelligence Research 16 (2002). 321–357</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>Laplace</strong> – Any Double value.</p></li>
-<li><p><strong>Verbose</strong> – Boolean value.</p></li>
+<li><p><strong>X</strong> – Matrix of minority class samples</p></li>
+<li><p><strong>mask</strong> – 0/1 mask vector where 0 represent numeric value and 1 represent categorical value</p></li>
+<li><p><strong>s</strong> – Amount of SMOTE (percentage of oversampling), integral multiple of 100</p></li>
+<li><p><strong>k</strong> – Number of nearest neighbor</p></li>
+<li><p><strong>verbose</strong> – if the algorithm should be verbose</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix of (N/100)-1 * nrow(X) synthetic minority class samples</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.outlier">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlier</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">opposite</span><span class="p">:</span> <span class="n">bool</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.ou [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.softmax">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">softmax</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">S</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.softmax" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>Performs softmax on the given input matrix.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>opposite</strong> – (1)TRUE for evaluating outlier from upper quartile range,</p>
+<dd class="field-odd"><p><strong>S</strong> – Inputs of shape (N, D).</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Outputs of shape (N, D).</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.outlierByArima">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByArima</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p"> [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.split">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">split</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kw [...]
+<dd><blockquote>
+<div><p>This function split input data X and Y into contiguous or samples train/test sets</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers, 1 = replace outliers as zeros</p></li>
-<li><p><strong>p</strong> – non-seasonal AR order</p></li>
-<li><p><strong>d</strong> – non-seasonal differencing order</p></li>
-<li><p><strong>q</strong> – non-seasonal MA order</p></li>
-<li><p><strong>P</strong> – seasonal AR order</p></li>
-<li><p><strong>D</strong> – seasonal differencing order</p></li>
-<li><p><strong>Q</strong> – seasonal MA order</p></li>
-<li><p><strong>s</strong> – period in terms of number of time-steps</p></li>
-<li><p><strong>solver</strong> – solver, is either “cg” or “jacobi”</p></li>
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>Y</strong> – Input Labels</p></li>
+<li><p><strong>f</strong> – Train set fraction [0,1]</p></li>
+<li><p><strong>cont</strong> – contiguous splits, otherwise sampled</p></li>
+<li><p><strong>seed</strong> – The seed to randomly select rows in sampled mode</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Train split of feature matrix</p>
</dd>
-</dl>
-</dd></dl>
-
-<dl class="py function">
-<dt id="systemds.operator.algorithm.outlierByIQR">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByIQR</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">k</span><span class="p">:</span> <span class="n">float</span></em>, <em class="sig-param"><span class="n">max_iterations</span><span class="p">:</span> <span [...]
-<dd><dl class="field-list simple">
-<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
-<li><p><strong>isIterative</strong> – iterative repair or single repair</p></li>
-<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,</p></li>
-<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteraition until all outliers are removed,</p></li>
-<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
-</ul>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Test split of feature matrix</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing meaning & matrix x with no outliers</p>
+<dd class="field-even"><p>Train split of label matrix</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Test split of label matrix</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.outlierByIQRApply">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierByIQRApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Q1</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Q3</span><span [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.splitBalanced">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">splitBalanced</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span clas [...]
+<dd><blockquote>
+<div><p>This functions split input data X and Y into contiguous balanced ratio
+Related to [SYSTEMDS-2902] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
-<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,</p></li>
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>Y</strong> – Input Labels</p></li>
+<li><p><strong>f</strong> – Train set fraction [0,1]</p></li>
+<li><p><strong>verbose</strong> – print available</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing meaning & matrix x with no outliers</p>
+<dd class="field-even"><p>Train split of feature matrix</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Test split of feature matrix</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Train split of label matrix</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Test split of label matrix</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.outlierBySd">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierBySd</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">max_iterations</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><spa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.stableMarriage">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">stableMarriage</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">P</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">A</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span cla [...]
+<dd><blockquote>
+<div><p>This script computes a solution for the stable marriage problem.</p>
+<p>result description:</p>
+<p>If cell [i,j] is non-zero, it means that acceptor i has matched with
+proposer j. Further, if cell [i,j] is non-zero, it holds the preference
+value that led to the match.
+Proposers.mtx:
+2.0,1.0,3.0
+1.0,2.0,3.0
+1.0,3.0,2.0</p>
+<p>Since ordered=TRUE, this means that proposer 1 (row 1) likes acceptor 2
+the most, followed by acceptor 1 and acceptor 3.
+If ordered=FALSE, this would mean that proposer 1 (row 1) likes acceptor 3
+the most (since the value at [1,3] is the row max),
+followed by acceptor 1 (2.0 preference value) and acceptor 2 (1.0 preference value).</p>
+<p>Acceptors.mtx:
+3.0,1.0,2.0
+2.0,1.0,3.0
+3.0,2.0,1.0</p>
+<p>Since ordered=TRUE, this means that acceptor 1 (row 1) likes proposer 3
+the most, followed by proposer 1 and proposer 2.
+If ordered=FALSE, this would mean that acceptor 1 (row 1) likes proposer 1
+the most (since the value at [1,1] is the row max),
+followed by proposer 3 (2.0 preference value) and proposer 2
+(1.0 preference value).</p>
+<p>Output.mtx (assuming ordered=TRUE):
+0.0,0.0,3.0
+0.0,3.0,0.0
+1.0,0.0,0.0</p>
+<p>Acceptor 1 has matched with proposer 3 (since [1,3] is non-zero) at a
+preference level of 3.0.
+Acceptor 2 has matched with proposer 2 (since [2,2] is non-zero) at a
+preference level of 3.0.
+Acceptor 3 has matched with proposer 1 (since [3,1] is non-zero) at a
+preference level of 1.0.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – threshold values 1, 2, 3 for 68%, 95%, 99.7% respectively (3-sigma rule)</p></li>
-<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers, 1 = replace outliers as zeros</p></li>
-<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteration until all outliers are removed,</p></li>
+<li><p><strong>P</strong> – proposer matrix P.
+It must be a square matrix with no zeros.</p></li>
+<li><p><strong>A</strong> – acceptor matrix A.
+It must be a square matrix with no zeros.</p></li>
+<li><p><strong>ordered</strong> – If true, P and A are assumed to be ordered,
+i.e. the leftmost value in a row is the most preferred partner’s index.
+i.e. the leftmost value in a row in P is the preference value for the acceptor with
+index 1 and vice-versa (higher is better).</p></li>
+<li><p><strong>verbose</strong> – if the algorithm should print verbosely</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Result Matrix</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.outlierBySdApply">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">outlierBySdApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">colMean</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">colSD</spa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.statsNA">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">statsNA</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span [...]
+<dd><blockquote>
+<div><p>The statsNA-function Print summary stats about the distribution of missing values in a univariate time series.</p>
+<dl class="simple">
+<dt>result matrix contains the following:</dt><dd><ol class="arabic simple">
+<li><p>Length of time series (including NAs)</p></li>
+<li><p>Number of Missing Values (NAs)</p></li>
+<li><p>Percentage of Missing Values (#2/#1)</p></li>
+<li><p>Number of Gaps (consisting of one or more consecutive NAs)</p></li>
+<li><p>Average Gap Size - Average size of consecutive NAs for the NA gaps</p></li>
+<li><p>Longest NA gap - Longest series of consecutive missing values</p></li>
+<li><p>Most frequent gap size - Most frequently occurring gap size</p></li>
+<li><p>Gap size accounting for most NAs</p></li>
+</ol>
+</dd>
+</dl>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – a constant used to discern outliers k*IQR</p></li>
-<li><p><strong>isIterative</strong> – iterative repair or single repair</p></li>
-<li><p><strong>repairMethod</strong> – values: 0 = delete rows having outliers,</p></li>
-<li><p><strong>max_iterations</strong> – values: 0 = arbitrary number of iteraition until all outliers are removed,</p></li>
-<li><p><strong>verbose</strong> – flag specifying if logging information should be printed</p></li>
+<li><p><strong>X</strong> – Numeric Vector (‘vector’) object containing NAs</p></li>
+<li><p><strong>bins</strong> – Split number for bin stats. Number of bins the time series gets
+divided into. For each bin information about amount/percentage of
+missing values is printed.</p></li>
+<li><p><strong>verbose</strong> – Print detailed information.
+For print_only = TRUE, the missing value stats are printed with
+more information (“Stats for Bins” and “overview NA series”).</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing meaning & matrix x with no outliers</p>
+<dd class="field-even"><p>Column vector where each row correspond to described values</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.pca">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pca</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DAG [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.steplm">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">steplm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">k [...]
+<dd><blockquote>
+<div><p>The steplm-function (stepwise linear regression) implements a classical forward feature selection method.
+This method iteratively runs what-if scenarios and greedily selects the next best feature
+until the Akaike information criterion (AIC) does not improve anymore. Each configuration trains a regression model
+via lm, which in turn calls either the closed form lmDS or iterative lmGC.</p>
+<div class="highlight-txt notranslate"><div class="highlight"><pre><span></span>return: Matrix of regression parameters (the betas) and its size depend on icpt input value:
+ OUTPUT SIZE: OUTPUT CONTENTS: HOW TO PREDICT Y FROM X AND B:
+icpt=0: ncol(X) x 1 Betas for X only Y ~ X %*% B[1:ncol(X), 1], or just X %*% B
+icpt=1: ncol(X)+1 x 1 Betas for X and intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
+icpt=2: ncol(X)+1 x 2 Col.1: betas for X & intercept Y ~ X %*% B[1:ncol(X), 1] + B[ncol(X)+1, 1]
+ Col.2: betas for shifted/rescaled X and intercept
+</pre></div>
+</div>
+<p>In addition, in the last run of linear regression some statistics are provided in CSV format, one comma-separated
+name-value pair per each line, as follows:</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>K</strong> – Number of reduced dimensions (i.e., columns)</p></li>
-<li><p><strong>Center</strong> – Indicates whether or not to center the feature matrix</p></li>
-<li><p><strong>Scale</strong> – Indicates whether or not to scale the feature matrix</p></li>
+<li><p><strong>X</strong> – Location (on HDFS) to read the matrix X of feature vectors</p></li>
+<li><p><strong>Y</strong> – Location (on HDFS) to read the 1-column matrix Y of response values</p></li>
+<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X:
+0 = no intercept, no shifting, no rescaling;
+1 = add intercept, but neither shift nor rescale X;
+2 = add intercept, shift & rescale X columns to mean = 0, variance = 1</p></li>
+<li><p><strong>reg</strong> – learning rate</p></li>
+<li><p><strong>tol</strong> – Tolerance threshold to train until achieved</p></li>
+<li><p><strong>maxi</strong> – maximum iterations 0 means until tolerance is reached</p></li>
+<li><p><strong>verbose</strong> – If the algorithm should be verbose</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix of regression parameters (the betas) and its size depend on icpt input value.</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Matrix of selected features ordered as computed by the algorithm.</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.pnmf">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">pnmf</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">rnk</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</spa [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.stratstats">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">stratstats</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</s [...]
+<dd><blockquote>
+<div><p>The stratstats.dml script computes common bivariate statistics, such as correlation, slope, and their p-value,
+in parallel for many pairs of input variables in the presence of a confounding categorical variable.</p>
+<p>Output contains:
+(1st covariante, 2nd covariante)
+40 columns containing the following information:
+Col 01: 1st covariate X-column number
+Col 02: 1st covariate global presence count
+Col 03: 1st covariate global mean
+Col 04: 1st covariate global standard deviation
+Col 05: 1st covariate stratified standard deviation
+Col 06: R-squared, 1st covariate vs. strata
+Col 07: adjusted R-squared, 1st covariate vs. strata
+Col 08: P-value, 1st covariate vs. strata
+Col 09-10: Reserved
+Col 11: 2nd covariate Y-column number
+Col 12: 2nd covariate global presence count
+Col 13: 2nd covariate global mean
+Col 14: 2nd covariate global standard deviation
+Col 15: 2nd covariate stratified standard deviation
+Col 16: R-squared, 2nd covariate vs. strata
+Col 17: adjusted R-squared, 2nd covariate vs. strata
+Col 18: P-value, 2nd covariate vs. strata
+Col 19-20: Reserved
+Col 21: Global 1st & 2nd covariate presence count
+Col 22: Global regression slope (2nd vs. 1st covariate)
+Col 23: Global regression slope standard deviation
+Col 24: Global correlation = +/- sqrt(R-squared)
+Col 25: Global residual standard deviation
+Col 26: Global R-squared
+Col 27: Global adjusted R-squared
+Col 28: Global P-value for hypothesis “slope = 0”
+Col 29-30: Reserved
+Col 31: Stratified 1st & 2nd covariate presence count
+Col 32: Stratified regression slope (2nd vs. 1st covariate)
+Col 33: Stratified regression slope standard deviation
+Col 34: Stratified correlation = +/- sqrt(R-squared)
+Col 35: Stratified residual standard deviation
+Col 36: Stratified R-squared
+Col 37: Stratified adjusted R-squared
+Col 38: Stratified P-value for hypothesis “slope = 0”
+Col 39: Number of strata with at least two counted points
+Col 40: Reserved</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>rnk</strong> – Number of components into which matrix X is to be factored.</p></li>
-<li><p><strong>maxi</strong> – Maximum number of conjugate gradient iterations.</p></li>
-<li><p><strong>verbose</strong> – If TRUE, ‘iter’ and ‘obj’ are printed.</p></li>
+<li><p><strong>X</strong> – Matrix X that has all 1-st covariates</p></li>
+<li><p><strong>Y</strong> – Matrix Y that has all 2-nd covariates
+the default value empty means “use X in place of Y”</p></li>
+<li><p><strong>S</strong> – Matrix S that has the stratum column
+the default value empty means “use X in place of S”</p></li>
+<li><p><strong>Xcid</strong> – 1-st covariate X-column indices
+the default value empty means “use columns 1 : ncol(X)”</p></li>
+<li><p><strong>Ycid</strong> – 2-nd covariate Y-column indices
+the default value empty means “use columns 1 : ncol(Y)”</p></li>
+<li><p><strong>Scid</strong> – Column index of the stratum column in S</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Output matrix, one row per each distinct pair</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.ppca">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">ppca</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DA [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.symmetricDifference">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">symmetricDifference</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" [...]
+<dd><blockquote>
+<div><p>Builtin function that implements symmetric difference set-operation on vectors</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – indicates dimension of the new vector space constructed from eigen vectors</p></li>
-<li><p><strong>maxi</strong> – maximum number of iterations until convergence</p></li>
-<li><p><strong>tolobj</strong> – objective function tolerance value to stop ppca algorithm</p></li>
-<li><p><strong>tolrecerr</strong> – reconstruction error tolerance value to stop the algorithm</p></li>
-<li><p><strong>verbose</strong> – verbose debug output</p></li>
+<li><p><strong>X</strong> – input vector</p></li>
+<li><p><strong>Y</strong> – input vector</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>vector with all elements in X and Y but not in both</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.randomForest">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">randomForest</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">R</span><span class= [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.tSNE">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">tSNE</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DA [...]
+<dd><blockquote>
+<div><p>This function performs dimensionality reduction using tSNE algorithm based on
+the paper: Visualizing Data using t-SNE, Maaten et. al.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>If</strong> – not provided by default all variables are assumed to be scale</p></li>
-<li><p><strong>bins</strong> – Number of equiheight bins per scale feature to choose thresholds</p></li>
-<li><p><strong>depth</strong> – Maximum depth of the learned tree</p></li>
-<li><p><strong>num_leaf</strong> – Number of samples when splitting stops and a leaf node is added</p></li>
-<li><p><strong>num_samples</strong> – Number of samples at which point we switch to in-memory subtree building</p></li>
-<li><p><strong>num_trees</strong> – Number of trees to be learned in the random forest model</p></li>
-<li><p><strong>subsamp_rate</strong> – Parameter controlling the size of each tree in the forest; samples are selected from a</p></li>
-<li><p><strong>Poisson</strong> – parameter subsamp_rate (the default value is 1.0)</p></li>
-<li><p><strong>feature_subset</strong> – Parameter that controls the number of feature used as candidates for splitting at each tree node</p></li>
-<li><p><strong>as</strong> – of number of features in the dataset;</p></li>
-<li><p><strong>by</strong> – root of features (i.e., feature_subset = 0.5) are used at each tree node</p></li>
-<li><p><strong>impurity</strong> – Impurity measure: entropy or Gini (the default)</p></li>
+<li><p><strong>X</strong> – Data Matrix of shape
+(number of data points, input dimensionality)</p></li>
+<li><p><strong>reduced_dims</strong> – Output dimensionality</p></li>
+<li><p><strong>perplexity</strong> – Perplexity Parameter</p></li>
+<li><p><strong>lr</strong> – Learning rate</p></li>
+<li><p><strong>momentum</strong> – Momentum Parameter</p></li>
+<li><p><strong>max_iter</strong> – Number of iterations</p></li>
+<li><p><strong>seed</strong> – The seed used for initial values.
+If set to -1 random seeds are selected.</p></li>
+<li><p><strong>is_verbose</strong> – Print debug information</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing tree and each row contains the following information: & that leaf node j is supposed to predict & subset of values & 7,8,… if j is categorical & stored at m[7,j] if the feature chosen for j is scale; & chosen for j is categorical rows 7,8,… depict the value subset chosen for j</p>
+<dd class="field-even"><p>Data Matrix of shape (number of data points, reduced_dims)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.scale">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">scale</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>D [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.toOneHot">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">toOneHot</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">numClasses</span><span class="p">:</span> <span class="n">int</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm. [...]
+<dd><blockquote>
+<div><p>The toOneHot-function encodes unordered categorical vector to multiple binary vectors.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>center</strong> – Indicates whether or not to center the feature matrix</p></li>
-<li><p><strong>scale</strong> – Indicates whether or not to scale the feature matrix</p></li>
+<li><p><strong>X</strong> – Vector with N integer entries between 1 and numClasses</p></li>
+<li><p><strong>numclasses</strong> – Number of columns, must be be greater than or equal to largest value in X</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>One-hot-encoded matrix with shape (N, numClasses)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.shortestPath">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">shortestPath</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">G</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">sourceNode</span><span class="p">:</span> <span class="n">int</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span c [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.tomeklink">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">tomeklink</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#sy [...]
+<dd><blockquote>
+<div><p>The tomekLink-function performs under sampling by removing Tomek’s links for imbalanced multi-class problems
+Computes TOMEK links and drops them from data matrix and label vector.
+Drops only the majority label and corresponding point of TOMEK links.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>The</strong> – G can be 0/1 (just specifying whether the nodes</p></li>
-<li><p><strong>are</strong> – not) or integer values (representing the weight</p></li>
-<li><p><strong>of</strong> – or the distances between nodes, 0 if not connected).</p></li>
-<li><p><strong>maxi</strong> – Integer max number of iterations accepted (0 for FALSE, i.e.</p></li>
-<li><p><strong>max</strong> – iterations not defined)</p></li>
-<li><p><strong>sourceNode</strong> – index to calculate the shortest paths to all other nodes.</p></li>
-<li><p><strong>verbose</strong> – flag for verbose debug output</p></li>
+<li><p><strong>X</strong> – Data Matrix (nxm)</p></li>
+<li><p><strong>y</strong> – Label Matrix (nx1), greater than zero</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing minimum distance shortest-path from vertex i to vertex j. & of the minimum distance is infinity, the two nodes are</p>
+<dd class="field-even"><p>Data Matrix without Tomek links</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>Labels corresponding to under sampled data</p>
+</dd>
+<dt class="field-even">Returns</dt>
+<dd class="field-even"><p>Indices of dropped rows/labels wrt input</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.sigmoid">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">sigmoid</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.sigmoid" title="Permalink to this definition">¶</a></dt>
-<dd><dl class="field-list simple">
-<dt class="field-odd">Returns</dt>
-<dd class="field-odd"><p>‘OperationNode’ containing meaning</p>
-</dd>
-</dl>
+<dt id="systemds.operator.algorithm.topk_cleaning">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">topk_cleaning</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">dataTrain</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">primitives</span><span class="p">:</span> <span class="n">systemds.operator.nodes.frame.Frame</span></em>, <em class="sig-param"><span class="n">parame [...]
+<dd><p>This function cleans top-K item (where K is given as input)for a given list of users.
+metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask</p>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.slicefinder">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">slicefinder</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">e</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class= [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.underSampling">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">underSampling</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">ratio</span><span c [...]
+<dd><blockquote>
+<div><p>Builtin to perform random under sampling on data.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>k</strong> – Number of subsets required</p></li>
-<li><p><strong>maxL</strong> – level L (conjunctions of L predicates), 0 unlimited</p></li>
-<li><p><strong>minSup</strong> – support (min number of rows per slice)</p></li>
-<li><p><strong>alpha</strong> – [0,1]: 0 only size, 1 only error</p></li>
-<li><p><strong>tpEval</strong> – for task-parallel slice evaluation,</p></li>
-<li><p><strong>tpBlksz</strong> – size for task-parallel execution (num slices)</p></li>
-<li><p><strong>selFeat</strong> – for removing one-hot-encoded features that don’t satisfy</p></li>
-<li><p><strong>the</strong> – constraint and/or have zero error</p></li>
-<li><p><strong>verbose</strong> – for verbose debug output</p></li>
+<li><p><strong>X</strong> – X data to sample from</p></li>
+<li><p><strong>Y</strong> – Y data to sample from it will sample the same rows from x.</p></li>
+<li><p><strong>ratio</strong> – The ratio to sample</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The under sample data X</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p>The under sample data Y</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.smote">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">smote</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n" [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.union">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">union</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#system [...]
+<dd><blockquote>
+<div><p>Builtin function that implements union operation on vectors</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>s</strong> – Amount of SMOTE (percentage of oversampling), integral multiple of 100</p></li>
-<li><p><strong>k</strong> – Number of nearest neighbour</p></li>
-<li><p><strong>verbose</strong> – if the algorithm should be verbose</p></li>
+<li><p><strong>X</strong> – input vector</p></li>
+<li><p><strong>Y</strong> – input vector</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>matrix with all unique rows existing in X and Y</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.split">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">split</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kw [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.unique">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">unique</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.unique" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>Builtin function that implements unique operation on vectors</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>f</strong> – Train set fraction [0,1]</p></li>
-<li><p><strong>cont</strong> – contiuous splits, otherwise sampled</p></li>
-<li><p><strong>seed</strong> – The seed to reandomly select rows in sampled mode</p></li>
-</ul>
+<dd class="field-odd"><p><strong>X</strong> – input vector</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>matrix with only unique rows</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.splitBalanced">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">splitBalanced</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span clas [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.univar">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">univar</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">types</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#s [...]
+<dd><blockquote>
+<div><p>Computes univariate statistics for all attributes in a given data set</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>f</strong> – Train set fraction [0,1]</p></li>
-<li><p><strong>verbose</strong> – print available</p></li>
+<li><p><strong>X</strong> – Input matrix of the shape (N, D)</p></li>
+<li><p><strong>TYPES</strong> – Matrix of the shape (1, D) with features types:
+1 for scale, 2 for nominal, 3 for ordinal</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>univariate statistics for all attributes</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.stableMarriage">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">stableMarriage</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">P</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">A</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span cla [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.vectorToCsv">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">vectorToCsv</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">mask</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.vectorToCsv" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This builtin function convert vector into csv string such as [1 0 0 1 1 0 1] = “1,4,5,7”
+Related to [SYSTEMDS-2662] dependency function for cleaning pipelines</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>It</strong> – a square matrix with no zeros.</p></li>
-<li><p><strong>It</strong> – a square matrix with no zeros.</p></li>
-<li><p><strong>ordered</strong> – If true, P and A are assumed to be ordered,</p></li>
-<li><p><strong>index</strong> – vice-versa (higher is better).</p></li>
-</ul>
+<dd class="field-odd"><p><strong>mask</strong> – Data vector (having 0 for excluded indexes)</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing to the match. & 1 (2.0 preference value) and acceptor 2 (1.0 preference value). & 3 (2.0 preference value) and proposer 2 & matched with proposer 3 (since [1,3] is non-zero) at a & 3.0. & matched with proposer 2 (since [2,2] is non-zero) at a & 3.0. & matched with proposer 1 (since [3,1] is non-zero) at a & 1.0.</p>
+<dd class="field-even"><p>indexes</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.statsNA">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">statsNA</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.winsorize">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">winsorize</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">verbose</span><span class="p">:</span> <span class="n">bool</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class= [...]
+<dd><blockquote>
+<div><p>The winsorize-function removes outliers from the data. It does so by computing upper and
+lower quartile range of the given data then it replaces any value that falls outside this range
+(less than lower quartile range or more than upper quartile range).</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>bins</strong> – Split number for bin stats. Number of bins the time series gets</p></li>
-<li><p><strong>missing</strong> – printed.</p></li>
-<li><p><strong>verbose</strong> – Print detailed information.</p></li>
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>verbose</strong> – To print output on screen</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix without outlier values</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.steplm">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">steplm</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">k [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.winsorizeApply">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">winsorizeApply</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">qLower</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">qUpper</span> [...]
+<dd><blockquote>
+<div><p>winsorizeApply takes the upper and lower quantile values per column, and
+remove outliers by replacing them with these upper and lower bound values.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
-<li><p><strong>icpt</strong> – Intercept presence, shifting and rescaling the columns of X:</p></li>
-<li><p><strong>reg</strong> – learning rate</p></li>
-<li><p><strong>tol</strong> – Tolerance threashold to train until achieved</p></li>
-<li><p><strong>maxi</strong> – maximum iterations 0 means until tolerange is reached</p></li>
-<li><p><strong>verbose</strong> – If the algorithm should be verbose</p></li>
+<li><p><strong>X</strong> – Input feature matrix</p></li>
+<li><p><strong>qLower</strong> – row vector of upper bounds per column</p></li>
+<li><p><strong>qUpper</strong> – row vector of lower bounds per column</p></li>
</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix without outlier values</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.tSNE">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">tSNE</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict<span class="p">[</span>str<span class="p">, </span>Union<span class="p">[</span>DA [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.xdummy1">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">xdummy1</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.xdummy1" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This builtin function is here for debugging purposes</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><ul class="simple">
-<li><p><strong>reduced_dims</strong> – Output dimensionality</p></li>
-<li><p><strong>perplexity</strong> – Perplexity Parameter</p></li>
-<li><p><strong>lr</strong> – Learning rate</p></li>
-<li><p><strong>momentum</strong> – Momentum Parameter</p></li>
-<li><p><strong>max_iter</strong> – Number of iterations</p></li>
-<li><p><strong>seed</strong> – The seed used for initial values.</p></li>
-<li><p><strong>If</strong> – -1 random seeds are selected.</p></li>
-<li><p><strong>is_verbose</strong> – Print debug information</p></li>
-</ul>
+<dd class="field-odd"><p><strong>X</strong> – test input</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>test result</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.toOneHot">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">toOneHot</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">numClasses</span><span class="p">:</span> <span class="n">int</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm. [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.xdummy2">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">xdummy2</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#systemds.operator.algorithm.xdummy2" title="Permalink to this definition">¶</a></dt>
+<dd><blockquote>
+<div><p>This builtin function is here for debugging purposes</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>numclasses</strong> – Number of columns, must be be greater than or equal to largest value in X</p>
+<dd class="field-odd"><p><strong>X</strong> – Debug input</p>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p><p>—</p>
+</p>
+</dd>
+<dt class="field-odd">Returns</dt>
+<dd class="field-odd"><p><p>—</p>
+</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.underSampling">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">underSampling</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">Y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">ratio</span><span c [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.xgboost">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">xgboost</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">y</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span><span class="n"> [...]
+<dd><blockquote>
+<div><p>XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+implementation supports classification and regression and is capable of working with categorical and scalar features.</p>
+<p>Output explained:
+(the first node is the init prediction) and each row contains
+the following information:
+M[1,j]: id of node j (in a complete binary tree)
+M[2,j]: tree id to which node j belongs
+M[3,j]: Offset (no. of columns) to left child of j if j is an internal node, otherwise 0
+M[4,j]: Feature index of the feature (scale feature id if the feature is
+scale or categorical feature id if the feature is categorical)
+that node j looks at if j is an internal node, otherwise 0
+M[5,j]: Type of the feature that node j looks at if j is an internal node.
+if leaf = 0, if scalar = 1, if categorical = 2
+M[6:,j]: If j is an internal node: Threshold the example’s feature value is
+compared to is stored at M[6,j] if the feature chosen for j is scale,
+otherwise if the feature chosen for j is categorical rows 6,7,… depict
+the value subset chosen for j
+If j is a leaf node 1 if j is impure and the number of samples at j > threshold, otherwise 0</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>ratio</strong> – The ratio to sample</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Feature matrix X; note that X needs to be both recoded and dummy coded</p></li>
+<li><p><strong>y</strong> – Label matrix y; note that y needs to be both recoded and dummy coded</p></li>
+<li><p><strong>R</strong> – Matrix R; 1xn vector which for each feature in X contains the following information
+- R[,1]: 1 (scalar feature)
+- R[,2]: 2 (categorical feature)
+Feature 1 is a scalar feature and features 2 is a categorical feature
+If R is not provided by default all variables are assumed to be scale (1)</p></li>
+<li><p><strong>sml_type</strong> – Supervised machine learning type: 1 = Regression(default), 2 = Classification</p></li>
+<li><p><strong>num_trees</strong> – Number of trees to be created in the xgboost model</p></li>
+<li><p><strong>learning_rate</strong> – Alias: eta. After each boosting step the learning rate controls the weights of the new predictions</p></li>
+<li><p><strong>max_depth</strong> – Maximum depth of a tree. Increasing this value will make the model more complex and more likely to overfit</p></li>
+<li><p><strong>lambda</strong> – L2 regularization term on weights. Increasing this value will make model more conservative and reduce amount of leaves of a tree</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>Matrix M where each column corresponds to a node in the learned tree</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.univar">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">univar</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">types</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em><span class="sig-paren">)</span><a class="headerlink" href="#s [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.xgboostPredictClassification">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">xgboostPredictClassification</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">M</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</ [...]
+<dd><blockquote>
+<div><p>XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+implementation supports classification and is capable of working with categorical features.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>1</strong> – 2 for nominal, 3 for ordinal</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors we want to predict (X_test)</p></li>
+<li><p><strong>M</strong> – The model created at xgboost</p></li>
+<li><p><strong>learning_rate</strong> – The learning rate used in the model</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The predictions of the samples using the given xgboost model. (y_prediction)</p>
</dd>
</dl>
</dd></dl>
<dl class="py function">
-<dt id="systemds.operator.algorithm.winsorize">
-<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">winsorize</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">verbose</span><span class="p">:</span> <span class="n">bool</span></em>, <em class="sig-param"><span class="o">**</span><span class="n">kwargs</span><span class= [...]
-<dd><dl class="field-list simple">
+<dt id="systemds.operator.algorithm.xgboostPredictRegression">
+<code class="sig-prename descclassname">systemds.operator.algorithm.</code><code class="sig-name descname">xgboostPredictRegression</code><span class="sig-paren">(</span><em class="sig-param"><span class="n">X</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="n">M</span><span class="p">:</span> <span class="n">systemds.operator.nodes.matrix.Matrix</span></em>, <em class="sig-param"><span class="o">**</span [...]
+<dd><blockquote>
+<div><p>XGBoost is a decision-tree-based ensemble Machine Learning algorithm that uses a gradient boosting. This xgboost
+implementation supports regression.</p>
+</div></blockquote>
+<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
-<dd class="field-odd"><p><strong>verbose</strong> – To print output on screen</p>
+<dd class="field-odd"><ul class="simple">
+<li><p><strong>X</strong> – Matrix of feature vectors we want to predict (X_test)</p></li>
+<li><p><strong>M</strong> – The model created at xgboost</p></li>
+<li><p><strong>learning_rate</strong> – The learning rate used in the model</p></li>
+</ul>
</dd>
<dt class="field-even">Returns</dt>
-<dd class="field-even"><p>‘OperationNode’ containing</p>
+<dd class="field-even"><p>The predictions of the samples using the given xgboost model. (y_prediction)</p>
</dd>
</dl>
</dd></dl>
diff --git a/docs/api/python/api/operator/node/scalar.html b/docs/api/python/api/operator/node/scalar.html
index 4dd7319c86..c5e83520a2 100644
--- a/docs/api/python/api/operator/node/scalar.html
+++ b/docs/api/python/api/operator/node/scalar.html
@@ -183,10 +183,10 @@
the recommended way is to use the methods defined on <code class="docutils literal notranslate"><span class="pre">SystemDSContext</span></code>.</p>
<dl class="py class">
<dt id="systemds.operator.Scalar">
-<em class="property">class </em><code class="sig-prename descclassname">systemds.operator.</code><code class="sig-name descname">Scalar</code><span class="sig-paren">(</span><em class="sig-param">sds_context: SystemDSContext</em>, <em class="sig-param">operation: str</em>, <em class="sig-param">unnamed_input_nodes: Iterable[Union[DAGNode</em>, <em class="sig-param">str</em>, <em class="sig-param">int</em>, <em class="sig-param">float</em>, <em class="sig-param">bool]] = None</em>, <em cl [...]
+<em class="property">class </em><code class="sig-prename descclassname">systemds.operator.</code><code class="sig-name descname">Scalar</code><span class="sig-paren">(</span><em class="sig-param">sds_context</em>, <em class="sig-param">operation: str</em>, <em class="sig-param">unnamed_input_nodes: Iterable[Union[DAGNode</em>, <em class="sig-param">str</em>, <em class="sig-param">int</em>, <em class="sig-param">float</em>, <em class="sig-param">bool]] = None</em>, <em class="sig-param">n [...]
<dd><dl class="py method">
<dt id="systemds.operator.Scalar.__init__">
-<code class="sig-name descname">__init__</code><span class="sig-paren">(</span><em class="sig-param">sds_context: SystemDSContext</em>, <em class="sig-param">operation: str</em>, <em class="sig-param">unnamed_input_nodes: Iterable[Union[DAGNode</em>, <em class="sig-param">str</em>, <em class="sig-param">int</em>, <em class="sig-param">float</em>, <em class="sig-param">bool]] = None</em>, <em class="sig-param">named_input_nodes: Dict[str</em>, <em class="sig-param">Union[DAGNode</em>, <em [...]
+<code class="sig-name descname">__init__</code><span class="sig-paren">(</span><em class="sig-param">sds_context</em>, <em class="sig-param">operation: str</em>, <em class="sig-param">unnamed_input_nodes: Iterable[Union[DAGNode</em>, <em class="sig-param">str</em>, <em class="sig-param">int</em>, <em class="sig-param">float</em>, <em class="sig-param">bool]] = None</em>, <em class="sig-param">named_input_nodes: Dict[str</em>, <em class="sig-param">Union[DAGNode</em>, <em class="sig-param [...]
<dd><p>Create general <cite>OperationNode</cite></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
diff --git a/docs/api/python/genindex.html b/docs/api/python/genindex.html
index de202fbc0f..7c3af1a423 100644
--- a/docs/api/python/genindex.html
+++ b/docs/api/python/genindex.html
@@ -197,6 +197,7 @@
| <a href="#U"><strong>U</strong></a>
| <a href="#V"><strong>V</strong></a>
| <a href="#W"><strong>W</strong></a>
+ | <a href="#X"><strong>X</strong></a>
</div>
<h2 id="_">_</h2>
@@ -244,13 +245,17 @@
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.als">als() (in module systemds.operator.algorithm)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.alsCG">alsCG() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.alsDS">alsDS() (in module systemds.operator.algorithm)</a>
+</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.alsPredict">alsPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.alsTopkPredict">alsTopkPredict() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.apply_pipeline">apply_pipeline() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.arima">arima() (in module systemds.operator.algorithm)</a>
</li>
@@ -268,16 +273,20 @@
<li><a href="api/operator/node/scalar.html#systemds.operator.Scalar.atan">(systemds.operator.Scalar method)</a>
</li>
</ul></li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.autoencoder_2layer">autoencoder_2layer() (in module systemds.operator.algorithm)</a>
+</li>
</ul></td>
</tr></table>
<h2 id="B">B</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.bivar">bivar() (in module systemds.operator.algorithm)</a>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.bandit">bandit() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.bivar">bivar() (in module systemds.operator.algorithm)</a>
+</li>
<li><a href="api/script_building/script.html#systemds.script_building.script.DMLScript.build_code">build_code() (systemds.script_building.script.DMLScript method)</a>
</li>
</ul></td>
@@ -333,6 +342,12 @@
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.confusionMatrix">confusionMatrix() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.cor">cor() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.correctTypos">correctTypos() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.correctTyposApply">correctTyposApply() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.cos">cos() (systemds.operator.Matrix method)</a>
@@ -372,17 +387,23 @@
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.decisionTree">decisionTree() (in module systemds.operator.algorithm)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.decisionTreePredict">decisionTreePredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.deepWalk">deepWalk() (in module systemds.operator.algorithm)</a>
+</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.denialConstraints">denialConstraints() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.dict">dict() (systemds.context.SystemDSContext method)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.discoverFD">discoverFD() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.dist">dist() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/script_building/script.html#systemds.script_building.script.DMLScript">DMLScript (class in systemds.script_building.script)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.dmv">dmv() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
</tr></table>
@@ -390,12 +411,14 @@
<h2 id="E">E</h2>
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.exception_and_close">exception_and_close() (systemds.context.SystemDSContext method)</a>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.ema">ema() (in module systemds.operator.algorithm)</a>
</li>
- <li><a href="api/script_building/script.html#systemds.script_building.script.DMLScript.execute">execute() (systemds.script_building.script.DMLScript method)</a>
+ <li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.exception_and_close">exception_and_close() (systemds.context.SystemDSContext method)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/script_building/script.html#systemds.script_building.script.DMLScript.execute">execute() (systemds.script_building.script.DMLScript method)</a>
+</li>
<li><a href="api/script_building/script.html#systemds.script_building.script.DMLScript.execute_with_lineage">execute_with_lineage() (systemds.script_building.script.DMLScript method)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.executePipeline">executePipeline() (in module systemds.operator.algorithm)</a>
@@ -407,13 +430,27 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.federated">federated() (systemds.context.SystemDSContext method)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.ffPredict">ffPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.ffTrain">ffTrain() (in module systemds.operator.algorithm)</a>
</li>
- <li><a href="api/operator/node/frame.html#systemds.operator.Frame">Frame (class in systemds.operator)</a>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.fit_pipeline">fit_pipeline() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.fixInvalidLengths">fixInvalidLengths() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.fixInvalidLengthsApply">fixInvalidLengthsApply() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/node/frame.html#systemds.operator.Frame">Frame (class in systemds.operator)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.frameSort">frameSort() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.frequencyEncode">frequencyEncode() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.frequencyEncodeApply">frequencyEncodeApply() (in module systemds.operator.algorithm)</a>
+</li>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.from_numpy">from_numpy() (systemds.context.SystemDSContext method)</a>
</li>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.from_pandas">from_pandas() (systemds.context.SystemDSContext method)</a>
@@ -440,19 +477,23 @@
</li>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.get_stderr">get_stderr() (systemds.context.SystemDSContext method)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.get_stdout">get_stdout() (systemds.context.SystemDSContext method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.getAccuracy">getAccuracy() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.glm">glm() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.glmPredict">glmPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.gmm">gmm() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.gmmPredict">gmmPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.gnmf">gnmf() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.gridSearch">gridSearch() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
</tr></table>
@@ -484,8 +525,6 @@
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.img_posterize">img_posterize() (in module systemds.operator.algorithm)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.img_rotate">img_rotate() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.img_sample_pairing">img_sample_pairing() (in module systemds.operator.algorithm)</a>
@@ -494,11 +533,29 @@
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.img_transform">img_transform() (in module systemds.operator.algorithm)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.img_translate">img_translate() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.impurityMeasures">impurityMeasures() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByFD">imputeByFD() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByFDApply">imputeByFDApply() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByMean">imputeByMean() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByMeanApply">imputeByMeanApply() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByMedian">imputeByMedian() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByMedianApply">imputeByMedianApply() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByMode">imputeByMode() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.imputeByModeApply">imputeByModeApply() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.intersect">intersect() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
</tr></table>
@@ -507,10 +564,18 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.km">km() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.kmeans">kmeans() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.kmeansPredict">kmeansPredict() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.kmeans">kmeans() (in module systemds.operator.algorithm)</a>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.knn">knn() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.knnbf">knnbf() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.knnGraph">knnGraph() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
</tr></table>
@@ -523,6 +588,8 @@
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.l2svmPredict">l2svmPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lasso">lasso() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lenetPredict">lenetPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lenetTrain">lenetTrain() (in module systemds.operator.algorithm)</a>
</li>
@@ -537,6 +604,8 @@
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lmCG">lmCG() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lmDS">lmDS() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.lmPredict">lmPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.logSumExp">logSumExp() (in module systemds.operator.algorithm)</a>
</li>
@@ -553,13 +622,21 @@
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.matrixProfile">matrixProfile() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.max">max() (systemds.operator.Matrix method)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.mcc">mcc() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.mdedup">mdedup() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.mean">mean() (systemds.operator.Matrix method)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.mice">mice() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.miceApply">miceApply() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.min">min() (systemds.operator.Matrix method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li>
module
@@ -575,9 +652,9 @@
<li><a href="api/utils/helpers.html#module-systemds.utils.helpers">systemds.utils.helpers</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.msvm">msvm() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.msvmPredict">msvmPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.multiLogReg">multiLogReg() (in module systemds.operator.algorithm)</a>
</li>
@@ -590,10 +667,16 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.na_locf">na_locf() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.naiveBayes">naiveBayes() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.naiveBayesPredict">naiveBayesPredict() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
- <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.naiveBayes">naiveBayes() (in module systemds.operator.algorithm)</a>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.normalize">normalize() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.normalizeApply">normalizeApply() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/utils/converters.html#systemds.utils.converters.numpy_to_matrix_block">numpy_to_matrix_block() (in module systemds.utils.converters)</a>
</li>
@@ -648,6 +731,10 @@
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.pca">pca() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.pcaInverse">pcaInverse() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.pcaTransform">pcaTransform() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.pnmf">pnmf() (in module systemds.operator.algorithm)</a>
</li>
@@ -696,8 +783,20 @@
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.scalar">scalar() (systemds.context.SystemDSContext method)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.scale">scale() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.scaleApply">scaleApply() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.scaleMinMax">scaleMinMax() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.selectByVarThresh">selectByVarThresh() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.seq">seq() (systemds.context.SystemDSContext method)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.setdiff">setdiff() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.sherlock">sherlock() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.sherlockPredict">sherlockPredict() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.shortestPath">shortestPath() (in module systemds.operator.algorithm)</a>
</li>
@@ -718,15 +817,17 @@
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.slicefinder">slicefinder() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.smote">smote() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.softmax">softmax() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/source.html#systemds.operator.Source">Source (class in systemds.operator)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/context/systemds_context.html#systemds.context.SystemDSContext.source">source() (systemds.context.SystemDSContext method)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.split">split() (in module systemds.operator.algorithm)</a>
</li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.splitBalanced">splitBalanced() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.stableMarriage">stableMarriage() (in module systemds.operator.algorithm)</a>
@@ -734,8 +835,12 @@
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.statsNA">statsNA() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.steplm">steplm() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.stratstats">stratstats() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.sum">sum() (systemds.operator.Matrix method)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.symmetricDifference">symmetricDifference() (in module systemds.operator.algorithm)</a>
</li>
<li>
systemds.operator.algorithm
@@ -794,17 +899,21 @@
<li><a href="api/operator/node/scalar.html#systemds.operator.Scalar.tanh">(systemds.operator.Scalar method)</a>
</li>
</ul></li>
- </ul></td>
- <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.to_one_hot">to_one_hot() (systemds.operator.Matrix method)</a>
</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.to_string">to_string() (systemds.operator.Matrix method)</a>
<ul>
<li><a href="api/operator/node/scalar.html#systemds.operator.Scalar.to_string">(systemds.operator.Scalar method)</a>
</li>
</ul></li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.tomeklink">tomeklink() (in module systemds.operator.algorithm)</a>
+</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.toOneHot">toOneHot() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.topk_cleaning">topk_cleaning() (in module systemds.operator.algorithm)</a>
</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.tSNE">tSNE() (in module systemds.operator.algorithm)</a>
</li>
@@ -815,9 +924,13 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.underSampling">underSampling() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.union">union() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.unique">unique() (in module systemds.operator.algorithm)</a>
+</li>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.univar">univar() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
@@ -827,6 +940,10 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/node/matrix.html#systemds.operator.Matrix.var">var() (systemds.operator.Matrix method)</a>
+</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.vectorToCsv">vectorToCsv() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
</tr></table>
@@ -835,14 +952,38 @@
<table style="width: 100%" class="indextable genindextable"><tr>
<td style="width: 33%; vertical-align: top;"><ul>
<li><a href="api/operator/algorithms.html#systemds.operator.algorithm.winsorize">winsorize() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.winsorizeApply">winsorizeApply() (in module systemds.operator.algorithm)</a>
</li>
</ul></td>
<td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.WoE">WoE() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.WoEApply">WoEApply() (in module systemds.operator.algorithm)</a>
+</li>
<li><a href="api/operator/operation_node.html#systemds.operator.OperationNode.write">write() (systemds.operator.OperationNode method)</a>
</li>
</ul></td>
</tr></table>
+<h2 id="X">X</h2>
+<table style="width: 100%" class="indextable genindextable"><tr>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.xdummy1">xdummy1() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.xdummy2">xdummy2() (in module systemds.operator.algorithm)</a>
+</li>
+ </ul></td>
+ <td style="width: 33%; vertical-align: top;"><ul>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.xgboost">xgboost() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.xgboostPredictClassification">xgboostPredictClassification() (in module systemds.operator.algorithm)</a>
+</li>
+ <li><a href="api/operator/algorithms.html#systemds.operator.algorithm.xgboostPredictRegression">xgboostPredictRegression() (in module systemds.operator.algorithm)</a>
+</li>
+ </ul></td>
+</tr></table>
+
</div>
diff --git a/docs/api/python/guide/federated.html b/docs/api/python/guide/federated.html
index 2ca1b782ec..61d545e162 100644
--- a/docs/api/python/guide/federated.html
+++ b/docs/api/python/guide/federated.html
@@ -93,6 +93,10 @@
<li class="toctree-l2"><a class="reference internal" href="#start-federated-worker">Start Federated worker</a></li>
<li class="toctree-l2"><a class="reference internal" href="#simple-aggregation-example">Simple Aggregation Example</a></li>
<li class="toctree-l2"><a class="reference internal" href="#multiple-federated-environments">Multiple Federated Environments</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#multi-tenant-federated-learning">Multi-tenant Federated Learning</a><ul>
+<li class="toctree-l3"><a class="reference internal" href="#limitations">Limitations</a></li>
+</ul>
+</li>
</ul>
</li>
<li class="toctree-l1"><a class="reference internal" href="algorithms_basics.html">Built-in Algorithms</a></li>
@@ -292,6 +296,29 @@ that you have:</p>
<p>a csv file, mtd file, and SystemDS Environment is set correctly.</p>
</div>
</div>
+<div class="section" id="multi-tenant-federated-learning">
+<h2>Multi-tenant Federated Learning<a class="headerlink" href="#multi-tenant-federated-learning" title="Permalink to this headline">¶</a></h2>
+<p>SystemDS supports Multi-tenant Federated Learning, meaning that multiple
+coordinators learn on shared federated workers. From another perspective,
+the federated worker allows multiple coordinators to perform model training
+simultaneously using the data from the respective federated site. This
+approach enables the worker to operate in a server-like mode, providing
+multiple tenants with the ability to learn on the federated data at the same
+time. Tenant isolation ensures that tenant-specific intermediate results are
+only accessible by the respective tenant.</p>
+<div class="section" id="limitations">
+<h3>Limitations<a class="headerlink" href="#limitations" title="Permalink to this headline">¶</a></h3>
+<p>Since the coordinators are differentiated by their IP address in combination
+with their process ID, the worker is not able to isolate coordinators which
+share the same IP address and the same process ID. This occurs, for example,
+when two coordinators are running behind a proxy (same IP address), where
+both coordinators coincidentally have the same process ID.</p>
+<p>A second limitation is showing up in networks using the Dynamic Host Protocol
+(DHCP). Since the federated worker identifies the coordinator based on the
+IP address, the worker does not re-identify the coordinator when its IP address
+has changed, i.e., when DHCP renews its IP address.</p>
+</div>
+</div>
</div>
diff --git a/docs/api/python/objects.inv b/docs/api/python/objects.inv
index ae1d4dc713..c92fc48f72 100644
Binary files a/docs/api/python/objects.inv and b/docs/api/python/objects.inv differ
diff --git a/docs/api/python/searchindex.js b/docs/api/python/searchindex.js
index 75be01e19e..547c2bb3bc 100644
--- a/docs/api/python/searchindex.js
+++ b/docs/api/python/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["api/context/systemds_context","api/operator/algorithms","api/operator/node/frame","api/operator/node/list","api/operator/node/matrix","api/operator/node/scalar","api/operator/node/source","api/operator/operation_node","api/script_building/dag","api/script_building/script","api/utils/converters","api/utils/helpers","getting_started/install","getting_started/simple_examples","guide/algorithms_basics","guide/federated","guide/python_end_to_end_tut","index"],envve [...]
\ No newline at end of file
+Search.setIndex({docnames:["api/context/systemds_context","api/operator/algorithms","api/operator/node/frame","api/operator/node/list","api/operator/node/matrix","api/operator/node/scalar","api/operator/node/source","api/operator/operation_node","api/script_building/dag","api/script_building/script","api/utils/converters","api/utils/helpers","getting_started/install","getting_started/simple_examples","guide/algorithms_basics","guide/federated","guide/python_end_to_end_tut","index"],envve [...]
\ No newline at end of file