You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2021/02/21 00:12:28 UTC

[systemds] branch master updated: [MINOR] Cleanup staging scripts (remove converted builtin functions)

This is an automated email from the ASF dual-hosted git repository.

mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new 58d46ea  [MINOR] Cleanup staging scripts (remove converted builtin functions)
58d46ea is described below

commit 58d46eaa1ae3384963425d6c166b9d2c39683429
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun Feb 21 01:10:59 2021 +0100

    [MINOR] Cleanup staging scripts (remove converted builtin functions)
    
    Additional cleanup PRs:
    Closes #1172.
---
 scripts/staging/PCA.dml                            | 117 ----
 scripts/staging/PPCA.dml                           | 160 -----
 scripts/staging/autoencoder-2layer.dml             | 248 -------
 scripts/staging/knn.dml                            | 779 ---------------------
 scripts/staging/regression/lasso/lasso.dml         | 113 ---
 scripts/staging/slicing/__init__.py                |  20 -
 scripts/staging/slicing/base/Bucket.py             | 170 -----
 scripts/staging/slicing/base/SparkNode.py          | 167 -----
 scripts/staging/slicing/base/__init__.py           |  20 -
 scripts/staging/slicing/base/node.py               | 210 ------
 scripts/staging/slicing/base/slicer.py             | 185 -----
 scripts/staging/slicing/base/top_k.py              |  58 --
 scripts/staging/slicing/base/union_slicer.py       | 162 -----
 scripts/staging/slicing/slicing.dml                | 311 --------
 .../slicing/spark_modules/join_data_parallel.py    | 123 ----
 .../staging/slicing/spark_modules/spark_slicer.py  | 103 ---
 .../slicing/spark_modules/spark_union_slicer.py    |  73 --
 .../slicing/spark_modules/union_data_parallel.py   | 124 ----
 scripts/staging/slicing/tests/__init__.py          |  20 -
 .../slicing/tests/classification/__init__.py       |  20 -
 .../slicing/tests/classification/compas_bd.py      | 136 ----
 .../slicing/tests/classification/sparked_adults.py | 118 ----
 .../slicing/tests/classification/test_adult.py     | 116 ---
 .../slicing/tests/classification/test_compas.py    | 105 ---
 .../slicing/tests/classification/test_iris.py      | 109 ---
 .../staging/slicing/tests/regression/__init__.py   |  20 -
 .../slicing/tests/regression/bd_spark_salary.py    | 130 ----
 .../slicing/tests/regression/spark_salary.py       | 157 -----
 .../slicing/tests/regression/test_insurance.py     | 103 ---
 .../slicing/tests/regression/test_salary.py        | 104 ---
 .../staging/slicing/tests/regression/toy_tests.py  | 428 -----------
 scripts/staging/slicing/utils/model_generator.py   |  77 --
 .../functions/codegenalg/Algorithm_AutoEncoder.dml | 221 +-----
 33 files changed, 6 insertions(+), 5001 deletions(-)

diff --git a/scripts/staging/PCA.dml b/scripts/staging/PCA.dml
deleted file mode 100644
index be7affc..0000000
--- a/scripts/staging/PCA.dml
+++ /dev/null
@@ -1,117 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#
-# This script performs Principal Component Analysis (PCA) on the given input data.
-#
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME   TYPE   DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# INPUT  String ---      Location to read the matrix A of feature vectors
-# K      Int    ---      Indicates dimension of the new vector space constructed from eigen vectors
-# CENTER Int    0        Indicates whether or not to center data
-# SCALE  Int    0        Indicates whether or not to scale data
-# OFMT   String ---      Output data format
-# PROJDATA Int  0	     This argument indicates if the data should be projected or not
-# MODEL  String ---      Location to already existing model: eigenvectors and eigenvalues
-# OUTPUT String /        Location to write output matrices (covariance matrix, new basis vectors,
-#                           and data projected onto new basis vectors)
-# hadoop jar SystemDS.jar -f PCA.dml -nvargs INPUT=INPUT_DIR/pca-1000x1000
-# OUTPUT=OUTPUT_DIR/pca-1000x1000-model PROJDATA=1 CENTER=1 SCALE=1
-# ---------------------------------------------------------------------------------------------
-
-A = read($INPUT);
-K = ifdef($K, ncol(A));
-ofmt = ifdef($OFMT, "CSV");
-projectData = ifdef($PROJDATA,0);
-model = ifdef($MODEL,"");
-center = ifdef($CENTER,0);
-scale = ifdef($SCALE,0);
-output = ifdef($OUTPUT,"/");
-
-# reuse existing model to project data
-if (model != "") {
-    evec_dominant = read(model+"/dominant.eigen.vectors");
-    }else{
-	model = output;
-}
-
-PCA = function(matrix[double] A, integer K, string ofmt, integer projectData, string model, integer center, integer scale, string output)
-    return(matrix[double] eval_dominant, matrix[double] evec_dominant) {
-
-    evec_dominant = matrix(0,cols=1,rows=1);
-
-    N = nrow(A);
-    D = ncol(A);
-
-    # perform z-scoring (centering and scaling)
-    if (center == 1) {
-        cm = colMeans(A);
-        A = A - cm;
-    }
-    if (scale == 1) {
-        cvars = (colSums (A^2));
-        if (center == 1){
-        cm = colMeans(A);
-            cvars = (cvars - N*(cm^2))/(N-1);
-        }
-        Azscored = (A)/sqrt(cvars);
-            A = Azscored;
-    }
-
-    # co-variance matrix
-    mu = colSums(A)/N;
-    C = (t(A) %*% A)/(N-1) - (N/(N-1))*t(mu) %*% mu;
-
-
-    # compute eigen vectors and values
-    [evalues, evectors] = eigen(C);
-
-    decreasing_Idx = order(target=evalues,by=1,decreasing=TRUE,index.return=TRUE);
-    diagmat = table(seq(1,D),decreasing_Idx);
-    # sorts eigenvalues by decreasing order
-    evalues = diagmat %*% evalues;
-    # sorts eigenvectors column-wise in the order of decreasing eigenvalues
-    evectors = evectors %*% diagmat;
-
-
-    # select K dominant eigen vectors
-    nvec = ncol(evectors);
-
-    eval_dominant = evalues[1:K, 1];
-    evec_dominant = evectors[,1:K];
-
-}
-
-[eval_dominant, evec_dominant] = PCA(A, K, ofmt, projectData, model, center, scale, output)
-
-# the square root of eigenvalues
-eval_stdev_dominant = sqrt(eval_dominant);
-write(eval_stdev_dominant, model+"/dominant.eigen.standard.deviations", format=ofmt);
-write(eval_dominant, model+"/dominant.eigen.values", format=ofmt);
-write(evec_dominant, model+"/dominant.eigen.vectors", format=ofmt);
-
-# Construct new data set by treating computed dominant eigenvectors as the basis vectors
-if (projectData == 1 | model != ""){
-	newA = A %*% evec_dominant;
-	write(newA, output+"/projected.data", format=ofmt);
-}
diff --git a/scripts/staging/PPCA.dml b/scripts/staging/PPCA.dml
deleted file mode 100644
index a501c60..0000000
--- a/scripts/staging/PPCA.dml
+++ /dev/null
@@ -1,160 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
- 
-# This script performs Probabilistic Principal Component Analysis (PCA) on the given input data. 
-# It is based on paper: sPCA: Scalable Principal Component Analysis for Big Data on Distributed 
-# Platforms. Tarek Elgamal et.al.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME   	TYPE   DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# X  	 	String ---      location to read the matrix X input matrix
-# k      	Int    ---      indicates dimension of the new vector space constructed from eigen vectors
-# tolobj 	Int    0.00001  objective function tolerance value to stop ppca algorithm
-# tolrecerr	Int    0.02     reconstruction error tolerance value to stop the algorithm	
-# iter   	Int    10       maximum number of iterations
-# fmt    	String 'text'   output format of results PPCA such as "text" or "csv"
-# hadoop jar SystemDS.jar -f PPCA.dml -nvargs X=/INPUT_DIR/X  C=/OUTPUT_DIR/C V=/OUTPUT_DIR/V k=2 tol=0.2 iter=100
-# ---------------------------------------------------------------------------------------------
-# OUTPUT PARAMETERS: 
-# ---------------------------------------------------------------------------------------------
-# NAME   TYPE   DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# C     	Matrix  ---     principal components
-# V      	Matrix  ---     eigenvalues / eigenvalues of principal components
-#
-
-X = read($X);
-
-fileC = $C;
-fileV = $V;
-
-k = ifdef($k, ncol(X));
-iter = ifdef($iter, 10);
-tolobj = ifdef($tolobj, 0.00001);
-tolrecerr = ifdef($tolrecerr, 0.02);
-fmt0 = ifdef($fmt, "text"); 
-
-n = nrow(X);
-m = ncol(X);
-
-#initializing principal components matrix
-C =  rand(rows=m, cols=k, pdf="normal");
-ss = rand(rows=1, cols=1, pdf="normal");
-ss = as.scalar(ss);
-ssPrev = ss;
-
-# best selected principle components - with the lowest reconstruction error 
-PC = C;
-
-# initilizing reconstruction error
-RE = tolrecerr+1;
-REBest = RE;
-
-Z = matrix(0,rows=1,cols=1);
-
-#Objective function value
-ObjRelChng = tolobj+1;
-
-# mean centered input matrix - dim -> [n,m]
-Xm = X - colMeans(X);
-
-#I -> k x k
-ITMP = matrix(1,rows=k,cols=1);
-I = diag(ITMP);
-
-i = 0;
-while (i < iter & ObjRelChng > tolobj & RE > tolrecerr){
-	#Estimation step - Covariance matrix 
-	#M -> k x k
-	M = t(C) %*% C + I*ss; 
-	
-	#Auxilary matrix with n latent variables 
-	# Z -> n x k		
-	Z = Xm %*% (C %*% inv(M)); 
-
-	#ZtZ -> k x k
-	ZtZ = t(Z) %*% Z + inv(M)*ss;
-	
-	#XtZ -> m x k
-	XtZ = t(Xm) %*% Z;
-	
-	#Maximization step
-	#C ->  m x k
-	ZtZ_sum = sum(ZtZ); #+n*inv(M)); 
-	C = XtZ/ZtZ_sum;
-
-	#ss2 -> 1 x 1
-	ss2 = trace(ZtZ * (t(C) %*% C));
-
-	#ss3 -> 1 x 1 
-	ss3 = sum((Z %*% t(C)) %*% t(Xm));
-	
-	#Frobenius norm of reconstruction error -> Euclidean norm 
-	#Fn -> 1 x 1	
-	Fn = sum(Xm*Xm);
-
-	#ss -> 1 x 1
-	ss = (Fn + ss2 - 2*ss3)/(n*m);
-
-   #calculating objective function relative change
-   ObjRelChng = abs(1 - ss/ssPrev);
-   #print("Objective Relative Change: " + ObjRelChng + ", Objective: " + ss);
-
-	#Reconstruction error
-	R = ((Z %*% t(C)) -  Xm);	
-
-	#calculate the error
-	#TODO rethink calculation of reconstruction error .... 
-	#1-Norm of reconstruction error - a big dense matrix 
-	#RE -> n x m
-	RE = abs(sum(R)/sum(Xm));	
-	if (RE < REBest){
-		PC = C;
-		REBest = RE;
-	}	
-	#print("ss: " + ss +" = Fn( "+ Fn +" ) + ss2( " + ss2  +" ) - 2*ss3( " + ss3 + " ), Reconstruction Error: " + RE);
-
-	ssPrev = ss;	
-	i = i+1;
-}
-print("Objective Relative Change: " + ObjRelChng);
-print ("Number of iterations: " + i + ", Reconstruction Err: " + REBest);
-
-# reconstructs data
-# RD -> n x k
-RD = X %*% PC;
-
-# calculate eigenvalues - principle component variance
-RDMean = colMeans(RD);
-V = t(colMeans(RD*RD) - (RDMean*RDMean));
-
-# sorting eigenvalues and eigenvectors in decreasing order
-V_decr_idx = order(target=V,by=1,decreasing=TRUE,index.return=TRUE);
-VF_decr = table(seq(1,nrow(V)),V_decr_idx);
-V = VF_decr %*% V;
-PC = PC %*% VF_decr;
-
-# writing principal components 
-write(PC, fileC, format=fmt0);
-# writing eigen values/pc variance
-write(V, fileV, format=fmt0);
diff --git a/scripts/staging/autoencoder-2layer.dml b/scripts/staging/autoencoder-2layer.dml
deleted file mode 100644
index 9eee8ae..0000000
--- a/scripts/staging/autoencoder-2layer.dml
+++ /dev/null
@@ -1,248 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# Trains a 2-layer autoencoder with minibatch SGD, momentum and step-size decay.
-# If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
-# Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
-# The script standardizes the input before training (can be turned off).
-# Also, it randomly reshuffles rows before training.
-# Currently, tanh is set to be the activation function. 
-# By re-implementing 'func' DML-bodied function, one can change the activation.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME   	TYPE     DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# X         String   ---      Filename where the input is stored
-# H1        Int      ---      Number of neurons in the 1st hidden layer
-# H2        Int      ---      Number of neurons in the 2nd hidden layer
-# EPOCH     Int      ---      Number of epochs to train for
-# fmt       String   'text'   Output format ("text", "csv", "binary" etc.)
-# OBJ       Boolean  FALSE    If TRUE, Computes objective function value (squared-loss)
-#                             at the end of each epoch. Note that, computing the full 
-#                             objective can take a lot of time. 
-# BATCH     Int      256      Mini-batch size (training parameter)
-# STEP      Double   1e-5     Initial step size (training parameter)
-# DECAY     Double   0.95     Decays step size after each epoch (training parameter)
-# MOMENTUM  Double   0.9      Momentum parameter (training parameter)
-# ---------------------------------------------------------------------------------------------
-#
-# OUTPUT PARAMETERS (all filenames):
-# ---------------------------------------------------------------------------------------------
-# NAME     TYPE   DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# W1_out   String   ---   File to store weights between input layer and 1st hidden layer
-# b1_out   String   ---   File to store bias between input layer and 1st hidden layer
-# W2_out   String   ---   File to store weights between 1st hidden layer and 2nd hidden layer
-# b2_out   String   ---   File to store bias between 1st hidden layer and 2nd hidden layer
-# W3_out   String   ---   File to store weights between 2nd hidden layer and 3rd hidden layer
-# b3_out   String   ---   File to store bias between 2nd hidden layer and 3rd hidden layer
-# W4_out   String   ---   File to store weights between 3rd hidden layer and output layer
-# b4_out   String   ---   File to store bias between 3rd hidden layer and output layer
-# HIDDEN   String   " "   File to store the hidden (2nd) layer representation if needed
-# ---------------------------------------------------------------------------------------------
-#
-# INVOCATION:
-# -f autoencoder_2layer.dml --nvargs X=<input file> H1=500 H2=2 EPOCH=1 fmt="csv" 
-#                                    W1_out=<weights from input to 1st hidden layer> b1_out=<bias from input to 1st hidden layer> 
-#									 W2_out=<weights from 1st hidden layer to 2nd hidden layer> b2_out=<bias from 1st hidden layer to 2nd hidden layer> 
-#									 W3_out=<weights from 2nd hidden layer to 3rd hidden layer> b3_out=<bias from 2nd hidden layer to 3rd hidden layer> 
-#									 W4_out=<weights from 3rd hidden layer to output> b4_out=<bias from 3rd hidden layer to output>
-#
-
-#implements tanh
-#to use another activation fn, implement a DML-bodied function with 
-#function name 'func' and comment out this one
-func = function(Matrix[Double] X) return(Matrix[Double] Y, Matrix[Double] Y_prime){
-     Y = (exp(2*X) - 1)/(exp(2*X) + 1)
-     Y_prime = 1 - Y^2
-}
-
-feedForward = function(Matrix[Double] X, 
-	               Matrix[Double] W1, Matrix[Double] b1, 
-	               Matrix[Double] W2, Matrix[Double] b2, 
-	               Matrix[Double] W3, Matrix[Double] b3, 
-	               Matrix[Double] W4, Matrix[Double] b4, 
-		       Matrix[Double] Y)
-      	      return(Matrix[Double] H1, Matrix[Double] H1_prime,
-	     	     Matrix[Double] H2, Matrix[Double] H2_prime,
-	     	     Matrix[Double] H3, Matrix[Double] H3_prime,
-	     	     Matrix[Double] Yhat, Matrix[Double] Yhat_prime,
-	     	     Matrix[Double] E){
-      H1_in = t(W1 %*% t(X) + b1)
-      [H1, H1_prime] = func(H1_in)
-
-      H2_in = t(W2 %*% t(H1) + b2)
-      [H2, H2_prime] = func(H2_in)
-
-      H3_in = t(W3 %*% t(H2) + b3)
-      [H3, H3_prime] = func(H3_in)
-
-      Yhat_in = t(W4 %*% t(H3) + b4)
-      [Yhat, Yhat_prime] = func(Yhat_in)
-      E = Yhat - Y
-}
-
-grad = function(Matrix[Double] X, 
-       		Matrix[Double] H1, Matrix[Double] H1_prime,
-       		Matrix[Double] H2, Matrix[Double] H2_prime,
-       		Matrix[Double] H3, Matrix[Double] H3_prime,
-		Matrix[Double] Yhat_prime,
-		Matrix[Double] E,
-		Matrix[Double] W1, Matrix[Double] W2, Matrix[Double] W3, Matrix[Double] W4)
-       return(Matrix[Double] W1_grad, Matrix[Double] b1_grad,
-              Matrix[Double] W2_grad, Matrix[Double] b2_grad,
-	      Matrix[Double] W3_grad, Matrix[Double] b3_grad,
-	      Matrix[Double] W4_grad, Matrix[Double] b4_grad){
-      #backprop
-      delta4 = E * Yhat_prime
-      delta3 = H3_prime * (delta4 %*% W4)
-      delta2 = H2_prime * (delta3 %*% W3)
-      delta1 = H1_prime * (delta2 %*% W2)
-
-      #compute gradients
-      b4_grad = t(colSums(delta4))
-      b3_grad = t(colSums(delta3))
-      b2_grad = t(colSums(delta2))
-      b1_grad = t(colSums(delta1))
-
-      W4_grad = t(delta4) %*% H3
-      W3_grad = t(delta3) %*% H2
-      W2_grad = t(delta2) %*% H1
-      W1_grad = t(delta1) %*% X
-}
-
-obj = function(Matrix[Double] E) return(Double val){
-    val = 0.5 * sum(E^2)
-}
-
-batch_size = ifdef($BATCH, 256)
-mu = ifdef($MOMENTUM, 0.9)
-step = ifdef($STEP, 1e-5)
-decay = ifdef($DECAY, 0.95)
-hfile = ifdef($HIDDEN, " ")
-fmt = ifdef($fmt, "text")
-full_obj = ifdef($OBJ, FALSE)
-
-X = read($X)
-num_hidden1 = $H1
-num_hidden2 = $H2
-max_epochs = $EPOCH
-
-n = nrow(X)
-m = ncol(X)
-
-#randomly reordering rows
-permut = table(seq(1,n,1), order(target=Rand(rows=n, cols=1, min=0, max=1, pdf="uniform"), by=1, index.return=TRUE), n, n)
-X = permut %*% X
-
-#z-transform, whitening operator is better
-means = colSums(X)/n
-stds = sqrt((colSums(X^2)/n - means*means)*n/(n-1)) + 1e-17
-X = (X - means)/stds
-
-W1 = sqrt(6)/sqrt(m + num_hidden1) * Rand(rows=num_hidden1, cols=m, min=-1, max=1, pdf="uniform")
-b1 = matrix(0, rows=num_hidden1, cols=1)
-W2 = sqrt(6)/sqrt(num_hidden1 + num_hidden2) * Rand(rows=num_hidden2, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b2 = matrix(0, rows=num_hidden2, cols=1)
-W3 = sqrt(6)/sqrt(num_hidden2 + num_hidden1) * Rand(rows=num_hidden1, cols=num_hidden2, min=-1, max=1, pdf="uniform")
-b3 = matrix(0, rows=num_hidden1, cols=1)
-W4 = sqrt(6)/sqrt(num_hidden2 + m) * Rand(rows=m, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b4 = matrix(0, rows=m, cols=1)
-
-upd_W1 = matrix(0, rows=nrow(W1), cols=ncol(W1))
-upd_b1 = matrix(0, rows=nrow(b1), cols=ncol(b1))
-upd_W2 = matrix(0, rows=nrow(W2), cols=ncol(W2))
-upd_b2 = matrix(0, rows=nrow(b2), cols=ncol(b2))
-upd_W3 = matrix(0, rows=nrow(W3), cols=ncol(W3))
-upd_b3 = matrix(0, rows=nrow(b3), cols=ncol(b3))
-upd_W4 = matrix(0, rows=nrow(W4), cols=ncol(W4))
-upd_b4 = matrix(0, rows=nrow(b4), cols=ncol(b4))
-
-if( full_obj ){
-      [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-      full_o = obj(full_E)
-      print("EPOCHS=" + 0 + " OBJ (FULL DATA): " + full_o)
-}
-
-iter = 0
-num_iters_per_epoch = ceil(n / batch_size)
-max_iterations = max_epochs * num_iters_per_epoch
-#print("num_iters_per_epoch=" + num_iters_per_epoch + " max_iterations=" + max_iterations)
-beg = 1
-while( iter < max_iterations ){
-      end = beg + batch_size - 1
-      if(end > n) end = n
-      X_batch = X[beg:end,]
-
-      [H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat, Yhat_prime, E] = feedForward(X_batch, W1, b1, W2, b2, W3, b3, W4, b4, X_batch)
-      [W1_grad, b1_grad, W2_grad, b2_grad, W3_grad, b3_grad, W4_grad, b4_grad] = grad(X_batch, H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat_prime, E, W1, W2, W3, W4)
-
-      o = obj(E)
-      print("epochs=%5.4f BATCH beg=%d end=%d obj=%f", (iter / num_iters_per_epoch), beg, end, o)
-
-      #update
-      local_step = step / nrow(X_batch)
-      upd_W1 = mu * upd_W1 - local_step * W1_grad
-      upd_b1 = mu * upd_b1 - local_step * b1
-      upd_W2 = mu * upd_W2 - local_step * W2_grad
-      upd_b2 = mu * upd_b2 - local_step * b2
-      upd_W3 = mu * upd_W3 - local_step * W3_grad
-      upd_b3 = mu * upd_b3 - local_step * b3
-      upd_W4 = mu * upd_W4 - local_step * W4_grad
-      upd_b4 = mu * upd_b4 - local_step * b4
-      W1 = W1 + upd_W1
-      b1 = b1 + upd_b1
-      W2 = W2 + upd_W2
-      b2 = b2 + upd_b2
-      W3 = W3 + upd_W3
-      b3 = b3 + upd_b3
-      W4 = W4 + upd_W4
-      b4 = b4 + upd_b4
-
-      iter = iter + 1
-      if(end == n) beg = 1
-      else beg = end + 1
-
-      if( iter %% num_iters_per_epoch == 0 ) step = step * decay
-
-      if( full_obj & iter %% num_iters_per_epoch == 0 ){
-      	  [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-          full_o = obj(full_E)
-      	  epochs = iter %/% num_iters_per_epoch
-          print("EPOCHS=" + epochs + " iter=" + iter + " OBJ (FULL DATA)=" + full_o)
-      }
-}
-
-write(W1, $W1_out, format=fmt)
-write(b1, $b1_out, format=fmt)
-write(W2, $W2_out, format=fmt)
-write(b2, $b2_out, format=fmt)
-write(W3, $W3_out, format=fmt)
-write(b3, $b3_out, format=fmt)
-write(W4, $W4_out, format=fmt)
-write(b4, $b4_out, format=fmt)
-
-if( hfile != " " ){
-      [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-      reordered_H = t(permut) %*% full_H2
-      write(reordered_H, hfile, format=fmt)
-}
diff --git a/scripts/staging/knn.dml b/scripts/staging/knn.dml
deleted file mode 100644
index ddf5b15..0000000
--- a/scripts/staging/knn.dml
+++ /dev/null
@@ -1,779 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# THIS SCRIPT IMPLEMENTS KNN( K Nearest Neighbor ) ALGORITHM
-#
-# INPUT   PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME    TYPE     DEFAULT     OPTIONAL     MEANING
-# ---------------------------------------------------------------------------------------------
-# X                     String   ---   N    Location to read the input matrix as features
-# T                     String   ---   N    Location to read the input matrix for nearest neighbor
-#                                           search
-# Y                     String   ---    Y   Location to read the input matrix as target
-# Y_T                   String   ---    Y   Location to read the target type of matrix Y whether
-#                                           columns in Y are continuous ( =1 ) or
-#                                           categorical ( =2 )
-# K                     String   ---    Y   Location to write the result of selected k value
-# PR                    String   ---    Y   Location to write the predicted value result,
-#                                           it's optional when Y is NOT specified.   
-# NNR                   String   ---    N   Location to write the nearest neighbor index result for T.   
-# trans_continuous      int      0      Y   Option flag for continuous feature transformed to [-1,1]:
-#                                           0 = do not transform continuous variable;
-#                                           1 = transform continuous variable;
-# select_k              int      0      Y   Use k selection algorithm to estimate k
-#                                           ( 1 means yes )
-# k_min                 int      1      Y   Min k value(  available if select_k = 1 )
-# k_max                 int      100    Y   Max k value(  available if select_k = 1 )
-# select_feature        int      0      Y   Use feature selection algorithm to select feature 
-#                                           ( 1 means yes )
-# START_SELECTED        String   ---    Y   Location to read feature selection initinal value
-# FEATURE_SELECTED      String   ---    Y   Location to write feature selection result
-# feature_max           int      10     Y   Max feature selection
-# interval              int      1000   Y   Interval value for K selecting (  available if select_k = 1 )
-# feature_importance    int      0      Y   Use feature importance algorithm to estimate each feature
-#                                           ( 1 means yes )
-# FEATURE_IMPORTANCE_VALUE String --- Y Location to write feature importance result
-# k_value               int      5      Y   k value for KNN, ignore if select_k enable
-# predict_con_tg        int      0      Y   Continuous  target predict function: mean(=0) or
-#                                           median(=1)
-# fmt                   String   "text" Y   Matrix output format for MB, ML, usually "text" or "csv"
-# ---------------------------------------------------------------------------------------------
-# OUTPUT: Matrix PR, Matrix NNR, Matrix FEATURE_IMPORTANCE_VALUE
-#
-# HOW TO INVOKE THIS SCRIPT - EXAMPLE:
-# hadoop jar SystemDS.jar -f knn.dml -nvargs X=%datapath%/train.csv
-#                       T=%datapath%/test.csv NNR=%tmpdir%/NNR.csv fmt=csv
-
-
-print( "start knn algorithm" );
-#input data
-TrainPath = $X;
-TestPath = $T;
-CLPath = ifdef( $Y," " );
-CLTypePath = ifdef( $Y_T," " );
-#output data
-NNRPath = $NNR;
-PRPath = ifdef( $PR," " );
-#parameter
-epsilon = ifdef( $epsilon,0 );
-distance_type = ifdef( $distance_type,1 );
-trans_continuous = ifdef( $trans_continuous,0 );
-predict_con_tg = ifdef($predict_con_tg,0);
-k_value = ifdef( $k_value,5 );
-FMT = ifdef( $fmt,"text" );
-#k select parameter
-select_k = ifdef( $select_k,0 );
-k_min = ifdef( $k_min,1 );
-k_max = ifdef( $k_max,100 );
-#feature select parameter
-select_feature = ifdef( $select_feature,0 );
-feature_max = ifdef( $feature_max,10 );
-#feature importance parameter
-feature_importance = ifdef ($feature_importance,0);
-
-
-#Read train data
-Train = read ( TrainPath );
-Test = read ( TestPath );
-
-#data prepare
-if( trans_continuous == 1 ){
-    print( "Do data prepare to transfer continuous variable to [-1,1] ..." );
-    Train = prepareKNNData( Train);
-    Test  = prepareKNNData( Test);
-}
-
-n_records = nrow( Train );
-n_features = ncol( Train );
-s_selected_k = 5;
-m_selected_feature = matrix(1,rows=1,cols=n_records);
-if( select_k == 1 | select_feature==1 ){
-    default_interval = as.integer( ceil( sqrt( n_records ) ) );
-    interval = ifdef( $interval,default_interval );
-    print( "interval is " + interval );
-    #parameter check
-    if(  CLPath == " " | CLTypePath == " "){
-        stop( "Y or Y_T must be set for k selection algorithm" );
-    }
-    #parameter re-define
-    if( select_k==1 ){
-        if(  k_max >= n_records  ){
-            k_max = n_records - 1;
-            print( "k_max should no greater than number of record, change k_max equal " +
-            "( number of record - 1 ) : " + k_max );
-        }
-        if(  k_max >= interval  ){
-            interval = k_max + 1;
-            # k_max should equal interval -1, because we drop self when search nn.
-            print( "interval should be no less than k_max, change interval equal : " + 
-            interval );
-        }
-        if(  k_max <= 1  ){
-            stop( "uncorrect k_max value"  );
-        }   
-        if(  k_min >= k_max ){
-            stop( "k_min >= k_max" );
-        }   
-    }
-    if( select_feature == 1 ){
-        if(  k_value >= n_records  ){
-            k_value = n_records - 1;
-            print( "k_value should be no greater than number of record, change k_value equal " +
-            "( number of record - 1 ) : " + k_value );
-        }
-        #Select feature only
-        StartSelectedPath = ifdef( $START_SELECTED," " );
-        SelectedPath = ifdef( $FEATURE_SELECTED," " );
-        if( StartSelectedPath == " " ){
-            m_start_selected_feature = matrix( 0,1,n_features );
-        }else{
-            m_start_selected_feature = read( StartSelectedPath );
-        }       
-    }
-    CL = read( CLPath );
-    CLType = read( CLTypePath );
-    type = as.integer( as.scalar( CLType[1,1] ) );
-    if( select_k == 1 & select_feature == 1){
-        #Combined k and feature selection
-        print("Start combined k and feature selection ...");
-        [m_selected_feature,s_selected_k] = 
-            getSelectedFeatureAndK( Train,CL,type,m_start_selected_feature,
-                feature_max,k_min,k_max,interval );
-    }else if( select_k == 1 ){
-        #Select k only
-        print("Start k select ...");
-        s_selected_k = getSelectedKBase( Train,CL,type,k_min,k_max,interval );
-    }else if( select_feature == 1 ){
-        #Select feature only
-        print("Start feature selection ... ");
-        [m_selected_feature,d_err] = 
-            getSelectedFeature( Train,CL,type,m_start_selected_feature,
-                feature_max,k_value,interval );
-    }
-}
-if( feature_importance == 1){
-    if(  k_value >= n_records  ){
-        k_value = n_records - 1;
-        print( "k_value should be no greater than number of record, make k_value equal " + 
-        "( number of record - 1 ) : " + k_value );
-    }
-    CL = read( CLPath );
-    CLType = read( CLTypePath );
-    type = as.integer( as.scalar( CLType[1,1] ) );  
-    FeatureImportanceExportPath = ifdef( $FEATURE_IMPORTANCE_VALUE," " );
-    if(FeatureImportanceExportPath != " "){
-        [m_feature_importance,m_norm_feature_importance] = 
-            getFeatureImportance(Train,CL,type,k_value);
-        #print( "feature importance is "  );
-        #tmp = matrixDisplay( m_feature_importance );
-        write ( m_feature_importance,FeatureImportanceExportPath,format=FMT );          
-    }
-}
-
-# knn search
-print("Start knn search ...")
-# apply k selection result
-if(select_k == 1){
-    k_value = s_selected_k
-}
-if(  k_value >= n_records ){
-    k_value = n_records - 1;
-    print( "k_value should be no greater than number of record, make k_value equal " + 
-        "( number of record - 1 )" );
-}
-print( "k value is " + k_value );
-# apply feature selection result
-if( select_feature == 1 ){
-    if( n_features > sum( m_selected_feature ) ){
-        Train = removeEmpty( target = Train,margin="cols",select = m_selected_feature);
-        Test = removeEmpty( target = Test,margin="cols",select = m_selected_feature );
-    }
-}
-
-NNR_matrix = naiveKNNsearch(Train,Test,k_value)
-write( NNR_matrix,NNRPath,format=FMT );
-
-# main part for knn predicted
-if( CLPath != " " ){
-    CL = read ( CLPath );
-    if(  CLTypePath == " " ){
-        stop( "Set Y but Miss Y_T for label type" );
-    }
-    CLType = read( CLTypePath );
-    i_cl_type = as.integer( as.scalar( CLType[1,1] ) );
-}
-CL_matrix = matrix( 0,nrow( Test ),1 );
-if( CLPath != " " ){
-    parfor( i in 1:nrow( NNR_matrix ) ,check=0){
-        NNR_tmp_matrix = matrix( 0,k_value,1 );
-        for( j in 1:k_value ){
-            NNR_tmp_matrix[j,1] = CL[as.scalar( NNR_matrix[i,j] ),1] ;
-        }
-        t_cl_value = 0.0 #test data label value
-        if( i_cl_type == 2 ){
-            t_cl_value = as.scalar( rowIndexMax( t( table( NNR_tmp_matrix,1 ) ) ) )
-        }else{
-            if ( predict_con_tg == 0){
-                t_cl_value = mean( NNR_tmp_matrix );
-            } else if(predict_con_tg == 1){
-                t_cl_value = median( NNR_tmp_matrix );
-            }
-        }
-        CL_matrix[i,1] = t_cl_value;
-    }
-}
-if( PRPath != " " ){
-    write( CL_matrix,PRPath,format=FMT );
-}
-
-#naive knn search implement
-naiveKNNsearch = function(
-    matrix[double] P,
-    matrix[double] Q,
-    integer K
-)return(
-    matrix[double] O
-){
-    num_records = nrow (P);
-    num_features = ncol (P);
-    num_queries = nrow (Q);
-    Qt = t(Q);
-    PQt = P %*% Qt;
-    P2 = rowSums (P ^ 2);
-    D = -2 * PQt + P2;
-    if (K == 1) {
-        Dt = t(D);
-        O = rowIndexMin (Dt);
-    } else {
-        O = matrix (0, rows = num_queries, cols = K);
-        parfor (i in 1:num_queries) {
-            D_sorted=order(target=D[,i], by=1, decreasing=FALSE, index.return=TRUE);
-            O[i,] = t(D_sorted[1:K,1]);
-        }   
-    }   
-}
-#naive knn search for predict value only implement
-naiveKNNsearchForPredict = function(
-    matrix[double] P,
-    matrix[double] Q,
-    matrix[double] L,
-    integer K
-)return(
-    matrix[double] OL
-){
-    num_records = nrow (P);
-    num_features = ncol (P);
-    num_queries = nrow (Q);
-    Qt = t(Q);
-    PQt = P %*% Qt;
-    P2 = rowSums (P ^ 2);
-    D = -2 * PQt + P2;
-    if (K == 1) {
-        Dt = t(D);
-        O = rowIndexMin (Dt);
-        OL = matrix (0, rows = num_queries, cols = 1)
-        parfor( i in 1:num_queries){
-            OL[i,] = L[as.scalar(O[i,1]),1]
-        }
-    } else {
-        OL = matrix (0, rows = num_queries, cols = K);
-        parfor (i in 1:num_queries) {
-            D_sorted=order(target=cbind(D[,i],L), by=1, decreasing=FALSE, index.return=FALSE);
-            OL[i,] = t(D_sorted[1:K,2]);
-        }   
-    }   
-}
-# prepareKNNData:
-#   Do data prepare - [-1,1] transform for continues variable 
-# Argument:
-# in_m_data                     input matrix as features
-prepareKNNData = function ( 
-    matrix[double] in_m_data
-     )
-return( 
-    matrix[double] out_m_data
-     )
-{
-    m_colmax = colMaxs(in_m_data);
-    m_colmin = colMins(in_m_data);
-    out_m_data = 2*( in_m_data - m_colmin )/( m_colmax - m_colmin ) - 1;
-}
-
-getKNeighbor = function (  matrix[double] in_m_data,
-        matrix[double] in_m_test_data,
-        matrix[double] in_m_cl, 
-        integer in_i_k_max   ) 
-    return (  matrix[double] out_m_neighbor_value  ) {
-    #to naive
-    m_search_result = naiveKNNsearchForPredict(in_m_data,in_m_test_data,in_m_cl,in_i_k_max + 1)
-    out_m_neighbor_value = m_search_result[ ,2:in_i_k_max + 1]
-}
-
-getErr_k = function (  matrix[double] in_m_neighbor_value,
-        matrix[double] in_m_cl,
-        integer in_i_cl_type , 
-        integer in_i_k_min  ) 
-    return (  matrix[double] out_m_err  ){
-    i_col = ncol( in_m_neighbor_value  );
-    i_row  = nrow( in_m_neighbor_value  );  
-    
-    out_m_err = matrix( 0,i_row,i_col - in_i_k_min + 1  );
-    if( in_i_cl_type == 2  ){
-        #category
-        m_correct = in_m_neighbor_value != in_m_cl[1:i_row,];
-    }else{
-        #continues
-        m_correct = (in_m_neighbor_value - in_m_cl[1:i_row,])^2;#ppred( in_m_neighbor_value,in_m_cl,"-"  );
-    }
-    parfor( i in 1:i_col-in_i_k_min+1 ,check = 0 ){
-        out_m_err[,i] = 
-            ( rowSums( m_correct[,1:in_i_k_min + i - 1]  ) / ( in_i_k_min + i - 1  )  );
-    }
-    #return err for each record and each k ( belong to range 1~max  );
-}
-/*
-eliminateModel = function (  matrix[double] in_j_err, matrix[double] in_j1_err   )
-    return(  boolean out_b_inactived   ){
-    out_b_inactived = FALSE;
-    i_row  = nrow( in_j_err  ); 
-    m_err_mean = as.scalar( colSums( in_j_err - in_j1_err  ) / i_row  );
-    m_err_variance = as.scalar( 
-        sqrt(  colSums( ( in_j_err - in_j1_err - m_err_mean  ) ^ 2  ) / ( i_row-1  )))
-    
-    #alpha, beta, gamma, delta
-    d_gamma = 0.001;
-    d_delta = 0.001;
-    
-    tmp_d_delta = cdf(  
-        target = ( -d_gamma - m_err_mean  ) / m_err_variance, dist="t",df=i_row-1  );
-    #print( "m_err_mean=" + m_err_mean + ",m_err_variance=" + 
-    #   m_err_variance + ",tmp_d_delta=" + tmp_d_delta  );
-    if( tmp_d_delta < d_delta  ){
-        out_b_inactived = TRUE;
-    }
-
-}*/
-eliminateModel = function (  double s_err_mean, double s_err_vars, integer i_row  )
-    return(  boolean out_b_inactived   ){
-    out_b_inactived = FALSE;
-    #alpha, beta, gamma, delta
-    d_gamma = 0.001;
-    d_delta = 0.001;
-    
-    tmp_d_delta = cdf(  
-        target = ( -d_gamma - s_err_mean  ) / s_err_vars, dist="t",df=i_row-1  );
-    #print( "m_err_mean=" + m_err_mean + ",m_err_variance=" + 
-    #   m_err_variance + ",tmp_d_delta=" + tmp_d_delta  );
-    if( tmp_d_delta < d_delta  ){
-        out_b_inactived = TRUE;
-    }
-
-}
-# getSelectedKBase:
-#   k selection algorithm with simple KNN algorithm.  
-#   Refer to ADD part "6.3 Optimized LOOCV + BRACE"
-# Argument:
-# in_m_data                     input matrix as features
-# in_m_data_target              input matrix as target value
-# in_i_is_categorical           1 = category , 0 = continuous
-# k_min                         minimun k
-# k_max                         maximun k
-# interval                      block size for BRACE algorithm
-#
-# Reture:
-# k                             output k value for k selection
-getSelectedKBase = function ( 
-    matrix[double] in_m_data,
-    matrix[double] in_m_data_target,
-    integer in_i_is_categorical, # 1 = category , 0 = continuous
-    integer k_min,
-    integer k_max,
-    integer interval  )
-return( 
-    integer k
-      )
-{
-    b_continue_loop = TRUE;
-    i_iter = 1;
-    i_record  = nrow( in_m_data  );
-
-    m_active_flag = matrix( 0,1,k_max-k_min+1  );
-    i_active_model_number = k_max - k_min + 1;
-    
-    m_iter_err_sum = matrix( 0,1,k_max-k_min+1);
-    m_iter_err_sum_squared = matrix( 0,1,k_max-k_min+1);
-    while( b_continue_loop  ){
-        # 1.build k-d tree? , or use hash method
-        # 2.search data to get k_max nearest neighbor
-        i_process_item = i_iter*interval;
-        if(  i_process_item >= i_record  ){
-            i_process_item = i_record;
-            b_continue_loop = FALSE;
-        }
-        i_process_begin_item = (i_iter - 1)*interval + 1
-		#i_process_begin_item = 1
-        i_process_end_item = i_process_item
-        
-        m_neighbor_value = getKNeighbor( in_m_data,in_m_data[i_process_begin_item:i_process_end_item,],in_m_data_target,k_max  );
-        # 3.get matrix of err from k_min to k_max 
-        m_err = getErr_k( m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical,k_min  );
-
-        # 4.check this matrix to drop unnessary record
-        m_active_flag_tmp = matrix( 0,1,ncol( m_err  )  );
-        
-        s_rows_number = i_process_item#nrow(m_err);
-            
-        m_iter_err_sum = colSums(m_err) + m_iter_err_sum
-        m_iter_err_sum_squared = colSums(m_err ^ 2) + m_iter_err_sum_squared
-        
-        m_err_mean = - outer(t(m_iter_err_sum), m_iter_err_sum , "-") / s_rows_number
-        m_err_vars = ( m_err_mean ^2 * s_rows_number -
-            2 * m_err_mean * m_iter_err_sum  + m_iter_err_sum_squared) / (s_rows_number-1)
-        m_err_vars = sqrt(m_err_vars)
-        
-        parfor( i in 1:ncol( m_err  )  ){
-            #m_err_i = m_err - m_err[,i];
-            #m_err_i_mean = colMeans(m_err_i);
-            #m_err_i_vars = colVars(m_err_i);
-            parfor( j in 1:ncol( m_err  ),check=0  ){
-                b_execute_block = TRUE;
-                if( j==i  ) b_execute_block = FALSE;
-                if( as.scalar( m_active_flag_tmp[1,i]  ) == 1  )  b_execute_block = FALSE; 
-                #i has dropped, ignore this case 
-                if( as.scalar( m_active_flag_tmp[1,j]  ) == 1  )  b_execute_block = FALSE; 
-                #j has dropped, ignore this case
-                if( b_execute_block  ){
-                    b_flag = eliminateModel( as.scalar(m_err_mean[i,j]),as.scalar(m_err_vars[i,j]),s_rows_number);
-                    if(  b_flag == TRUE  ){
-                        m_active_flag_tmp[1,i] = 1;
-                    }
-                }
-            }
-        }
-        
-        #m_tmp = ppred( colSums( m_active_flag_tmp  ),1,">="  );
-        m_active_flag =  m_active_flag + m_active_flag_tmp >= 1;
-        i_active_model_number = -sum( m_active_flag - 1  );
-        
-        # 5.break while check
-        if( i_active_model_number <= 1  ){
-            b_continue_loop = FALSE;
-        }
-        i_iter = i_iter + 1;
-        print("i_iter" + i_iter)
-    }
-
-    k = 0;
-    if(  i_active_model_number == 0   ){
-        print( "All k kick out, use min of range " + k_min  );
-        k = k_min;
-    }else if( i_active_model_number == 1  ){
-        k = k_min + as.integer( as.scalar( rowIndexMin( m_active_flag  )  )  ) - 1;
-        print( "Get k, which value is " + k  );
-    }else{
-        m_err_for_order = 
-            cbind( t( m_iter_err_sum ), matrix( seq( k_min,k_max,1  ),k_max-k_min+1,1  )  );
-        m_err_for_order = removeEmpty( 
-            target=m_err_for_order * t( m_active_flag == 0 ),margin="rows"  );
-        for( i in 1:nrow( m_err_for_order  )  ){
-            print( "k:" + as.scalar( m_err_for_order[i,2]  ) + 
-                ",err:" + as.scalar( m_err_for_order[i,1]  )  );
-        }
-        m_err_order = order( target=m_err_for_order,by=1,decreasing=FALSE,index.return=FALSE  );
-        k = as.integer( as.scalar( m_err_order[1,2]  )  );
-        print( "Get minimum LOOCV error, which value is " + k  );
-    }
-}
-
-getErr = function (  matrix[double] in_m_neighbor_value,
-        matrix[double] in_m_cl,
-        integer in_i_cl_type ) 
-    return (  matrix[double] out_m_err ){
-    i_col = ncol( in_m_neighbor_value );
-    i_row  = nrow( in_m_neighbor_value );   
-    
-    if( in_i_cl_type == 2 ){
-        #category
-        m_correct = in_m_neighbor_value != in_m_cl[1:i_row,];
-    }else{
-        #continues
-        m_correct = (in_m_neighbor_value - in_m_cl[1:i_row,])^2;#ppred( in_m_neighbor_value,in_m_cl,"-" );
-    }
-    out_m_err = ( rowSums( m_correct[,1:i_col] )/( i_col ) );
-}
-
-# getSelectedFeature:
-#   feature selection algorithm.  
-#   Refer to ADD part "7.1 FS+BRACE"
-# Argument:
-# in_m_data                     input matrix as features
-# in_m_data_target              input matrix as target value
-# in_i_is_categorical           1 = category , 0 = continuous
-# in_m_init_selected            S.user, initial selected feature which use specified
-# in_i_max_select               J, max feature selected
-# k_value                       k
-# interval                      block size for BRACE algorithm
-#
-# Reture:
-# out_m_selected_feature        output matrix for feature selection
-# out_d_min_LOOCV               output err
-
-getSelectedFeature = function ( 
-    matrix[double] in_m_data,
-    matrix[double] in_m_data_target,
-    integer in_i_is_categorical, # 1 = category , 0 = continuous
-    matrix[double] in_m_init_selected,
-    integer in_i_max_select,
-    integer k_value,
-    integer interval )
-return( 
-    matrix[double] out_m_selected_feature,
-    double out_d_min_LOOCV
-     )
-{
-    i_n_record  = nrow( in_m_data );
-    i_n_column = ncol( in_m_data );
-    m_main_selected_flag = in_m_init_selected;
-    b_no_feature_selected = TRUE;
-    if( sum( in_m_init_selected ) >= 1 ){
-        b_no_feature_selected = FALSE;
-    }
-    d_max_err_value = ( max( in_m_data_target ) - min( in_m_data_target ) ) * 100;
-    b_continue_main_loop = TRUE;    #level 1 while loop flag
-    d_min_LOOCV = Inf;
-    while( b_continue_main_loop ){
-        m_feature_selected_flag = m_main_selected_flag;
-        m_this_model_selected_flag = TRUE;
-        i_index_min_LOOCV = -1; # flag for which model win in BRACE algorith
-        b_selected_morethan_one = FALSE;
-        b_continue_loop = TRUE; #level 2 while loop flag
-        i_iter = 1;
-        m_iter_err_sum = matrix( 0,1,i_n_column+1 );
-        m_iter_err_sum_squared = matrix( 0,1,i_n_column+1 );
-        while( b_continue_loop ){
-            #print( "*************" + i_iter + "*************" );
-            i_process_item = i_iter*interval;
-            if(  i_process_item >= i_n_record ){
-                i_process_item = i_n_record;
-                b_continue_loop = FALSE;
-            }
-            i_process_begin_item = (i_iter - 1)*interval + 1
-            #i_process_begin_item = 1
-            i_process_end_item = i_process_item
-            m_err = matrix( d_max_err_value,i_process_end_item - i_process_begin_item + 1,i_n_column+1 );
-            if( b_no_feature_selected == TRUE ){
-                parfor( i in 1:i_n_column ,check=0){
-                    if( as.scalar( m_feature_selected_flag[1,i] ) != 1 ){
-                        m_tmp_process_data = in_m_data[,i];
-                        m_neighbor_value = getKNeighbor( 
-                            m_tmp_process_data,m_tmp_process_data[i_process_begin_item:i_process_end_item,],
-                            in_m_data_target,k_value );                            
-                        m_tmp_err = getErr( 
-                            m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
-                        m_err[,i] = m_tmp_err;
-                    }
-                }
-            }else{
-                #Use m_main_selected_flag but not m_feature_selected_flag,
-                # m_main_selected_flag: which feature are init selected
-                # m_feature_selected_flag: which feature are dropped & init selected
-                m_tmp_data = removeEmpty( target=in_m_data ,margin="cols", select = m_main_selected_flag);
-                if( m_this_model_selected_flag == TRUE ){
-                    m_neighbor_value = getKNeighbor( 
-                        m_tmp_data,m_tmp_data[i_process_begin_item:i_process_end_item,],in_m_data_target, k_value );
-                    
-                    m_tmp_err = getErr( m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
-                    m_err[,i_n_column+1] = m_tmp_err;
-                }
-                parfor( i in 1:i_n_column ,check=0){
-                    if( as.scalar( m_feature_selected_flag[1,i] ) != 1 ){
-                        m_tmp_process_data = cbind( m_tmp_data,in_m_data[,i] );
-                        m_neighbor_value = getKNeighbor( 
-                            m_tmp_process_data,m_tmp_process_data[i_process_begin_item:i_process_end_item,],in_m_data_target,k_value );
-                        m_tmp_err = getErr( 
-                            m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
-                        m_err[,i] = m_tmp_err;
-                    }
-                }
-            }
-            if( m_this_model_selected_flag == TRUE ){
-                m_active_flag_tmp = cbind( m_feature_selected_flag,matrix( 0,1,1 ) );
-            }else{
-                m_active_flag_tmp = cbind( m_feature_selected_flag,matrix( 1,1,1 ) );
-            }
-            
-            s_rows_number = i_process_item#nrow(m_err);
-            
-            m_iter_err_sum = colSums(m_err) + m_iter_err_sum
-            m_iter_err_sum_squared = colSums(m_err ^ 2) + m_iter_err_sum_squared
-            
-            m_err_mean = - outer(t(m_iter_err_sum), m_iter_err_sum , "-") / s_rows_number
-            m_err_vars = ( m_err_mean ^2 * s_rows_number -
-                    2 * m_err_mean * m_iter_err_sum  + m_iter_err_sum_squared) / (s_rows_number-1)
-            m_err_vars = sqrt(m_err_vars)
-            
-            parfor( i in 1:ncol( m_err ) ){
-                #m_err_i_mean = (m_iter_err_sum - m_iter_err_sum[,i])/s_rows_number
-                #m_err_i_vars = (m_iter_err_sum_squared + m_err_i_mean ^2 * s_rows_number -
-                #   2 * m_iter_err_sum * m_err_i_mean) / (s_rows_number-1)
-                #m_err_i_vars = sqrt(m_err_i_vars)
-                #m_err_i_mean = m_err_mean[i,]
-                #m_err_i_vars = m_err_vars[i,]
-                parfor( j in 1:ncol( m_err ) ,check=0){
-                    b_execute_block = TRUE;
-                    if( j==i ) b_execute_block = FALSE;
-                    if( as.scalar( m_active_flag_tmp[1,i] ) == 1 )  b_execute_block = FALSE; 
-                    #i has dropped, ignore this case 
-                    if( as.scalar( m_active_flag_tmp[1,j] ) == 1 )  b_execute_block = FALSE; 
-                    #j has dropped, ignore this case
-                    if( b_execute_block ){
-                        #print( "i=" + i + ",j=" + j );
-                        b_flag = eliminateModel( as.scalar(m_err_mean[i,j]),as.scalar(m_err_vars[i,j]),s_rows_number);
-                        if(  b_flag == TRUE ){
-                            m_active_flag_tmp[1,i] = 1;
-                        }
-                    }
-                }
-            }
-            #We mark bit to 1 for selected feature before current loop, 
-            #and mark bit to 1 also for dropped feature in current loop
-                        
-            if( sum( m_active_flag_tmp != 1 ) > 1 ){
-                b_selected_morethan_one = TRUE;
-            }
-            m_col_sums_err = m_iter_err_sum #colSums( m_err );
-            i_index_min_LOOCV = as.scalar( rowIndexMin( m_col_sums_err ) );
-            d_min_LOOCV = as.scalar( m_col_sums_err[1,i_index_min_LOOCV] );
-            i_index_min_LOOCV = i_index_min_LOOCV%% ( i_n_column+1 )
-            
-            if( sum( m_active_flag_tmp != 1 ) <= 1 ){
-                b_continue_loop = FALSE;
-            }
-            if( as.scalar( m_active_flag_tmp[1,i_n_column+1] ) == 1 ){
-                m_this_model_selected_flag = FALSE;
-            }
-            m_feature_selected_flag = m_active_flag_tmp[,1:i_n_column];
-            i_iter = i_iter + 1;            
-        }
-        #select current model, jump out.
-        if( i_index_min_LOOCV == 0 ){
-            b_continue_main_loop = FALSE;
-            print( "Select Current model" );
-        }else{
-            print( "select feature " + i_index_min_LOOCV + ", change bit value to 1" );
-            m_main_selected_flag[1,i_index_min_LOOCV] = 1;
-            b_no_feature_selected = FALSE;
-        }
-        if( sum( m_main_selected_flag - in_m_init_selected ) >= in_i_max_select ){
-            #select more than 10
-            b_continue_main_loop = FALSE;
-        }
-        if( sum( m_main_selected_flag ) == i_n_column ){
-            #all selected
-            b_continue_main_loop = FALSE;
-        }
-    }
-    out_m_selected_feature = m_main_selected_flag;
-    out_d_min_LOOCV = d_min_LOOCV;
-}
-
-# getSelectedFeatureAndK:
-#   Combine k and feature selection algorithm.  
-#   Refer to ADD part "8.Combined k and feature selection"
-# Argument:
-# in_m_data                     input matrix as features
-# in_m_data_target              input matrix as target value
-# in_i_is_categorical           1 = category , 0 = continuous
-# in_m_init_selected            S.user, initial selected feature which use specified
-# in_i_max_select               J, max feature selected
-# k_min                         minimun k
-# k_max                         maximun k
-# interval                      block size for BRACE algorithm
-#
-# Reture:
-# out_m_selected_feature        output matrix for feature selection
-# out_i_selected_k              output k value for k selection
-getSelectedFeatureAndK = function ( 
-    matrix[double] in_m_data,
-    matrix[double] in_m_data_target,
-    integer in_i_is_categorical, # 1 = category , 0 = continuous
-    matrix[double] in_m_init_selected,
-    integer in_i_max_select,
-    integer k_min,
-    integer k_max,
-    integer interval )
-return( 
-    matrix[double] out_m_selected_feature,
-    integer out_i_selected_k
-     )
-{
-    m_err = matrix( 0,1,k_max-k_min+1 );
-    m_feature = matrix( 0,k_max-k_min+1,ncol( in_m_data ) );
-    
-    #Step 1. For each k in [k_min,k_max] ( k_min has default value 1, k_max has default value 100 )
-    #in parallel select relevant features using FS+BRACE or schemata search described in Section 7.
-    parfor( i in k_min:k_max,check=0 ){
-        #print( " ************** " + i  + " ****************" );
-        [m_selected_feature,d_err] = 
-            getSelectedFeature( in_m_data,in_m_data_target,in_i_is_categorical
-            ,in_m_init_selected,in_i_max_select,i,interval );
-        
-        m_err[1,i] = d_err;
-        m_feature[i,] = m_selected_feature;
-        #print( "err:" + d_err );
-    }
-    
-    #Step 2. Output the combination of features and k with the smallest LOOCV error.
-    i_min_err_index = as.integer( as.scalar( rowIndexMin( m_err ) ) );
-    out_m_selected_feature = m_feature[i_min_err_index,];
-    out_i_selected_k = i_min_err_index + k_min - 1;
-    #print( "min err k is " + out_i_selected_k );
-}
-
-getFeatureImportance = function ( 
-    matrix[double] in_m_data,
-    matrix[double] in_m_data_target,
-    integer in_i_is_categorical, # 1 = category , 0 = continuous
-    integer k_value)
-return( 
-    matrix[double] out_m_feature_importance,
-    matrix[double] out_m_norm_feature_importance
-     )
-{
-    n_feature = ncol(in_m_data)
-    n_record = nrow(in_m_data)
-    if(n_feature <= 1){
-        stop("can't estimate feature importance when ncol = 1")
-    }
-    m_err = matrix( 0,n_record,n_feature);
-    parfor(i_feature in 1:n_feature, check=0){
-        m_feature_select = matrix(1,1,n_feature)
-        m_feature_select[1,i_feature] = 0;
-        m_in_tmp_data = removeEmpty(target=in_m_data,margin="cols", select= m_feature_select)
-        m_neighbor_value = getKNeighbor( m_in_tmp_data,m_in_tmp_data,in_m_data_target,k_value);
-        m_tmp_err = getErr( m_neighbor_value,in_m_data_target ,in_i_is_categorical );
-        m_err[,i_feature] = m_tmp_err
-    }
-    out_m_feature_importance = colSums( m_err );
-    out_m_norm_feature_importance = 
-        out_m_feature_importance / as.scalar(rowSums(out_m_feature_importance))
-}
\ No newline at end of file
diff --git a/scripts/staging/regression/lasso/lasso.dml b/scripts/staging/regression/lasso/lasso.dml
deleted file mode 100644
index 476ffbe..0000000
--- a/scripts/staging/regression/lasso/lasso.dml
+++ /dev/null
@@ -1,113 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-# 
-#   http://www.apache.org/licenses/LICENSE-2.0
-# 
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#uses the sparsa algorithm to perform lasso regression
-
-X = read($X)
-y = read($Y)
-n = nrow(X)
-m = ncol(X)
-
-#params
-tol = 10^(-15)
-M = 5
-tau = 1
-maxiter = 1000
-
-#constants
-eta = 2
-sigma = 0.01
-alpha_min = 10^(-30)
-alpha_max = 10^(30)
-
-#init
-alpha = 1
-w = Rand(rows=m, cols=1, min=0, max=1, pdf="uniform")
-history = -1*10^30 * matrix(1, rows=M, cols=1)
-
-r = X %*% w - y
-g = t(X) %*% r
-obj = 0.5 * sum(r*r) + tau*sum(abs(w))
-
-print("Initial OBJ=" + obj)
-
-history[M,1] = obj
-
-inactive_set = matrix(1, rows=m, cols=1)
-iter = 0
-continue = TRUE
-while(iter < maxiter & continue) {
-	dw = matrix(0, rows=m, cols=1)
-	dg = matrix(0, rows=m, cols=1)
-	relChangeObj = -1.0
-	
-	inner_iter = 0
-	inner_continue = TRUE
-	inner_maxiter = 100
-	while(inner_iter < inner_maxiter & inner_continue) {
-		u = w - g/alpha
-		lambda = tau/alpha
-		
-		wnew = sign(u) * (abs(u) - lambda) * ((abs(u) - lambda) > 0)
-		dw = wnew - w
-		dw2 = sum(dw*dw)
-		
-		r = X %*% wnew - y
-		gnew = t(X) %*% r
-		objnew = 0.5 * sum(r*r) + tau*sum(abs(wnew))		
-		obj_threshold = max(history) - 0.5*sigma*alpha*dw2
-		
-		if(objnew <= obj_threshold) {
-			w = wnew
-			dg = gnew - g
-			g = gnew
-			inner_continue = FALSE
-			
-			history[1:(M-1),] = history[2:M,]
-			history[M,1] = objnew
-			relChangeObj = abs(objnew - obj)/obj
-			obj = objnew
-		}
-		else 
-			alpha = eta*alpha
-	
-		inner_iter = inner_iter + 1
-	}
-	
-	if(inner_continue) 
-		print("Inner loop did not converge")
-	
-	alphanew = sum(dw*dg)/sum(dw*dw)
-	alpha = max(alpha_min, min(alpha_max, alphanew))
-	
-	old_inactive_set = inactive_set
-	inactive_set = w != 0
-	diff = sum(abs(old_inactive_set - inactive_set))
-
-	if(diff == 0 & relChangeObj < tol) 
-		continue = FALSE
-
-	num_inactive = sum(w != 0)
-	print("ITER=" + iter + " OBJ=" + obj + " relative change=" + relChangeObj + " num_inactive=" + num_inactive)
-	iter = iter + 1
-}
-
-write(w, $model)
diff --git a/scripts/staging/slicing/__init__.py b/scripts/staging/slicing/__init__.py
deleted file mode 100644
index e66abb4..0000000
--- a/scripts/staging/slicing/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
diff --git a/scripts/staging/slicing/base/Bucket.py b/scripts/staging/slicing/base/Bucket.py
deleted file mode 100644
index dc8402e..0000000
--- a/scripts/staging/slicing/base/Bucket.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class Bucket:
-
-    key: []
-    attributes: []
-    name: ""
-    error: float
-    e_upper: float
-    size: float
-    sum_error: float
-    max_tuple_error: float
-    s_upper: float
-    s_lower: float
-    e_max: float
-    e_max_upper: float
-    score: float
-    c_upper: float
-    parents: []
-    x_size: int
-    loss: float
-    w: float
-
-    def __init__(self, node, cur_lvl, w, x_size, loss):
-        self.attributes = []
-        self.parents = []
-        self.sum_error = 0
-        self.size = 0
-        self.s_upper = 0
-        self.s_lower = 0
-        self.score = 0
-        self.error = 0
-        self.max_tuple_error = 0
-        self.x_size = x_size
-        self.w = w
-        self.loss = loss
-        if cur_lvl == 0:
-            self.key = node
-            self.attributes.append(node)
-        else:
-            self.attributes = node.attributes
-            self.key = node.attributes
-        self.name = self.make_name()
-        self.__hash__()
-
-    def __hash__(self):
-        return hash(self.name)
-
-    def __add__(self, other):
-        self.size += other.size
-        self.sum_error += other.sum_error
-        return self
-
-    def combine_with(self, other):
-        self.size = max(self.size, other.size)
-        self.sum_error = max(self.sum_error, other.sum_error)
-        return self
-
-    def minimize_bounds(self, other):
-        minimized = min(self.s_upper, other.s_upper)
-        self.s_upper = minimized
-        minimized = min(other.s_lower, self.s_lower)
-        self.s_lower = minimized
-        minimized = min(other.e_upper, self.e_upper)
-        self.e_upper = minimized
-        minimized = min(other.e_max_upper, self.e_max_upper)
-        self.e_max_upper = minimized
-        c_upper = self.calc_c_upper(self.w, self.x_size, self.loss)
-        minimized = min(c_upper, self.c_upper)
-        self.c_upper = minimized
-
-    def __eq__(self, other):
-        return (
-            self.__hash__ == other.__hash__ and self.key == other.key)
-
-    def update_metrics(self, row, loss_type):
-        self.size += 1
-        if loss_type == 0:
-            self.sum_error += row[2]
-            if row[2] > self.max_tuple_error:
-                self.max_tuple_error = row[2]
-        else:
-            if row[2] != 0:
-                self.sum_error += 1
-
-    def calc_error(self):
-        if self.size != 0:
-            self.error = self.sum_error / self.size
-        else:
-            self.error = 0
-        self.e_max = self.max_tuple_error
-        self.e_max_upper = self.e_max
-        self.e_upper = self.error
-
-    def check_constraint(self, top_k, x_size, alpha):
-        return self.score >= top_k.min_score and self.size >= x_size / alpha
-
-    def make_name(self):
-        name = ""
-        for attribute in self.attributes:
-            name = name + str(attribute) + " && "
-        name = name[0: len(name) - 4]
-        return name
-
-    def calc_bounds(self, w, x_size, loss):
-        self.s_upper = self.calc_s_upper()
-        self.s_lower = self.calc_s_lower(x_size)
-        self.e_upper = self.calc_e_upper()
-        self.e_max_upper = self.calc_e_max_upper()
-        self.c_upper = self.calc_c_upper(w, x_size, loss)
-
-    def check_bounds(self, x_size, alpha, top_k):
-        return self.s_upper >= x_size / alpha and self.c_upper >= top_k.min_score
-
-    def calc_s_upper(self):
-        cur_min = self.parents[0].size
-        for parent in self.parents:
-            cur_min = min(cur_min, parent.s_upper)
-        return cur_min
-
-    def calc_e_max_upper(self):
-        e_max_min = self.parents[0].e_max_upper
-        for parent in self.parents:
-            e_max_min = min(e_max_min, parent.e_max_upper)
-        return e_max_min
-
-    def calc_s_lower(self, x_size):
-        size_value = x_size
-        for parent in self.parents:
-            size_value = size_value - (size_value - parent.s_lower)
-        return max(size_value, 1)
-
-    def calc_e_upper(self):
-        prev_e_uppers = []
-        for parent in self.parents:
-            prev_e_uppers.append(parent.e_upper)
-        return float(min(prev_e_uppers))
-
-    def calc_c_upper(self, w, x_size, loss):
-        upper_score = w * (self.e_upper / self.s_lower) / (loss / x_size) + (
-                    1 - w) * self.s_upper
-        return float(upper_score)
-
-    def print_debug(self, topk):
-        print("new node has been created: " + self.make_name() + "\n")
-        print("s_upper = " + str(self.s_upper))
-        print("s_lower = " + str(self.s_lower))
-        print("e_upper = " + str(self.e_upper))
-        print("c_upper = " + str(self.c_upper))
-        print("current topk min score = " + str(topk.min_score))
-        print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/SparkNode.py b/scripts/staging/slicing/base/SparkNode.py
deleted file mode 100644
index 81bfcb9..0000000
--- a/scripts/staging/slicing/base/SparkNode.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class SparkNode:
-    error: float
-    name: ""
-    attributes: []
-    parents: []
-    children: []
-    size: int
-    loss: float
-    score: float
-    e_max: float
-    s_upper: int
-    s_lower: int
-    e_upper: float
-    c_upper: float
-    e_max_upper: float
-    key: ""
-
-    def __init__(self, loss, preds):
-        if loss:
-            self.error = loss,
-        if preds:
-            self.preds = preds
-        self.parents = []
-        self.attributes = []
-        self.size = 0
-        self.score = 0
-        self.loss = 0
-        self.s_upper = 0
-        self.s_lower = 1
-        self.key = ''
-
-    def calc_c_upper(self, w):
-        upper_score = w * (self.e_upper / self.s_lower) / (self.error[0] / self.preds[0][0].size) + (1 - w) * self.s_upper
-        return float(upper_score)
-
-    def make_slice_mask(self):
-        mask = []
-        for feature in self.attributes:
-            mask.append(feature)
-        return mask
-
-    def process_slice(self, loss_type):
-        mask = self.make_slice_mask()
-        if loss_type == 0:
-            self.calc_l2(mask)
-        elif loss_type == 1:
-            self.calc_class(mask)
-
-    def calc_class(self, mask):
-        self.e_max = 1
-        filtered = self.filter_by_mask(mask)
-        self.size = len(filtered)
-        mistakes = len(list(filter(lambda row: row[1] == 0, filtered)))
-        if self.size != 0:
-            self.loss = mistakes / self.size
-        else:
-            self.loss = 0
-        self.e_upper = self.loss
-
-    def calc_l2(self, mask):
-        max_tuple_error = 0
-        sum_error = 0
-        filtered = self.filter_by_mask(mask)
-        self.size = len(filtered)
-        for row in filtered:
-            if row[1] > max_tuple_error:
-                max_tuple_error = row[1]
-            sum_error += row[1]
-        self.e_max = max_tuple_error
-        self.e_upper = max_tuple_error
-        self.e_max_upper = max_tuple_error
-        if self.size != 0:
-            self.loss = sum_error/self.size
-        else:
-            self.loss = 0
-
-    def filter_by_mask(self, mask):
-        return list(filter(lambda row: all(attr in row[0].indices for attr in mask), self.preds))
-
-    def calc_s_upper(self, cur_lvl):
-        cur_min = self.parents[0].size
-        for parent in self.parents:
-            if cur_lvl == 1:
-                cur_min = min(cur_min, parent.size)
-            else:
-                cur_min = min(cur_min, parent.s_upper)
-        return cur_min
-
-    def calc_e_max_upper(self, cur_lvl):
-        if cur_lvl == 1:
-            e_max_min = self.parents[0].e_max
-        else:
-            e_max_min = self.parents[0].e_max_upper
-        for parent in self.parents:
-            if cur_lvl == 1:
-                e_max_min = min(e_max_min, parent.e_max)
-            else:
-                e_max_min = min(e_max_min, parent.e_max_upper)
-        return e_max_min
-
-    def calc_s_lower(self):
-        size_value = len(self.preds)
-        for parent in self.parents:
-            size_value = size_value - (size_value - parent.s_lower)
-        return max(size_value, 1)
-
-    def calc_e_upper(self):
-        prev_e_uppers = []
-        for parent in self.parents:
-            prev_e_uppers.append(parent.e_upper)
-        return float(min(prev_e_uppers))
-
-    def calc_bounds(self, cur_lvl, w):
-        self.s_upper = self.calc_s_upper(cur_lvl)
-        self.s_lower = self.calc_s_lower()
-        self.e_upper = self.calc_e_upper()
-        self.e_max_upper = self.calc_e_max_upper(cur_lvl)
-        self.c_upper = self.calc_c_upper(w)
-
-    def make_name(self):
-        name = ""
-        for attribute in self.attributes:
-            name = name + str(attribute) + " && "
-        name = name[0: len(name) - 4]
-        return name
-
-    def make_key(self):
-        return self.name
-
-    def check_constraint(self, top_k, x_size, alpha, cur_min):
-        return self.score >= cur_min and self.size >= x_size / alpha
-
-    def check_bounds(self, top_k, x_size, alpha):
-        return self.s_upper >= x_size / alpha and self.c_upper >= top_k.min_score
-
-    def print_debug(self, topk, level):
-        print("new node has been created: " + self.make_name() + "\n")
-        if level >= 1:
-            print("s_upper = " + str(self.s_upper))
-            print("s_lower = " + str(self.s_lower))
-            print("e_upper = " + str(self.e_upper))
-            print("c_upper = " + str(self.c_upper))
-        print("size = " + str(self.size))
-        print("score = " + str(self.score))
-        print("current topk min score = " + str(topk.min_score))
-        print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/__init__.py b/scripts/staging/slicing/base/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/base/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/base/node.py b/scripts/staging/slicing/base/node.py
deleted file mode 100644
index f7b553b..0000000
--- a/scripts/staging/slicing/base/node.py
+++ /dev/null
@@ -1,210 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-
-class Node:
-    error: float
-    name: ""
-    attributes: []
-    parents: []
-    children: []
-    size: int
-    loss: float
-    score: float
-    e_max: float
-    s_upper: int
-    s_lower: int
-    e_upper: float
-    c_upper: float
-    e_max_upper: float
-    key: ""
-
-    def __init__(self, complete_x, loss, x_size, y_test, preds):
-        self.error = loss,
-        self.parents = []
-        self.attributes = []
-        self.size = 0
-        self.s_upper = 0
-        self.s_lower = 0
-        self.score = 0
-        self.complete_x = complete_x
-        self.loss = 0
-        self.e_upper = 0
-        self.e_max_upper = 0
-        self.c_upper = 0
-        self.x_size = x_size
-        self.preds = preds
-        self.s_lower = 1
-        self.y_test = y_test
-        self.key = ''
-
-    def calc_c_upper(self, w):
-        upper_score = w * (self.e_upper / self.s_lower) / (float(self.error[0]) / self.x_size) + (1 - w) * self.s_upper
-        return float(upper_score)
-
-    def make_slice_mask(self):
-        mask = []
-        for feature in self.attributes:
-            mask.append(feature[1])
-        return mask
-
-    def process_slice(self, loss_type):
-        mask = self.make_slice_mask()
-        if loss_type == 0:
-            self.calc_l2(mask)
-        if loss_type == 1:
-            self.calc_class(mask)
-
-    def calc_class(self, mask):
-        self.e_max = 1
-        mistakes = 0
-        filtered = self.filter_by_mask(mask)
-        for row in filtered:
-            if self.y_test[row[0]][1] != self.preds[row[0]][1]:
-                mistakes = mistakes + 1
-        self.size = len(filtered)
-        if self.size != 0:
-            self.loss = mistakes / self.size
-        else:
-            self.loss = 0
-        self.e_upper = self.loss
-
-    def calc_l2(self, mask):
-        filtered = self.filter_by_mask(mask)
-        max_tuple_error = 0
-        sum_error = 0
-        size = len(filtered)
-        for row in filtered:
-            if float(self.preds[row[0]][1]) > max_tuple_error:
-                    max_tuple_error = float(self.preds[row[0]][1])
-            sum_error += float(self.preds[row[0]][1])
-        self.e_max = max_tuple_error
-        self.e_upper = max_tuple_error
-        if size != 0:
-            self.loss = sum_error/size
-        else:
-            self.loss = 0
-        self.size = size
-
-    def filter_by_mask(self, mask):
-        return list(filter(lambda row: all(row[1][attr] == 1 for attr in mask), self.complete_x))
-
-    def calc_s_upper(self, cur_lvl):
-        if cur_lvl == 1:
-            cur_min = self.parents[0].size
-            for parent in self.parents:
-                cur_min = min(cur_min, parent.size)
-        else:
-            cur_min = self.parents[0].s_upper
-            for parent in self.parents:
-                cur_min = min(cur_min, parent.s_upper)
-        return cur_min
-
-    def calc_e_max_upper(self, cur_lvl):
-        if cur_lvl == 1:
-            e_max_min = self.parents[0].e_max
-        else:
-            e_max_min = self.parents[0].e_max_upper
-        for parent in self.parents:
-            if cur_lvl == 1:
-                e_max_min = min(e_max_min, parent.e_max)
-            else:
-                e_max_min = min(e_max_min, parent.e_max_upper)
-        return e_max_min
-
-    def calc_s_lower(self, cur_lvl):
-        size_value = self.x_size
-        for parent in self.parents:
-            size_value = size_value - (self.x_size - parent.s_lower)
-        return max(size_value, 1)
-
-    def calc_e_upper(self):
-        prev_e_uppers = []
-        for parent in self.parents:
-            prev_e_uppers.append(parent.e_upper)
-        return float(min(prev_e_uppers))
-
-    def calc_bounds(self, cur_lvl, w):
-        self.s_upper = self.calc_s_upper(cur_lvl)
-        self.s_lower = self.calc_s_lower(cur_lvl)
-        self.e_upper = self.calc_e_upper()
-        self.e_max_upper = self.calc_e_max_upper(cur_lvl)
-        self.c_upper = self.calc_c_upper(w)
-
-    def make_name(self):
-        name = ""
-        for attribute in self.attributes:
-            name = name + str(attribute[0]) + " && "
-        name = name[0: len(name) - 4]
-        return name
-
-    def make_key(self, new_id):
-        return new_id, self.name
-
-    def check_constraint(self, top_k, x_size, alpha, cur_min):
-        return self.score >= cur_min and self.size >= x_size / alpha
-
-    def check_bounds(self, x_size, alpha, cur_min):
-        return self.s_upper >= x_size / alpha and self.c_upper >= cur_min
-
-    def update_bounds(self, s_upper, s_lower, e_upper, e_max_upper, w):
-        if self.s_upper:
-            minimized = min(s_upper, self.s_upper)
-            self.s_upper = minimized
-        else:
-            self.s_upper = s_upper
-
-        if self.s_lower:
-            minimized = min(s_lower, self.s_lower)
-            self.s_lower = minimized
-        else:
-            self.s_lower = s_lower
-
-        if self.e_upper:
-            minimized = min(e_upper, self.e_upper)
-            self.e_upper = minimized
-        else:
-            self.e_upper = e_upper
-
-        if self.e_max_upper:
-            minimized = min(e_max_upper, self.e_max_upper)
-            self.e_max_upper = minimized
-        else:
-            self.e_max_upper = e_max_upper
-
-        c_upper = self.calc_c_upper(w)
-        if self.c_upper:
-            minimized = min(c_upper, self.c_upper)
-            self.c_upper = minimized
-        else:
-            self.c_upper = c_upper
-
-    def print_debug(self, topk, level):
-        print("new node has been created: " + self.make_name() + "\n")
-        if level >= 1:
-            print("s_upper = " + str(self.s_upper))
-            print("s_lower = " + str(self.s_lower))
-            print("e_upper = " + str(self.e_upper))
-            print("c_upper = " + str(self.c_upper))
-        print("size = " + str(self.size))
-        print("score = " + str(self.score))
-        print("current topk min score = " + str(topk.min_score))
-        print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/slicer.py b/scripts/staging/slicing/base/slicer.py
deleted file mode 100644
index 77851b6..0000000
--- a/scripts/staging/slicing/base/slicer.py
+++ /dev/null
@@ -1,185 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-import matplotlib.pyplot as plt
-
-
-# optimization function calculation:
-# fi - error of subset, si - its size
-# f - error of complete set, x_size - its size
-# w - "significance" weight of error constraint
-def opt_fun(fi, si, f, x_size, w):
-    formula = w * (fi/f) + (1 - w) * (si/x_size)
-    return float(formula)
-
-
-# slice_name_nonsense function defines if combination of nodes on current level is fine or impossible:
-# there is dependency between common nodes' attributes number and current level is such that:
-# commons == cur_lvl - 1
-# valid combination example: node ABC + node BCD (on 4th level) // three attributes nodes have two common attributes
-# invalid combination example: node ABC + CDE (on 4th level) // result node - ABCDE (absurd for 4th level)
-def slice_name_nonsense(node_i, node_j, cur_lvl):
-    attr1 = list(map(lambda x: x[0].split("_")[0], node_i.attributes))
-    key1 = node_i.key[0]
-    attr2 = list(map(lambda x: x[0].split("_")[0], node_j.attributes))
-    key2 = node_j.key[0]
-    commons = len(list(set(attr1) & set(attr2)))
-    return commons == cur_lvl - 1 and key1 < key2
-
-
-def union(lst1, lst2):
-    final_list = sorted(set(list(lst1) + list(lst2)))
-    return final_list
-
-
-def make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, top_k, alpha, w):
-    first_level = {}
-    counter = 0
-    all_nodes = {}
-    # First level slices are enumerated in a "classic way" (getting data and not analyzing bounds
-    for feature in all_features:
-        new_node = Node(complete_x, loss, x_size, y_test, errors)
-        new_node.parents = [(feature, counter)]
-        new_node.attributes.append((feature, counter))
-        new_node.name = new_node.make_name()
-        new_id = len(all_nodes)
-        new_node.key = new_node.make_key(new_id)
-        all_nodes[new_node.key] = new_node
-        new_node.process_slice(loss_type)
-        new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
-        new_node.c_upper = new_node.score
-        first_level[new_node.key] = new_node
-        new_node.print_debug(top_k, 0)
-        # constraints for 1st level nodes to be problematic candidates
-        if new_node.check_constraint(top_k, x_size, alpha, 1):
-            # this method updates top k slices if needed
-            top_k.add_new_top_slice(new_node)
-        counter = counter + 1
-    return first_level, all_nodes
-
-
-def join_enum(node_i, prev_lvl, complete_x, loss, x_size, y_test, errors, debug, alpha, w, loss_type, b_update, cur_lvl,
-              all_nodes, top_k, cur_lvl_nodes):
-    for node_j in prev_lvl:
-        flag = slice_name_nonsense(prev_lvl[node_i], prev_lvl[node_j], cur_lvl)
-        required_number = len(union(prev_lvl[node_i].attributes, prev_lvl[node_j].attributes))
-        if flag and required_number == cur_lvl + 1:
-            new_node = Node(complete_x, loss, x_size, y_test, errors)
-            parents_set = set(new_node.parents)
-            parents_set.add(prev_lvl[node_i])
-            parents_set.add(prev_lvl[node_j])
-            new_node.parents = list(parents_set)
-            parent1_attr = prev_lvl[node_i].attributes
-            parent2_attr = prev_lvl[node_j].attributes
-            new_node_attr = union(parent1_attr, parent2_attr)
-            new_node.attributes = new_node_attr
-            new_node.name = new_node.make_name()
-            new_id = len(all_nodes)
-            new_node.key = new_node.make_key(new_id)
-            if new_node.key[1] in cur_lvl_nodes:
-                existing_item = cur_lvl_nodes[new_node.key[1]]
-                parents_set = set(existing_item.parents)
-                existing_item.parents = parents_set
-                if b_update:
-                    s_upper = new_node.calc_s_upper(cur_lvl)
-                    s_lower = new_node.calc_s_lower(cur_lvl)
-                    e_upper = new_node.calc_e_upper()
-                    e_max_upper = new_node.calc_e_max_upper(cur_lvl)
-                    new_node.update_bounds(s_upper, s_lower, e_upper, e_max_upper, w)
-            else:
-                new_node.calc_bounds(cur_lvl, w)
-                all_nodes[new_node.key[0]] = new_node
-                # check if concrete data should be extracted or not (only for those that have score upper
-                # big enough and if size of subset is big enough
-                to_slice = new_node.check_bounds(x_size, alpha, top_k.min_score)
-                if to_slice:
-                    new_node.process_slice(loss_type)
-                    new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
-                    # we decide to add node to current level nodes (in order to make new combinations
-                    # on the next one or not basing on its score value
-                    if new_node.check_constraint(top_k, x_size, alpha, top_k.min_score) and new_node.key not in top_k.keys:
-                        top_k.add_new_top_slice(new_node)
-                    cur_lvl_nodes[new_node.key[1]] = new_node
-                if debug:
-                    new_node.print_debug(top_k, cur_lvl)
-    return cur_lvl_nodes, all_nodes
-
-
-# alpha is size significance coefficient (required for optimization function)
-# verbose option is for returning debug info while creating slices and printing it (in console)
-# k is number of top-slices we want to receive as a result (maximum output, if less all of subsets will be printed)
-# w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-# loss_type = 0 (in case of regression model)
-# loss_type = 1 (in case of classification model)
-def process(all_features, complete_x, loss, x_size, y_test, errors, debug, alpha, k, w, loss_type, b_update):
-    levels = []
-    top_k = Topk(k)
-    first_level = make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, top_k, alpha, w)
-    candidates = []
-    pruned = []
-    indexes = []
-    indexes.append(1)
-    candidates.append(len(first_level[0]))
-    pruned.append(len(first_level[0]))
-    all_nodes = first_level[1]
-    levels.append(first_level[0])
-    # cur_lvl - index of current level, correlates with number of slice forming features
-    cur_lvl = 1  # currently filled level after first init iteration
-    print()
-    print("Current topk are: ")
-    top_k.print_topk()
-    # combining each candidate of previous level with every till it becomes useless (one node can't make a pair)
-    while len(levels[cur_lvl - 1]) > 1:
-        cur_lvl_nodes = {}
-        prev_lvl = levels[cur_lvl - 1]
-        level_candidates = len(prev_lvl) * (len(prev_lvl) - 1)
-        candidates.append(level_candidates)
-        for node_i in prev_lvl:
-            partial = join_enum(node_i, prev_lvl, complete_x, loss, x_size, y_test, errors, debug, alpha, w, loss_type,
-                                b_update, cur_lvl, all_nodes, top_k, cur_lvl_nodes)
-            cur_lvl_nodes = partial[0]
-            all_nodes = partial[1]
-        cur_lvl = cur_lvl + 1
-        indexes.append(cur_lvl)
-        levels.append(cur_lvl_nodes)
-        print("Level " + str(cur_lvl) + " had " + str(candidates) +
-              " candidates but after pruning only " + str(len(cur_lvl_nodes)) + " go to the next level")
-        pruned.append(len(cur_lvl_nodes))
-        print()
-        print("Current topk are: ")
-        top_k.print_topk()
-    plt.plot(indexes, candidates, 'r--',
-             indexes, pruned, 'g--')
-    plt.xlabel('Level')
-    plt.ylabel('Number of slices')
-    plt.show()
-    print("Program stopped at level " + str(cur_lvl))
-    print()
-    print("Selected slices are: ")
-    top_k.print_topk()
-    print("candidates:")
-    print(candidates)
-    print(">>>>>>>>>")
-    print("pruned:")
-    print(pruned)
-    return top_k
diff --git a/scripts/staging/slicing/base/top_k.py b/scripts/staging/slicing/base/top_k.py
deleted file mode 100644
index 38c6545..0000000
--- a/scripts/staging/slicing/base/top_k.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class Topk:
-    k: int
-    min_score: float
-    slices: []
-    keys: []
-
-    def __init__(self, k):
-        self.k = k
-        self.slices = []
-        self.keys = []
-        self.min_score = 1
-
-    def top_k_min_score(self):
-        self.slices.sort(key=lambda x: x.score, reverse=True)
-        self.min_score = self.slices[len(self.slices) - 1].score
-        return self.min_score
-
-    def add_new_top_slice(self, new_top_slice):
-        self.min_score = new_top_slice.score
-        if len(self.slices) < self.k:
-            self.slices.append(new_top_slice)
-            self.keys.append(new_top_slice.key)
-            return self.top_k_min_score()
-        else:
-            self.slices[len(self.slices) - 1] = new_top_slice
-            self.keys[len(self.slices) - 1] = new_top_slice.key
-            return self.top_k_min_score()
-
-    def print_topk(self):
-        for candidate in self.slices:
-            print(candidate.name + ": " + "score = " + str(candidate.score) + "; size = " + str(candidate.size))
-
-    def buckets_top_k(self, cur_lvl_slices, x_size, alpha, cur_min):
-        for bucket in cur_lvl_slices:
-            if bucket.size > x_size / alpha and bucket.score >= cur_min:
-                self.add_new_top_slice(bucket)
-        return self
diff --git a/scripts/staging/slicing/base/union_slicer.py b/scripts/staging/slicing/base/union_slicer.py
deleted file mode 100644
index 8ec64d5..0000000
--- a/scripts/staging/slicing/base/union_slicer.py
+++ /dev/null
@@ -1,162 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-from slicing.base.slicer import opt_fun, union
-import matplotlib.pyplot as plt
-
-
-def check_attributes(left_node, right_node):
-    attr1 = list(map(lambda x: x[0].split("_")[0], left_node.attributes))
-    attr2 = list(map(lambda x: x[0].split("_")[0], right_node.attributes))
-    if set(attr1).intersection(set(attr2)):
-        return False
-    return True
-
-
-def make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, w, alpha, top_k):
-    all_nodes = {}
-    counter = 0
-    first_level = []
-    for feature in all_features:
-        new_node = Node(complete_x, loss, x_size, y_test, errors)
-        new_node.parents = [(feature, counter)]
-        new_node.attributes.append((feature, counter))
-        new_node.name = new_node.make_name()
-        new_id = len(all_nodes)
-        new_node.key = new_node.make_key(new_id)
-        all_nodes[new_node.key] = new_node
-        new_node.process_slice(loss_type)
-        # for first level nodes all bounds are strict as concrete metrics
-        new_node.s_upper = new_node.size
-        new_node.s_lower = 0
-        new_node.e_upper = new_node.loss
-        new_node.e_max_upper = new_node.e_max
-        new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
-        new_node.c_upper = new_node.score
-        first_level.append(new_node)
-        new_node.print_debug(top_k, 0)
-        # constraints for 1st level nodes to be problematic candidates
-        if new_node.score > 1 and new_node.size >= x_size / alpha:
-            # this method updates top k slices if needed
-            top_k.add_new_top_slice(new_node)
-        counter = counter + 1
-    return first_level, all_nodes
-
-
-def process(all_features, complete_x, loss, x_size, y_test, errors, debug, alpha, k, w, loss_type, b_update):
-    top_k = Topk(k)
-    # First level slices are enumerated in a "classic way" (getting data and not analyzing bounds
-    levels = []
-    first_level = make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, w, alpha, top_k)
-    # double appending of first level nodes in order to enumerating second level in the same way as others
-    levels.append((first_level[0], len(all_features)))
-    candidates = []
-    pruned = []
-    indexes = []
-    indexes.append(1)
-    candidates.append(len(first_level[0]))
-    pruned.append(len(first_level[0]))
-    all_nodes = first_level[1]
-    # cur_lvl - index of current level, correlates with number of slice forming features
-    cur_lvl = 1  # level that is planned to be filled later
-    cur_lvl_nodes = first_level
-    # currently for debug
-    print("Level 1 had " + str(len(all_features)) + " candidates")
-    print()
-    print("Current topk are: ")
-    top_k.print_topk()
-    # DPSize algorithm approach of previous levels nodes combinations and updating bounds for those that already exist
-    while len(cur_lvl_nodes) > 0:
-        cur_lvl_nodes = []
-        count = 0
-        prev_lvl = levels[cur_lvl - 1]
-        for left in range(int(cur_lvl / 2) + 1):
-            right = cur_lvl - 1 - left
-            for node_i in range(len(levels[left][0])):
-                for node_j in range(len(levels[right][0])):
-                    flag = check_attributes(levels[left][0][node_i], levels[right][0][node_j])
-                    required_number = len(union(levels[left][0][node_i].attributes, levels[right][0][node_j].attributes))
-                    if flag and required_number == cur_lvl + 1:
-                        new_node = Node(complete_x, loss, x_size, y_test, errors)
-                        parents_set = set(new_node.parents)
-                        parents_set.add(levels[left][0][node_i])
-                        parents_set.add(levels[right][0][node_j])
-                        new_node.parents = list(parents_set)
-                        parent1_attr = levels[left][0][node_i].attributes
-                        parent2_attr = levels[right][0][node_j].attributes
-                        new_node_attr = union(parent1_attr, parent2_attr)
-                        new_node.attributes = new_node_attr
-                        new_node.name = new_node.make_name()
-                        new_id = len(all_nodes)
-                        new_node.key = new_node.make_key(new_id)
-                        if new_node.key[1] in all_nodes:
-                            existing_item = all_nodes[new_node.key[1]]
-                            parents_set = set(existing_item.parents)
-                            existing_item.parents = parents_set
-                            if b_update:
-                                s_upper = new_node.calc_s_upper(cur_lvl)
-                                s_lower = new_node.calc_s_lower(cur_lvl)
-                                e_upper = new_node.calc_e_upper()
-                                e_max_upper = new_node.calc_e_max_upper(cur_lvl)
-                                new_node.update_bounds(s_upper, s_lower, e_upper, e_max_upper, w)
-                        else:
-                            new_node.calc_bounds(cur_lvl, w)
-                            all_nodes[new_node.key[1]] = new_node
-                            # check if concrete data should be extracted or not (only for those that have score upper
-                            # big enough and if size of subset is big enough
-                            to_slice = new_node.check_bounds(x_size, alpha, top_k.min_score)
-                            if to_slice:
-                                new_node.process_slice(loss_type)
-                                new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
-                                # we decide to add node to current level nodes (in order to make new combinations
-                                # on the next one or not basing on its score value
-                                if new_node.check_constraint(top_k, x_size, alpha, top_k.min_score) and new_node.key \
-                                        not in top_k.keys:
-                                    top_k.add_new_top_slice(new_node)
-                                cur_lvl_nodes.append(new_node)
-                            if debug:
-                                new_node.print_debug(top_k, cur_lvl)
-            count = count + levels[left][1] * levels[right][1]
-        cur_lvl = cur_lvl + 1
-        indexes.append(cur_lvl)
-        print("Level " + str(cur_lvl) + " had " + str(count) +
-              " candidates but after pruning only " + str(len(cur_lvl_nodes)) + " go to the next level")
-        levels.append((cur_lvl_nodes, count))
-        candidates.append(count)
-        pruned.append(len(cur_lvl_nodes))
-        top_k.print_topk()
-    plt.plot(indexes, candidates, 'r--',
-             indexes, pruned, 'g--')
-    plt.xlabel('Level')
-    plt.ylabel('Number of slices')
-    plt.show()
-    print("Program stopped at level " + str(cur_lvl))
-    print()
-    print("Selected slices are: ")
-    top_k.print_topk()
-    print("candidates:")
-    print(candidates)
-    print(">>>>>>>>>")
-    print("pruned:")
-    print(pruned)
-    return top_k
diff --git a/scripts/staging/slicing/slicing.dml b/scripts/staging/slicing/slicing.dml
deleted file mode 100644
index 8faf416..0000000
--- a/scripts/staging/slicing/slicing.dml
+++ /dev/null
@@ -1,311 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#-------------------------------------------------------------
-# X         Input matrix (integer encoded [1..v])
-# e         error vector (classification accuracy, l2 norm, etc)
-# k         top-K subsets / slices
-# minSup    minimum support (min number of rows per slice)
-# alpha     weight [0,1]: 0 only size, 1 only error
-# dpEval    flag for data-parallel slice evaluation, 
-#           otherwise task-parallel
-# verbose   flag for verbose debug output 
-# ------------------------------------------------------------
-# TK        top-k slices (k x ncol(X) if successful) 
-# TKC       score, size, error of slices (k x 3)
-# ------------------------------------------------------------
-
-slicing = function(Matrix[Double] X, Matrix[Double] e, Integer k = 4, Integer minSup = 32, Double alpha = 0.5, Boolean dpEval = FALSE, Boolean verbose = FALSE) 
-  return(Matrix[Double] TK, Matrix[Double] TKC)
-{
-  m = nrow(X);
-  n = ncol(X);
-
-  # prepare offset vectors and one-hot encoded X
-  fdom = colMaxs(X);
-  foffb = t(cumsum(t(fdom))) - fdom;
-  foffe = t(cumsum(t(fdom)))
-  rix = matrix(seq(1,m)%*%matrix(1,1,n), m*n, 1)
-  cix = matrix(X + foffb, m*n, 1);
-  X2 = table(rix, cix); #one-hot encoded
-
-  # initialize statistics and basic slices
-  n2 = ncol(X2);     # one-hot encoded features
-  eAvg = sum(e) / m; # average error
-  [S, R] = createAndScoreBasicSlices(X2, e, eAvg, minSup, alpha, verbose); 
-
-  # initialize top-k
-  [TK, TKC] = maintainTopK(S, R, matrix(0, 0, n2), matrix(0, 0, 3), k, minSup);
-
-  if( verbose ) {
-    [maxsc, minsc] = analyzeTopK(TKC);
-    print("SliceFinder: initial top-K: count="+nrow(TK)+", max="+maxsc+", min="+minsc)
-  }
-
-  # lattice enumeration w/ size/error pruning, one iteration per level
-  # termination condition (max #feature levels)
-  level = 1;
-  while( nrow(S) > 0 & sum(S) > 0 & level < n ) {
-    level = level + 1;
-
-    # enumerate candidate join pairs, incl size/error pruning 
-    nrS = nrow(S);
-    S = getPairedCandidates(S, R, TK, TKC, k, level, eAvg, minSup, alpha, n2, foffb, foffe); 
-
-    if(verbose) {
-      print("\nSliceFinder: level "+level+":")
-      print(" -- generated paired slice candidates: "+nrS+" -> "+nrow(S));      
-    }
-
-    # extract and evaluate candidate slices
-    if( dpEval ) { #data-parallel
-      R = evalSlice(X2, e, eAvg, t(S), level, alpha);
-    }
-    else { # task-parallel
-      R = matrix(0, nrow(S), 3)
-      parfor( i in 1:nrow(S) )
-        R[i,] = evalSlice(X2, e, eAvg, t(S[i,]), level, alpha);
-    }
-
-    # maintain top-k after evaluation
-    [TK, TKC] = maintainTopK(S, R, TK, TKC, k, minSup);
-
-    if(verbose) {
-      [maxsc, minsc] = analyzeTopK(TKC);
-      valid = as.integer(sum(R[,3]>=minSup));
-      print(" -- valid slices after eval: "+valid+"/"+nrow(S));
-      print(" -- top-K: count="+nrow(TK)+", max="+maxsc+", min="+minsc);
-    }
-  }
-
-  TK = decodeTopK(TK, foffb, foffe);
-
-  if( verbose ) {
-    print("SliceFinder: terminated at level "+level+":\n"
-      + toString(TK) + "\n" + toString(TKC));
-  }
-}
-
-createAndScoreBasicSlices = function(Matrix[Double] X2, Matrix[Double] e, Double eAvg, 
-  Double minSup, Double alpha, Boolean verbose)
-  return(Matrix[Double] S, Matrix[Double] R)
-{
-  n2 = ncol(X2);
-  cCnts = t(colSums(X2)); # column counts
-  err = t(t(e) %*% X2)    # total error vector
-
-  if( verbose ) {
-    drop = as.integer(sum(cCnts < minSup));
-    print("SliceFinder: dropping "+drop+"/"+n2+" features below minSup = "+minSup+".");
-  }
-
-  # working set of active slices (#attr x #slices) and top k
-  selCols = (cCnts >= minSup);
-  attr = removeEmpty(target=seq(1,n2), margin="rows", select=selCols);
-  ss = removeEmpty(target=cCnts, margin="rows", select=selCols);
-  se = removeEmpty(target=err, margin="rows", select=selCols);
-  S = table(seq(1,nrow(attr)), attr, nrow(attr), n2);
-
-  # score 1-slices and create initial top-k 
-  sc = score(ss, se, eAvg, alpha, nrow(X2));
-  R = cbind(sc, se, ss);
-}
-
-score = function(Matrix[Double] ss, Matrix[Double] se, Double eAvg, Double alpha, Integer n)
-  return(Matrix[Double] sc)
-{
-  sc = alpha * ((se/ss) / eAvg - 1) - (1-alpha) * (n/ss - 1);
-}
-
-scoreUB = function(Matrix[Double] ss, Matrix[Double] se, Double eAvg, Integer minSup, Double alpha, Integer n)
-  return(Matrix[Double] sc)
-{
-  sc = alpha * ((se/minSup) / eAvg - 1) - (1-alpha) * (n/ss - 1);
-}
-
-
-maintainTopK = function(Matrix[Double] S, Matrix[Double] R, Matrix[Double] TK, Matrix[Double] TKC, Integer k, Integer minSup) 
-  return(Matrix[Double] TK, Matrix[Double] TKC)
-{
-  # prune invalid minSup and scores
-  I = (R[,1] > 0) & (R[,3] >= minSup);
-
-  if( sum(I)!=0 ) {
-    S = removeEmpty(target=S, margin="rows", select=I);
-    R = removeEmpty(target=R, margin="rows", select=I);
-
-    # evaluated candidated and previous top-k
-    slices = rbind(TK, S);
-    scores = rbind(TKC, R);
-
-    # extract top-k
-    IX = order(target=scores, by=1, decreasing=TRUE, index.return=TRUE);
-    IX = IX[1:min(k,nrow(IX)),];
-    P = table(seq(1,nrow(IX)), IX, nrow(IX), nrow(slices));
-    TK = P %*% slices;
-    TKC = P %*% scores;
-  }
-}
-
-analyzeTopK = function(Matrix[Double] TKC) return(Double maxsc, Double minsc) {
-  maxsc = -Inf;
-  minsc = -Inf;
-  if( nrow(TKC)>0 ) {
-    maxsc = as.scalar(TKC[1,1]);
-    minsc = as.scalar(TKC[nrow(TKC),1]);  
-  }
-}
-
-getPairedCandidates = function(Matrix[Double] S, Matrix[Double] R, Matrix[Double] TK, 
-    Matrix[Double] TKC, Integer k, Integer level, Double eAvg, Integer minSup, Double alpha, 
-    Integer n2, Matrix[Double] foffb, Matrix[Double] foffe)
-  return(Matrix[Double] P) 
-{
-  # prune invalid slices (possible without affecting overall
-  # pruning effectiveness due to handling of missing parents)
-  pI = (R[,3] >= minSup);
-  S = removeEmpty(target=S, margin="rows", select=pI)
-  R = removeEmpty(target=R, margin="rows", select=pI)
-
-  # join compatible slices (without self)
-  join = S %*% t(S) == (level-2)
-  I = upper.tri(target=join, diag=FALSE, values=TRUE);
-  
-  # pair construction
-  nr = nrow(I); nc = ncol(I);
-  rix = matrix(I * seq(1,nr), nr*nc, 1);
-  cix = matrix(I * t(seq(1,nc)), nr*nc, 1);
-  rix = removeEmpty(target=rix, margin="rows");
-  cix = removeEmpty(target=cix, margin="rows");
-  
-  P = matrix(0,0,ncol(S))
-  if( sum(rix)!=0 ) {
-    P1 = table(seq(1,nrow(rix)), rix, nrow(rix), nrow(S));
-    P2 = table(seq(1,nrow(cix)), cix, nrow(rix), nrow(S));
-    P12 = P1 + P2; # combined slice
-    P = (P12 %*% S) != 0;
-    ss = min(P1 %*% R[,3], P2 %*% R[,3])
-    se = min(P1 %*% R[,2], P2 %*% R[,2])
-
-    # prune invalid self joins (>1 bit per feature)
-    I = matrix(1, nrow(P), 1);
-    for( j in 1:ncol(foffb) ) {
-      beg = as.scalar(foffb[1,j])+1;
-      end = as.scalar(foffe[1,j]);
-      I = I & (rowSums(P[,beg:end]) <= 1);
-    }
-    P12 = removeEmpty(target=P12, margin="rows", select=I)
-    P = removeEmpty(target=P, margin="rows", select=I);
-    ss = removeEmpty(target=ss, margin="rows", select=I);
-    se = removeEmpty(target=se, margin="rows", select=I);
-
-    # prepare IDs for deduplication and pruning
-    ID = matrix(0, nrow(P), 1);
-    dom = foffe-foffb+1;
-    for( j in 1:ncol(dom) ) {
-      beg = as.scalar(foffb[1,j])+1;
-      end = as.scalar(foffe[1,j]);
-      I = rowIndexMax(P[,beg:end]) * rowMaxs(P[,beg:end]);
-      prod = 1;
-      if(j<ncol(dom))
-        prod = prod(dom[1,(j+1):ncol(dom)])
-      ID = ID + I * prod;
-    }
-
-    # ID transformation to avoid exceeding INT_MAX and
-    # and to void creating huge sparse intermediates
-    [ID, M] = transformencode(target=as.frame(ID), spec="{ids:true,recode:[1]}")
-
-    # size pruning, with rowMin-rowMax transform 
-    # to avoid densification (ignored zeros)
-    map = table(ID, seq(1,nrow(P)), max(ID), nrow(P))
-    ubSizes = 1/rowMaxs(map * (1/t(ss)));
-    ubSizes = replace(target=ubSizes, pattern=Inf, replacement=0);
-    fSizes = (ubSizes >= minSup)
-
-    # error pruning
-    ubError = 1/rowMaxs(map * (1/t(se)));
-    ubError = replace(target=ubError, pattern=Inf, replacement=0);
-    ubScores = scoreUB(ubSizes, ubError, eAvg, minSup, alpha, n2);
-    [maxsc, minsc] = analyzeTopK(TKC);
-    fScores = (ubScores > minsc & ubScores > 0) 
-
-    # missing parents pruning
-    numParents = rowSums((map %*% P12) != 0) 
-    fParents = (numParents == level);
-
-    # apply all pruning 
-    map = map * (fSizes & fScores & fParents);
-    
-    # deduplication of join outputs
-    Dedup = removeEmpty(target=map, margin="rows") != 0
-    P = (Dedup %*% P) != 0
-  }
-}
-
-evalSlice = function(Matrix[Double] X, Matrix[Double] e, Double eAvg, Matrix[Double] tS, Integer l, Double alpha) 
-  return(Matrix[Double] R)
-{
-  I = (X %*% tS) == l; # slice indicator
-  ss = t(colSums(I));  # absolute slice size (nnz)
-  se = t(t(e) %*% I);  # absolute slice error 
-
-  # score of relative error and relative size
-  sc = score(ss, se, eAvg, alpha, nrow(X));
-  R = cbind(sc, se, ss);
-}
-
-decodeTopK = function(Matrix[Double] TK, Matrix[Double] foffb, Matrix[Double] foffe)
-  return(Matrix[Double] TK) 
-{
-  R = matrix(1, nrow(TK), ncol(foffb));
-  if( nrow(TK) > 0 ) {
-    parfor( j in 1:ncol(foffb) ) {
-      beg = as.scalar(foffb[1,j])+1;
-      end = as.scalar(foffe[1,j]);
-      I = rowSums(TK[,beg:end]) * rowIndexMax(TK[,beg:end]);
-      R[, j] = I;
-    }
-  }
-  TK = R;
-}
-
-Forig = read("./Salaries.csv", data_type="frame", format="csv", header=TRUE);
-
-F = Forig[,1:ncol(Forig)-1];
-y = as.matrix(Forig[,ncol(Forig)]);
-
-# data preparation
-jspec= "{ ids:true, recode:[1,2,3,6], bin:[{id:4, method:equi-width, numbins:14},{id:5, method:equi-width, numbins:12}]}"
-[X,M] = transformencode(target=F, spec=jspec);
-X = X[,2:ncol(X)]
-
-X = rbind(X,X)
-X = cbind(X,X)
-y = rbind(y,y)
-
-# learn model
-B = lm(X=X, y=y, verbose=FALSE);
-yhat = X %*% B;
-e = (y-yhat)^2;
-
-# call slice finding
-[S,C] = slicing(X=X, e=e, k=10, alpha=0.95, minSup=32, verbose=TRUE);
diff --git a/scripts/staging/slicing/spark_modules/join_data_parallel.py b/scripts/staging/slicing/spark_modules/join_data_parallel.py
deleted file mode 100644
index f55b0ca..0000000
--- a/scripts/staging/slicing/spark_modules/join_data_parallel.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.Bucket import Bucket
-from slicing.base.SparkNode import SparkNode
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import approved_join_slice
-
-
-def rows_mapper(row, buckets, loss_type):
-    # filtered = dict(filter(lambda bucket: all(attr in row[1] for attr in bucket[1].attributes), buckets.items()))
-    filtered = dict(filter(lambda bucket: all(attr in row[0] for attr in bucket[1].attributes), buckets.items()))
-    for item in filtered:
-        filtered[item].update_metrics(row, loss_type)
-    return filtered
-
-
-def join_enum(cur_lvl_nodes, cur_lvl, x_size, alpha, top_k, w, loss):
-    buckets = {}
-    for node_i in range(len(cur_lvl_nodes)):
-        for node_j in range(node_i + 1, len(cur_lvl_nodes)):
-            flag = approved_join_slice(cur_lvl_nodes[node_i], cur_lvl_nodes[node_j], cur_lvl)
-            if flag:
-                node = SparkNode(None, None)
-                node.attributes = list(set(cur_lvl_nodes[node_i].attributes) | set(cur_lvl_nodes[node_j].attributes))
-                bucket = Bucket(node, cur_lvl, w, x_size, loss)
-                bucket.parents.append(cur_lvl_nodes[node_i])
-                bucket.parents.append(cur_lvl_nodes[node_j])
-                bucket.calc_bounds(w, x_size, loss)
-                if bucket.check_bounds(x_size, alpha, top_k):
-                    buckets[bucket.name] = bucket
-    return buckets
-
-
-def combiner(a):
-    return a
-
-
-def merge_values(a, b):
-    a.combine_with(b)
-    return a
-
-
-def merge_combiners(a, b):
-    a + b
-    return a
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type):
-    top_k = Topk(k)
-    cur_lvl = 0
-    cur_lvl_nodes = list(all_features)
-    pred_pandas = predictions.toPandas()
-    x_size = len(pred_pandas)
-    b_topk = SparkContext.broadcast(sc, top_k)
-    b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
-    buckets = {}
-    for node in cur_lvl_nodes:
-        bucket = Bucket(node, cur_lvl, w, x_size, loss)
-        buckets[bucket.name] = bucket
-    b_buckets = SparkContext.broadcast(sc, buckets)
-    rows = predictions.rdd.map(lambda row: (row[1].indices, row[2]))\
-        .map(lambda item: list(item))
-    mapped = rows.map(lambda row: rows_mapper(row, b_buckets.value, loss_type))
-    flattened = mapped.flatMap(lambda line: (line.items()))
-    reduced = flattened.combineByKey(combiner, merge_values, merge_combiners)
-    cur_lvl_nodes = reduced.values()\
-        .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket, loss, w, x_size, b_cur_lvl.value))
-    if debug:
-        cur_lvl_nodes.map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
-    cur_lvl = 1
-    prev_level = cur_lvl_nodes.collect()
-    top_k = top_k.buckets_top_k(prev_level, x_size, alpha, 1)
-    while len(prev_level) > 0:
-        b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
-        b_topk = SparkContext.broadcast(sc, top_k)
-        cur_min = top_k.min_score
-        b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
-        top_k.print_topk()
-        buckets = join_enum(prev_level, cur_lvl, x_size, alpha, top_k, w, loss)
-        b_buckets = SparkContext.broadcast(sc, buckets)
-        to_slice = dict(filter(lambda bucket: bucket[1].check_bounds(x_size, alpha, top_k), buckets.items()))
-        b_to_slice = SparkContext.broadcast(sc, to_slice)
-        mapped = rows.map(lambda row: rows_mapper(row, b_to_slice.value, loss_type))
-        flattened = mapped.flatMap(lambda line: (line.items()))
-        to_process = flattened.combineByKey(combiner, merge_values, merge_combiners)
-        if debug:
-            to_process.values().map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
-        prev_level = to_process\
-            .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket[1], loss, w, x_size, b_cur_lvl.value))\
-            .collect()
-        cur_lvl += 1
-        top_k = top_k.buckets_top_k(prev_level, x_size, alpha, cur_min)
-        print("Level " + str(cur_lvl) + " had " + str(
-            len(b_cur_lvl_nodes.value * (len(prev_level) - 1)))+" candidates but after pruning only " +
-              str(len(prev_level)) + " go to the next level")
-        top_k.print_topk()
-    print()
-    print("Program stopped at level " + str(cur_lvl - 1))
-    print("Selected slices are: ")
-    top_k.print_topk()
-    return None
diff --git a/scripts/staging/slicing/spark_modules/spark_slicer.py b/scripts/staging/slicing/spark_modules/spark_slicer.py
deleted file mode 100644
index 9da5b98..0000000
--- a/scripts/staging/slicing/spark_modules/spark_slicer.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.SparkNode import SparkNode
-from slicing.base.slicer import union, opt_fun
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import update_top_k
-
-
-def join_enum_fun(node_a, list_b, predictions, f_l2, debug, alpha, w, loss_type, cur_lvl, top_k):
-    x_size = len(predictions)
-    nodes = []
-    for node_i in range(len(list_b)):
-        flag = spark_utils.approved_join_slice(node_i, node_a, cur_lvl)
-        if not flag:
-            new_node = SparkNode(predictions, f_l2)
-            parents_set = set(new_node.parents)
-            parents_set.add(node_i)
-            parents_set.add(node_a)
-            new_node.parents = list(parents_set)
-            parent1_attr = node_a.attributes
-            parent2_attr = list_b[node_i].attributes
-            new_node_attr = union(parent1_attr, parent2_attr)
-            new_node.attributes = new_node_attr
-            new_node.name = new_node.make_name()
-            new_node.calc_bounds(cur_lvl, w)
-            # check if concrete data should be extracted or not (only for those that have score upper
-            # and if size of subset is big enough
-            to_slice = new_node.check_bounds(top_k, x_size, alpha)
-            if to_slice:
-                new_node.process_slice(loss_type)
-                new_node.score = opt_fun(new_node.loss, new_node.size, f_l2, x_size, w)
-                # we decide to add node to current level nodes (in order to make new combinations
-                # on the next one or not basing on its score value
-                if new_node.check_constraint(top_k, x_size, alpha) and new_node.key not in top_k.keys:
-                    top_k.add_new_top_slice(new_node)
-                nodes.append(new_node)
-            if debug:
-                new_node.print_debug(top_k, cur_lvl)
-    return nodes
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type, enumerator):
-    top_k = Topk(k)
-    cur_lvl = 0
-    levels = []
-    first_level = {}
-    all_features = list(all_features)
-    first_tasks = sc.parallelize(all_features)
-    b_topk = SparkContext.broadcast(sc, top_k)
-    init_slices = first_tasks.mapPartitions(lambda features: spark_utils.make_first_level(features, predictions, loss,
-                                                                                          b_topk.value, w, loss_type)) \
-        .map(lambda node: (node.key, node)).collect()
-    first_level.update(init_slices)
-    update_top_k(first_level, top_k, alpha, predictions, 1)
-    prev_level = SparkContext.broadcast(sc, first_level)
-    levels.append(prev_level)
-    cur_lvl = cur_lvl + 1
-    top_k.print_topk()
-    # checking the first partition of level. if not empty then processing otherwise no elements were added to this level
-    while len(levels[cur_lvl - 1].value) > 0:
-        nodes_list = {}
-        b_topk = SparkContext.broadcast(sc, top_k)
-        cur_min = top_k.min_score
-        partitions = sc.parallelize(levels[cur_lvl - 1].value.values())
-        mapped = partitions.mapPartitions(lambda nodes: spark_utils.nodes_enum(nodes, levels[cur_lvl - 1].value.values(),
-                                                                               predictions, loss, b_topk.value, alpha, k, w,
-                                                                               loss_type, cur_lvl, debug, enumerator, cur_min))
-        flattened = mapped.flatMap(lambda node: node)
-        nodes_list.update(flattened.map(lambda node: (node.key, node)).distinct().collect())
-        prev_level = SparkContext.broadcast(sc, nodes_list)
-        levels.append(prev_level)
-        update_top_k(nodes_list, top_k, alpha, predictions, cur_min)
-        cur_lvl = cur_lvl + 1
-        b_topk.value.print_topk()
-        print("Level " + str(cur_lvl) + " had " + str(len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
-              " candidates but after pruning only " + str(len(nodes_list)) + " go to the next level")
-    print("Program stopped at level " + str(cur_lvl - 1))
-    print()
-    print("Selected slices are: ")
-    top_k.print_topk()
-    return top_k
diff --git a/scripts/staging/slicing/spark_modules/spark_union_slicer.py b/scripts/staging/slicing/spark_modules/spark_union_slicer.py
deleted file mode 100644
index 94172de..0000000
--- a/scripts/staging/slicing/spark_modules/spark_union_slicer.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-
-from pyspark import SparkContext
-
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import update_top_k
-
-
-def process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type, enumerator):
-    top_k = Topk(k)
-    cur_lvl = 0
-    levels = []
-    all_features = list(all_features)
-    first_level = {}
-    first_tasks = sc.parallelize(all_features)
-    b_topk = SparkContext.broadcast(sc, top_k)
-    init_slices = first_tasks.mapPartitions(lambda features: spark_utils.make_first_level(features, predictions, loss,
-                                                                                          b_topk.value, w, loss_type)) \
-        .map(lambda node: (node.key, node)) \
-        .collect()
-    first_level.update(init_slices)
-    update_top_k(first_level, top_k, alpha, predictions, 1)
-    prev_level = SparkContext.broadcast(sc, first_level)
-    levels.append(prev_level)
-    cur_lvl = 1
-    top_k.print_topk()
-    while len(levels[cur_lvl - 1].value) > 0:
-        cur_lvl_res = {}
-        b_topk = SparkContext.broadcast(sc, top_k)
-        cur_min = top_k.min_score
-        for left in range(int(cur_lvl / 2) + 1):
-            right = cur_lvl - left - 1
-            partitions = sc.parallelize(levels[left].value.values())
-            mapped = partitions.mapPartitions(lambda nodes: spark_utils.nodes_enum(nodes, levels[right].value.values(),
-                                                                                   predictions, loss, b_topk.value, alpha, k,
-                                                                                   w, loss_type, cur_lvl, debug,
-                                                                                   enumerator, cur_min))
-            flattened = mapped.flatMap(lambda node: node)
-            partial = flattened.map(lambda node: (node.key, node)).collect()
-            cur_lvl_res.update(partial)
-        prev_level = SparkContext.broadcast(sc, cur_lvl_res)
-        levels.append(prev_level)
-        update_top_k(cur_lvl_res, top_k, alpha, predictions, cur_min)
-        cur_lvl = cur_lvl + 1
-        top_k.print_topk()
-        print("Level " + str(cur_lvl) + " had " + str(len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
-              " candidates but after pruning only " + str(len(prev_level.value)) + " go to the next level")
-    print("Program stopped at level " + str(cur_lvl - 1))
-    print()
-    print("Selected slices are: ")
-    top_k.print_topk()
-    return top_k
diff --git a/scripts/staging/slicing/spark_modules/union_data_parallel.py b/scripts/staging/slicing/spark_modules/union_data_parallel.py
deleted file mode 100644
index 5ea70cc..0000000
--- a/scripts/staging/slicing/spark_modules/union_data_parallel.py
+++ /dev/null
@@ -1,124 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.Bucket import Bucket
-from slicing.base.SparkNode import SparkNode
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils, join_data_parallel
-from slicing.spark_modules.join_data_parallel import rows_mapper, combiner
-from slicing.spark_modules.spark_utils import approved_union_slice
-
-
-def merge_values(a, b):
-    a.minimize_bounds(b)
-    return a
-
-
-def merge_combiners(a, b):
-    a + b
-    return a
-
-
-def union_enum(left_level, right_level, x_size, alpha, top_k, w, loss, cur_lvl):
-    buckets = {}
-    for node_i in range(len(left_level)):
-        for node_j in range(len(right_level)):
-            flag = approved_union_slice(left_level[node_i], right_level[node_j])
-            if flag:
-                node = SparkNode(None, None)
-                node.attributes = list(set(left_level[node_i].attributes) | set(right_level[node_j].attributes))
-                bucket = Bucket(node, cur_lvl, w, x_size, loss)
-                bucket.parents.append(left_level[node_i])
-                bucket.parents.append(right_level[node_j])
-                bucket.calc_bounds(w, x_size, loss)
-                if bucket.check_bounds(x_size, alpha, top_k):
-                    buckets[bucket.name] = bucket
-    return buckets
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type):
-    top_k = Topk(k)
-    cur_lvl = 0
-    levels = []
-    cur_lvl_nodes = list(all_features)
-    pred_pandas = predictions.toPandas()
-    x_size = len(pred_pandas)
-    b_topk = SparkContext.broadcast(sc, top_k)
-    b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
-    buckets = {}
-    for node in cur_lvl_nodes:
-        bucket = Bucket(node, cur_lvl, w, x_size, loss)
-        buckets[bucket.name] = bucket
-    b_buckets = SparkContext.broadcast(sc, buckets)
-    # rows = predictions.rdd.map(lambda row: (row[0], row[1].indices, row[2])) \
-    #     .map(lambda item: (item[0], item[1].tolist(), item[2]))
-    rows = predictions.rdd.map(lambda row: row[1].indices) \
-        .map(lambda item: list(item))
-    mapped = rows.map(lambda row: rows_mapper(row, b_buckets.value, loss_type))
-    flattened = mapped.flatMap(lambda line: (line.items()))
-    reduced = flattened.combineByKey(combiner, join_data_parallel.merge_values, join_data_parallel.merge_combiners)
-    cur_lvl_nodes = reduced.values() \
-        .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket, loss, w, x_size, b_cur_lvl.value))
-    if debug:
-        cur_lvl_nodes.map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
-    cur_lvl = 1
-    prev_level = cur_lvl_nodes.collect()
-    b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
-    levels.append(b_cur_lvl_nodes)
-    top_k = top_k.buckets_top_k(prev_level, x_size, alpha, 1)
-    while len(prev_level) > 0:
-        b_topk = SparkContext.broadcast(sc, top_k)
-        cur_min = top_k.min_score
-        b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
-        top_k.print_topk()
-        buckets = []
-        for left in range(int(cur_lvl / 2) + 1):
-            right = cur_lvl - left - 1
-            nodes = union_enum(levels[left].value, levels[right].value, x_size, alpha, top_k, w, loss, cur_lvl)
-            buckets.append(nodes)
-        b_buckets = sc.parallelize(buckets)
-        all_buckets = b_buckets.flatMap(lambda line: (line.items()))
-        combined = dict(all_buckets.combineByKey(combiner, merge_values, merge_combiners).collect())
-        b_buckets = SparkContext.broadcast(sc, combined)
-        to_slice = dict(filter(lambda bucket: bucket[1].check_bounds(x_size, alpha, top_k), combined.items()))
-        b_to_slice = SparkContext.broadcast(sc, to_slice)
-        mapped = rows.map(lambda row: rows_mapper(row, b_to_slice.value, loss_type))
-        flattened = mapped.flatMap(lambda line: (line.items()))
-        partial = flattened.combineByKey(combiner, join_data_parallel.merge_values, join_data_parallel.merge_combiners)
-        prev_level = partial\
-            .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket[1], loss, w, x_size, b_cur_lvl.value)).collect()
-        top_k = top_k.buckets_top_k(prev_level, x_size, alpha, cur_min)
-        b_topk = SparkContext.broadcast(sc, top_k)
-        if debug:
-            partial.values().map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
-        print("Level " + str(cur_lvl) + " had " + str(
-            len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
-              " candidates but after pruning only " + str(len(prev_level)) + " go to the next level")
-        print("Program stopped at level " + str(cur_lvl))
-        b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
-        levels.append(b_cur_lvl_nodes)
-        cur_lvl += 1
-    print()
-    print("Selected slices are: ")
-    top_k.print_topk()
-    return None
diff --git a/scripts/staging/slicing/tests/__init__.py b/scripts/staging/slicing/tests/__init__.py
deleted file mode 100644
index e66abb4..0000000
--- a/scripts/staging/slicing/tests/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/classification/__init__.py b/scripts/staging/slicing/tests/classification/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/tests/classification/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/classification/compas_bd.py b/scripts/staging/slicing/tests/classification/compas_bd.py
deleted file mode 100644
index 3f1a5b3..0000000
--- a/scripts/staging/slicing/tests/classification/compas_bd.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, join_data_parallel, union_data_parallel
-
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-        dataset = args[8]
-    else:
-        k = 10
-        w = 0.1
-        alpha = 4
-        b_update = True
-        debug = True
-        loss_type = 0
-        dataset = 'compas-scores-two-years.csv'
-        enumerator = "union"
-
-    conf = SparkConf().setAppName("salary_test").setMaster('local[8]')
-    num_partitions = 8
-    model_type = "classification"
-    label = "is_recid"
-    sparkContext = SparkContext(conf=conf)
-    sqlContext = SQLContext(sparkContext)
-    dataset_df = sqlContext.read\
-        .option("badRecordsPath", "/tmp/badRecordsPath")\
-        .csv(dataset, header='true',
-                                     inferSchema='true')
-    # fileRDD = sparkContext.textFile(dataset, num_partitions)
-    # header = fileRDD.first()
-    # head_split = header.split(",")
-    # fileRDD = fileRDD.filter(lambda line: line != header)
-    # data = fileRDD.map(lambda row: row.split(","))
-    # dataset_df = sqlContext.createDataFrame(data, head_split)
-
-    cat_features = ["sex", "age_cat", "race", "decile_score10", "c_charge_degree", "c_charge_desc"]
-    stages = []
-    dataset_df = dataset_df.withColumn("id", sf.monotonically_increasing_id())
-    dataset_df = dataset_df.withColumn('target', dataset_df[label].cast("int"))
-    dataset_df = dataset_df.withColumn('model_type', sf.lit(1))
-    for feature in cat_features:
-        string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
-        encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
-        encoder.setDropLast(False)
-        stages += [string_indexer, encoder]
-    assembler_inputs = [feature + "_vec" for feature in cat_features]
-    assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-    stages += [assembler]
-    assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-    stages += [assembler_final]
-
-    pipeline = Pipeline(stages=stages)
-    pipeline_model = pipeline.fit(dataset_df)
-    dataset_transformed = pipeline_model.transform(dataset_df)
-    df_transform_fin = dataset_transformed.select('id', 'features', 'target', 'model_type').toPandas()
-
-    cat = 0
-    counter = 0
-    decode_dict = {}
-    for feature in cat_features:
-        colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
-        colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
-        for item in colIdx:
-            decode_dict[counter] = (cat, item, colIdx[item])
-            counter = counter + 1
-        cat = cat + 1
-
-    train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
-    train_df = sqlContext.createDataFrame(train)
-    test_df = sqlContext.createDataFrame(test)
-
-    rf = RandomForestClassifier(featuresCol='features', labelCol="target", numTrees=10)
-    rf_model = rf.fit(train_df)
-    predictions = rf_model.transform(test_df)
-    # Select example rows to display.
-    predictions.select("id", "features", "target", "prediction", "model_type")
-    # Select (prediction, true label) and compute test error
-    evaluator = MulticlassClassificationEvaluator(
-        labelCol="target", predictionCol="prediction", metricName="accuracy")
-    accuracy = evaluator.evaluate(predictions)
-    loss = 1.0 - accuracy
-    pred_df_fin = predictions.withColumn('error',
-                                         spark_utils.calc_loss(predictions["target"], predictions['prediction'],
-                                                               predictions['model_type']))
-    predictions = pred_df_fin.select('id', 'features', 'error').repartition(num_partitions)
-    all_features = range(predictions.toPandas().values[0][0])
-
-    if enumerator == "join":
-        join_data_parallel.parallel_process(all_features, predictions, loss, sparkContext, debug=debug, alpha=alpha, k=k,
-                                            w=w, loss_type=loss_type)
-    elif enumerator == "union":
-        union_data_parallel.parallel_process(all_features, predictions, loss, sparkContext, debug=debug, alpha=alpha, k=k,
-                                             w=w, loss_type=loss_type)
diff --git a/scripts/staging/slicing/tests/classification/sparked_adults.py b/scripts/staging/slicing/tests/classification/sparked_adults.py
deleted file mode 100644
index 9c37fef..0000000
--- a/scripts/staging/slicing/tests/classification/sparked_adults.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-
-from slicing.sparked import sparked_utils, sparked_slicer, sparked_union_slicer
-
-
-ten_binner = sf.udf(lambda arg: int(arg / 10))
-fnlwgt_binner = sf.udf(lambda arg: int(arg // 100000))
-edu_binner = sf.udf(lambda arg: int(arg / 5))
-cap_gain_binner = sf.udf(lambda arg: int(arg / 1000))
-
-conf = SparkConf().setAppName("adults_test").setMaster('local[2]')
-num_partitions = 2
-model_type = "classification"
-label = 'Income'
-sparkContext = SparkContext(conf=conf)
-sqlContext = SQLContext(sparkContext)
-dataset_df = sqlContext.read.csv('/slicing/datasets/adult.csv', header='true', inferSchema='true')
-# initializing stages of main transformation pipeline
-stages = []
-dataset_df = dataset_df.withColumn('age_bin', ten_binner(dataset_df['Age']))
-dataset_df = dataset_df.withColumn('fnlwgt_bin', fnlwgt_binner(dataset_df['fnlwgt']))
-dataset_df = dataset_df.withColumn('edu_num_bin', edu_binner(dataset_df['EducationNum']))
-dataset_df = dataset_df.withColumn('cap_gain_bin', cap_gain_binner(dataset_df['CapitalGain']))
-dataset_df = dataset_df.withColumn('hours_per_w_bin', ten_binner(dataset_df['HoursPerWeek']))
-dataset_df = dataset_df.withColumn('model_type', sf.lit(1))
-
-# list of categorical features for further hot-encoding
-cat_features = ["age_bin", "WorkClass", "fnlwgt_bin", "Education", "edu_num_bin",
-        "MaritalStatus", "Occupation", "Relationship", "Race", "Gender",
-        "cap_gain_bin", "CapitalLoss", "hours_per_w_bin", "NativeCountry"]
-
-# hot encoding categorical features
-for feature in cat_features:
-    string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index")
-    encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
-    encoder.setDropLast(False)
-    stages += [string_indexer, encoder]
-assembler_inputs = [feature + "_vec" for feature in cat_features]
-assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-stages += [assembler]
-assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-label_indexer = StringIndexer(inputCol=label, outputCol=label+"_idx")
-stages += [assembler_final]
-stages += [label_indexer]
-pipeline = Pipeline(stages=stages)
-pipeline_model = pipeline.fit(dataset_df)
-dataset_transformed = pipeline_model.transform(dataset_df)
-cat_dict = []
-decode_dict = {}
-counter = 0
-cat = 0
-for feature in cat_features:
-    colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
-    colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
-    for item in colIdx:
-        decode_dict[counter] = (cat, item, colIdx[item])
-        counter = counter + 1
-    cat = cat + 1
-df_transform_fin = dataset_transformed.select('features', label+"_idx", 'model_type')
-splits = df_transform_fin.randomSplit([0.8, 0.2], seed=1234)
-train_df = splits[0]
-test_df = splits[1]
-rf = RandomForestClassifier(featuresCol='features', labelCol=label+"_idx", numTrees=10)
-rf_model = rf.fit(train_df)
-predictions = rf_model.transform(test_df)
-# Select example rows to display.
-predictions.select("features", label+"_idx", "prediction", "model_type")
-# Select (prediction, true label) and compute test error
-evaluator = MulticlassClassificationEvaluator(
-    labelCol=label+"_idx", predictionCol="prediction", metricName="accuracy")
-accuracy = evaluator.evaluate(predictions)
-loss = 1.0 - accuracy
-pred_df_fin = predictions.withColumn('error', sparked_utils.calc_loss(predictions[label+"_idx"],
-                                                                      predictions['prediction'],
-                                                                      predictions['model_type']))
-predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
-all_features = range(predictions.toPandas().values[0][0].size)
-predictions = predictions.collect()
-k = 10
-SparkContext.broadcast(sparkContext, loss)
-SparkContext.broadcast(sparkContext, predictions)
-SparkContext.broadcast(sparkContext, all_features)
-SparkContext.broadcast(sparkContext, decode_dict)
-enumerator = "join"
-if enumerator == "join":
-    sparked_slicer.parallel_process(all_features, predictions, loss, sparkContext, debug=True, alpha=6, k=k, w=0.5,
-                                    loss_type=0, enumerator="join")
-elif enumerator == "union":
-    sparked_union_slicer.process(all_features, predictions, loss, sparkContext, debug=True, alpha=4, k=k, w=0.5, loss_type=0,
-                                 enumerator="union")
-
diff --git a/scripts/staging/slicing/tests/classification/test_adult.py b/scripts/staging/slicing/tests/classification/test_adult.py
deleted file mode 100644
index cacb715..0000000
--- a/scripts/staging/slicing/tests/classification/test_adult.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.preprocessing import OneHotEncoder
-from sklearn import svm
-
-from slicing.base import slicer as slicer, union_slicer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn import preprocessing
-from sklearn.model_selection import train_test_split
-
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 4
-        b_update = True
-        debug = True
-        loss_type = 1
-        enumerator = "union"
-    dataset = pd.read_csv('/slicing/datasets/adult.csv')
-    attributes_amount = len(dataset.values[0])
-    x = dataset.iloc[:, 0:attributes_amount - 1].values
-    y = dataset.iloc[:, attributes_amount - 1]
-    le = preprocessing.LabelEncoder()
-    le.fit(y)
-    y = le.transform(y)
-    complete_x = []
-    complete_y = []
-    counter = 0
-    all_indexes = []
-    not_encoded_columns = [
-        "Age", "WorkClass", "fnlwgt", "Education", "EducationNum",
-        "MaritalStatus", "Occupation", "Relationship", "Race", "Gender",
-        "CapitalGain", "CapitalLoss", "HoursPerWeek", "NativeCountry", "Income"
-    ]
-    for row in x:
-            row[0] = int(row[0] / 10)
-            row[2] = int(row[2]) // 100000
-            row[4] = int(row[4] / 5)
-            row[10] = int(row[10] / 1000)
-            row[12] = int(row[12] / 10)
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x).toarray()
-    all_features = enc.get_feature_names()
-    x_size = len(complete_x)
-    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
-    for item in x_test:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    x_size = counter
-    clf = svm.SVC()
-    clf.fit(x_train, y_train)
-
-    # alpha is size significance coefficient
-    # verbose option is for returning debug info while creating slices and printing it
-    # k is number of top-slices we want
-    # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-    # loss_type = 0 (l2 in case of regression model
-    # loss_type = 1 (cross entropy in case of classification model)
-    preds = clf.predict(x_test)
-    predictions = []
-    counter = 0
-    mistakes = 0
-    for pred in preds:
-        predictions.append((counter, pred))
-        if y_test[counter] != pred:
-            mistakes = mistakes + 1
-        counter = counter + 1
-    lossF = mistakes / counter
-
-    # enumerator <union>/<join> indicates an approach of next level slices combination process:
-    # in case of <join> in order to create new node of current level slicer
-    # combines only nodes of previous layer with each other
-    # <union> case implementation is based on DPSize algorithm
-    if enumerator == "join":
-        slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
-                       w=w, loss_type=loss_type, b_update=b_update)
-    elif enumerator == "union":
-        union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha,
-                             k=k, w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/classification/test_compas.py b/scripts/staging/slicing/tests/classification/test_compas.py
deleted file mode 100644
index 7fbb6fc..0000000
--- a/scripts/staging/slicing/tests/classification/test_compas.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LinearRegression
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-import sys
-
-from slicing.base import slicer, union_slicer
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 4
-        b_update = True
-        debug = True
-        loss_type = 0
-        enumerator = "union"
-    file_name = 'slicing/datasets/real/compas/compas-test.csv'
-    dataset = pd.read_csv(file_name)
-    attributes_amount = len(dataset.values[0])
-    y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-    # starting with one not including id field
-    x = dataset.iloc[:, 0:attributes_amount - 1].values
-    # hot encoding of categorical features
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x).toarray()
-    complete_x = []
-    complete_y = []
-    counter = 0
-    all_features = enc.get_feature_names()
-    # train model on a whole dataset
-    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
-    for item in x_test:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    x_size = counter
-    clf = RandomForestClassifier(n_jobs=2, random_state=0)
-    clf.fit(x_train, y_train)
-    RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
-                           max_depth=None, max_features='auto', max_leaf_nodes=None,
-                           min_impurity_split=1e-07, min_samples_leaf=1,
-                           min_samples_split=2, min_weight_fraction_leaf=0.0,
-                           n_estimators=10, n_jobs=2, oob_score=False, random_state=0,
-                           verbose=0, warm_start=False)
-    preds = clf.predict(x_test)
-    predictions = []
-    counter = 0
-    mistakes = 0
-    for pred in preds:
-        predictions.append((counter, pred))
-        if y_test[counter] != pred:
-            mistakes = mistakes + 1
-        counter = counter + 1
-    lossF = mistakes / counter
-    # alpha is size significance coefficient
-    # verbose option is for returning debug info while creating slices and printing it
-    # k is number of top-slices we want
-    # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
-    # enumerator <union>/<join> indicates an approach of next level slices combination process:
-    # in case of <join> in order to create new node of current level slicer
-    # combines only nodes of previous layer with each other
-    # <union> case implementation is based on DPSize algorithm
-    if enumerator == "join":
-        slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
-                       w=w, loss_type=loss_type, b_update=b_update)
-    elif enumerator == "union":
-        union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha,
-                             k=k, w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/classification/test_iris.py b/scripts/staging/slicing/tests/classification/test_iris.py
deleted file mode 100644
index 2bd0c09..0000000
--- a/scripts/staging/slicing/tests/classification/test_iris.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.preprocessing import OneHotEncoder
-
-from slicing.base import slicer as slicer, union_slicer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn import preprocessing
-from sklearn.model_selection import train_test_split
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 6
-        b_update = True
-        debug = True
-        loss_type = 1
-        enumerator = "join"
-    dataset = pd.read_csv('/slicing/datasets/iris.csv')
-    attributes_amount = len(dataset.values[0])
-    x = dataset.iloc[:, 0:attributes_amount - 1].values
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x).toarray()
-    y = dataset.iloc[:, attributes_amount - 1]
-    le = preprocessing.LabelEncoder()
-    le.fit(y)
-    y = le.transform(y)
-    complete_x = []
-    complete_y = []
-    counter = 0
-    all_indexes = []
-    all_features = enc.get_feature_names()
-    x_size = len(complete_x)
-    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
-    for item in x_test:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    x_size = counter
-    clf = RandomForestClassifier(n_jobs=2, random_state=0)
-    clf.fit(x_train, y_train)
-    RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
-                max_depth=None, max_features='auto', max_leaf_nodes=None,
-                min_impurity_split=1e-07, min_samples_leaf=1,
-                min_samples_split=2, min_weight_fraction_leaf=0.0,
-                n_estimators=10, n_jobs=2, oob_score=False, random_state=0,
-                verbose=0, warm_start=False)
-
-    # alpha is size significance coefficient
-    # verbose option is for returning debug info while creating slices and printing it
-    # k is number of top-slices we want
-    # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-    # loss_type = 0 (l2 in case of regression model
-    # loss_type = 1 (cross entropy in case of classification model)
-    preds = clf.predict(x_test)
-    predictions = []
-    counter = 0
-    mistakes = 0
-    for pred in preds:
-        predictions.append((counter, pred))
-        if y_test[counter] != pred:
-            mistakes = mistakes + 1
-        counter = counter + 1
-    lossF = mistakes / counter
-
-    # enumerator <union>/<join> indicates an approach of next level slices combination process:
-    # in case of <join> in order to create new node of current level slicer
-    # combines only nodes of previous layer with each other
-    # <union> case implementation is based on DPSize algorithm
-    if enumerator == "join":
-        slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
-                       w=w, loss_type=1, b_update=b_update)
-    elif enumerator == "union":
-        union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
-                             w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/__init__.py b/scripts/staging/slicing/tests/regression/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/tests/regression/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/regression/bd_spark_salary.py b/scripts/staging/slicing/tests/regression/bd_spark_salary.py
deleted file mode 100644
index 1889292..0000000
--- a/scripts/staging/slicing/tests/regression/bd_spark_salary.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, join_data_parallel, union_data_parallel
-
-binner = udf(lambda arg: int(int(arg) // 5))
-
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-        dataset = args[8]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 6
-        b_update = True
-        debug = True
-        loss_type = 0
-        dataset = 'slicing/datasets/parallel_data/salaries/rows1000.csv'
-        enumerator = "join"
-
-    conf = SparkConf().setAppName("salary_test").setMaster('local[4]')
-    num_partitions = 8
-    model_type = "regression"
-    label = 'salary'
-    sparkContext = SparkContext(conf=conf)
-    sqlContext = SQLContext(sparkContext)
-    fileRDD = sparkContext.textFile(dataset, num_partitions)
-    header = fileRDD.first()
-    head_split = header.split(",")
-    fileRDD = fileRDD.filter(lambda line: line != header)
-    data = fileRDD.map(lambda row: row.split(","))
-    dataset_df = sqlContext.createDataFrame(data, head_split)
-
-    cat_features = ["rank", "discipline", "sincephd_bin", "service_bin", "sex"]
-    # initializing stages of main transformation pipeline
-    stages = []
-    dataset_df = dataset_df.withColumn("id", sf.monotonically_increasing_id())
-    # bining numeric features by local binner udf function (specified for current dataset if needed)
-    dataset_df = dataset_df.withColumn('sincephd_bin', binner(dataset_df['sincephd']))
-    dataset_df = dataset_df.withColumn('service_bin', binner(dataset_df['service']))
-    dataset_df = dataset_df.withColumn('model_type', sf.lit(0))
-    dataset_df = dataset_df.drop('sincephd', 'service')
-    dataset_df = dataset_df.withColumn('target', dataset_df[label].cast("int"))
-    # hot encoding categorical features
-    for feature in cat_features:
-        string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index")
-        encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
-        encoder.setDropLast(False)
-        stages += [string_indexer, encoder]
-    assembler_inputs = [feature + "_vec" for feature in cat_features]
-    assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-    stages += [assembler]
-    assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-    stages += [assembler_final]
-
-    pipeline = Pipeline(stages=stages)
-    pipeline_model = pipeline.fit(dataset_df)
-    dataset_transformed = pipeline_model.transform(dataset_df)
-    df_transform_fin = dataset_transformed.select('id', 'features', 'target', 'model_type').toPandas()
-
-    cat = 0
-    counter = 0
-    decode_dict = {}
-    for feature in cat_features:
-        colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
-        colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
-        for item in colIdx:
-            decode_dict[counter] = (cat, item, colIdx[item])
-            counter = counter + 1
-        cat = cat + 1
-
-    train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
-    train_df = sqlContext.createDataFrame(train)
-    test_df = sqlContext.createDataFrame(test)
-    lr = LinearRegression(featuresCol='features', labelCol='target', maxIter=10, regParam=0.3, elasticNetParam=0.8)
-    lr_model = lr.fit(train_df)
-    eval = lr_model.evaluate(test_df)
-    f_l2 = eval.meanSquaredError
-    pred = eval.predictions
-    pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred['target'], pred['prediction'], pred['model_type']))
-    predictions = pred_df_fin.select('id', 'features', 'error').repartition(num_partitions)
-    converter = IndexToString(inputCol='features', outputCol='cats')
-    all_features = list(decode_dict.keys())
-    if enumerator == "join":
-        join_data_parallel.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha,
-                                            k=k, w=w, loss_type=loss_type)
-    elif enumerator == "union":
-        union_data_parallel.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha,
-                                             k=k, w=w, loss_type=loss_type)
diff --git a/scripts/staging/slicing/tests/regression/spark_salary.py b/scripts/staging/slicing/tests/regression/spark_salary.py
deleted file mode 100644
index be9868f..0000000
--- a/scripts/staging/slicing/tests/regression/spark_salary.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, spark_slicer, spark_union_slicer
-
-
-binner = udf(lambda arg: int(arg / 5))
-
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-        dataset = args[8]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 10
-        b_update = True
-        debug = True
-        loss_type = 0
-        enumerator = "union"
-        dataset = 'slicing/datasets/salaries.csv'
-        # dataset = 'slicing/datasets/parallel_attr/salaries/attr3000.csv'
-
-    conf = SparkConf().setAppName("salary_test").setMaster('local[4]')
-    num_partitions = 4
-    model_type = "regression"
-    label = 'salary'
-    sparkContext = SparkContext(conf=conf)
-    sqlContext = SQLContext(sparkContext)
-    dataset_df = sqlContext.read.csv(dataset, header='true',
-                                     inferSchema='true')
-    # initializing stages of main transformation pipeline
-    stages = []
-    # list of categorical features for further hot-encoding
-    cat_features = ["rank", "discipline", "sincephd_bin", "service_bin", "sex"]  # base
-    # cat_features = ["rank", "discipline", "sex"]  # 10
-    # cat_features = ["rank", "discipline", "sincephd", "sex"]  # 50
-    # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "start", "dept", "card"]  # 100
-    # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
-    #                  "uni", "dept", "race", "code", "job", "size", "previous", "stock"]  # 1013
-    # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
-    #                     "uni", "dept", "origin", "children", "bmi", "code", "job", "size", "previous", "stock", "market",
-    #                     "freq", "smoker", "region", "WorkClass", "Education", "EducationNum", "MaritalStatus",
-    #                     "Occupation", "Relationship", "color", "sepal_length", "sepal_width", "petal_length",
-    #                     "petal_width", "variety", "symboling", "normalized-losses", "make",	"fuel-type", "aspiration",
-    #                     "num-of-doors",	"body-style", "drive-wheels", "engine-location", "wheel-base", 	"length", "width",
-    #                     "height", "curb-weight", "engine-type", "num-of-cylinders", "engine-size", "fuel-system", "bore",
-    #                     "stroke", "compression-ratio", "horsepower", "peak-rpm", "city-mpg", "highway-mpg",	"price",
-    #                     "codeNumber", "clump", "cellsize", "cellshape", "adhesion", "epitel", "nuclei", "chromatin",
-    #                     "nucleoli", "mitoses", "class", "elevation", "aspect", "slope", "distToHydr"]
-
-#     cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
-#                     "uni", "dept", "origin", "children", "bmi", "code", "job", "size", "previous", "stock", "market",
-#                     "freq", "smoker", "region", "WorkClass", "Education", "EducationNum", "MaritalStatus",
-#                     "Occupation", "Relationship", "color", "sepal_length", "sepal_width", "petal_length",
-#                     "petal_width", "variety", "symboling", "normalized-losses", "make",	"fuel-type", "aspiration",
-#                     "num-of-doors",	"body-style", "drive-wheels", "engine-location", "wheel-base", 	"length", "width",
-#                     "height", "curb-weight", "engine-type", "num-of-cylinders", "engine-size", "fuel-system", "bore",
-#                     "stroke", "compression-ratio", "horsepower", "peak-rpm", "city-mpg", "highway-mpg",	"price",
-#                     "codeNumber", "clump", "cellsize", "cellshape", "adhesion", "epitel", "nuclei", "chromatin",
-#                     "nucleoli", "mitoses", "class", "elevation", "aspect", "slope", "distToHydr", "distToRoad",
-#                     "hillshadeMorning", "hillSahdeNoon", "hillSadeAfternoon", "distToFire", "DSName", "T", "N", "p",
-#                     "k", "Bin", "Cost",	"Sdratio", "correl", "cancor1",	"cancor2", "fract1", "fract2", "skewness",
-#                     "kurtosis",	"Hc", "Hx", "Mcx", "EnAttr", "NSRation", "Alg", "error"]   # 3000
-
-    # lines till 87 only for base case
-    # removing column with ID field
-    # dataset_df = dataset_df.drop('_c0')
-    # bining numeric features by local binner udf function (specified for current dataset if needed)
-    dataset_df = dataset_df.withColumn('sincephd_bin', binner(dataset_df['sincephd']))
-    dataset_df = dataset_df.withColumn('service_bin', binner(dataset_df['service']))
-    dataset_df = dataset_df.drop('sincephd', 'service')
-    dataset_df = dataset_df.withColumn('model_type', sf.lit(0))
-    # hot encoding categorical features
-    for feature in cat_features:
-        string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
-        encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
-        encoder.setDropLast(False)
-        stages += [string_indexer, encoder]
-    assembler_inputs = [feature + "_vec" for feature in cat_features]
-    assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-    stages += [assembler]
-    assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-    stages += [assembler_final]
-    pipeline = Pipeline(stages=stages)
-    pipeline_model = pipeline.fit(dataset_df)
-    dataset_transformed = pipeline_model.transform(dataset_df)
-    df_transform_fin = dataset_transformed.select('features', label, 'model_type').toPandas()
-    train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
-    train_df = sqlContext.createDataFrame(train)
-    test_df = sqlContext.createDataFrame(test)
-    decode_dict = {}
-    counter = 0
-    cat = 0
-    for feature in cat_features:
-        colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
-        colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
-        for item in colIdx:
-            decode_dict[counter] = (cat, item, colIdx[item])
-            counter = counter + 1
-        cat = cat + 1
-    lr = LinearRegression(featuresCol='features', labelCol=label, maxIter=10, regParam=0.3, elasticNetParam=0.8)
-    lr_model = lr.fit(train_df)
-    eval = lr_model.evaluate(test_df)
-    f_l2 = eval.meanSquaredError
-    pred = eval.predictions
-    pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred[label], pred['prediction'], pred['model_type']))
-    predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
-    converter = IndexToString(inputCol='features', outputCol='cats')
-    all_features = list(decode_dict.keys())
-    predictions = predictions.collect()
-    if enumerator == "join":
-        spark_slicer.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha, k=k, w=w,
-                                      loss_type=loss_type, enumerator=enumerator)
-    elif enumerator == "union":
-        spark_union_slicer.process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha, k=k, w=w,
-                                   loss_type=loss_type, enumerator=enumerator)
diff --git a/scripts/staging/slicing/tests/regression/test_insurance.py b/scripts/staging/slicing/tests/regression/test_insurance.py
deleted file mode 100644
index 64b58c4..0000000
--- a/scripts/staging/slicing/tests/regression/test_insurance.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.linear_model import LinearRegression
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-
-from slicing.base import slicer as slicer, union_slicer
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 4
-        b_update = True
-        debug = True
-        loss_type = 0
-        enumerator = "union"
-    file_name = '/slicing/datasets/insurance.csv'
-    dataset = pd.read_csv(file_name)
-    attributes_amount = len(dataset.values[0])
-    # for now working with regression datasets, assuming that target attribute is the last one
-    # currently non-categorical features are not supported and should be binned
-    y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-    # starting with one not including id field
-    x = dataset.iloc[:, 0:attributes_amount - 1].values
-    # list of numerical columns
-    non_categorical = [1, 3]
-    for row in x:
-        for attribute in non_categorical:
-            # <attribute - 2> as we already excluded from x id column
-            row[attribute - 1] = int(row[attribute - 1] / 5)
-    # hot encoding of categorical features
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x).toarray()
-    complete_x = []
-    complete_y = []
-    counter = 0
-    all_features = enc.get_feature_names()
-    # train model on a whole dataset
-    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
-    for item in x_test:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    x_size = counter
-    model = LinearRegression()
-    model.fit(x_train, y_train)
-    preds = (model.predict(x_test) - y_test) ** 2
-    f_l2 = sum(preds)/x_size
-    errors = []
-    counter = 0
-    for pred in preds:
-        errors.append((counter, pred))
-        counter = counter + 1
-    # alpha is size significance coefficient
-    # verbose option is for returning debug info while creating slices and printing it
-    # k is number of top-slices we want
-    # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
-    # enumerator <union>/<join> indicates an approach of next level slices combination process:
-    # in case of <join> in order to create new node of current level slicer
-    # combines only nodes of previous layer with each other
-    # <union> case implementation is based on DPSize algorithm
-    if enumerator == "join":
-        slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k,
-                       w=w, loss_type=loss_type, b_update=b_update)
-    elif enumerator == "union":
-        union_slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k,
-                             w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/test_salary.py b/scripts/staging/slicing/tests/regression/test_salary.py
deleted file mode 100644
index 92e0055..0000000
--- a/scripts/staging/slicing/tests/regression/test_salary.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn.linear_model import LinearRegression
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-import sys
-
-from slicing.base import slicer, union_slicer
-
-if __name__ == "__main__":
-    args = sys.argv
-    if len(args) > 1:
-        k = int(args[1])
-        w = float(args[2].replace(',', '.'))
-        alpha = int(args[3])
-        if args[4] == "True":
-            b_update = True
-        else:
-            b_update = False
-        debug = args[5]
-        loss_type = int(args[6])
-        enumerator = args[7]
-    else:
-        k = 10
-        w = 0.5
-        alpha = 4
-        b_update = True
-        debug = True
-        loss_type = 0
-        enumerator = "union"
-    file_name = '/slicing/datasets/salaries.csv'
-    dataset = pd.read_csv(file_name)
-    attributes_amount = len(dataset.values[0])
-    # for now working with regression datasets, assuming that target attribute is the last one
-    # currently non-categorical features are not supported and should be binned
-    y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-    # starting with one not including id field
-    x = dataset.iloc[:, 1:attributes_amount - 1].values
-    # list of numerical columns
-    non_categorical = [4, 5]
-    for row in x:
-        for attribute in non_categorical:
-            # <attribute - 2> as we already excluded from x id column
-            row[attribute - 2] = int(row[attribute - 2] / 5)
-    # hot encoding of categorical features
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x).toarray()
-    complete_x = []
-    complete_y = []
-    counter = 0
-    all_features = enc.get_feature_names()
-    # train model on a whole dataset
-    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
-    for item in x_test:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    x_size = counter
-    model = LinearRegression()
-    model.fit(x_train, y_train)
-    predictions = model.predict(x_test)
-    f_l2 = mean_squared_error(y_test, predictions)
-    preds = (model.predict(x_test) - y_test) ** 2
-    errors = []
-    counter = 0
-    for pred in preds:
-        errors.append((counter, pred))
-        counter = counter + 1
-    # alpha is size significance coefficient
-    # verbose option is for returning debug info while creating slices and printing it
-    # k is number of top-slices we want
-    # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
-    # enumerator <union>/<join> indicates an approach of next level slices combination process:
-    # in case of <join> in order to create new node of current level slicer
-    # combines only nodes of previous layer with each other
-    # <union> case implementation is based on DPSize algorithm
-    if enumerator == "join":
-        slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k, w=w,
-                       loss_type=loss_type, b_update=b_update)
-    elif enumerator == "union":
-        union_slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k, w=w,
-                             loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/toy_tests.py b/scripts/staging/slicing/tests/regression/toy_tests.py
deleted file mode 100644
index 876aa8b..0000000
--- a/scripts/staging/slicing/tests/regression/toy_tests.py
+++ /dev/null
@@ -1,428 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import unittest
-
-import pandas as pd
-from pyspark import SparkConf, SparkContext, SQLContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-import pyspark.sql.functions as sf
-from sklearn import linear_model
-from sklearn.metrics import mean_squared_error, r2_score
-from sklearn.preprocessing import OneHotEncoder
-
-import slicing.base.slicer as slicer
-from slicing.base import union_slicer
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils, spark_slicer, spark_union_slicer
-
-
-class SliceTests(unittest.TestCase):
-    loss_type = 0
-    # x, y = m.generate_dataset(10, 100)
-    train_dataset = pd.read_csv("toy_train.csv")
-    attributes_amount = len(train_dataset.values[0])
-    model = linear_model.LinearRegression()
-    y_train = train_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-    x_train = train_dataset.iloc[:, 0:attributes_amount - 1].values
-    model.fit(x_train, y_train)
-    test_dataset = pd.read_csv("toy.csv")
-    y_test = test_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-    x_test = test_dataset.iloc[:, 0:attributes_amount - 1].values
-    y_pred = model.predict(x_test)
-    print("Mean squared error: %.2f"
-          % mean_squared_error(y_test, y_pred))
-    print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
-    # Now that we have trained the model, we can print the coefficient of x that it has predicted
-    print('Coefficients: \n', model.coef_)
-    enc = OneHotEncoder(handle_unknown='ignore')
-    x = enc.fit_transform(x_test).toarray()
-    complete_x = []
-    complete_y = []
-    counter = 0
-    for item in x:
-        complete_x.append((counter, item))
-        complete_y.append((counter, y_test[counter]))
-        counter = counter + 1
-    all_features = enc.get_feature_names()
-    loss = mean_squared_error(y_test, y_pred)
-    devs = (y_pred - y_test) ** 2
-    errors = []
-    counter = 0
-    for pred in devs:
-        errors.append((counter, pred))
-        counter = counter + 1
-    k = 5
-    w = 0.5
-    alpha = 4
-    top_k = Topk(k)
-    debug = True
-    b_update = True
-    first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
-                                          loss_type, top_k, alpha, w)
-    first_level_nodes = first_level[0]
-    slice_member = first_level_nodes[(7, 'x2_2')]
-
-    def test_attr_spark(self):
-        conf = SparkConf().setAppName("toy_test").setMaster('local[2]')
-        num_partitions = 2
-        enumerator = "join"
-        model_type = "regression"
-        label = 'target'
-        sparkContext = SparkContext(conf=conf)
-        sqlContext = SQLContext(sparkContext)
-        train_df = sqlContext.read.csv("toy_train.csv", header='true',
-                            inferSchema='true')
-        test_df = sqlContext.read.csv("toy.csv", header='true',
-                            inferSchema='true')
-        # initializing stages of main transformation pipeline
-        stages = []
-        # list of categorical features for further hot-encoding
-        cat_features = ['a', 'b', 'c']
-        for feature in cat_features:
-            string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
-            encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
-            encoder.setDropLast(False)
-            stages += [string_indexer, encoder]
-        assembler_inputs = [feature + "_vec" for feature in cat_features]
-        assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-        stages += [assembler]
-        assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-        stages += [assembler_final]
-        pipeline = Pipeline(stages=stages)
-        train_pipeline_model = pipeline.fit(train_df)
-        test_pipeline_model = pipeline.fit(test_df)
-        train_df_transformed = train_pipeline_model.transform(train_df)
-        test_df_transformed = test_pipeline_model.transform(test_df)
-        train_df_transformed = train_df_transformed.withColumn('model_type', sf.lit(0))
-        test_df_transformed = test_df_transformed.withColumn('model_type', sf.lit(0))
-        decode_dict = {}
-        counter = 0
-        cat = 0
-        for feature in cat_features:
-            colIdx = test_df_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
-            colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
-            for item in colIdx:
-                decode_dict[counter] = (cat, item, colIdx[item], counter)
-                counter = counter + 1
-            cat = cat + 1
-        train_df_transform_fin = train_df_transformed.select('features', label, 'model_type')
-        test_df_transform_fin = test_df_transformed.select('features', label, 'model_type')
-        lr = LinearRegression(featuresCol='features', labelCol=label, maxIter=10, regParam=0.0, elasticNetParam=0.8)
-        lr_model = lr.fit(train_df_transform_fin)
-        eval = lr_model.evaluate(test_df_transform_fin)
-        f_l2 = eval.meanSquaredError
-        pred = eval.predictions
-        pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred[label], pred['prediction'], pred['model_type']))
-        predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
-        converter = IndexToString(inputCol='features', outputCol='cats')
-        all_features = list(decode_dict)
-        predictions = predictions.collect()
-        spark_join = spark_slicer.parallel_process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha,
-                                      k=self.k, w=self.w, loss_type=self.loss_type, enumerator="join")
-        spark_union = spark_union_slicer.process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha,
-                                      k=self.k, w=self.w, loss_type=self.loss_type, enumerator="union")
-        self.assertEqual(3, len(spark_join.slices))
-        print("check1")
-        self.assertEqual(spark_join.min_score, spark_union.min_score)
-        print("check2")
-        self.assertEqual(spark_join.keys, spark_union.keys)
-        print("check3")
-        self.assertEqual(len(spark_join.slices), len(spark_union.slices))
-        print("check4")
-        idx = -1
-        for sliced in spark_join.slices:
-            idx += 1
-            self.assertEqual(sliced.score, spark_union.slices[idx].score)
-        print("check5")
-
-    def test_features_number(self):
-        self.assertEqual(len(self.all_features), 9)
-        print("check 1")
-
-    def test_base_first_level(self):
-        self.assertEqual(9, len(self.first_level_nodes))
-        print("check 2")
-
-    def test_parents_first(self):
-        self.assertIn(('x2_2', 7), self.slice_member.parents)
-        print("check 3")
-
-    def test_name(self):
-        self.assertEqual('x2_2', self.slice_member.make_name())
-        print("check 4")
-
-    def test_size(self):
-        self.assertEqual(36, self.slice_member.size)
-        print("check 5")
-
-    def test_e_upper(self):
-        self.assertEqual(81, self.slice_member.e_upper)
-        print("check 6")
-
-    def test_loss(self):
-        self.assertEqual(22, int(self.slice_member.loss))
-        print("check 7")
-
-    def test_opt_fun(self):
-        self.slice_member.score = slicer.opt_fun(self.slice_member.loss, self.slice_member.size, self.loss, len(self.x_test), self.w)
-        print("check 8")
-
-    def test_score(self):
-        self.assertEqual(1.2673015873015872, self.slice_member.score)
-        print("check 9")
-
-    def test_base_join_enum(self):
-        cur_lvl_nodes = {}
-        all_nodes = {}
-        b_update = True
-        cur_lvl = 1
-        slice_index = (2, 'x0_3')
-        combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
-                               len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
-                               self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
-        self.assertEqual(6, len(combined[0]))
-        print("check1")
-
-    def test_parents_second(self):
-        cur_lvl_nodes = {}
-        all_nodes = {}
-        b_update = True
-        cur_lvl = 1
-        slice_index = (2, 'x0_3')
-        combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
-                                    len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
-                                    self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
-        parent1 = combined[0][('x0_3 && x1_3')]
-        parent2 = combined[0][('x0_3 && x2_2')]
-        new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
-        new_node.parents = [parent1, parent2]
-        parent1_attr = parent1.attributes
-        parent2_attr = parent2.attributes
-        new_node_attr = slicer.union(parent1_attr, parent2_attr)
-        self.assertEqual(new_node_attr, [('x0_3', 2), ('x1_3', 5), ('x2_2', 7)])
-        print("check2")
-
-    def test_nonsense(self):
-        cur_lvl_nodes = {}
-        all_nodes = {}
-        b_update = True
-        cur_lvl = 1
-        slice_index = (2, 'x0_3')
-        combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
-                                    len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
-                                    self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
-        parent1 = combined[0][('x0_3 && x1_3')]
-        parent2 = combined[0][('x0_3 && x2_2')]
-        new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
-        new_node.parents = [parent1, parent2]
-        parent1_attr = parent1.attributes
-        parent2_attr = parent2.attributes
-        new_node_attr = slicer.union(parent1_attr, parent2_attr)
-        new_node.attributes = new_node_attr
-        new_node.name = new_node.make_name()
-        flagTrue = slicer.slice_name_nonsense(parent1, parent2, 2)
-        self.assertEqual(True, flagTrue)
-        print("check3")
-
-    def test_non_nonsense(self):
-        cur_lvl_nodes = {}
-        all_nodes = {}
-        b_update = True
-        cur_lvl = 1
-        slice_index = (2, 'x0_3')
-        parent3 = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
-        parent3.parents = [self.first_level_nodes[(4, 'x1_2')], self.first_level_nodes[(7, 'x2_2')]]
-        parent3.attributes = [('x1_2', 4), ('x2_2', 7)]
-        combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
-                                    len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
-                                    self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
-        parent2 = combined[0]['x0_3 && x2_3']
-        parent3.key = (8, 'x1_2 && x2_2')
-        flag_nonsense = slicer.slice_name_nonsense(parent2, parent3, 2)
-        self.assertEqual(True, flag_nonsense)
-        print("check4")
-
-    def test_uppers(self):
-        cur_lvl_nodes = {}
-        all_nodes = {}
-        b_update = True
-        cur_lvl = 1
-        slice_index = (2, 'x0_3')
-        parent3 = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
-        parent3.parents = [self.first_level_nodes[(4, 'x1_2')], self.first_level_nodes[(7, 'x2_2')]]
-        parent3.attributes = [('x1_2', 4), ('x2_2', 7)]
-        combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
-                                    len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
-                                    self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
-        parent1 = combined[0]['x0_3 && x1_3']
-        parent2 = combined[0]['x0_3 && x2_3']
-        new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
-        new_node.parents = [parent1, parent2]
-        new_node.calc_bounds(2, self.w)
-        self.assertEqual(25, new_node.s_upper)
-        print("check5")
-        self.assertEqual(398, int(new_node.c_upper))
-        print("check6")
-
-    def test_topk_slicing(self):
-        join_top_k = slicer.process(self.all_features, self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors,
-                       self.debug, self.alpha, self.k, self.w, self.loss_type, self.b_update)
-        union_top_k = union_slicer.process(self.all_features, self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors,
-                       self.debug, self.alpha, self.k, self.w, self.loss_type, self.b_update)
-        self.assertEqual(join_top_k.min_score, union_top_k.min_score)
-        print("check1")
-        self.assertEqual(join_top_k.keys, union_top_k.keys)
-        print("check2")
-        self.assertEqual(len(join_top_k.slices), len(union_top_k.slices))
-        print("check3")
-        idx = -1
-        for sliced in join_top_k.slices:
-            idx += 1
-            self.assertEqual(sliced.score, union_top_k.slices[idx].score)
-        print("check4")
-
-    def test_extreme_target(self):
-        test_dataset = pd.read_csv("/home/lana/diploma/project/slicing/datasets/toy_extreme_change.csv")
-        y_test = test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
-        x_test = test_dataset.iloc[:, 0:self.attributes_amount - 1].values
-        y_pred = self.model.predict(x_test)
-        print("Mean squared error: %.2f"
-              % mean_squared_error(y_test, y_pred))
-        print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
-        # Now that we have trained the model, we can print the coefficient of x that it has predicted
-        print('Coefficients: \n', self.model.coef_)
-        enc = OneHotEncoder(handle_unknown='ignore')
-        x = enc.fit_transform(x_test).toarray()
-        complete_x = []
-        complete_y = []
-        counter = 0
-        for item in x:
-            complete_x.append((counter, item))
-            complete_y.append((counter, y_test[counter]))
-            counter = counter + 1
-        all_features = enc.get_feature_names()
-        loss = mean_squared_error(y_test, y_pred)
-        devs = (y_pred - y_test) ** 2
-        errors = []
-        counter = 0
-        for pred in devs:
-            errors.append((counter, pred))
-            counter = counter + 1
-        k = 5
-        w = 0.5
-        alpha = 4
-        top_k = Topk(k)
-        debug = True
-        b_update = True
-        first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
-                                              self.loss_type, top_k, alpha, w)
-        first_level_nodes = first_level[0]
-        slice_member = first_level_nodes[(7, 'x2_2')]
-        self.assertGreater(slice_member.loss, self.slice_member.loss)
-        print("check 1")
-        self.assertGreater(slice_member.score, self.slice_member.score)
-        print("check 2")
-
-    def test_error_significance(self):
-        y_test = self.test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
-        x_test = self.test_dataset.iloc[:, 0:self.attributes_amount - 1].values
-        y_pred = self.model.predict(x_test)
-        print("Mean squared error: %.2f"
-              % mean_squared_error(y_test, y_pred))
-        print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
-        # Now that we have trained the model, we can print the coefficient of x that it has predicted
-        print('Coefficients: \n', self.model.coef_)
-        enc = OneHotEncoder(handle_unknown='ignore')
-        x = enc.fit_transform(x_test).toarray()
-        complete_x = []
-        complete_y = []
-        counter = 0
-        for item in x:
-            complete_x.append((counter, item))
-            complete_y.append((counter, y_test[counter]))
-            counter = counter + 1
-        all_features = enc.get_feature_names()
-        loss = mean_squared_error(y_test, y_pred)
-        devs = (y_pred - y_test) ** 2
-        errors = []
-        counter = 0
-        for pred in devs:
-            errors.append((counter, pred))
-            counter = counter + 1
-        k = 5
-        # Maximized size significance
-        w = 0
-        alpha = 4
-        top_k = Topk(k)
-        debug = True
-        b_update = True
-        first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
-                                              self.loss_type, top_k, alpha, w)
-        first_level_nodes = first_level[0]
-        slice_member = first_level_nodes[(7, 'x2_2')]
-        self.assertGreater(self.slice_member.score, slice_member.score)
-
-    def test_size_significance(self):
-        y_test = self.test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
-        x_test = self.test_dataset.iloc[:, 0:self.attributes_amount - 1].values
-        y_pred = self.model.predict(x_test)
-        print("Mean squared error: %.2f"
-                  % mean_squared_error(y_test, y_pred))
-        print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
-        # Now that we have trained the model, we can print the coefficient of x that it has predicted
-        print('Coefficients: \n', self.model.coef_)
-        enc = OneHotEncoder(handle_unknown='ignore')
-        x = enc.fit_transform(x_test).toarray()
-        complete_x = []
-        complete_y = []
-        counter = 0
-        for item in x:
-            complete_x.append((counter, item))
-            complete_y.append((counter, y_test[counter]))
-            counter = counter + 1
-        all_features = enc.get_feature_names()
-        loss = mean_squared_error(y_test, y_pred)
-        devs = (y_pred - y_test) ** 2
-        errors = []
-        counter = 0
-        for pred in devs:
-            errors.append((counter, pred))
-            counter = counter + 1
-        k = 5
-        # Maximized size significance
-        w = 1
-        alpha = 4
-        top_k = Topk(k)
-        debug = True
-        b_update = True
-        first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
-                                                  self.loss_type, top_k, alpha, w)
-        first_level_nodes = first_level[0]
-        slice_member = first_level_nodes[(7, 'x2_2')]
-        self.assertGreater(slice_member.score, self.slice_member.score)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/scripts/staging/slicing/utils/model_generator.py b/scripts/staging/slicing/utils/model_generator.py
deleted file mode 100644
index 8f80b74..0000000
--- a/scripts/staging/slicing/utils/model_generator.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn import linear_model
-from sklearn.datasets import make_regression
-import numpy as np
-from sklearn.metrics import mean_squared_error, r2_score
-
-
-def generate_dataset(beta, n):
-    # Generate x as an array of `n` samples which can take a value between 0 and 5
-    x = np.random.random_integers(1, 3, (n, 3))
-    # Calculate `y` according to the equation discussed
-    coeff = [2, 3, 1]
-    coeff_mat = np.array(coeff).reshape(len(coeff), 1)
-    y = np.matmul(x, coeff_mat)
-    dataset = np.append(x, y, axis=1)
-    np.savetxt("toy_train.csv", dataset, delimiter=",", fmt='%s')
-    return x, y
-
-
-if __name__ == '__main__':
-    def main():
-        x, y = generate_dataset(10, 100)
-        x_train = x
-        y_train = y
-        dataset = np.append(x, y, axis=1)
-        np.savetxt("toy_train.csv", dataset, delimiter=",", fmt='%s')
-        model = linear_model.LinearRegression()
-
-        # Train the model using the training data that we created
-        model.fit(x_train, y_train)
-        # Now that we have trained the model, we can print the coefficient of x that it has predicted
-        print('Coefficients: \n', model.coef_)
-
-        # We then use the model to make predictions based on the test values of x
-        test_dataset = pd.read_csv("../datasets/toy.csv")
-        attributes_amount = len(test_dataset.values[0])
-        y_test = test_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
-        x_test = test_dataset.iloc[:, 0:attributes_amount - 1].values
-        y_pred = model.predict(x_test)
-
-        # Now, we can calculate the models accuracy metrics based on what the actual value of y was
-        print("Mean squared error: %.2f"
-              % mean_squared_error(y_test, y_pred))
-        print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
-        # Coefficients: [[2. 3. 1.]]
-        # Mean squared error: 3.00
-        # r_2 statistic: 0.72
-
-        # dataset = np.append(x, y, axis=1)
-        # np.savetxt("toy.csv", dataset, delimiter=",", fmt='%s')
-
-        return model, x_test, y_test, y_pred
-
-
-
-
diff --git a/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml b/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
index c0624a1..9e3aa93 100644
--- a/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
+++ b/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
@@ -19,221 +19,18 @@
 #
 #-------------------------------------------------------------
 
-# Trains a 2-layer autoencoder with minibatch SGD, momentum and step-size decay.
-# If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
-# Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
-# The script standardizes the input before training (can be turned off).
-# Also, it randomly reshuffles rows before training.
-# Currently, tanh is set to be the activation function.
-# By re-implementing 'func' DML-bodied function, one can change the activation.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME      TYPE     DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# X         String   ---      Filename where the input is stored
-# H1        Int      ---      Number of neurons in the 1st hidden layer
-# H2        Int      ---      Number of neurons in the 2nd hidden layer
-# EPOCH     Int      ---      Number of epochs to train for
-# fmt       String   'text'   Output format ("text", "csv", "binary" etc.)
-# OBJ       Boolean  FALSE    If TRUE, Computes objective function value (squared-loss)
-#                             at the end of each epoch. Note that, computing the full
-#                             objective can take a lot of time.
-# BATCH     Int      256      Mini-batch size (training parameter)
-# STEP      Double   1e-5     Initial step size (training parameter)
-# DECAY     Double   0.95     Decays step size after each epoch (training parameter)
-# MOMENTUM  Double   0.9      Momentum parameter (training parameter)
-# ---------------------------------------------------------------------------------------------
-#
-# OUTPUT PARAMETERS (all filenames):
-# ---------------------------------------------------------------------------------------------
-# NAME     TYPE   DEFAULT  MEANING
-# ---------------------------------------------------------------------------------------------
-# W1_out   String   ---   File to store weights between input layer and 1st hidden layer
-# b1_out   String   ---   File to store bias between input layer and 1st hidden layer
-# W2_out   String   ---   File to store weights between 1st hidden layer and 2nd hidden layer
-# b2_out   String   ---   File to store bias between 1st hidden layer and 2nd hidden layer
-# W3_out   String   ---   File to store weights between 2nd hidden layer and 3rd hidden layer
-# b3_out   String   ---   File to store bias between 2nd hidden layer and 3rd hidden layer
-# W4_out   String   ---   File to store weights between 3rd hidden layer and output layer
-# b4_out   String   ---   File to store bias between 3rd hidden layer and output layer
-# HIDDEN   String   " "   File to store the hidden (2nd) layer representation if needed
-# ---------------------------------------------------------------------------------------------
-#
-# INVOCATION:
-# -f autoencoder_2layer.dml --nvargs X=<input file> H1=500 H2=2 EPOCH=1 fmt="csv"
-#      W1_out=<weights from input to 1st hidden layer> b1_out=<bias from input to 1st hidden layer>
-#      W2_out=<weights from 1st hidden layer to 2nd hidden layer> b2_out=<bias from 1st hidden layer to 2nd hidden layer>
-#      W3_out=<weights from 2nd hidden layer to 3rd hidden layer> b3_out=<bias from 2nd hidden layer to 3rd hidden layer>
-#      W4_out=<weights from 3rd hidden layer to output> b4_out=<bias from 3rd hidden layer to output>
-#
-
-# NOTE: This is a copy of the `scripts/staging/autoencoder-2layer.dml` with rand()
-# replaced with random matrices generated in the corresponding test class.
-
-#implements tanh
-#to use another activation fn, implement a DML-bodied function with
-#function name 'func' and comment out this one
-func = function(Matrix[Double] X) return(Matrix[Double] Y, Matrix[Double] Y_prime){
-  Y = tanh(X)
-  Y_prime = 1 - Y^2
-}
-
-feedForward = function(Matrix[Double] X, Matrix[Double] W1, Matrix[Double] b1,
-    Matrix[Double] W2, Matrix[Double] b2, Matrix[Double] W3, Matrix[Double] b3,
-    Matrix[Double] W4, Matrix[Double] b4, Matrix[Double] Y)
-  return(Matrix[Double] H1, Matrix[Double] H1_prime, Matrix[Double] H2, 
-    Matrix[Double] H2_prime, Matrix[Double] H3, Matrix[Double] H3_prime,
-    Matrix[Double] Yhat, Matrix[Double] Yhat_prime, Matrix[Double] E)
-{
-  H1_in = t(W1 %*% t(X) + b1)
-  [H1, H1_prime] = func(H1_in)
-
-  H2_in = t(W2 %*% t(H1) + b2)
-  [H2, H2_prime] = func(H2_in)
-
-  H3_in = t(W3 %*% t(H2) + b3)
-  [H3, H3_prime] = func(H3_in)
-
-  Yhat_in = t(W4 %*% t(H3) + b4)
-  [Yhat, Yhat_prime] = func(Yhat_in)
-  E = Yhat - Y
-}
-
-grad = function(Matrix[Double] X, Matrix[Double] H1, Matrix[Double] H1_prime,
-    Matrix[Double] H2, Matrix[Double] H2_prime, Matrix[Double] H3,
-    Matrix[Double] H3_prime, Matrix[Double] Yhat_prime, Matrix[Double] E,
-    Matrix[Double] W1, Matrix[Double] W2, Matrix[Double] W3, Matrix[Double] W4)
-  return(Matrix[Double] W1_grad, Matrix[Double] b1_grad, Matrix[Double] W2_grad,
-    Matrix[Double] b2_grad, Matrix[Double] W3_grad, Matrix[Double] b3_grad,
-    Matrix[Double] W4_grad, Matrix[Double] b4_grad)
-{
-  #backprop
-  delta4 = E * Yhat_prime
-  delta3 = H3_prime * (delta4 %*% W4)
-  delta2 = H2_prime * (delta3 %*% W3)
-  delta1 = H1_prime * (delta2 %*% W2)
-
-  #compute gradients
-  b4_grad = t(colSums(delta4))
-  b3_grad = t(colSums(delta3))
-  b2_grad = t(colSums(delta2))
-  b1_grad = t(colSums(delta1))
-
-  W4_grad = t(delta4) %*% H3
-  W3_grad = t(delta3) %*% H2
-  W2_grad = t(delta2) %*% H1
-  W1_grad = t(delta1) %*% X
-}
-
-obj = function(Matrix[Double] E) return(Double val){
-  val = 0.5 * sum(E^2)
-}
-
 batch_size = ifdef($BATCH, 256)
 mu = ifdef($MOMENTUM, 0.9)
 step = ifdef($STEP, 1e-5)
 decay = ifdef($DECAY, 0.95)
-hfile = ifdef($HIDDEN, " ")
-fmt = ifdef($fmt, "text")
 full_obj = ifdef($OBJ, FALSE)
+fmt = ifdef($fmt, "text")
 
-X = read($X)
-num_hidden1 = $H1
-num_hidden2 = $H2
-max_epochs = $EPOCH
-W1_rand = read($W1_rand)
-W2_rand = read($W2_rand)
-W3_rand = read($W3_rand)
-W4_rand = read($W4_rand)
-order_rand = read($order_rand)
-
-n = nrow(X)
-m = ncol(X)
-
-#randomly reordering rows
-#permut = table(seq(1,n,1), order(target=Rand(rows=n, cols=1, min=0, max=1, pdf="uniform"), by=1, index.return=TRUE), n, n)
-permut = table(seq(1,n,1), order(target=order_rand, by=1, index.return=TRUE), n, n)
-X = permut %*% X
-
-#z-transform, whitening operator is better
-means = colSums(X)/n
-stds = sqrt((colSums(X^2)/n - means*means)*n/(n-1)) + 1e-17
-X = (X - means)/stds
-
-W1 = sqrt(6)/sqrt(m + num_hidden1) * W1_rand #Rand(rows=num_hidden1, cols=m, min=-1, max=1, pdf="uniform")
-b1 = matrix(0, rows=num_hidden1, cols=1)
-W2 = sqrt(6)/sqrt(num_hidden1 + num_hidden2) * W2_rand #Rand(rows=num_hidden2, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b2 = matrix(0, rows=num_hidden2, cols=1)
-W3 = sqrt(6)/sqrt(num_hidden2 + num_hidden1) * W3_rand #Rand(rows=num_hidden1, cols=num_hidden2, min=-1, max=1, pdf="uniform")
-b3 = matrix(0, rows=num_hidden1, cols=1)
-W4 = sqrt(6)/sqrt(num_hidden2 + m) * W4_rand #Rand(rows=m, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b4 = matrix(0, rows=m, cols=1)
-
-upd_W1 = matrix(0, rows=nrow(W1), cols=ncol(W1))
-upd_b1 = matrix(0, rows=nrow(b1), cols=ncol(b1))
-upd_W2 = matrix(0, rows=nrow(W2), cols=ncol(W2))
-upd_b2 = matrix(0, rows=nrow(b2), cols=ncol(b2))
-upd_W3 = matrix(0, rows=nrow(W3), cols=ncol(W3))
-upd_b3 = matrix(0, rows=nrow(b3), cols=ncol(b3))
-upd_W4 = matrix(0, rows=nrow(W4), cols=ncol(W4))
-upd_b4 = matrix(0, rows=nrow(b4), cols=ncol(b4))
-
-if( full_obj ){
-  [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, 
-    full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-  full_o = obj(full_E)
-  print("EPOCHS=" + 0 + " OBJ (FULL DATA): " + full_o)
-}
-
-iter = 0
-num_iters_per_epoch = ceil(n / batch_size)
-max_iterations = max_epochs * num_iters_per_epoch
-#print("num_iters_per_epoch=" + num_iters_per_epoch + " max_iterations=" + max_iterations)
-beg = 1
-while( iter < max_iterations ) {
-  end = beg + batch_size - 1
-  if(end > n) end = n
-  X_batch = X[beg:end,]
-
-  [H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat, Yhat_prime, E] = feedForward(X_batch, W1, b1, W2, b2, W3, b3, W4, b4, X_batch)
-  [W1_grad, b1_grad, W2_grad, b2_grad, W3_grad, b3_grad, W4_grad, b4_grad] = grad(X_batch, H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat_prime, E, W1, W2, W3, W4)
-
-  o = obj(E)
-  print("epochs=%5.4f BATCH beg=%d end=%d obj=%f", (iter / num_iters_per_epoch), beg, end, o)
-
-  #update
-  local_step = step / nrow(X_batch)
-  upd_W1 = mu * upd_W1 - local_step * W1_grad
-  upd_b1 = mu * upd_b1 - local_step * b1
-  upd_W2 = mu * upd_W2 - local_step * W2_grad
-  upd_b2 = mu * upd_b2 - local_step * b2
-  upd_W3 = mu * upd_W3 - local_step * W3_grad
-  upd_b3 = mu * upd_b3 - local_step * b3
-  upd_W4 = mu * upd_W4 - local_step * W4_grad
-  upd_b4 = mu * upd_b4 - local_step * b4
-  W1 = W1 + upd_W1
-  b1 = b1 + upd_b1
-  W2 = W2 + upd_W2
-  b2 = b2 + upd_b2
-  W3 = W3 + upd_W3
-  b3 = b3 + upd_b3
-  W4 = W4 + upd_W4
-  b4 = b4 + upd_b4
-
-  iter = iter + 1
-  if(end == n) beg = 1
-  else beg = end + 1
-
-  if( iter %% num_iters_per_epoch == 0 ) step = step * decay
-
-  if( full_obj & iter %% num_iters_per_epoch == 0 ){
-    [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-    full_o = obj(full_E)
-    epochs = iter %/% num_iters_per_epoch
-    print("EPOCHS=" + epochs + " iter=" + iter + " OBJ (FULL DATA)=" + full_o)
-  }
-}
+[W1, b1, W2, b2, W3, b3, W4, b4, hidden] = autoencoder_2layer(
+  X = read($X), W1_rand = read($W1_rand), W2_rand = read($W2_rand), 
+  W3_rand = read($W3_rand), W4_rand = read($W4_rand), order_rand = read($order_rand),
+  num_hidden1 = $H1, num_hidden2 = $H2, max_epochs = $EPOCH, full_obj = full_obj, 
+  batch_size = batch_size, step = step, decay = decay, mu = mu)
 
 write(W1, $W1_out, format=fmt)
 write(b1, $b1_out, format=fmt)
@@ -243,9 +40,3 @@ write(W3, $W3_out, format=fmt)
 write(b3, $b3_out, format=fmt)
 write(W4, $W4_out, format=fmt)
 write(b4, $b4_out, format=fmt)
-
-if( hfile != " " ){
-  [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
-  reordered_H = t(permut) %*% full_H2
-  write(reordered_H, hfile, format=fmt)
-}