You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemds.apache.org by mb...@apache.org on 2021/02/21 00:12:28 UTC
[systemds] branch master updated: [MINOR] Cleanup staging scripts
(remove converted builtin functions)
This is an automated email from the ASF dual-hosted git repository.
mboehm7 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/master by this push:
new 58d46ea [MINOR] Cleanup staging scripts (remove converted builtin functions)
58d46ea is described below
commit 58d46eaa1ae3384963425d6c166b9d2c39683429
Author: Matthias Boehm <mb...@gmail.com>
AuthorDate: Sun Feb 21 01:10:59 2021 +0100
[MINOR] Cleanup staging scripts (remove converted builtin functions)
Additional cleanup PRs:
Closes #1172.
---
scripts/staging/PCA.dml | 117 ----
scripts/staging/PPCA.dml | 160 -----
scripts/staging/autoencoder-2layer.dml | 248 -------
scripts/staging/knn.dml | 779 ---------------------
scripts/staging/regression/lasso/lasso.dml | 113 ---
scripts/staging/slicing/__init__.py | 20 -
scripts/staging/slicing/base/Bucket.py | 170 -----
scripts/staging/slicing/base/SparkNode.py | 167 -----
scripts/staging/slicing/base/__init__.py | 20 -
scripts/staging/slicing/base/node.py | 210 ------
scripts/staging/slicing/base/slicer.py | 185 -----
scripts/staging/slicing/base/top_k.py | 58 --
scripts/staging/slicing/base/union_slicer.py | 162 -----
scripts/staging/slicing/slicing.dml | 311 --------
.../slicing/spark_modules/join_data_parallel.py | 123 ----
.../staging/slicing/spark_modules/spark_slicer.py | 103 ---
.../slicing/spark_modules/spark_union_slicer.py | 73 --
.../slicing/spark_modules/union_data_parallel.py | 124 ----
scripts/staging/slicing/tests/__init__.py | 20 -
.../slicing/tests/classification/__init__.py | 20 -
.../slicing/tests/classification/compas_bd.py | 136 ----
.../slicing/tests/classification/sparked_adults.py | 118 ----
.../slicing/tests/classification/test_adult.py | 116 ---
.../slicing/tests/classification/test_compas.py | 105 ---
.../slicing/tests/classification/test_iris.py | 109 ---
.../staging/slicing/tests/regression/__init__.py | 20 -
.../slicing/tests/regression/bd_spark_salary.py | 130 ----
.../slicing/tests/regression/spark_salary.py | 157 -----
.../slicing/tests/regression/test_insurance.py | 103 ---
.../slicing/tests/regression/test_salary.py | 104 ---
.../staging/slicing/tests/regression/toy_tests.py | 428 -----------
scripts/staging/slicing/utils/model_generator.py | 77 --
.../functions/codegenalg/Algorithm_AutoEncoder.dml | 221 +-----
33 files changed, 6 insertions(+), 5001 deletions(-)
diff --git a/scripts/staging/PCA.dml b/scripts/staging/PCA.dml
deleted file mode 100644
index be7affc..0000000
--- a/scripts/staging/PCA.dml
+++ /dev/null
@@ -1,117 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#
-# This script performs Principal Component Analysis (PCA) on the given input data.
-#
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# INPUT String --- Location to read the matrix A of feature vectors
-# K Int --- Indicates dimension of the new vector space constructed from eigen vectors
-# CENTER Int 0 Indicates whether or not to center data
-# SCALE Int 0 Indicates whether or not to scale data
-# OFMT String --- Output data format
-# PROJDATA Int 0 This argument indicates if the data should be projected or not
-# MODEL String --- Location to already existing model: eigenvectors and eigenvalues
-# OUTPUT String / Location to write output matrices (covariance matrix, new basis vectors,
-# and data projected onto new basis vectors)
-# hadoop jar SystemDS.jar -f PCA.dml -nvargs INPUT=INPUT_DIR/pca-1000x1000
-# OUTPUT=OUTPUT_DIR/pca-1000x1000-model PROJDATA=1 CENTER=1 SCALE=1
-# ---------------------------------------------------------------------------------------------
-
-A = read($INPUT);
-K = ifdef($K, ncol(A));
-ofmt = ifdef($OFMT, "CSV");
-projectData = ifdef($PROJDATA,0);
-model = ifdef($MODEL,"");
-center = ifdef($CENTER,0);
-scale = ifdef($SCALE,0);
-output = ifdef($OUTPUT,"/");
-
-# reuse existing model to project data
-if (model != "") {
- evec_dominant = read(model+"/dominant.eigen.vectors");
- }else{
- model = output;
-}
-
-PCA = function(matrix[double] A, integer K, string ofmt, integer projectData, string model, integer center, integer scale, string output)
- return(matrix[double] eval_dominant, matrix[double] evec_dominant) {
-
- evec_dominant = matrix(0,cols=1,rows=1);
-
- N = nrow(A);
- D = ncol(A);
-
- # perform z-scoring (centering and scaling)
- if (center == 1) {
- cm = colMeans(A);
- A = A - cm;
- }
- if (scale == 1) {
- cvars = (colSums (A^2));
- if (center == 1){
- cm = colMeans(A);
- cvars = (cvars - N*(cm^2))/(N-1);
- }
- Azscored = (A)/sqrt(cvars);
- A = Azscored;
- }
-
- # co-variance matrix
- mu = colSums(A)/N;
- C = (t(A) %*% A)/(N-1) - (N/(N-1))*t(mu) %*% mu;
-
-
- # compute eigen vectors and values
- [evalues, evectors] = eigen(C);
-
- decreasing_Idx = order(target=evalues,by=1,decreasing=TRUE,index.return=TRUE);
- diagmat = table(seq(1,D),decreasing_Idx);
- # sorts eigenvalues by decreasing order
- evalues = diagmat %*% evalues;
- # sorts eigenvectors column-wise in the order of decreasing eigenvalues
- evectors = evectors %*% diagmat;
-
-
- # select K dominant eigen vectors
- nvec = ncol(evectors);
-
- eval_dominant = evalues[1:K, 1];
- evec_dominant = evectors[,1:K];
-
-}
-
-[eval_dominant, evec_dominant] = PCA(A, K, ofmt, projectData, model, center, scale, output)
-
-# the square root of eigenvalues
-eval_stdev_dominant = sqrt(eval_dominant);
-write(eval_stdev_dominant, model+"/dominant.eigen.standard.deviations", format=ofmt);
-write(eval_dominant, model+"/dominant.eigen.values", format=ofmt);
-write(evec_dominant, model+"/dominant.eigen.vectors", format=ofmt);
-
-# Construct new data set by treating computed dominant eigenvectors as the basis vectors
-if (projectData == 1 | model != ""){
- newA = A %*% evec_dominant;
- write(newA, output+"/projected.data", format=ofmt);
-}
diff --git a/scripts/staging/PPCA.dml b/scripts/staging/PPCA.dml
deleted file mode 100644
index a501c60..0000000
--- a/scripts/staging/PPCA.dml
+++ /dev/null
@@ -1,160 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# This script performs Probabilistic Principal Component Analysis (PCA) on the given input data.
-# It is based on paper: sPCA: Scalable Principal Component Analysis for Big Data on Distributed
-# Platforms. Tarek Elgamal et.al.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# X String --- location to read the matrix X input matrix
-# k Int --- indicates dimension of the new vector space constructed from eigen vectors
-# tolobj Int 0.00001 objective function tolerance value to stop ppca algorithm
-# tolrecerr Int 0.02 reconstruction error tolerance value to stop the algorithm
-# iter Int 10 maximum number of iterations
-# fmt String 'text' output format of results PPCA such as "text" or "csv"
-# hadoop jar SystemDS.jar -f PPCA.dml -nvargs X=/INPUT_DIR/X C=/OUTPUT_DIR/C V=/OUTPUT_DIR/V k=2 tol=0.2 iter=100
-# ---------------------------------------------------------------------------------------------
-# OUTPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# C Matrix --- principal components
-# V Matrix --- eigenvalues / eigenvalues of principal components
-#
-
-X = read($X);
-
-fileC = $C;
-fileV = $V;
-
-k = ifdef($k, ncol(X));
-iter = ifdef($iter, 10);
-tolobj = ifdef($tolobj, 0.00001);
-tolrecerr = ifdef($tolrecerr, 0.02);
-fmt0 = ifdef($fmt, "text");
-
-n = nrow(X);
-m = ncol(X);
-
-#initializing principal components matrix
-C = rand(rows=m, cols=k, pdf="normal");
-ss = rand(rows=1, cols=1, pdf="normal");
-ss = as.scalar(ss);
-ssPrev = ss;
-
-# best selected principle components - with the lowest reconstruction error
-PC = C;
-
-# initilizing reconstruction error
-RE = tolrecerr+1;
-REBest = RE;
-
-Z = matrix(0,rows=1,cols=1);
-
-#Objective function value
-ObjRelChng = tolobj+1;
-
-# mean centered input matrix - dim -> [n,m]
-Xm = X - colMeans(X);
-
-#I -> k x k
-ITMP = matrix(1,rows=k,cols=1);
-I = diag(ITMP);
-
-i = 0;
-while (i < iter & ObjRelChng > tolobj & RE > tolrecerr){
- #Estimation step - Covariance matrix
- #M -> k x k
- M = t(C) %*% C + I*ss;
-
- #Auxilary matrix with n latent variables
- # Z -> n x k
- Z = Xm %*% (C %*% inv(M));
-
- #ZtZ -> k x k
- ZtZ = t(Z) %*% Z + inv(M)*ss;
-
- #XtZ -> m x k
- XtZ = t(Xm) %*% Z;
-
- #Maximization step
- #C -> m x k
- ZtZ_sum = sum(ZtZ); #+n*inv(M));
- C = XtZ/ZtZ_sum;
-
- #ss2 -> 1 x 1
- ss2 = trace(ZtZ * (t(C) %*% C));
-
- #ss3 -> 1 x 1
- ss3 = sum((Z %*% t(C)) %*% t(Xm));
-
- #Frobenius norm of reconstruction error -> Euclidean norm
- #Fn -> 1 x 1
- Fn = sum(Xm*Xm);
-
- #ss -> 1 x 1
- ss = (Fn + ss2 - 2*ss3)/(n*m);
-
- #calculating objective function relative change
- ObjRelChng = abs(1 - ss/ssPrev);
- #print("Objective Relative Change: " + ObjRelChng + ", Objective: " + ss);
-
- #Reconstruction error
- R = ((Z %*% t(C)) - Xm);
-
- #calculate the error
- #TODO rethink calculation of reconstruction error ....
- #1-Norm of reconstruction error - a big dense matrix
- #RE -> n x m
- RE = abs(sum(R)/sum(Xm));
- if (RE < REBest){
- PC = C;
- REBest = RE;
- }
- #print("ss: " + ss +" = Fn( "+ Fn +" ) + ss2( " + ss2 +" ) - 2*ss3( " + ss3 + " ), Reconstruction Error: " + RE);
-
- ssPrev = ss;
- i = i+1;
-}
-print("Objective Relative Change: " + ObjRelChng);
-print ("Number of iterations: " + i + ", Reconstruction Err: " + REBest);
-
-# reconstructs data
-# RD -> n x k
-RD = X %*% PC;
-
-# calculate eigenvalues - principle component variance
-RDMean = colMeans(RD);
-V = t(colMeans(RD*RD) - (RDMean*RDMean));
-
-# sorting eigenvalues and eigenvectors in decreasing order
-V_decr_idx = order(target=V,by=1,decreasing=TRUE,index.return=TRUE);
-VF_decr = table(seq(1,nrow(V)),V_decr_idx);
-V = VF_decr %*% V;
-PC = PC %*% VF_decr;
-
-# writing principal components
-write(PC, fileC, format=fmt0);
-# writing eigen values/pc variance
-write(V, fileV, format=fmt0);
diff --git a/scripts/staging/autoencoder-2layer.dml b/scripts/staging/autoencoder-2layer.dml
deleted file mode 100644
index 9eee8ae..0000000
--- a/scripts/staging/autoencoder-2layer.dml
+++ /dev/null
@@ -1,248 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# Trains a 2-layer autoencoder with minibatch SGD, momentum and step-size decay.
-# If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
-# Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
-# The script standardizes the input before training (can be turned off).
-# Also, it randomly reshuffles rows before training.
-# Currently, tanh is set to be the activation function.
-# By re-implementing 'func' DML-bodied function, one can change the activation.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# X String --- Filename where the input is stored
-# H1 Int --- Number of neurons in the 1st hidden layer
-# H2 Int --- Number of neurons in the 2nd hidden layer
-# EPOCH Int --- Number of epochs to train for
-# fmt String 'text' Output format ("text", "csv", "binary" etc.)
-# OBJ Boolean FALSE If TRUE, Computes objective function value (squared-loss)
-# at the end of each epoch. Note that, computing the full
-# objective can take a lot of time.
-# BATCH Int 256 Mini-batch size (training parameter)
-# STEP Double 1e-5 Initial step size (training parameter)
-# DECAY Double 0.95 Decays step size after each epoch (training parameter)
-# MOMENTUM Double 0.9 Momentum parameter (training parameter)
-# ---------------------------------------------------------------------------------------------
-#
-# OUTPUT PARAMETERS (all filenames):
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# W1_out String --- File to store weights between input layer and 1st hidden layer
-# b1_out String --- File to store bias between input layer and 1st hidden layer
-# W2_out String --- File to store weights between 1st hidden layer and 2nd hidden layer
-# b2_out String --- File to store bias between 1st hidden layer and 2nd hidden layer
-# W3_out String --- File to store weights between 2nd hidden layer and 3rd hidden layer
-# b3_out String --- File to store bias between 2nd hidden layer and 3rd hidden layer
-# W4_out String --- File to store weights between 3rd hidden layer and output layer
-# b4_out String --- File to store bias between 3rd hidden layer and output layer
-# HIDDEN String " " File to store the hidden (2nd) layer representation if needed
-# ---------------------------------------------------------------------------------------------
-#
-# INVOCATION:
-# -f autoencoder_2layer.dml --nvargs X=<input file> H1=500 H2=2 EPOCH=1 fmt="csv"
-# W1_out=<weights from input to 1st hidden layer> b1_out=<bias from input to 1st hidden layer>
-# W2_out=<weights from 1st hidden layer to 2nd hidden layer> b2_out=<bias from 1st hidden layer to 2nd hidden layer>
-# W3_out=<weights from 2nd hidden layer to 3rd hidden layer> b3_out=<bias from 2nd hidden layer to 3rd hidden layer>
-# W4_out=<weights from 3rd hidden layer to output> b4_out=<bias from 3rd hidden layer to output>
-#
-
-#implements tanh
-#to use another activation fn, implement a DML-bodied function with
-#function name 'func' and comment out this one
-func = function(Matrix[Double] X) return(Matrix[Double] Y, Matrix[Double] Y_prime){
- Y = (exp(2*X) - 1)/(exp(2*X) + 1)
- Y_prime = 1 - Y^2
-}
-
-feedForward = function(Matrix[Double] X,
- Matrix[Double] W1, Matrix[Double] b1,
- Matrix[Double] W2, Matrix[Double] b2,
- Matrix[Double] W3, Matrix[Double] b3,
- Matrix[Double] W4, Matrix[Double] b4,
- Matrix[Double] Y)
- return(Matrix[Double] H1, Matrix[Double] H1_prime,
- Matrix[Double] H2, Matrix[Double] H2_prime,
- Matrix[Double] H3, Matrix[Double] H3_prime,
- Matrix[Double] Yhat, Matrix[Double] Yhat_prime,
- Matrix[Double] E){
- H1_in = t(W1 %*% t(X) + b1)
- [H1, H1_prime] = func(H1_in)
-
- H2_in = t(W2 %*% t(H1) + b2)
- [H2, H2_prime] = func(H2_in)
-
- H3_in = t(W3 %*% t(H2) + b3)
- [H3, H3_prime] = func(H3_in)
-
- Yhat_in = t(W4 %*% t(H3) + b4)
- [Yhat, Yhat_prime] = func(Yhat_in)
- E = Yhat - Y
-}
-
-grad = function(Matrix[Double] X,
- Matrix[Double] H1, Matrix[Double] H1_prime,
- Matrix[Double] H2, Matrix[Double] H2_prime,
- Matrix[Double] H3, Matrix[Double] H3_prime,
- Matrix[Double] Yhat_prime,
- Matrix[Double] E,
- Matrix[Double] W1, Matrix[Double] W2, Matrix[Double] W3, Matrix[Double] W4)
- return(Matrix[Double] W1_grad, Matrix[Double] b1_grad,
- Matrix[Double] W2_grad, Matrix[Double] b2_grad,
- Matrix[Double] W3_grad, Matrix[Double] b3_grad,
- Matrix[Double] W4_grad, Matrix[Double] b4_grad){
- #backprop
- delta4 = E * Yhat_prime
- delta3 = H3_prime * (delta4 %*% W4)
- delta2 = H2_prime * (delta3 %*% W3)
- delta1 = H1_prime * (delta2 %*% W2)
-
- #compute gradients
- b4_grad = t(colSums(delta4))
- b3_grad = t(colSums(delta3))
- b2_grad = t(colSums(delta2))
- b1_grad = t(colSums(delta1))
-
- W4_grad = t(delta4) %*% H3
- W3_grad = t(delta3) %*% H2
- W2_grad = t(delta2) %*% H1
- W1_grad = t(delta1) %*% X
-}
-
-obj = function(Matrix[Double] E) return(Double val){
- val = 0.5 * sum(E^2)
-}
-
-batch_size = ifdef($BATCH, 256)
-mu = ifdef($MOMENTUM, 0.9)
-step = ifdef($STEP, 1e-5)
-decay = ifdef($DECAY, 0.95)
-hfile = ifdef($HIDDEN, " ")
-fmt = ifdef($fmt, "text")
-full_obj = ifdef($OBJ, FALSE)
-
-X = read($X)
-num_hidden1 = $H1
-num_hidden2 = $H2
-max_epochs = $EPOCH
-
-n = nrow(X)
-m = ncol(X)
-
-#randomly reordering rows
-permut = table(seq(1,n,1), order(target=Rand(rows=n, cols=1, min=0, max=1, pdf="uniform"), by=1, index.return=TRUE), n, n)
-X = permut %*% X
-
-#z-transform, whitening operator is better
-means = colSums(X)/n
-stds = sqrt((colSums(X^2)/n - means*means)*n/(n-1)) + 1e-17
-X = (X - means)/stds
-
-W1 = sqrt(6)/sqrt(m + num_hidden1) * Rand(rows=num_hidden1, cols=m, min=-1, max=1, pdf="uniform")
-b1 = matrix(0, rows=num_hidden1, cols=1)
-W2 = sqrt(6)/sqrt(num_hidden1 + num_hidden2) * Rand(rows=num_hidden2, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b2 = matrix(0, rows=num_hidden2, cols=1)
-W3 = sqrt(6)/sqrt(num_hidden2 + num_hidden1) * Rand(rows=num_hidden1, cols=num_hidden2, min=-1, max=1, pdf="uniform")
-b3 = matrix(0, rows=num_hidden1, cols=1)
-W4 = sqrt(6)/sqrt(num_hidden2 + m) * Rand(rows=m, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b4 = matrix(0, rows=m, cols=1)
-
-upd_W1 = matrix(0, rows=nrow(W1), cols=ncol(W1))
-upd_b1 = matrix(0, rows=nrow(b1), cols=ncol(b1))
-upd_W2 = matrix(0, rows=nrow(W2), cols=ncol(W2))
-upd_b2 = matrix(0, rows=nrow(b2), cols=ncol(b2))
-upd_W3 = matrix(0, rows=nrow(W3), cols=ncol(W3))
-upd_b3 = matrix(0, rows=nrow(b3), cols=ncol(b3))
-upd_W4 = matrix(0, rows=nrow(W4), cols=ncol(W4))
-upd_b4 = matrix(0, rows=nrow(b4), cols=ncol(b4))
-
-if( full_obj ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- full_o = obj(full_E)
- print("EPOCHS=" + 0 + " OBJ (FULL DATA): " + full_o)
-}
-
-iter = 0
-num_iters_per_epoch = ceil(n / batch_size)
-max_iterations = max_epochs * num_iters_per_epoch
-#print("num_iters_per_epoch=" + num_iters_per_epoch + " max_iterations=" + max_iterations)
-beg = 1
-while( iter < max_iterations ){
- end = beg + batch_size - 1
- if(end > n) end = n
- X_batch = X[beg:end,]
-
- [H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat, Yhat_prime, E] = feedForward(X_batch, W1, b1, W2, b2, W3, b3, W4, b4, X_batch)
- [W1_grad, b1_grad, W2_grad, b2_grad, W3_grad, b3_grad, W4_grad, b4_grad] = grad(X_batch, H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat_prime, E, W1, W2, W3, W4)
-
- o = obj(E)
- print("epochs=%5.4f BATCH beg=%d end=%d obj=%f", (iter / num_iters_per_epoch), beg, end, o)
-
- #update
- local_step = step / nrow(X_batch)
- upd_W1 = mu * upd_W1 - local_step * W1_grad
- upd_b1 = mu * upd_b1 - local_step * b1
- upd_W2 = mu * upd_W2 - local_step * W2_grad
- upd_b2 = mu * upd_b2 - local_step * b2
- upd_W3 = mu * upd_W3 - local_step * W3_grad
- upd_b3 = mu * upd_b3 - local_step * b3
- upd_W4 = mu * upd_W4 - local_step * W4_grad
- upd_b4 = mu * upd_b4 - local_step * b4
- W1 = W1 + upd_W1
- b1 = b1 + upd_b1
- W2 = W2 + upd_W2
- b2 = b2 + upd_b2
- W3 = W3 + upd_W3
- b3 = b3 + upd_b3
- W4 = W4 + upd_W4
- b4 = b4 + upd_b4
-
- iter = iter + 1
- if(end == n) beg = 1
- else beg = end + 1
-
- if( iter %% num_iters_per_epoch == 0 ) step = step * decay
-
- if( full_obj & iter %% num_iters_per_epoch == 0 ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- full_o = obj(full_E)
- epochs = iter %/% num_iters_per_epoch
- print("EPOCHS=" + epochs + " iter=" + iter + " OBJ (FULL DATA)=" + full_o)
- }
-}
-
-write(W1, $W1_out, format=fmt)
-write(b1, $b1_out, format=fmt)
-write(W2, $W2_out, format=fmt)
-write(b2, $b2_out, format=fmt)
-write(W3, $W3_out, format=fmt)
-write(b3, $b3_out, format=fmt)
-write(W4, $W4_out, format=fmt)
-write(b4, $b4_out, format=fmt)
-
-if( hfile != " " ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- reordered_H = t(permut) %*% full_H2
- write(reordered_H, hfile, format=fmt)
-}
diff --git a/scripts/staging/knn.dml b/scripts/staging/knn.dml
deleted file mode 100644
index ddf5b15..0000000
--- a/scripts/staging/knn.dml
+++ /dev/null
@@ -1,779 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-# THIS SCRIPT IMPLEMENTS KNN( K Nearest Neighbor ) ALGORITHM
-#
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT OPTIONAL MEANING
-# ---------------------------------------------------------------------------------------------
-# X String --- N Location to read the input matrix as features
-# T String --- N Location to read the input matrix for nearest neighbor
-# search
-# Y String --- Y Location to read the input matrix as target
-# Y_T String --- Y Location to read the target type of matrix Y whether
-# columns in Y are continuous ( =1 ) or
-# categorical ( =2 )
-# K String --- Y Location to write the result of selected k value
-# PR String --- Y Location to write the predicted value result,
-# it's optional when Y is NOT specified.
-# NNR String --- N Location to write the nearest neighbor index result for T.
-# trans_continuous int 0 Y Option flag for continuous feature transformed to [-1,1]:
-# 0 = do not transform continuous variable;
-# 1 = transform continuous variable;
-# select_k int 0 Y Use k selection algorithm to estimate k
-# ( 1 means yes )
-# k_min int 1 Y Min k value( available if select_k = 1 )
-# k_max int 100 Y Max k value( available if select_k = 1 )
-# select_feature int 0 Y Use feature selection algorithm to select feature
-# ( 1 means yes )
-# START_SELECTED String --- Y Location to read feature selection initinal value
-# FEATURE_SELECTED String --- Y Location to write feature selection result
-# feature_max int 10 Y Max feature selection
-# interval int 1000 Y Interval value for K selecting ( available if select_k = 1 )
-# feature_importance int 0 Y Use feature importance algorithm to estimate each feature
-# ( 1 means yes )
-# FEATURE_IMPORTANCE_VALUE String --- Y Location to write feature importance result
-# k_value int 5 Y k value for KNN, ignore if select_k enable
-# predict_con_tg int 0 Y Continuous target predict function: mean(=0) or
-# median(=1)
-# fmt String "text" Y Matrix output format for MB, ML, usually "text" or "csv"
-# ---------------------------------------------------------------------------------------------
-# OUTPUT: Matrix PR, Matrix NNR, Matrix FEATURE_IMPORTANCE_VALUE
-#
-# HOW TO INVOKE THIS SCRIPT - EXAMPLE:
-# hadoop jar SystemDS.jar -f knn.dml -nvargs X=%datapath%/train.csv
-# T=%datapath%/test.csv NNR=%tmpdir%/NNR.csv fmt=csv
-
-
-print( "start knn algorithm" );
-#input data
-TrainPath = $X;
-TestPath = $T;
-CLPath = ifdef( $Y," " );
-CLTypePath = ifdef( $Y_T," " );
-#output data
-NNRPath = $NNR;
-PRPath = ifdef( $PR," " );
-#parameter
-epsilon = ifdef( $epsilon,0 );
-distance_type = ifdef( $distance_type,1 );
-trans_continuous = ifdef( $trans_continuous,0 );
-predict_con_tg = ifdef($predict_con_tg,0);
-k_value = ifdef( $k_value,5 );
-FMT = ifdef( $fmt,"text" );
-#k select parameter
-select_k = ifdef( $select_k,0 );
-k_min = ifdef( $k_min,1 );
-k_max = ifdef( $k_max,100 );
-#feature select parameter
-select_feature = ifdef( $select_feature,0 );
-feature_max = ifdef( $feature_max,10 );
-#feature importance parameter
-feature_importance = ifdef ($feature_importance,0);
-
-
-#Read train data
-Train = read ( TrainPath );
-Test = read ( TestPath );
-
-#data prepare
-if( trans_continuous == 1 ){
- print( "Do data prepare to transfer continuous variable to [-1,1] ..." );
- Train = prepareKNNData( Train);
- Test = prepareKNNData( Test);
-}
-
-n_records = nrow( Train );
-n_features = ncol( Train );
-s_selected_k = 5;
-m_selected_feature = matrix(1,rows=1,cols=n_records);
-if( select_k == 1 | select_feature==1 ){
- default_interval = as.integer( ceil( sqrt( n_records ) ) );
- interval = ifdef( $interval,default_interval );
- print( "interval is " + interval );
- #parameter check
- if( CLPath == " " | CLTypePath == " "){
- stop( "Y or Y_T must be set for k selection algorithm" );
- }
- #parameter re-define
- if( select_k==1 ){
- if( k_max >= n_records ){
- k_max = n_records - 1;
- print( "k_max should no greater than number of record, change k_max equal " +
- "( number of record - 1 ) : " + k_max );
- }
- if( k_max >= interval ){
- interval = k_max + 1;
- # k_max should equal interval -1, because we drop self when search nn.
- print( "interval should be no less than k_max, change interval equal : " +
- interval );
- }
- if( k_max <= 1 ){
- stop( "uncorrect k_max value" );
- }
- if( k_min >= k_max ){
- stop( "k_min >= k_max" );
- }
- }
- if( select_feature == 1 ){
- if( k_value >= n_records ){
- k_value = n_records - 1;
- print( "k_value should be no greater than number of record, change k_value equal " +
- "( number of record - 1 ) : " + k_value );
- }
- #Select feature only
- StartSelectedPath = ifdef( $START_SELECTED," " );
- SelectedPath = ifdef( $FEATURE_SELECTED," " );
- if( StartSelectedPath == " " ){
- m_start_selected_feature = matrix( 0,1,n_features );
- }else{
- m_start_selected_feature = read( StartSelectedPath );
- }
- }
- CL = read( CLPath );
- CLType = read( CLTypePath );
- type = as.integer( as.scalar( CLType[1,1] ) );
- if( select_k == 1 & select_feature == 1){
- #Combined k and feature selection
- print("Start combined k and feature selection ...");
- [m_selected_feature,s_selected_k] =
- getSelectedFeatureAndK( Train,CL,type,m_start_selected_feature,
- feature_max,k_min,k_max,interval );
- }else if( select_k == 1 ){
- #Select k only
- print("Start k select ...");
- s_selected_k = getSelectedKBase( Train,CL,type,k_min,k_max,interval );
- }else if( select_feature == 1 ){
- #Select feature only
- print("Start feature selection ... ");
- [m_selected_feature,d_err] =
- getSelectedFeature( Train,CL,type,m_start_selected_feature,
- feature_max,k_value,interval );
- }
-}
-if( feature_importance == 1){
- if( k_value >= n_records ){
- k_value = n_records - 1;
- print( "k_value should be no greater than number of record, make k_value equal " +
- "( number of record - 1 ) : " + k_value );
- }
- CL = read( CLPath );
- CLType = read( CLTypePath );
- type = as.integer( as.scalar( CLType[1,1] ) );
- FeatureImportanceExportPath = ifdef( $FEATURE_IMPORTANCE_VALUE," " );
- if(FeatureImportanceExportPath != " "){
- [m_feature_importance,m_norm_feature_importance] =
- getFeatureImportance(Train,CL,type,k_value);
- #print( "feature importance is " );
- #tmp = matrixDisplay( m_feature_importance );
- write ( m_feature_importance,FeatureImportanceExportPath,format=FMT );
- }
-}
-
-# knn search
-print("Start knn search ...")
-# apply k selection result
-if(select_k == 1){
- k_value = s_selected_k
-}
-if( k_value >= n_records ){
- k_value = n_records - 1;
- print( "k_value should be no greater than number of record, make k_value equal " +
- "( number of record - 1 )" );
-}
-print( "k value is " + k_value );
-# apply feature selection result
-if( select_feature == 1 ){
- if( n_features > sum( m_selected_feature ) ){
- Train = removeEmpty( target = Train,margin="cols",select = m_selected_feature);
- Test = removeEmpty( target = Test,margin="cols",select = m_selected_feature );
- }
-}
-
-NNR_matrix = naiveKNNsearch(Train,Test,k_value)
-write( NNR_matrix,NNRPath,format=FMT );
-
-# main part for knn predicted
-if( CLPath != " " ){
- CL = read ( CLPath );
- if( CLTypePath == " " ){
- stop( "Set Y but Miss Y_T for label type" );
- }
- CLType = read( CLTypePath );
- i_cl_type = as.integer( as.scalar( CLType[1,1] ) );
-}
-CL_matrix = matrix( 0,nrow( Test ),1 );
-if( CLPath != " " ){
- parfor( i in 1:nrow( NNR_matrix ) ,check=0){
- NNR_tmp_matrix = matrix( 0,k_value,1 );
- for( j in 1:k_value ){
- NNR_tmp_matrix[j,1] = CL[as.scalar( NNR_matrix[i,j] ),1] ;
- }
- t_cl_value = 0.0 #test data label value
- if( i_cl_type == 2 ){
- t_cl_value = as.scalar( rowIndexMax( t( table( NNR_tmp_matrix,1 ) ) ) )
- }else{
- if ( predict_con_tg == 0){
- t_cl_value = mean( NNR_tmp_matrix );
- } else if(predict_con_tg == 1){
- t_cl_value = median( NNR_tmp_matrix );
- }
- }
- CL_matrix[i,1] = t_cl_value;
- }
-}
-if( PRPath != " " ){
- write( CL_matrix,PRPath,format=FMT );
-}
-
-#naive knn search implement
-naiveKNNsearch = function(
- matrix[double] P,
- matrix[double] Q,
- integer K
-)return(
- matrix[double] O
-){
- num_records = nrow (P);
- num_features = ncol (P);
- num_queries = nrow (Q);
- Qt = t(Q);
- PQt = P %*% Qt;
- P2 = rowSums (P ^ 2);
- D = -2 * PQt + P2;
- if (K == 1) {
- Dt = t(D);
- O = rowIndexMin (Dt);
- } else {
- O = matrix (0, rows = num_queries, cols = K);
- parfor (i in 1:num_queries) {
- D_sorted=order(target=D[,i], by=1, decreasing=FALSE, index.return=TRUE);
- O[i,] = t(D_sorted[1:K,1]);
- }
- }
-}
-#naive knn search for predict value only implement
-naiveKNNsearchForPredict = function(
- matrix[double] P,
- matrix[double] Q,
- matrix[double] L,
- integer K
-)return(
- matrix[double] OL
-){
- num_records = nrow (P);
- num_features = ncol (P);
- num_queries = nrow (Q);
- Qt = t(Q);
- PQt = P %*% Qt;
- P2 = rowSums (P ^ 2);
- D = -2 * PQt + P2;
- if (K == 1) {
- Dt = t(D);
- O = rowIndexMin (Dt);
- OL = matrix (0, rows = num_queries, cols = 1)
- parfor( i in 1:num_queries){
- OL[i,] = L[as.scalar(O[i,1]),1]
- }
- } else {
- OL = matrix (0, rows = num_queries, cols = K);
- parfor (i in 1:num_queries) {
- D_sorted=order(target=cbind(D[,i],L), by=1, decreasing=FALSE, index.return=FALSE);
- OL[i,] = t(D_sorted[1:K,2]);
- }
- }
-}
-# prepareKNNData:
-# Do data prepare - [-1,1] transform for continues variable
-# Argument:
-# in_m_data input matrix as features
-prepareKNNData = function (
- matrix[double] in_m_data
- )
-return(
- matrix[double] out_m_data
- )
-{
- m_colmax = colMaxs(in_m_data);
- m_colmin = colMins(in_m_data);
- out_m_data = 2*( in_m_data - m_colmin )/( m_colmax - m_colmin ) - 1;
-}
-
-getKNeighbor = function ( matrix[double] in_m_data,
- matrix[double] in_m_test_data,
- matrix[double] in_m_cl,
- integer in_i_k_max )
- return ( matrix[double] out_m_neighbor_value ) {
- #to naive
- m_search_result = naiveKNNsearchForPredict(in_m_data,in_m_test_data,in_m_cl,in_i_k_max + 1)
- out_m_neighbor_value = m_search_result[ ,2:in_i_k_max + 1]
-}
-
-getErr_k = function ( matrix[double] in_m_neighbor_value,
- matrix[double] in_m_cl,
- integer in_i_cl_type ,
- integer in_i_k_min )
- return ( matrix[double] out_m_err ){
- i_col = ncol( in_m_neighbor_value );
- i_row = nrow( in_m_neighbor_value );
-
- out_m_err = matrix( 0,i_row,i_col - in_i_k_min + 1 );
- if( in_i_cl_type == 2 ){
- #category
- m_correct = in_m_neighbor_value != in_m_cl[1:i_row,];
- }else{
- #continues
- m_correct = (in_m_neighbor_value - in_m_cl[1:i_row,])^2;#ppred( in_m_neighbor_value,in_m_cl,"-" );
- }
- parfor( i in 1:i_col-in_i_k_min+1 ,check = 0 ){
- out_m_err[,i] =
- ( rowSums( m_correct[,1:in_i_k_min + i - 1] ) / ( in_i_k_min + i - 1 ) );
- }
- #return err for each record and each k ( belong to range 1~max );
-}
-/*
-eliminateModel = function ( matrix[double] in_j_err, matrix[double] in_j1_err )
- return( boolean out_b_inactived ){
- out_b_inactived = FALSE;
- i_row = nrow( in_j_err );
- m_err_mean = as.scalar( colSums( in_j_err - in_j1_err ) / i_row );
- m_err_variance = as.scalar(
- sqrt( colSums( ( in_j_err - in_j1_err - m_err_mean ) ^ 2 ) / ( i_row-1 )))
-
- #alpha, beta, gamma, delta
- d_gamma = 0.001;
- d_delta = 0.001;
-
- tmp_d_delta = cdf(
- target = ( -d_gamma - m_err_mean ) / m_err_variance, dist="t",df=i_row-1 );
- #print( "m_err_mean=" + m_err_mean + ",m_err_variance=" +
- # m_err_variance + ",tmp_d_delta=" + tmp_d_delta );
- if( tmp_d_delta < d_delta ){
- out_b_inactived = TRUE;
- }
-
-}*/
-eliminateModel = function ( double s_err_mean, double s_err_vars, integer i_row )
- return( boolean out_b_inactived ){
- out_b_inactived = FALSE;
- #alpha, beta, gamma, delta
- d_gamma = 0.001;
- d_delta = 0.001;
-
- tmp_d_delta = cdf(
- target = ( -d_gamma - s_err_mean ) / s_err_vars, dist="t",df=i_row-1 );
- #print( "m_err_mean=" + m_err_mean + ",m_err_variance=" +
- # m_err_variance + ",tmp_d_delta=" + tmp_d_delta );
- if( tmp_d_delta < d_delta ){
- out_b_inactived = TRUE;
- }
-
-}
-# getSelectedKBase:
-# k selection algorithm with simple KNN algorithm.
-# Refer to ADD part "6.3 Optimized LOOCV + BRACE"
-# Argument:
-# in_m_data input matrix as features
-# in_m_data_target input matrix as target value
-# in_i_is_categorical 1 = category , 0 = continuous
-# k_min minimun k
-# k_max maximun k
-# interval block size for BRACE algorithm
-#
-# Reture:
-# k output k value for k selection
-getSelectedKBase = function (
- matrix[double] in_m_data,
- matrix[double] in_m_data_target,
- integer in_i_is_categorical, # 1 = category , 0 = continuous
- integer k_min,
- integer k_max,
- integer interval )
-return(
- integer k
- )
-{
- b_continue_loop = TRUE;
- i_iter = 1;
- i_record = nrow( in_m_data );
-
- m_active_flag = matrix( 0,1,k_max-k_min+1 );
- i_active_model_number = k_max - k_min + 1;
-
- m_iter_err_sum = matrix( 0,1,k_max-k_min+1);
- m_iter_err_sum_squared = matrix( 0,1,k_max-k_min+1);
- while( b_continue_loop ){
- # 1.build k-d tree? , or use hash method
- # 2.search data to get k_max nearest neighbor
- i_process_item = i_iter*interval;
- if( i_process_item >= i_record ){
- i_process_item = i_record;
- b_continue_loop = FALSE;
- }
- i_process_begin_item = (i_iter - 1)*interval + 1
- #i_process_begin_item = 1
- i_process_end_item = i_process_item
-
- m_neighbor_value = getKNeighbor( in_m_data,in_m_data[i_process_begin_item:i_process_end_item,],in_m_data_target,k_max );
- # 3.get matrix of err from k_min to k_max
- m_err = getErr_k( m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical,k_min );
-
- # 4.check this matrix to drop unnessary record
- m_active_flag_tmp = matrix( 0,1,ncol( m_err ) );
-
- s_rows_number = i_process_item#nrow(m_err);
-
- m_iter_err_sum = colSums(m_err) + m_iter_err_sum
- m_iter_err_sum_squared = colSums(m_err ^ 2) + m_iter_err_sum_squared
-
- m_err_mean = - outer(t(m_iter_err_sum), m_iter_err_sum , "-") / s_rows_number
- m_err_vars = ( m_err_mean ^2 * s_rows_number -
- 2 * m_err_mean * m_iter_err_sum + m_iter_err_sum_squared) / (s_rows_number-1)
- m_err_vars = sqrt(m_err_vars)
-
- parfor( i in 1:ncol( m_err ) ){
- #m_err_i = m_err - m_err[,i];
- #m_err_i_mean = colMeans(m_err_i);
- #m_err_i_vars = colVars(m_err_i);
- parfor( j in 1:ncol( m_err ),check=0 ){
- b_execute_block = TRUE;
- if( j==i ) b_execute_block = FALSE;
- if( as.scalar( m_active_flag_tmp[1,i] ) == 1 ) b_execute_block = FALSE;
- #i has dropped, ignore this case
- if( as.scalar( m_active_flag_tmp[1,j] ) == 1 ) b_execute_block = FALSE;
- #j has dropped, ignore this case
- if( b_execute_block ){
- b_flag = eliminateModel( as.scalar(m_err_mean[i,j]),as.scalar(m_err_vars[i,j]),s_rows_number);
- if( b_flag == TRUE ){
- m_active_flag_tmp[1,i] = 1;
- }
- }
- }
- }
-
- #m_tmp = ppred( colSums( m_active_flag_tmp ),1,">=" );
- m_active_flag = m_active_flag + m_active_flag_tmp >= 1;
- i_active_model_number = -sum( m_active_flag - 1 );
-
- # 5.break while check
- if( i_active_model_number <= 1 ){
- b_continue_loop = FALSE;
- }
- i_iter = i_iter + 1;
- print("i_iter" + i_iter)
- }
-
- k = 0;
- if( i_active_model_number == 0 ){
- print( "All k kick out, use min of range " + k_min );
- k = k_min;
- }else if( i_active_model_number == 1 ){
- k = k_min + as.integer( as.scalar( rowIndexMin( m_active_flag ) ) ) - 1;
- print( "Get k, which value is " + k );
- }else{
- m_err_for_order =
- cbind( t( m_iter_err_sum ), matrix( seq( k_min,k_max,1 ),k_max-k_min+1,1 ) );
- m_err_for_order = removeEmpty(
- target=m_err_for_order * t( m_active_flag == 0 ),margin="rows" );
- for( i in 1:nrow( m_err_for_order ) ){
- print( "k:" + as.scalar( m_err_for_order[i,2] ) +
- ",err:" + as.scalar( m_err_for_order[i,1] ) );
- }
- m_err_order = order( target=m_err_for_order,by=1,decreasing=FALSE,index.return=FALSE );
- k = as.integer( as.scalar( m_err_order[1,2] ) );
- print( "Get minimum LOOCV error, which value is " + k );
- }
-}
-
-getErr = function ( matrix[double] in_m_neighbor_value,
- matrix[double] in_m_cl,
- integer in_i_cl_type )
- return ( matrix[double] out_m_err ){
- i_col = ncol( in_m_neighbor_value );
- i_row = nrow( in_m_neighbor_value );
-
- if( in_i_cl_type == 2 ){
- #category
- m_correct = in_m_neighbor_value != in_m_cl[1:i_row,];
- }else{
- #continues
- m_correct = (in_m_neighbor_value - in_m_cl[1:i_row,])^2;#ppred( in_m_neighbor_value,in_m_cl,"-" );
- }
- out_m_err = ( rowSums( m_correct[,1:i_col] )/( i_col ) );
-}
-
-# getSelectedFeature:
-# feature selection algorithm.
-# Refer to ADD part "7.1 FS+BRACE"
-# Argument:
-# in_m_data input matrix as features
-# in_m_data_target input matrix as target value
-# in_i_is_categorical 1 = category , 0 = continuous
-# in_m_init_selected S.user, initial selected feature which use specified
-# in_i_max_select J, max feature selected
-# k_value k
-# interval block size for BRACE algorithm
-#
-# Reture:
-# out_m_selected_feature output matrix for feature selection
-# out_d_min_LOOCV output err
-
-getSelectedFeature = function (
- matrix[double] in_m_data,
- matrix[double] in_m_data_target,
- integer in_i_is_categorical, # 1 = category , 0 = continuous
- matrix[double] in_m_init_selected,
- integer in_i_max_select,
- integer k_value,
- integer interval )
-return(
- matrix[double] out_m_selected_feature,
- double out_d_min_LOOCV
- )
-{
- i_n_record = nrow( in_m_data );
- i_n_column = ncol( in_m_data );
- m_main_selected_flag = in_m_init_selected;
- b_no_feature_selected = TRUE;
- if( sum( in_m_init_selected ) >= 1 ){
- b_no_feature_selected = FALSE;
- }
- d_max_err_value = ( max( in_m_data_target ) - min( in_m_data_target ) ) * 100;
- b_continue_main_loop = TRUE; #level 1 while loop flag
- d_min_LOOCV = Inf;
- while( b_continue_main_loop ){
- m_feature_selected_flag = m_main_selected_flag;
- m_this_model_selected_flag = TRUE;
- i_index_min_LOOCV = -1; # flag for which model win in BRACE algorith
- b_selected_morethan_one = FALSE;
- b_continue_loop = TRUE; #level 2 while loop flag
- i_iter = 1;
- m_iter_err_sum = matrix( 0,1,i_n_column+1 );
- m_iter_err_sum_squared = matrix( 0,1,i_n_column+1 );
- while( b_continue_loop ){
- #print( "*************" + i_iter + "*************" );
- i_process_item = i_iter*interval;
- if( i_process_item >= i_n_record ){
- i_process_item = i_n_record;
- b_continue_loop = FALSE;
- }
- i_process_begin_item = (i_iter - 1)*interval + 1
- #i_process_begin_item = 1
- i_process_end_item = i_process_item
- m_err = matrix( d_max_err_value,i_process_end_item - i_process_begin_item + 1,i_n_column+1 );
- if( b_no_feature_selected == TRUE ){
- parfor( i in 1:i_n_column ,check=0){
- if( as.scalar( m_feature_selected_flag[1,i] ) != 1 ){
- m_tmp_process_data = in_m_data[,i];
- m_neighbor_value = getKNeighbor(
- m_tmp_process_data,m_tmp_process_data[i_process_begin_item:i_process_end_item,],
- in_m_data_target,k_value );
- m_tmp_err = getErr(
- m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
- m_err[,i] = m_tmp_err;
- }
- }
- }else{
- #Use m_main_selected_flag but not m_feature_selected_flag,
- # m_main_selected_flag: which feature are init selected
- # m_feature_selected_flag: which feature are dropped & init selected
- m_tmp_data = removeEmpty( target=in_m_data ,margin="cols", select = m_main_selected_flag);
- if( m_this_model_selected_flag == TRUE ){
- m_neighbor_value = getKNeighbor(
- m_tmp_data,m_tmp_data[i_process_begin_item:i_process_end_item,],in_m_data_target, k_value );
-
- m_tmp_err = getErr( m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
- m_err[,i_n_column+1] = m_tmp_err;
- }
- parfor( i in 1:i_n_column ,check=0){
- if( as.scalar( m_feature_selected_flag[1,i] ) != 1 ){
- m_tmp_process_data = cbind( m_tmp_data,in_m_data[,i] );
- m_neighbor_value = getKNeighbor(
- m_tmp_process_data,m_tmp_process_data[i_process_begin_item:i_process_end_item,],in_m_data_target,k_value );
- m_tmp_err = getErr(
- m_neighbor_value,in_m_data_target[i_process_begin_item:i_process_end_item,],in_i_is_categorical );
- m_err[,i] = m_tmp_err;
- }
- }
- }
- if( m_this_model_selected_flag == TRUE ){
- m_active_flag_tmp = cbind( m_feature_selected_flag,matrix( 0,1,1 ) );
- }else{
- m_active_flag_tmp = cbind( m_feature_selected_flag,matrix( 1,1,1 ) );
- }
-
- s_rows_number = i_process_item#nrow(m_err);
-
- m_iter_err_sum = colSums(m_err) + m_iter_err_sum
- m_iter_err_sum_squared = colSums(m_err ^ 2) + m_iter_err_sum_squared
-
- m_err_mean = - outer(t(m_iter_err_sum), m_iter_err_sum , "-") / s_rows_number
- m_err_vars = ( m_err_mean ^2 * s_rows_number -
- 2 * m_err_mean * m_iter_err_sum + m_iter_err_sum_squared) / (s_rows_number-1)
- m_err_vars = sqrt(m_err_vars)
-
- parfor( i in 1:ncol( m_err ) ){
- #m_err_i_mean = (m_iter_err_sum - m_iter_err_sum[,i])/s_rows_number
- #m_err_i_vars = (m_iter_err_sum_squared + m_err_i_mean ^2 * s_rows_number -
- # 2 * m_iter_err_sum * m_err_i_mean) / (s_rows_number-1)
- #m_err_i_vars = sqrt(m_err_i_vars)
- #m_err_i_mean = m_err_mean[i,]
- #m_err_i_vars = m_err_vars[i,]
- parfor( j in 1:ncol( m_err ) ,check=0){
- b_execute_block = TRUE;
- if( j==i ) b_execute_block = FALSE;
- if( as.scalar( m_active_flag_tmp[1,i] ) == 1 ) b_execute_block = FALSE;
- #i has dropped, ignore this case
- if( as.scalar( m_active_flag_tmp[1,j] ) == 1 ) b_execute_block = FALSE;
- #j has dropped, ignore this case
- if( b_execute_block ){
- #print( "i=" + i + ",j=" + j );
- b_flag = eliminateModel( as.scalar(m_err_mean[i,j]),as.scalar(m_err_vars[i,j]),s_rows_number);
- if( b_flag == TRUE ){
- m_active_flag_tmp[1,i] = 1;
- }
- }
- }
- }
- #We mark bit to 1 for selected feature before current loop,
- #and mark bit to 1 also for dropped feature in current loop
-
- if( sum( m_active_flag_tmp != 1 ) > 1 ){
- b_selected_morethan_one = TRUE;
- }
- m_col_sums_err = m_iter_err_sum #colSums( m_err );
- i_index_min_LOOCV = as.scalar( rowIndexMin( m_col_sums_err ) );
- d_min_LOOCV = as.scalar( m_col_sums_err[1,i_index_min_LOOCV] );
- i_index_min_LOOCV = i_index_min_LOOCV%% ( i_n_column+1 )
-
- if( sum( m_active_flag_tmp != 1 ) <= 1 ){
- b_continue_loop = FALSE;
- }
- if( as.scalar( m_active_flag_tmp[1,i_n_column+1] ) == 1 ){
- m_this_model_selected_flag = FALSE;
- }
- m_feature_selected_flag = m_active_flag_tmp[,1:i_n_column];
- i_iter = i_iter + 1;
- }
- #select current model, jump out.
- if( i_index_min_LOOCV == 0 ){
- b_continue_main_loop = FALSE;
- print( "Select Current model" );
- }else{
- print( "select feature " + i_index_min_LOOCV + ", change bit value to 1" );
- m_main_selected_flag[1,i_index_min_LOOCV] = 1;
- b_no_feature_selected = FALSE;
- }
- if( sum( m_main_selected_flag - in_m_init_selected ) >= in_i_max_select ){
- #select more than 10
- b_continue_main_loop = FALSE;
- }
- if( sum( m_main_selected_flag ) == i_n_column ){
- #all selected
- b_continue_main_loop = FALSE;
- }
- }
- out_m_selected_feature = m_main_selected_flag;
- out_d_min_LOOCV = d_min_LOOCV;
-}
-
-# getSelectedFeatureAndK:
-# Combine k and feature selection algorithm.
-# Refer to ADD part "8.Combined k and feature selection"
-# Argument:
-# in_m_data input matrix as features
-# in_m_data_target input matrix as target value
-# in_i_is_categorical 1 = category , 0 = continuous
-# in_m_init_selected S.user, initial selected feature which use specified
-# in_i_max_select J, max feature selected
-# k_min minimun k
-# k_max maximun k
-# interval block size for BRACE algorithm
-#
-# Reture:
-# out_m_selected_feature output matrix for feature selection
-# out_i_selected_k output k value for k selection
-getSelectedFeatureAndK = function (
- matrix[double] in_m_data,
- matrix[double] in_m_data_target,
- integer in_i_is_categorical, # 1 = category , 0 = continuous
- matrix[double] in_m_init_selected,
- integer in_i_max_select,
- integer k_min,
- integer k_max,
- integer interval )
-return(
- matrix[double] out_m_selected_feature,
- integer out_i_selected_k
- )
-{
- m_err = matrix( 0,1,k_max-k_min+1 );
- m_feature = matrix( 0,k_max-k_min+1,ncol( in_m_data ) );
-
- #Step 1. For each k in [k_min,k_max] ( k_min has default value 1, k_max has default value 100 )
- #in parallel select relevant features using FS+BRACE or schemata search described in Section 7.
- parfor( i in k_min:k_max,check=0 ){
- #print( " ************** " + i + " ****************" );
- [m_selected_feature,d_err] =
- getSelectedFeature( in_m_data,in_m_data_target,in_i_is_categorical
- ,in_m_init_selected,in_i_max_select,i,interval );
-
- m_err[1,i] = d_err;
- m_feature[i,] = m_selected_feature;
- #print( "err:" + d_err );
- }
-
- #Step 2. Output the combination of features and k with the smallest LOOCV error.
- i_min_err_index = as.integer( as.scalar( rowIndexMin( m_err ) ) );
- out_m_selected_feature = m_feature[i_min_err_index,];
- out_i_selected_k = i_min_err_index + k_min - 1;
- #print( "min err k is " + out_i_selected_k );
-}
-
-getFeatureImportance = function (
- matrix[double] in_m_data,
- matrix[double] in_m_data_target,
- integer in_i_is_categorical, # 1 = category , 0 = continuous
- integer k_value)
-return(
- matrix[double] out_m_feature_importance,
- matrix[double] out_m_norm_feature_importance
- )
-{
- n_feature = ncol(in_m_data)
- n_record = nrow(in_m_data)
- if(n_feature <= 1){
- stop("can't estimate feature importance when ncol = 1")
- }
- m_err = matrix( 0,n_record,n_feature);
- parfor(i_feature in 1:n_feature, check=0){
- m_feature_select = matrix(1,1,n_feature)
- m_feature_select[1,i_feature] = 0;
- m_in_tmp_data = removeEmpty(target=in_m_data,margin="cols", select= m_feature_select)
- m_neighbor_value = getKNeighbor( m_in_tmp_data,m_in_tmp_data,in_m_data_target,k_value);
- m_tmp_err = getErr( m_neighbor_value,in_m_data_target ,in_i_is_categorical );
- m_err[,i_feature] = m_tmp_err
- }
- out_m_feature_importance = colSums( m_err );
- out_m_norm_feature_importance =
- out_m_feature_importance / as.scalar(rowSums(out_m_feature_importance))
-}
\ No newline at end of file
diff --git a/scripts/staging/regression/lasso/lasso.dml b/scripts/staging/regression/lasso/lasso.dml
deleted file mode 100644
index 476ffbe..0000000
--- a/scripts/staging/regression/lasso/lasso.dml
+++ /dev/null
@@ -1,113 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#uses the sparsa algorithm to perform lasso regression
-
-X = read($X)
-y = read($Y)
-n = nrow(X)
-m = ncol(X)
-
-#params
-tol = 10^(-15)
-M = 5
-tau = 1
-maxiter = 1000
-
-#constants
-eta = 2
-sigma = 0.01
-alpha_min = 10^(-30)
-alpha_max = 10^(30)
-
-#init
-alpha = 1
-w = Rand(rows=m, cols=1, min=0, max=1, pdf="uniform")
-history = -1*10^30 * matrix(1, rows=M, cols=1)
-
-r = X %*% w - y
-g = t(X) %*% r
-obj = 0.5 * sum(r*r) + tau*sum(abs(w))
-
-print("Initial OBJ=" + obj)
-
-history[M,1] = obj
-
-inactive_set = matrix(1, rows=m, cols=1)
-iter = 0
-continue = TRUE
-while(iter < maxiter & continue) {
- dw = matrix(0, rows=m, cols=1)
- dg = matrix(0, rows=m, cols=1)
- relChangeObj = -1.0
-
- inner_iter = 0
- inner_continue = TRUE
- inner_maxiter = 100
- while(inner_iter < inner_maxiter & inner_continue) {
- u = w - g/alpha
- lambda = tau/alpha
-
- wnew = sign(u) * (abs(u) - lambda) * ((abs(u) - lambda) > 0)
- dw = wnew - w
- dw2 = sum(dw*dw)
-
- r = X %*% wnew - y
- gnew = t(X) %*% r
- objnew = 0.5 * sum(r*r) + tau*sum(abs(wnew))
- obj_threshold = max(history) - 0.5*sigma*alpha*dw2
-
- if(objnew <= obj_threshold) {
- w = wnew
- dg = gnew - g
- g = gnew
- inner_continue = FALSE
-
- history[1:(M-1),] = history[2:M,]
- history[M,1] = objnew
- relChangeObj = abs(objnew - obj)/obj
- obj = objnew
- }
- else
- alpha = eta*alpha
-
- inner_iter = inner_iter + 1
- }
-
- if(inner_continue)
- print("Inner loop did not converge")
-
- alphanew = sum(dw*dg)/sum(dw*dw)
- alpha = max(alpha_min, min(alpha_max, alphanew))
-
- old_inactive_set = inactive_set
- inactive_set = w != 0
- diff = sum(abs(old_inactive_set - inactive_set))
-
- if(diff == 0 & relChangeObj < tol)
- continue = FALSE
-
- num_inactive = sum(w != 0)
- print("ITER=" + iter + " OBJ=" + obj + " relative change=" + relChangeObj + " num_inactive=" + num_inactive)
- iter = iter + 1
-}
-
-write(w, $model)
diff --git a/scripts/staging/slicing/__init__.py b/scripts/staging/slicing/__init__.py
deleted file mode 100644
index e66abb4..0000000
--- a/scripts/staging/slicing/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
diff --git a/scripts/staging/slicing/base/Bucket.py b/scripts/staging/slicing/base/Bucket.py
deleted file mode 100644
index dc8402e..0000000
--- a/scripts/staging/slicing/base/Bucket.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class Bucket:
-
- key: []
- attributes: []
- name: ""
- error: float
- e_upper: float
- size: float
- sum_error: float
- max_tuple_error: float
- s_upper: float
- s_lower: float
- e_max: float
- e_max_upper: float
- score: float
- c_upper: float
- parents: []
- x_size: int
- loss: float
- w: float
-
- def __init__(self, node, cur_lvl, w, x_size, loss):
- self.attributes = []
- self.parents = []
- self.sum_error = 0
- self.size = 0
- self.s_upper = 0
- self.s_lower = 0
- self.score = 0
- self.error = 0
- self.max_tuple_error = 0
- self.x_size = x_size
- self.w = w
- self.loss = loss
- if cur_lvl == 0:
- self.key = node
- self.attributes.append(node)
- else:
- self.attributes = node.attributes
- self.key = node.attributes
- self.name = self.make_name()
- self.__hash__()
-
- def __hash__(self):
- return hash(self.name)
-
- def __add__(self, other):
- self.size += other.size
- self.sum_error += other.sum_error
- return self
-
- def combine_with(self, other):
- self.size = max(self.size, other.size)
- self.sum_error = max(self.sum_error, other.sum_error)
- return self
-
- def minimize_bounds(self, other):
- minimized = min(self.s_upper, other.s_upper)
- self.s_upper = minimized
- minimized = min(other.s_lower, self.s_lower)
- self.s_lower = minimized
- minimized = min(other.e_upper, self.e_upper)
- self.e_upper = minimized
- minimized = min(other.e_max_upper, self.e_max_upper)
- self.e_max_upper = minimized
- c_upper = self.calc_c_upper(self.w, self.x_size, self.loss)
- minimized = min(c_upper, self.c_upper)
- self.c_upper = minimized
-
- def __eq__(self, other):
- return (
- self.__hash__ == other.__hash__ and self.key == other.key)
-
- def update_metrics(self, row, loss_type):
- self.size += 1
- if loss_type == 0:
- self.sum_error += row[2]
- if row[2] > self.max_tuple_error:
- self.max_tuple_error = row[2]
- else:
- if row[2] != 0:
- self.sum_error += 1
-
- def calc_error(self):
- if self.size != 0:
- self.error = self.sum_error / self.size
- else:
- self.error = 0
- self.e_max = self.max_tuple_error
- self.e_max_upper = self.e_max
- self.e_upper = self.error
-
- def check_constraint(self, top_k, x_size, alpha):
- return self.score >= top_k.min_score and self.size >= x_size / alpha
-
- def make_name(self):
- name = ""
- for attribute in self.attributes:
- name = name + str(attribute) + " && "
- name = name[0: len(name) - 4]
- return name
-
- def calc_bounds(self, w, x_size, loss):
- self.s_upper = self.calc_s_upper()
- self.s_lower = self.calc_s_lower(x_size)
- self.e_upper = self.calc_e_upper()
- self.e_max_upper = self.calc_e_max_upper()
- self.c_upper = self.calc_c_upper(w, x_size, loss)
-
- def check_bounds(self, x_size, alpha, top_k):
- return self.s_upper >= x_size / alpha and self.c_upper >= top_k.min_score
-
- def calc_s_upper(self):
- cur_min = self.parents[0].size
- for parent in self.parents:
- cur_min = min(cur_min, parent.s_upper)
- return cur_min
-
- def calc_e_max_upper(self):
- e_max_min = self.parents[0].e_max_upper
- for parent in self.parents:
- e_max_min = min(e_max_min, parent.e_max_upper)
- return e_max_min
-
- def calc_s_lower(self, x_size):
- size_value = x_size
- for parent in self.parents:
- size_value = size_value - (size_value - parent.s_lower)
- return max(size_value, 1)
-
- def calc_e_upper(self):
- prev_e_uppers = []
- for parent in self.parents:
- prev_e_uppers.append(parent.e_upper)
- return float(min(prev_e_uppers))
-
- def calc_c_upper(self, w, x_size, loss):
- upper_score = w * (self.e_upper / self.s_lower) / (loss / x_size) + (
- 1 - w) * self.s_upper
- return float(upper_score)
-
- def print_debug(self, topk):
- print("new node has been created: " + self.make_name() + "\n")
- print("s_upper = " + str(self.s_upper))
- print("s_lower = " + str(self.s_lower))
- print("e_upper = " + str(self.e_upper))
- print("c_upper = " + str(self.c_upper))
- print("current topk min score = " + str(topk.min_score))
- print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/SparkNode.py b/scripts/staging/slicing/base/SparkNode.py
deleted file mode 100644
index 81bfcb9..0000000
--- a/scripts/staging/slicing/base/SparkNode.py
+++ /dev/null
@@ -1,167 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class SparkNode:
- error: float
- name: ""
- attributes: []
- parents: []
- children: []
- size: int
- loss: float
- score: float
- e_max: float
- s_upper: int
- s_lower: int
- e_upper: float
- c_upper: float
- e_max_upper: float
- key: ""
-
- def __init__(self, loss, preds):
- if loss:
- self.error = loss,
- if preds:
- self.preds = preds
- self.parents = []
- self.attributes = []
- self.size = 0
- self.score = 0
- self.loss = 0
- self.s_upper = 0
- self.s_lower = 1
- self.key = ''
-
- def calc_c_upper(self, w):
- upper_score = w * (self.e_upper / self.s_lower) / (self.error[0] / self.preds[0][0].size) + (1 - w) * self.s_upper
- return float(upper_score)
-
- def make_slice_mask(self):
- mask = []
- for feature in self.attributes:
- mask.append(feature)
- return mask
-
- def process_slice(self, loss_type):
- mask = self.make_slice_mask()
- if loss_type == 0:
- self.calc_l2(mask)
- elif loss_type == 1:
- self.calc_class(mask)
-
- def calc_class(self, mask):
- self.e_max = 1
- filtered = self.filter_by_mask(mask)
- self.size = len(filtered)
- mistakes = len(list(filter(lambda row: row[1] == 0, filtered)))
- if self.size != 0:
- self.loss = mistakes / self.size
- else:
- self.loss = 0
- self.e_upper = self.loss
-
- def calc_l2(self, mask):
- max_tuple_error = 0
- sum_error = 0
- filtered = self.filter_by_mask(mask)
- self.size = len(filtered)
- for row in filtered:
- if row[1] > max_tuple_error:
- max_tuple_error = row[1]
- sum_error += row[1]
- self.e_max = max_tuple_error
- self.e_upper = max_tuple_error
- self.e_max_upper = max_tuple_error
- if self.size != 0:
- self.loss = sum_error/self.size
- else:
- self.loss = 0
-
- def filter_by_mask(self, mask):
- return list(filter(lambda row: all(attr in row[0].indices for attr in mask), self.preds))
-
- def calc_s_upper(self, cur_lvl):
- cur_min = self.parents[0].size
- for parent in self.parents:
- if cur_lvl == 1:
- cur_min = min(cur_min, parent.size)
- else:
- cur_min = min(cur_min, parent.s_upper)
- return cur_min
-
- def calc_e_max_upper(self, cur_lvl):
- if cur_lvl == 1:
- e_max_min = self.parents[0].e_max
- else:
- e_max_min = self.parents[0].e_max_upper
- for parent in self.parents:
- if cur_lvl == 1:
- e_max_min = min(e_max_min, parent.e_max)
- else:
- e_max_min = min(e_max_min, parent.e_max_upper)
- return e_max_min
-
- def calc_s_lower(self):
- size_value = len(self.preds)
- for parent in self.parents:
- size_value = size_value - (size_value - parent.s_lower)
- return max(size_value, 1)
-
- def calc_e_upper(self):
- prev_e_uppers = []
- for parent in self.parents:
- prev_e_uppers.append(parent.e_upper)
- return float(min(prev_e_uppers))
-
- def calc_bounds(self, cur_lvl, w):
- self.s_upper = self.calc_s_upper(cur_lvl)
- self.s_lower = self.calc_s_lower()
- self.e_upper = self.calc_e_upper()
- self.e_max_upper = self.calc_e_max_upper(cur_lvl)
- self.c_upper = self.calc_c_upper(w)
-
- def make_name(self):
- name = ""
- for attribute in self.attributes:
- name = name + str(attribute) + " && "
- name = name[0: len(name) - 4]
- return name
-
- def make_key(self):
- return self.name
-
- def check_constraint(self, top_k, x_size, alpha, cur_min):
- return self.score >= cur_min and self.size >= x_size / alpha
-
- def check_bounds(self, top_k, x_size, alpha):
- return self.s_upper >= x_size / alpha and self.c_upper >= top_k.min_score
-
- def print_debug(self, topk, level):
- print("new node has been created: " + self.make_name() + "\n")
- if level >= 1:
- print("s_upper = " + str(self.s_upper))
- print("s_lower = " + str(self.s_lower))
- print("e_upper = " + str(self.e_upper))
- print("c_upper = " + str(self.c_upper))
- print("size = " + str(self.size))
- print("score = " + str(self.score))
- print("current topk min score = " + str(topk.min_score))
- print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/__init__.py b/scripts/staging/slicing/base/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/base/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/base/node.py b/scripts/staging/slicing/base/node.py
deleted file mode 100644
index f7b553b..0000000
--- a/scripts/staging/slicing/base/node.py
+++ /dev/null
@@ -1,210 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-
-class Node:
- error: float
- name: ""
- attributes: []
- parents: []
- children: []
- size: int
- loss: float
- score: float
- e_max: float
- s_upper: int
- s_lower: int
- e_upper: float
- c_upper: float
- e_max_upper: float
- key: ""
-
- def __init__(self, complete_x, loss, x_size, y_test, preds):
- self.error = loss,
- self.parents = []
- self.attributes = []
- self.size = 0
- self.s_upper = 0
- self.s_lower = 0
- self.score = 0
- self.complete_x = complete_x
- self.loss = 0
- self.e_upper = 0
- self.e_max_upper = 0
- self.c_upper = 0
- self.x_size = x_size
- self.preds = preds
- self.s_lower = 1
- self.y_test = y_test
- self.key = ''
-
- def calc_c_upper(self, w):
- upper_score = w * (self.e_upper / self.s_lower) / (float(self.error[0]) / self.x_size) + (1 - w) * self.s_upper
- return float(upper_score)
-
- def make_slice_mask(self):
- mask = []
- for feature in self.attributes:
- mask.append(feature[1])
- return mask
-
- def process_slice(self, loss_type):
- mask = self.make_slice_mask()
- if loss_type == 0:
- self.calc_l2(mask)
- if loss_type == 1:
- self.calc_class(mask)
-
- def calc_class(self, mask):
- self.e_max = 1
- mistakes = 0
- filtered = self.filter_by_mask(mask)
- for row in filtered:
- if self.y_test[row[0]][1] != self.preds[row[0]][1]:
- mistakes = mistakes + 1
- self.size = len(filtered)
- if self.size != 0:
- self.loss = mistakes / self.size
- else:
- self.loss = 0
- self.e_upper = self.loss
-
- def calc_l2(self, mask):
- filtered = self.filter_by_mask(mask)
- max_tuple_error = 0
- sum_error = 0
- size = len(filtered)
- for row in filtered:
- if float(self.preds[row[0]][1]) > max_tuple_error:
- max_tuple_error = float(self.preds[row[0]][1])
- sum_error += float(self.preds[row[0]][1])
- self.e_max = max_tuple_error
- self.e_upper = max_tuple_error
- if size != 0:
- self.loss = sum_error/size
- else:
- self.loss = 0
- self.size = size
-
- def filter_by_mask(self, mask):
- return list(filter(lambda row: all(row[1][attr] == 1 for attr in mask), self.complete_x))
-
- def calc_s_upper(self, cur_lvl):
- if cur_lvl == 1:
- cur_min = self.parents[0].size
- for parent in self.parents:
- cur_min = min(cur_min, parent.size)
- else:
- cur_min = self.parents[0].s_upper
- for parent in self.parents:
- cur_min = min(cur_min, parent.s_upper)
- return cur_min
-
- def calc_e_max_upper(self, cur_lvl):
- if cur_lvl == 1:
- e_max_min = self.parents[0].e_max
- else:
- e_max_min = self.parents[0].e_max_upper
- for parent in self.parents:
- if cur_lvl == 1:
- e_max_min = min(e_max_min, parent.e_max)
- else:
- e_max_min = min(e_max_min, parent.e_max_upper)
- return e_max_min
-
- def calc_s_lower(self, cur_lvl):
- size_value = self.x_size
- for parent in self.parents:
- size_value = size_value - (self.x_size - parent.s_lower)
- return max(size_value, 1)
-
- def calc_e_upper(self):
- prev_e_uppers = []
- for parent in self.parents:
- prev_e_uppers.append(parent.e_upper)
- return float(min(prev_e_uppers))
-
- def calc_bounds(self, cur_lvl, w):
- self.s_upper = self.calc_s_upper(cur_lvl)
- self.s_lower = self.calc_s_lower(cur_lvl)
- self.e_upper = self.calc_e_upper()
- self.e_max_upper = self.calc_e_max_upper(cur_lvl)
- self.c_upper = self.calc_c_upper(w)
-
- def make_name(self):
- name = ""
- for attribute in self.attributes:
- name = name + str(attribute[0]) + " && "
- name = name[0: len(name) - 4]
- return name
-
- def make_key(self, new_id):
- return new_id, self.name
-
- def check_constraint(self, top_k, x_size, alpha, cur_min):
- return self.score >= cur_min and self.size >= x_size / alpha
-
- def check_bounds(self, x_size, alpha, cur_min):
- return self.s_upper >= x_size / alpha and self.c_upper >= cur_min
-
- def update_bounds(self, s_upper, s_lower, e_upper, e_max_upper, w):
- if self.s_upper:
- minimized = min(s_upper, self.s_upper)
- self.s_upper = minimized
- else:
- self.s_upper = s_upper
-
- if self.s_lower:
- minimized = min(s_lower, self.s_lower)
- self.s_lower = minimized
- else:
- self.s_lower = s_lower
-
- if self.e_upper:
- minimized = min(e_upper, self.e_upper)
- self.e_upper = minimized
- else:
- self.e_upper = e_upper
-
- if self.e_max_upper:
- minimized = min(e_max_upper, self.e_max_upper)
- self.e_max_upper = minimized
- else:
- self.e_max_upper = e_max_upper
-
- c_upper = self.calc_c_upper(w)
- if self.c_upper:
- minimized = min(c_upper, self.c_upper)
- self.c_upper = minimized
- else:
- self.c_upper = c_upper
-
- def print_debug(self, topk, level):
- print("new node has been created: " + self.make_name() + "\n")
- if level >= 1:
- print("s_upper = " + str(self.s_upper))
- print("s_lower = " + str(self.s_lower))
- print("e_upper = " + str(self.e_upper))
- print("c_upper = " + str(self.c_upper))
- print("size = " + str(self.size))
- print("score = " + str(self.score))
- print("current topk min score = " + str(topk.min_score))
- print("-------------------------------------------------------------------------------------")
diff --git a/scripts/staging/slicing/base/slicer.py b/scripts/staging/slicing/base/slicer.py
deleted file mode 100644
index 77851b6..0000000
--- a/scripts/staging/slicing/base/slicer.py
+++ /dev/null
@@ -1,185 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-import matplotlib.pyplot as plt
-
-
-# optimization function calculation:
-# fi - error of subset, si - its size
-# f - error of complete set, x_size - its size
-# w - "significance" weight of error constraint
-def opt_fun(fi, si, f, x_size, w):
- formula = w * (fi/f) + (1 - w) * (si/x_size)
- return float(formula)
-
-
-# slice_name_nonsense function defines if combination of nodes on current level is fine or impossible:
-# there is dependency between common nodes' attributes number and current level is such that:
-# commons == cur_lvl - 1
-# valid combination example: node ABC + node BCD (on 4th level) // three attributes nodes have two common attributes
-# invalid combination example: node ABC + CDE (on 4th level) // result node - ABCDE (absurd for 4th level)
-def slice_name_nonsense(node_i, node_j, cur_lvl):
- attr1 = list(map(lambda x: x[0].split("_")[0], node_i.attributes))
- key1 = node_i.key[0]
- attr2 = list(map(lambda x: x[0].split("_")[0], node_j.attributes))
- key2 = node_j.key[0]
- commons = len(list(set(attr1) & set(attr2)))
- return commons == cur_lvl - 1 and key1 < key2
-
-
-def union(lst1, lst2):
- final_list = sorted(set(list(lst1) + list(lst2)))
- return final_list
-
-
-def make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, top_k, alpha, w):
- first_level = {}
- counter = 0
- all_nodes = {}
- # First level slices are enumerated in a "classic way" (getting data and not analyzing bounds
- for feature in all_features:
- new_node = Node(complete_x, loss, x_size, y_test, errors)
- new_node.parents = [(feature, counter)]
- new_node.attributes.append((feature, counter))
- new_node.name = new_node.make_name()
- new_id = len(all_nodes)
- new_node.key = new_node.make_key(new_id)
- all_nodes[new_node.key] = new_node
- new_node.process_slice(loss_type)
- new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
- new_node.c_upper = new_node.score
- first_level[new_node.key] = new_node
- new_node.print_debug(top_k, 0)
- # constraints for 1st level nodes to be problematic candidates
- if new_node.check_constraint(top_k, x_size, alpha, 1):
- # this method updates top k slices if needed
- top_k.add_new_top_slice(new_node)
- counter = counter + 1
- return first_level, all_nodes
-
-
-def join_enum(node_i, prev_lvl, complete_x, loss, x_size, y_test, errors, debug, alpha, w, loss_type, b_update, cur_lvl,
- all_nodes, top_k, cur_lvl_nodes):
- for node_j in prev_lvl:
- flag = slice_name_nonsense(prev_lvl[node_i], prev_lvl[node_j], cur_lvl)
- required_number = len(union(prev_lvl[node_i].attributes, prev_lvl[node_j].attributes))
- if flag and required_number == cur_lvl + 1:
- new_node = Node(complete_x, loss, x_size, y_test, errors)
- parents_set = set(new_node.parents)
- parents_set.add(prev_lvl[node_i])
- parents_set.add(prev_lvl[node_j])
- new_node.parents = list(parents_set)
- parent1_attr = prev_lvl[node_i].attributes
- parent2_attr = prev_lvl[node_j].attributes
- new_node_attr = union(parent1_attr, parent2_attr)
- new_node.attributes = new_node_attr
- new_node.name = new_node.make_name()
- new_id = len(all_nodes)
- new_node.key = new_node.make_key(new_id)
- if new_node.key[1] in cur_lvl_nodes:
- existing_item = cur_lvl_nodes[new_node.key[1]]
- parents_set = set(existing_item.parents)
- existing_item.parents = parents_set
- if b_update:
- s_upper = new_node.calc_s_upper(cur_lvl)
- s_lower = new_node.calc_s_lower(cur_lvl)
- e_upper = new_node.calc_e_upper()
- e_max_upper = new_node.calc_e_max_upper(cur_lvl)
- new_node.update_bounds(s_upper, s_lower, e_upper, e_max_upper, w)
- else:
- new_node.calc_bounds(cur_lvl, w)
- all_nodes[new_node.key[0]] = new_node
- # check if concrete data should be extracted or not (only for those that have score upper
- # big enough and if size of subset is big enough
- to_slice = new_node.check_bounds(x_size, alpha, top_k.min_score)
- if to_slice:
- new_node.process_slice(loss_type)
- new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
- # we decide to add node to current level nodes (in order to make new combinations
- # on the next one or not basing on its score value
- if new_node.check_constraint(top_k, x_size, alpha, top_k.min_score) and new_node.key not in top_k.keys:
- top_k.add_new_top_slice(new_node)
- cur_lvl_nodes[new_node.key[1]] = new_node
- if debug:
- new_node.print_debug(top_k, cur_lvl)
- return cur_lvl_nodes, all_nodes
-
-
-# alpha is size significance coefficient (required for optimization function)
-# verbose option is for returning debug info while creating slices and printing it (in console)
-# k is number of top-slices we want to receive as a result (maximum output, if less all of subsets will be printed)
-# w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-# loss_type = 0 (in case of regression model)
-# loss_type = 1 (in case of classification model)
-def process(all_features, complete_x, loss, x_size, y_test, errors, debug, alpha, k, w, loss_type, b_update):
- levels = []
- top_k = Topk(k)
- first_level = make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, top_k, alpha, w)
- candidates = []
- pruned = []
- indexes = []
- indexes.append(1)
- candidates.append(len(first_level[0]))
- pruned.append(len(first_level[0]))
- all_nodes = first_level[1]
- levels.append(first_level[0])
- # cur_lvl - index of current level, correlates with number of slice forming features
- cur_lvl = 1 # currently filled level after first init iteration
- print()
- print("Current topk are: ")
- top_k.print_topk()
- # combining each candidate of previous level with every till it becomes useless (one node can't make a pair)
- while len(levels[cur_lvl - 1]) > 1:
- cur_lvl_nodes = {}
- prev_lvl = levels[cur_lvl - 1]
- level_candidates = len(prev_lvl) * (len(prev_lvl) - 1)
- candidates.append(level_candidates)
- for node_i in prev_lvl:
- partial = join_enum(node_i, prev_lvl, complete_x, loss, x_size, y_test, errors, debug, alpha, w, loss_type,
- b_update, cur_lvl, all_nodes, top_k, cur_lvl_nodes)
- cur_lvl_nodes = partial[0]
- all_nodes = partial[1]
- cur_lvl = cur_lvl + 1
- indexes.append(cur_lvl)
- levels.append(cur_lvl_nodes)
- print("Level " + str(cur_lvl) + " had " + str(candidates) +
- " candidates but after pruning only " + str(len(cur_lvl_nodes)) + " go to the next level")
- pruned.append(len(cur_lvl_nodes))
- print()
- print("Current topk are: ")
- top_k.print_topk()
- plt.plot(indexes, candidates, 'r--',
- indexes, pruned, 'g--')
- plt.xlabel('Level')
- plt.ylabel('Number of slices')
- plt.show()
- print("Program stopped at level " + str(cur_lvl))
- print()
- print("Selected slices are: ")
- top_k.print_topk()
- print("candidates:")
- print(candidates)
- print(">>>>>>>>>")
- print("pruned:")
- print(pruned)
- return top_k
diff --git a/scripts/staging/slicing/base/top_k.py b/scripts/staging/slicing/base/top_k.py
deleted file mode 100644
index 38c6545..0000000
--- a/scripts/staging/slicing/base/top_k.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-class Topk:
- k: int
- min_score: float
- slices: []
- keys: []
-
- def __init__(self, k):
- self.k = k
- self.slices = []
- self.keys = []
- self.min_score = 1
-
- def top_k_min_score(self):
- self.slices.sort(key=lambda x: x.score, reverse=True)
- self.min_score = self.slices[len(self.slices) - 1].score
- return self.min_score
-
- def add_new_top_slice(self, new_top_slice):
- self.min_score = new_top_slice.score
- if len(self.slices) < self.k:
- self.slices.append(new_top_slice)
- self.keys.append(new_top_slice.key)
- return self.top_k_min_score()
- else:
- self.slices[len(self.slices) - 1] = new_top_slice
- self.keys[len(self.slices) - 1] = new_top_slice.key
- return self.top_k_min_score()
-
- def print_topk(self):
- for candidate in self.slices:
- print(candidate.name + ": " + "score = " + str(candidate.score) + "; size = " + str(candidate.size))
-
- def buckets_top_k(self, cur_lvl_slices, x_size, alpha, cur_min):
- for bucket in cur_lvl_slices:
- if bucket.size > x_size / alpha and bucket.score >= cur_min:
- self.add_new_top_slice(bucket)
- return self
diff --git a/scripts/staging/slicing/base/union_slicer.py b/scripts/staging/slicing/base/union_slicer.py
deleted file mode 100644
index 8ec64d5..0000000
--- a/scripts/staging/slicing/base/union_slicer.py
+++ /dev/null
@@ -1,162 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-from slicing.base.slicer import opt_fun, union
-import matplotlib.pyplot as plt
-
-
-def check_attributes(left_node, right_node):
- attr1 = list(map(lambda x: x[0].split("_")[0], left_node.attributes))
- attr2 = list(map(lambda x: x[0].split("_")[0], right_node.attributes))
- if set(attr1).intersection(set(attr2)):
- return False
- return True
-
-
-def make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, w, alpha, top_k):
- all_nodes = {}
- counter = 0
- first_level = []
- for feature in all_features:
- new_node = Node(complete_x, loss, x_size, y_test, errors)
- new_node.parents = [(feature, counter)]
- new_node.attributes.append((feature, counter))
- new_node.name = new_node.make_name()
- new_id = len(all_nodes)
- new_node.key = new_node.make_key(new_id)
- all_nodes[new_node.key] = new_node
- new_node.process_slice(loss_type)
- # for first level nodes all bounds are strict as concrete metrics
- new_node.s_upper = new_node.size
- new_node.s_lower = 0
- new_node.e_upper = new_node.loss
- new_node.e_max_upper = new_node.e_max
- new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
- new_node.c_upper = new_node.score
- first_level.append(new_node)
- new_node.print_debug(top_k, 0)
- # constraints for 1st level nodes to be problematic candidates
- if new_node.score > 1 and new_node.size >= x_size / alpha:
- # this method updates top k slices if needed
- top_k.add_new_top_slice(new_node)
- counter = counter + 1
- return first_level, all_nodes
-
-
-def process(all_features, complete_x, loss, x_size, y_test, errors, debug, alpha, k, w, loss_type, b_update):
- top_k = Topk(k)
- # First level slices are enumerated in a "classic way" (getting data and not analyzing bounds
- levels = []
- first_level = make_first_level(all_features, complete_x, loss, x_size, y_test, errors, loss_type, w, alpha, top_k)
- # double appending of first level nodes in order to enumerating second level in the same way as others
- levels.append((first_level[0], len(all_features)))
- candidates = []
- pruned = []
- indexes = []
- indexes.append(1)
- candidates.append(len(first_level[0]))
- pruned.append(len(first_level[0]))
- all_nodes = first_level[1]
- # cur_lvl - index of current level, correlates with number of slice forming features
- cur_lvl = 1 # level that is planned to be filled later
- cur_lvl_nodes = first_level
- # currently for debug
- print("Level 1 had " + str(len(all_features)) + " candidates")
- print()
- print("Current topk are: ")
- top_k.print_topk()
- # DPSize algorithm approach of previous levels nodes combinations and updating bounds for those that already exist
- while len(cur_lvl_nodes) > 0:
- cur_lvl_nodes = []
- count = 0
- prev_lvl = levels[cur_lvl - 1]
- for left in range(int(cur_lvl / 2) + 1):
- right = cur_lvl - 1 - left
- for node_i in range(len(levels[left][0])):
- for node_j in range(len(levels[right][0])):
- flag = check_attributes(levels[left][0][node_i], levels[right][0][node_j])
- required_number = len(union(levels[left][0][node_i].attributes, levels[right][0][node_j].attributes))
- if flag and required_number == cur_lvl + 1:
- new_node = Node(complete_x, loss, x_size, y_test, errors)
- parents_set = set(new_node.parents)
- parents_set.add(levels[left][0][node_i])
- parents_set.add(levels[right][0][node_j])
- new_node.parents = list(parents_set)
- parent1_attr = levels[left][0][node_i].attributes
- parent2_attr = levels[right][0][node_j].attributes
- new_node_attr = union(parent1_attr, parent2_attr)
- new_node.attributes = new_node_attr
- new_node.name = new_node.make_name()
- new_id = len(all_nodes)
- new_node.key = new_node.make_key(new_id)
- if new_node.key[1] in all_nodes:
- existing_item = all_nodes[new_node.key[1]]
- parents_set = set(existing_item.parents)
- existing_item.parents = parents_set
- if b_update:
- s_upper = new_node.calc_s_upper(cur_lvl)
- s_lower = new_node.calc_s_lower(cur_lvl)
- e_upper = new_node.calc_e_upper()
- e_max_upper = new_node.calc_e_max_upper(cur_lvl)
- new_node.update_bounds(s_upper, s_lower, e_upper, e_max_upper, w)
- else:
- new_node.calc_bounds(cur_lvl, w)
- all_nodes[new_node.key[1]] = new_node
- # check if concrete data should be extracted or not (only for those that have score upper
- # big enough and if size of subset is big enough
- to_slice = new_node.check_bounds(x_size, alpha, top_k.min_score)
- if to_slice:
- new_node.process_slice(loss_type)
- new_node.score = opt_fun(new_node.loss, new_node.size, loss, x_size, w)
- # we decide to add node to current level nodes (in order to make new combinations
- # on the next one or not basing on its score value
- if new_node.check_constraint(top_k, x_size, alpha, top_k.min_score) and new_node.key \
- not in top_k.keys:
- top_k.add_new_top_slice(new_node)
- cur_lvl_nodes.append(new_node)
- if debug:
- new_node.print_debug(top_k, cur_lvl)
- count = count + levels[left][1] * levels[right][1]
- cur_lvl = cur_lvl + 1
- indexes.append(cur_lvl)
- print("Level " + str(cur_lvl) + " had " + str(count) +
- " candidates but after pruning only " + str(len(cur_lvl_nodes)) + " go to the next level")
- levels.append((cur_lvl_nodes, count))
- candidates.append(count)
- pruned.append(len(cur_lvl_nodes))
- top_k.print_topk()
- plt.plot(indexes, candidates, 'r--',
- indexes, pruned, 'g--')
- plt.xlabel('Level')
- plt.ylabel('Number of slices')
- plt.show()
- print("Program stopped at level " + str(cur_lvl))
- print()
- print("Selected slices are: ")
- top_k.print_topk()
- print("candidates:")
- print(candidates)
- print(">>>>>>>>>")
- print("pruned:")
- print(pruned)
- return top_k
diff --git a/scripts/staging/slicing/slicing.dml b/scripts/staging/slicing/slicing.dml
deleted file mode 100644
index 8faf416..0000000
--- a/scripts/staging/slicing/slicing.dml
+++ /dev/null
@@ -1,311 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-#-------------------------------------------------------------
-# X Input matrix (integer encoded [1..v])
-# e error vector (classification accuracy, l2 norm, etc)
-# k top-K subsets / slices
-# minSup minimum support (min number of rows per slice)
-# alpha weight [0,1]: 0 only size, 1 only error
-# dpEval flag for data-parallel slice evaluation,
-# otherwise task-parallel
-# verbose flag for verbose debug output
-# ------------------------------------------------------------
-# TK top-k slices (k x ncol(X) if successful)
-# TKC score, size, error of slices (k x 3)
-# ------------------------------------------------------------
-
-slicing = function(Matrix[Double] X, Matrix[Double] e, Integer k = 4, Integer minSup = 32, Double alpha = 0.5, Boolean dpEval = FALSE, Boolean verbose = FALSE)
- return(Matrix[Double] TK, Matrix[Double] TKC)
-{
- m = nrow(X);
- n = ncol(X);
-
- # prepare offset vectors and one-hot encoded X
- fdom = colMaxs(X);
- foffb = t(cumsum(t(fdom))) - fdom;
- foffe = t(cumsum(t(fdom)))
- rix = matrix(seq(1,m)%*%matrix(1,1,n), m*n, 1)
- cix = matrix(X + foffb, m*n, 1);
- X2 = table(rix, cix); #one-hot encoded
-
- # initialize statistics and basic slices
- n2 = ncol(X2); # one-hot encoded features
- eAvg = sum(e) / m; # average error
- [S, R] = createAndScoreBasicSlices(X2, e, eAvg, minSup, alpha, verbose);
-
- # initialize top-k
- [TK, TKC] = maintainTopK(S, R, matrix(0, 0, n2), matrix(0, 0, 3), k, minSup);
-
- if( verbose ) {
- [maxsc, minsc] = analyzeTopK(TKC);
- print("SliceFinder: initial top-K: count="+nrow(TK)+", max="+maxsc+", min="+minsc)
- }
-
- # lattice enumeration w/ size/error pruning, one iteration per level
- # termination condition (max #feature levels)
- level = 1;
- while( nrow(S) > 0 & sum(S) > 0 & level < n ) {
- level = level + 1;
-
- # enumerate candidate join pairs, incl size/error pruning
- nrS = nrow(S);
- S = getPairedCandidates(S, R, TK, TKC, k, level, eAvg, minSup, alpha, n2, foffb, foffe);
-
- if(verbose) {
- print("\nSliceFinder: level "+level+":")
- print(" -- generated paired slice candidates: "+nrS+" -> "+nrow(S));
- }
-
- # extract and evaluate candidate slices
- if( dpEval ) { #data-parallel
- R = evalSlice(X2, e, eAvg, t(S), level, alpha);
- }
- else { # task-parallel
- R = matrix(0, nrow(S), 3)
- parfor( i in 1:nrow(S) )
- R[i,] = evalSlice(X2, e, eAvg, t(S[i,]), level, alpha);
- }
-
- # maintain top-k after evaluation
- [TK, TKC] = maintainTopK(S, R, TK, TKC, k, minSup);
-
- if(verbose) {
- [maxsc, minsc] = analyzeTopK(TKC);
- valid = as.integer(sum(R[,3]>=minSup));
- print(" -- valid slices after eval: "+valid+"/"+nrow(S));
- print(" -- top-K: count="+nrow(TK)+", max="+maxsc+", min="+minsc);
- }
- }
-
- TK = decodeTopK(TK, foffb, foffe);
-
- if( verbose ) {
- print("SliceFinder: terminated at level "+level+":\n"
- + toString(TK) + "\n" + toString(TKC));
- }
-}
-
-createAndScoreBasicSlices = function(Matrix[Double] X2, Matrix[Double] e, Double eAvg,
- Double minSup, Double alpha, Boolean verbose)
- return(Matrix[Double] S, Matrix[Double] R)
-{
- n2 = ncol(X2);
- cCnts = t(colSums(X2)); # column counts
- err = t(t(e) %*% X2) # total error vector
-
- if( verbose ) {
- drop = as.integer(sum(cCnts < minSup));
- print("SliceFinder: dropping "+drop+"/"+n2+" features below minSup = "+minSup+".");
- }
-
- # working set of active slices (#attr x #slices) and top k
- selCols = (cCnts >= minSup);
- attr = removeEmpty(target=seq(1,n2), margin="rows", select=selCols);
- ss = removeEmpty(target=cCnts, margin="rows", select=selCols);
- se = removeEmpty(target=err, margin="rows", select=selCols);
- S = table(seq(1,nrow(attr)), attr, nrow(attr), n2);
-
- # score 1-slices and create initial top-k
- sc = score(ss, se, eAvg, alpha, nrow(X2));
- R = cbind(sc, se, ss);
-}
-
-score = function(Matrix[Double] ss, Matrix[Double] se, Double eAvg, Double alpha, Integer n)
- return(Matrix[Double] sc)
-{
- sc = alpha * ((se/ss) / eAvg - 1) - (1-alpha) * (n/ss - 1);
-}
-
-scoreUB = function(Matrix[Double] ss, Matrix[Double] se, Double eAvg, Integer minSup, Double alpha, Integer n)
- return(Matrix[Double] sc)
-{
- sc = alpha * ((se/minSup) / eAvg - 1) - (1-alpha) * (n/ss - 1);
-}
-
-
-maintainTopK = function(Matrix[Double] S, Matrix[Double] R, Matrix[Double] TK, Matrix[Double] TKC, Integer k, Integer minSup)
- return(Matrix[Double] TK, Matrix[Double] TKC)
-{
- # prune invalid minSup and scores
- I = (R[,1] > 0) & (R[,3] >= minSup);
-
- if( sum(I)!=0 ) {
- S = removeEmpty(target=S, margin="rows", select=I);
- R = removeEmpty(target=R, margin="rows", select=I);
-
- # evaluated candidated and previous top-k
- slices = rbind(TK, S);
- scores = rbind(TKC, R);
-
- # extract top-k
- IX = order(target=scores, by=1, decreasing=TRUE, index.return=TRUE);
- IX = IX[1:min(k,nrow(IX)),];
- P = table(seq(1,nrow(IX)), IX, nrow(IX), nrow(slices));
- TK = P %*% slices;
- TKC = P %*% scores;
- }
-}
-
-analyzeTopK = function(Matrix[Double] TKC) return(Double maxsc, Double minsc) {
- maxsc = -Inf;
- minsc = -Inf;
- if( nrow(TKC)>0 ) {
- maxsc = as.scalar(TKC[1,1]);
- minsc = as.scalar(TKC[nrow(TKC),1]);
- }
-}
-
-getPairedCandidates = function(Matrix[Double] S, Matrix[Double] R, Matrix[Double] TK,
- Matrix[Double] TKC, Integer k, Integer level, Double eAvg, Integer minSup, Double alpha,
- Integer n2, Matrix[Double] foffb, Matrix[Double] foffe)
- return(Matrix[Double] P)
-{
- # prune invalid slices (possible without affecting overall
- # pruning effectiveness due to handling of missing parents)
- pI = (R[,3] >= minSup);
- S = removeEmpty(target=S, margin="rows", select=pI)
- R = removeEmpty(target=R, margin="rows", select=pI)
-
- # join compatible slices (without self)
- join = S %*% t(S) == (level-2)
- I = upper.tri(target=join, diag=FALSE, values=TRUE);
-
- # pair construction
- nr = nrow(I); nc = ncol(I);
- rix = matrix(I * seq(1,nr), nr*nc, 1);
- cix = matrix(I * t(seq(1,nc)), nr*nc, 1);
- rix = removeEmpty(target=rix, margin="rows");
- cix = removeEmpty(target=cix, margin="rows");
-
- P = matrix(0,0,ncol(S))
- if( sum(rix)!=0 ) {
- P1 = table(seq(1,nrow(rix)), rix, nrow(rix), nrow(S));
- P2 = table(seq(1,nrow(cix)), cix, nrow(rix), nrow(S));
- P12 = P1 + P2; # combined slice
- P = (P12 %*% S) != 0;
- ss = min(P1 %*% R[,3], P2 %*% R[,3])
- se = min(P1 %*% R[,2], P2 %*% R[,2])
-
- # prune invalid self joins (>1 bit per feature)
- I = matrix(1, nrow(P), 1);
- for( j in 1:ncol(foffb) ) {
- beg = as.scalar(foffb[1,j])+1;
- end = as.scalar(foffe[1,j]);
- I = I & (rowSums(P[,beg:end]) <= 1);
- }
- P12 = removeEmpty(target=P12, margin="rows", select=I)
- P = removeEmpty(target=P, margin="rows", select=I);
- ss = removeEmpty(target=ss, margin="rows", select=I);
- se = removeEmpty(target=se, margin="rows", select=I);
-
- # prepare IDs for deduplication and pruning
- ID = matrix(0, nrow(P), 1);
- dom = foffe-foffb+1;
- for( j in 1:ncol(dom) ) {
- beg = as.scalar(foffb[1,j])+1;
- end = as.scalar(foffe[1,j]);
- I = rowIndexMax(P[,beg:end]) * rowMaxs(P[,beg:end]);
- prod = 1;
- if(j<ncol(dom))
- prod = prod(dom[1,(j+1):ncol(dom)])
- ID = ID + I * prod;
- }
-
- # ID transformation to avoid exceeding INT_MAX and
- # and to void creating huge sparse intermediates
- [ID, M] = transformencode(target=as.frame(ID), spec="{ids:true,recode:[1]}")
-
- # size pruning, with rowMin-rowMax transform
- # to avoid densification (ignored zeros)
- map = table(ID, seq(1,nrow(P)), max(ID), nrow(P))
- ubSizes = 1/rowMaxs(map * (1/t(ss)));
- ubSizes = replace(target=ubSizes, pattern=Inf, replacement=0);
- fSizes = (ubSizes >= minSup)
-
- # error pruning
- ubError = 1/rowMaxs(map * (1/t(se)));
- ubError = replace(target=ubError, pattern=Inf, replacement=0);
- ubScores = scoreUB(ubSizes, ubError, eAvg, minSup, alpha, n2);
- [maxsc, minsc] = analyzeTopK(TKC);
- fScores = (ubScores > minsc & ubScores > 0)
-
- # missing parents pruning
- numParents = rowSums((map %*% P12) != 0)
- fParents = (numParents == level);
-
- # apply all pruning
- map = map * (fSizes & fScores & fParents);
-
- # deduplication of join outputs
- Dedup = removeEmpty(target=map, margin="rows") != 0
- P = (Dedup %*% P) != 0
- }
-}
-
-evalSlice = function(Matrix[Double] X, Matrix[Double] e, Double eAvg, Matrix[Double] tS, Integer l, Double alpha)
- return(Matrix[Double] R)
-{
- I = (X %*% tS) == l; # slice indicator
- ss = t(colSums(I)); # absolute slice size (nnz)
- se = t(t(e) %*% I); # absolute slice error
-
- # score of relative error and relative size
- sc = score(ss, se, eAvg, alpha, nrow(X));
- R = cbind(sc, se, ss);
-}
-
-decodeTopK = function(Matrix[Double] TK, Matrix[Double] foffb, Matrix[Double] foffe)
- return(Matrix[Double] TK)
-{
- R = matrix(1, nrow(TK), ncol(foffb));
- if( nrow(TK) > 0 ) {
- parfor( j in 1:ncol(foffb) ) {
- beg = as.scalar(foffb[1,j])+1;
- end = as.scalar(foffe[1,j]);
- I = rowSums(TK[,beg:end]) * rowIndexMax(TK[,beg:end]);
- R[, j] = I;
- }
- }
- TK = R;
-}
-
-Forig = read("./Salaries.csv", data_type="frame", format="csv", header=TRUE);
-
-F = Forig[,1:ncol(Forig)-1];
-y = as.matrix(Forig[,ncol(Forig)]);
-
-# data preparation
-jspec= "{ ids:true, recode:[1,2,3,6], bin:[{id:4, method:equi-width, numbins:14},{id:5, method:equi-width, numbins:12}]}"
-[X,M] = transformencode(target=F, spec=jspec);
-X = X[,2:ncol(X)]
-
-X = rbind(X,X)
-X = cbind(X,X)
-y = rbind(y,y)
-
-# learn model
-B = lm(X=X, y=y, verbose=FALSE);
-yhat = X %*% B;
-e = (y-yhat)^2;
-
-# call slice finding
-[S,C] = slicing(X=X, e=e, k=10, alpha=0.95, minSup=32, verbose=TRUE);
diff --git a/scripts/staging/slicing/spark_modules/join_data_parallel.py b/scripts/staging/slicing/spark_modules/join_data_parallel.py
deleted file mode 100644
index f55b0ca..0000000
--- a/scripts/staging/slicing/spark_modules/join_data_parallel.py
+++ /dev/null
@@ -1,123 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.Bucket import Bucket
-from slicing.base.SparkNode import SparkNode
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import approved_join_slice
-
-
-def rows_mapper(row, buckets, loss_type):
- # filtered = dict(filter(lambda bucket: all(attr in row[1] for attr in bucket[1].attributes), buckets.items()))
- filtered = dict(filter(lambda bucket: all(attr in row[0] for attr in bucket[1].attributes), buckets.items()))
- for item in filtered:
- filtered[item].update_metrics(row, loss_type)
- return filtered
-
-
-def join_enum(cur_lvl_nodes, cur_lvl, x_size, alpha, top_k, w, loss):
- buckets = {}
- for node_i in range(len(cur_lvl_nodes)):
- for node_j in range(node_i + 1, len(cur_lvl_nodes)):
- flag = approved_join_slice(cur_lvl_nodes[node_i], cur_lvl_nodes[node_j], cur_lvl)
- if flag:
- node = SparkNode(None, None)
- node.attributes = list(set(cur_lvl_nodes[node_i].attributes) | set(cur_lvl_nodes[node_j].attributes))
- bucket = Bucket(node, cur_lvl, w, x_size, loss)
- bucket.parents.append(cur_lvl_nodes[node_i])
- bucket.parents.append(cur_lvl_nodes[node_j])
- bucket.calc_bounds(w, x_size, loss)
- if bucket.check_bounds(x_size, alpha, top_k):
- buckets[bucket.name] = bucket
- return buckets
-
-
-def combiner(a):
- return a
-
-
-def merge_values(a, b):
- a.combine_with(b)
- return a
-
-
-def merge_combiners(a, b):
- a + b
- return a
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type):
- top_k = Topk(k)
- cur_lvl = 0
- cur_lvl_nodes = list(all_features)
- pred_pandas = predictions.toPandas()
- x_size = len(pred_pandas)
- b_topk = SparkContext.broadcast(sc, top_k)
- b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
- buckets = {}
- for node in cur_lvl_nodes:
- bucket = Bucket(node, cur_lvl, w, x_size, loss)
- buckets[bucket.name] = bucket
- b_buckets = SparkContext.broadcast(sc, buckets)
- rows = predictions.rdd.map(lambda row: (row[1].indices, row[2]))\
- .map(lambda item: list(item))
- mapped = rows.map(lambda row: rows_mapper(row, b_buckets.value, loss_type))
- flattened = mapped.flatMap(lambda line: (line.items()))
- reduced = flattened.combineByKey(combiner, merge_values, merge_combiners)
- cur_lvl_nodes = reduced.values()\
- .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket, loss, w, x_size, b_cur_lvl.value))
- if debug:
- cur_lvl_nodes.map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
- cur_lvl = 1
- prev_level = cur_lvl_nodes.collect()
- top_k = top_k.buckets_top_k(prev_level, x_size, alpha, 1)
- while len(prev_level) > 0:
- b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
- b_topk = SparkContext.broadcast(sc, top_k)
- cur_min = top_k.min_score
- b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
- top_k.print_topk()
- buckets = join_enum(prev_level, cur_lvl, x_size, alpha, top_k, w, loss)
- b_buckets = SparkContext.broadcast(sc, buckets)
- to_slice = dict(filter(lambda bucket: bucket[1].check_bounds(x_size, alpha, top_k), buckets.items()))
- b_to_slice = SparkContext.broadcast(sc, to_slice)
- mapped = rows.map(lambda row: rows_mapper(row, b_to_slice.value, loss_type))
- flattened = mapped.flatMap(lambda line: (line.items()))
- to_process = flattened.combineByKey(combiner, merge_values, merge_combiners)
- if debug:
- to_process.values().map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
- prev_level = to_process\
- .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket[1], loss, w, x_size, b_cur_lvl.value))\
- .collect()
- cur_lvl += 1
- top_k = top_k.buckets_top_k(prev_level, x_size, alpha, cur_min)
- print("Level " + str(cur_lvl) + " had " + str(
- len(b_cur_lvl_nodes.value * (len(prev_level) - 1)))+" candidates but after pruning only " +
- str(len(prev_level)) + " go to the next level")
- top_k.print_topk()
- print()
- print("Program stopped at level " + str(cur_lvl - 1))
- print("Selected slices are: ")
- top_k.print_topk()
- return None
diff --git a/scripts/staging/slicing/spark_modules/spark_slicer.py b/scripts/staging/slicing/spark_modules/spark_slicer.py
deleted file mode 100644
index 9da5b98..0000000
--- a/scripts/staging/slicing/spark_modules/spark_slicer.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.SparkNode import SparkNode
-from slicing.base.slicer import union, opt_fun
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import update_top_k
-
-
-def join_enum_fun(node_a, list_b, predictions, f_l2, debug, alpha, w, loss_type, cur_lvl, top_k):
- x_size = len(predictions)
- nodes = []
- for node_i in range(len(list_b)):
- flag = spark_utils.approved_join_slice(node_i, node_a, cur_lvl)
- if not flag:
- new_node = SparkNode(predictions, f_l2)
- parents_set = set(new_node.parents)
- parents_set.add(node_i)
- parents_set.add(node_a)
- new_node.parents = list(parents_set)
- parent1_attr = node_a.attributes
- parent2_attr = list_b[node_i].attributes
- new_node_attr = union(parent1_attr, parent2_attr)
- new_node.attributes = new_node_attr
- new_node.name = new_node.make_name()
- new_node.calc_bounds(cur_lvl, w)
- # check if concrete data should be extracted or not (only for those that have score upper
- # and if size of subset is big enough
- to_slice = new_node.check_bounds(top_k, x_size, alpha)
- if to_slice:
- new_node.process_slice(loss_type)
- new_node.score = opt_fun(new_node.loss, new_node.size, f_l2, x_size, w)
- # we decide to add node to current level nodes (in order to make new combinations
- # on the next one or not basing on its score value
- if new_node.check_constraint(top_k, x_size, alpha) and new_node.key not in top_k.keys:
- top_k.add_new_top_slice(new_node)
- nodes.append(new_node)
- if debug:
- new_node.print_debug(top_k, cur_lvl)
- return nodes
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type, enumerator):
- top_k = Topk(k)
- cur_lvl = 0
- levels = []
- first_level = {}
- all_features = list(all_features)
- first_tasks = sc.parallelize(all_features)
- b_topk = SparkContext.broadcast(sc, top_k)
- init_slices = first_tasks.mapPartitions(lambda features: spark_utils.make_first_level(features, predictions, loss,
- b_topk.value, w, loss_type)) \
- .map(lambda node: (node.key, node)).collect()
- first_level.update(init_slices)
- update_top_k(first_level, top_k, alpha, predictions, 1)
- prev_level = SparkContext.broadcast(sc, first_level)
- levels.append(prev_level)
- cur_lvl = cur_lvl + 1
- top_k.print_topk()
- # checking the first partition of level. if not empty then processing otherwise no elements were added to this level
- while len(levels[cur_lvl - 1].value) > 0:
- nodes_list = {}
- b_topk = SparkContext.broadcast(sc, top_k)
- cur_min = top_k.min_score
- partitions = sc.parallelize(levels[cur_lvl - 1].value.values())
- mapped = partitions.mapPartitions(lambda nodes: spark_utils.nodes_enum(nodes, levels[cur_lvl - 1].value.values(),
- predictions, loss, b_topk.value, alpha, k, w,
- loss_type, cur_lvl, debug, enumerator, cur_min))
- flattened = mapped.flatMap(lambda node: node)
- nodes_list.update(flattened.map(lambda node: (node.key, node)).distinct().collect())
- prev_level = SparkContext.broadcast(sc, nodes_list)
- levels.append(prev_level)
- update_top_k(nodes_list, top_k, alpha, predictions, cur_min)
- cur_lvl = cur_lvl + 1
- b_topk.value.print_topk()
- print("Level " + str(cur_lvl) + " had " + str(len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
- " candidates but after pruning only " + str(len(nodes_list)) + " go to the next level")
- print("Program stopped at level " + str(cur_lvl - 1))
- print()
- print("Selected slices are: ")
- top_k.print_topk()
- return top_k
diff --git a/scripts/staging/slicing/spark_modules/spark_union_slicer.py b/scripts/staging/slicing/spark_modules/spark_union_slicer.py
deleted file mode 100644
index 94172de..0000000
--- a/scripts/staging/slicing/spark_modules/spark_union_slicer.py
+++ /dev/null
@@ -1,73 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-
-from pyspark import SparkContext
-
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils
-from slicing.spark_modules.spark_utils import update_top_k
-
-
-def process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type, enumerator):
- top_k = Topk(k)
- cur_lvl = 0
- levels = []
- all_features = list(all_features)
- first_level = {}
- first_tasks = sc.parallelize(all_features)
- b_topk = SparkContext.broadcast(sc, top_k)
- init_slices = first_tasks.mapPartitions(lambda features: spark_utils.make_first_level(features, predictions, loss,
- b_topk.value, w, loss_type)) \
- .map(lambda node: (node.key, node)) \
- .collect()
- first_level.update(init_slices)
- update_top_k(first_level, top_k, alpha, predictions, 1)
- prev_level = SparkContext.broadcast(sc, first_level)
- levels.append(prev_level)
- cur_lvl = 1
- top_k.print_topk()
- while len(levels[cur_lvl - 1].value) > 0:
- cur_lvl_res = {}
- b_topk = SparkContext.broadcast(sc, top_k)
- cur_min = top_k.min_score
- for left in range(int(cur_lvl / 2) + 1):
- right = cur_lvl - left - 1
- partitions = sc.parallelize(levels[left].value.values())
- mapped = partitions.mapPartitions(lambda nodes: spark_utils.nodes_enum(nodes, levels[right].value.values(),
- predictions, loss, b_topk.value, alpha, k,
- w, loss_type, cur_lvl, debug,
- enumerator, cur_min))
- flattened = mapped.flatMap(lambda node: node)
- partial = flattened.map(lambda node: (node.key, node)).collect()
- cur_lvl_res.update(partial)
- prev_level = SparkContext.broadcast(sc, cur_lvl_res)
- levels.append(prev_level)
- update_top_k(cur_lvl_res, top_k, alpha, predictions, cur_min)
- cur_lvl = cur_lvl + 1
- top_k.print_topk()
- print("Level " + str(cur_lvl) + " had " + str(len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
- " candidates but after pruning only " + str(len(prev_level.value)) + " go to the next level")
- print("Program stopped at level " + str(cur_lvl - 1))
- print()
- print("Selected slices are: ")
- top_k.print_topk()
- return top_k
diff --git a/scripts/staging/slicing/spark_modules/union_data_parallel.py b/scripts/staging/slicing/spark_modules/union_data_parallel.py
deleted file mode 100644
index 5ea70cc..0000000
--- a/scripts/staging/slicing/spark_modules/union_data_parallel.py
+++ /dev/null
@@ -1,124 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkContext
-
-from slicing.base.Bucket import Bucket
-from slicing.base.SparkNode import SparkNode
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils, join_data_parallel
-from slicing.spark_modules.join_data_parallel import rows_mapper, combiner
-from slicing.spark_modules.spark_utils import approved_union_slice
-
-
-def merge_values(a, b):
- a.minimize_bounds(b)
- return a
-
-
-def merge_combiners(a, b):
- a + b
- return a
-
-
-def union_enum(left_level, right_level, x_size, alpha, top_k, w, loss, cur_lvl):
- buckets = {}
- for node_i in range(len(left_level)):
- for node_j in range(len(right_level)):
- flag = approved_union_slice(left_level[node_i], right_level[node_j])
- if flag:
- node = SparkNode(None, None)
- node.attributes = list(set(left_level[node_i].attributes) | set(right_level[node_j].attributes))
- bucket = Bucket(node, cur_lvl, w, x_size, loss)
- bucket.parents.append(left_level[node_i])
- bucket.parents.append(right_level[node_j])
- bucket.calc_bounds(w, x_size, loss)
- if bucket.check_bounds(x_size, alpha, top_k):
- buckets[bucket.name] = bucket
- return buckets
-
-
-def parallel_process(all_features, predictions, loss, sc, debug, alpha, k, w, loss_type):
- top_k = Topk(k)
- cur_lvl = 0
- levels = []
- cur_lvl_nodes = list(all_features)
- pred_pandas = predictions.toPandas()
- x_size = len(pred_pandas)
- b_topk = SparkContext.broadcast(sc, top_k)
- b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
- buckets = {}
- for node in cur_lvl_nodes:
- bucket = Bucket(node, cur_lvl, w, x_size, loss)
- buckets[bucket.name] = bucket
- b_buckets = SparkContext.broadcast(sc, buckets)
- # rows = predictions.rdd.map(lambda row: (row[0], row[1].indices, row[2])) \
- # .map(lambda item: (item[0], item[1].tolist(), item[2]))
- rows = predictions.rdd.map(lambda row: row[1].indices) \
- .map(lambda item: list(item))
- mapped = rows.map(lambda row: rows_mapper(row, b_buckets.value, loss_type))
- flattened = mapped.flatMap(lambda line: (line.items()))
- reduced = flattened.combineByKey(combiner, join_data_parallel.merge_values, join_data_parallel.merge_combiners)
- cur_lvl_nodes = reduced.values() \
- .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket, loss, w, x_size, b_cur_lvl.value))
- if debug:
- cur_lvl_nodes.map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
- cur_lvl = 1
- prev_level = cur_lvl_nodes.collect()
- b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
- levels.append(b_cur_lvl_nodes)
- top_k = top_k.buckets_top_k(prev_level, x_size, alpha, 1)
- while len(prev_level) > 0:
- b_topk = SparkContext.broadcast(sc, top_k)
- cur_min = top_k.min_score
- b_cur_lvl = SparkContext.broadcast(sc, cur_lvl)
- top_k.print_topk()
- buckets = []
- for left in range(int(cur_lvl / 2) + 1):
- right = cur_lvl - left - 1
- nodes = union_enum(levels[left].value, levels[right].value, x_size, alpha, top_k, w, loss, cur_lvl)
- buckets.append(nodes)
- b_buckets = sc.parallelize(buckets)
- all_buckets = b_buckets.flatMap(lambda line: (line.items()))
- combined = dict(all_buckets.combineByKey(combiner, merge_values, merge_combiners).collect())
- b_buckets = SparkContext.broadcast(sc, combined)
- to_slice = dict(filter(lambda bucket: bucket[1].check_bounds(x_size, alpha, top_k), combined.items()))
- b_to_slice = SparkContext.broadcast(sc, to_slice)
- mapped = rows.map(lambda row: rows_mapper(row, b_to_slice.value, loss_type))
- flattened = mapped.flatMap(lambda line: (line.items()))
- partial = flattened.combineByKey(combiner, join_data_parallel.merge_values, join_data_parallel.merge_combiners)
- prev_level = partial\
- .map(lambda bucket: spark_utils.calc_bucket_metrics(bucket[1], loss, w, x_size, b_cur_lvl.value)).collect()
- top_k = top_k.buckets_top_k(prev_level, x_size, alpha, cur_min)
- b_topk = SparkContext.broadcast(sc, top_k)
- if debug:
- partial.values().map(lambda bucket: bucket.print_debug(b_topk.value)).collect()
- print("Level " + str(cur_lvl) + " had " + str(
- len(levels[cur_lvl - 1].value) * (len(levels[cur_lvl - 1].value) - 1)) +
- " candidates but after pruning only " + str(len(prev_level)) + " go to the next level")
- print("Program stopped at level " + str(cur_lvl))
- b_cur_lvl_nodes = SparkContext.broadcast(sc, prev_level)
- levels.append(b_cur_lvl_nodes)
- cur_lvl += 1
- print()
- print("Selected slices are: ")
- top_k.print_topk()
- return None
diff --git a/scripts/staging/slicing/tests/__init__.py b/scripts/staging/slicing/tests/__init__.py
deleted file mode 100644
index e66abb4..0000000
--- a/scripts/staging/slicing/tests/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# -------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# -------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/classification/__init__.py b/scripts/staging/slicing/tests/classification/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/tests/classification/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/classification/compas_bd.py b/scripts/staging/slicing/tests/classification/compas_bd.py
deleted file mode 100644
index 3f1a5b3..0000000
--- a/scripts/staging/slicing/tests/classification/compas_bd.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, join_data_parallel, union_data_parallel
-
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- dataset = args[8]
- else:
- k = 10
- w = 0.1
- alpha = 4
- b_update = True
- debug = True
- loss_type = 0
- dataset = 'compas-scores-two-years.csv'
- enumerator = "union"
-
- conf = SparkConf().setAppName("salary_test").setMaster('local[8]')
- num_partitions = 8
- model_type = "classification"
- label = "is_recid"
- sparkContext = SparkContext(conf=conf)
- sqlContext = SQLContext(sparkContext)
- dataset_df = sqlContext.read\
- .option("badRecordsPath", "/tmp/badRecordsPath")\
- .csv(dataset, header='true',
- inferSchema='true')
- # fileRDD = sparkContext.textFile(dataset, num_partitions)
- # header = fileRDD.first()
- # head_split = header.split(",")
- # fileRDD = fileRDD.filter(lambda line: line != header)
- # data = fileRDD.map(lambda row: row.split(","))
- # dataset_df = sqlContext.createDataFrame(data, head_split)
-
- cat_features = ["sex", "age_cat", "race", "decile_score10", "c_charge_degree", "c_charge_desc"]
- stages = []
- dataset_df = dataset_df.withColumn("id", sf.monotonically_increasing_id())
- dataset_df = dataset_df.withColumn('target', dataset_df[label].cast("int"))
- dataset_df = dataset_df.withColumn('model_type', sf.lit(1))
- for feature in cat_features:
- string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
- encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
- encoder.setDropLast(False)
- stages += [string_indexer, encoder]
- assembler_inputs = [feature + "_vec" for feature in cat_features]
- assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
- stages += [assembler]
- assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
- stages += [assembler_final]
-
- pipeline = Pipeline(stages=stages)
- pipeline_model = pipeline.fit(dataset_df)
- dataset_transformed = pipeline_model.transform(dataset_df)
- df_transform_fin = dataset_transformed.select('id', 'features', 'target', 'model_type').toPandas()
-
- cat = 0
- counter = 0
- decode_dict = {}
- for feature in cat_features:
- colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
- colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
- for item in colIdx:
- decode_dict[counter] = (cat, item, colIdx[item])
- counter = counter + 1
- cat = cat + 1
-
- train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
- train_df = sqlContext.createDataFrame(train)
- test_df = sqlContext.createDataFrame(test)
-
- rf = RandomForestClassifier(featuresCol='features', labelCol="target", numTrees=10)
- rf_model = rf.fit(train_df)
- predictions = rf_model.transform(test_df)
- # Select example rows to display.
- predictions.select("id", "features", "target", "prediction", "model_type")
- # Select (prediction, true label) and compute test error
- evaluator = MulticlassClassificationEvaluator(
- labelCol="target", predictionCol="prediction", metricName="accuracy")
- accuracy = evaluator.evaluate(predictions)
- loss = 1.0 - accuracy
- pred_df_fin = predictions.withColumn('error',
- spark_utils.calc_loss(predictions["target"], predictions['prediction'],
- predictions['model_type']))
- predictions = pred_df_fin.select('id', 'features', 'error').repartition(num_partitions)
- all_features = range(predictions.toPandas().values[0][0])
-
- if enumerator == "join":
- join_data_parallel.parallel_process(all_features, predictions, loss, sparkContext, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type)
- elif enumerator == "union":
- union_data_parallel.parallel_process(all_features, predictions, loss, sparkContext, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type)
diff --git a/scripts/staging/slicing/tests/classification/sparked_adults.py b/scripts/staging/slicing/tests/classification/sparked_adults.py
deleted file mode 100644
index 9c37fef..0000000
--- a/scripts/staging/slicing/tests/classification/sparked_adults.py
+++ /dev/null
@@ -1,118 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.evaluation import MulticlassClassificationEvaluator
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-
-from slicing.sparked import sparked_utils, sparked_slicer, sparked_union_slicer
-
-
-ten_binner = sf.udf(lambda arg: int(arg / 10))
-fnlwgt_binner = sf.udf(lambda arg: int(arg // 100000))
-edu_binner = sf.udf(lambda arg: int(arg / 5))
-cap_gain_binner = sf.udf(lambda arg: int(arg / 1000))
-
-conf = SparkConf().setAppName("adults_test").setMaster('local[2]')
-num_partitions = 2
-model_type = "classification"
-label = 'Income'
-sparkContext = SparkContext(conf=conf)
-sqlContext = SQLContext(sparkContext)
-dataset_df = sqlContext.read.csv('/slicing/datasets/adult.csv', header='true', inferSchema='true')
-# initializing stages of main transformation pipeline
-stages = []
-dataset_df = dataset_df.withColumn('age_bin', ten_binner(dataset_df['Age']))
-dataset_df = dataset_df.withColumn('fnlwgt_bin', fnlwgt_binner(dataset_df['fnlwgt']))
-dataset_df = dataset_df.withColumn('edu_num_bin', edu_binner(dataset_df['EducationNum']))
-dataset_df = dataset_df.withColumn('cap_gain_bin', cap_gain_binner(dataset_df['CapitalGain']))
-dataset_df = dataset_df.withColumn('hours_per_w_bin', ten_binner(dataset_df['HoursPerWeek']))
-dataset_df = dataset_df.withColumn('model_type', sf.lit(1))
-
-# list of categorical features for further hot-encoding
-cat_features = ["age_bin", "WorkClass", "fnlwgt_bin", "Education", "edu_num_bin",
- "MaritalStatus", "Occupation", "Relationship", "Race", "Gender",
- "cap_gain_bin", "CapitalLoss", "hours_per_w_bin", "NativeCountry"]
-
-# hot encoding categorical features
-for feature in cat_features:
- string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index")
- encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
- encoder.setDropLast(False)
- stages += [string_indexer, encoder]
-assembler_inputs = [feature + "_vec" for feature in cat_features]
-assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
-stages += [assembler]
-assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
-label_indexer = StringIndexer(inputCol=label, outputCol=label+"_idx")
-stages += [assembler_final]
-stages += [label_indexer]
-pipeline = Pipeline(stages=stages)
-pipeline_model = pipeline.fit(dataset_df)
-dataset_transformed = pipeline_model.transform(dataset_df)
-cat_dict = []
-decode_dict = {}
-counter = 0
-cat = 0
-for feature in cat_features:
- colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
- colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
- for item in colIdx:
- decode_dict[counter] = (cat, item, colIdx[item])
- counter = counter + 1
- cat = cat + 1
-df_transform_fin = dataset_transformed.select('features', label+"_idx", 'model_type')
-splits = df_transform_fin.randomSplit([0.8, 0.2], seed=1234)
-train_df = splits[0]
-test_df = splits[1]
-rf = RandomForestClassifier(featuresCol='features', labelCol=label+"_idx", numTrees=10)
-rf_model = rf.fit(train_df)
-predictions = rf_model.transform(test_df)
-# Select example rows to display.
-predictions.select("features", label+"_idx", "prediction", "model_type")
-# Select (prediction, true label) and compute test error
-evaluator = MulticlassClassificationEvaluator(
- labelCol=label+"_idx", predictionCol="prediction", metricName="accuracy")
-accuracy = evaluator.evaluate(predictions)
-loss = 1.0 - accuracy
-pred_df_fin = predictions.withColumn('error', sparked_utils.calc_loss(predictions[label+"_idx"],
- predictions['prediction'],
- predictions['model_type']))
-predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
-all_features = range(predictions.toPandas().values[0][0].size)
-predictions = predictions.collect()
-k = 10
-SparkContext.broadcast(sparkContext, loss)
-SparkContext.broadcast(sparkContext, predictions)
-SparkContext.broadcast(sparkContext, all_features)
-SparkContext.broadcast(sparkContext, decode_dict)
-enumerator = "join"
-if enumerator == "join":
- sparked_slicer.parallel_process(all_features, predictions, loss, sparkContext, debug=True, alpha=6, k=k, w=0.5,
- loss_type=0, enumerator="join")
-elif enumerator == "union":
- sparked_union_slicer.process(all_features, predictions, loss, sparkContext, debug=True, alpha=4, k=k, w=0.5, loss_type=0,
- enumerator="union")
-
diff --git a/scripts/staging/slicing/tests/classification/test_adult.py b/scripts/staging/slicing/tests/classification/test_adult.py
deleted file mode 100644
index cacb715..0000000
--- a/scripts/staging/slicing/tests/classification/test_adult.py
+++ /dev/null
@@ -1,116 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.preprocessing import OneHotEncoder
-from sklearn import svm
-
-from slicing.base import slicer as slicer, union_slicer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn import preprocessing
-from sklearn.model_selection import train_test_split
-
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- else:
- k = 10
- w = 0.5
- alpha = 4
- b_update = True
- debug = True
- loss_type = 1
- enumerator = "union"
- dataset = pd.read_csv('/slicing/datasets/adult.csv')
- attributes_amount = len(dataset.values[0])
- x = dataset.iloc[:, 0:attributes_amount - 1].values
- y = dataset.iloc[:, attributes_amount - 1]
- le = preprocessing.LabelEncoder()
- le.fit(y)
- y = le.transform(y)
- complete_x = []
- complete_y = []
- counter = 0
- all_indexes = []
- not_encoded_columns = [
- "Age", "WorkClass", "fnlwgt", "Education", "EducationNum",
- "MaritalStatus", "Occupation", "Relationship", "Race", "Gender",
- "CapitalGain", "CapitalLoss", "HoursPerWeek", "NativeCountry", "Income"
- ]
- for row in x:
- row[0] = int(row[0] / 10)
- row[2] = int(row[2]) // 100000
- row[4] = int(row[4] / 5)
- row[10] = int(row[10] / 1000)
- row[12] = int(row[12] / 10)
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x).toarray()
- all_features = enc.get_feature_names()
- x_size = len(complete_x)
- x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=0)
- for item in x_test:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- x_size = counter
- clf = svm.SVC()
- clf.fit(x_train, y_train)
-
- # alpha is size significance coefficient
- # verbose option is for returning debug info while creating slices and printing it
- # k is number of top-slices we want
- # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
- # loss_type = 0 (l2 in case of regression model
- # loss_type = 1 (cross entropy in case of classification model)
- preds = clf.predict(x_test)
- predictions = []
- counter = 0
- mistakes = 0
- for pred in preds:
- predictions.append((counter, pred))
- if y_test[counter] != pred:
- mistakes = mistakes + 1
- counter = counter + 1
- lossF = mistakes / counter
-
- # enumerator <union>/<join> indicates an approach of next level slices combination process:
- # in case of <join> in order to create new node of current level slicer
- # combines only nodes of previous layer with each other
- # <union> case implementation is based on DPSize algorithm
- if enumerator == "join":
- slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type, b_update=b_update)
- elif enumerator == "union":
- union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha,
- k=k, w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/classification/test_compas.py b/scripts/staging/slicing/tests/classification/test_compas.py
deleted file mode 100644
index 7fbb6fc..0000000
--- a/scripts/staging/slicing/tests/classification/test_compas.py
+++ /dev/null
@@ -1,105 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.linear_model import LinearRegression
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-import sys
-
-from slicing.base import slicer, union_slicer
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- else:
- k = 10
- w = 0.5
- alpha = 4
- b_update = True
- debug = True
- loss_type = 0
- enumerator = "union"
- file_name = 'slicing/datasets/real/compas/compas-test.csv'
- dataset = pd.read_csv(file_name)
- attributes_amount = len(dataset.values[0])
- y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- # starting with one not including id field
- x = dataset.iloc[:, 0:attributes_amount - 1].values
- # hot encoding of categorical features
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- all_features = enc.get_feature_names()
- # train model on a whole dataset
- x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
- for item in x_test:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- x_size = counter
- clf = RandomForestClassifier(n_jobs=2, random_state=0)
- clf.fit(x_train, y_train)
- RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
- max_depth=None, max_features='auto', max_leaf_nodes=None,
- min_impurity_split=1e-07, min_samples_leaf=1,
- min_samples_split=2, min_weight_fraction_leaf=0.0,
- n_estimators=10, n_jobs=2, oob_score=False, random_state=0,
- verbose=0, warm_start=False)
- preds = clf.predict(x_test)
- predictions = []
- counter = 0
- mistakes = 0
- for pred in preds:
- predictions.append((counter, pred))
- if y_test[counter] != pred:
- mistakes = mistakes + 1
- counter = counter + 1
- lossF = mistakes / counter
- # alpha is size significance coefficient
- # verbose option is for returning debug info while creating slices and printing it
- # k is number of top-slices we want
- # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
- # enumerator <union>/<join> indicates an approach of next level slices combination process:
- # in case of <join> in order to create new node of current level slicer
- # combines only nodes of previous layer with each other
- # <union> case implementation is based on DPSize algorithm
- if enumerator == "join":
- slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type, b_update=b_update)
- elif enumerator == "union":
- union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha,
- k=k, w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/classification/test_iris.py b/scripts/staging/slicing/tests/classification/test_iris.py
deleted file mode 100644
index 2bd0c09..0000000
--- a/scripts/staging/slicing/tests/classification/test_iris.py
+++ /dev/null
@@ -1,109 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.preprocessing import OneHotEncoder
-
-from slicing.base import slicer as slicer, union_slicer
-from sklearn.ensemble import RandomForestClassifier
-from sklearn import preprocessing
-from sklearn.model_selection import train_test_split
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- else:
- k = 10
- w = 0.5
- alpha = 6
- b_update = True
- debug = True
- loss_type = 1
- enumerator = "join"
- dataset = pd.read_csv('/slicing/datasets/iris.csv')
- attributes_amount = len(dataset.values[0])
- x = dataset.iloc[:, 0:attributes_amount - 1].values
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x).toarray()
- y = dataset.iloc[:, attributes_amount - 1]
- le = preprocessing.LabelEncoder()
- le.fit(y)
- y = le.transform(y)
- complete_x = []
- complete_y = []
- counter = 0
- all_indexes = []
- all_features = enc.get_feature_names()
- x_size = len(complete_x)
- x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
- for item in x_test:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- x_size = counter
- clf = RandomForestClassifier(n_jobs=2, random_state=0)
- clf.fit(x_train, y_train)
- RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',
- max_depth=None, max_features='auto', max_leaf_nodes=None,
- min_impurity_split=1e-07, min_samples_leaf=1,
- min_samples_split=2, min_weight_fraction_leaf=0.0,
- n_estimators=10, n_jobs=2, oob_score=False, random_state=0,
- verbose=0, warm_start=False)
-
- # alpha is size significance coefficient
- # verbose option is for returning debug info while creating slices and printing it
- # k is number of top-slices we want
- # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
- # loss_type = 0 (l2 in case of regression model
- # loss_type = 1 (cross entropy in case of classification model)
- preds = clf.predict(x_test)
- predictions = []
- counter = 0
- mistakes = 0
- for pred in preds:
- predictions.append((counter, pred))
- if y_test[counter] != pred:
- mistakes = mistakes + 1
- counter = counter + 1
- lossF = mistakes / counter
-
- # enumerator <union>/<join> indicates an approach of next level slices combination process:
- # in case of <join> in order to create new node of current level slicer
- # combines only nodes of previous layer with each other
- # <union> case implementation is based on DPSize algorithm
- if enumerator == "join":
- slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=1, b_update=b_update)
- elif enumerator == "union":
- union_slicer.process(all_features, complete_x, lossF, x_size, complete_y, predictions, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/__init__.py b/scripts/staging/slicing/tests/regression/__init__.py
deleted file mode 100644
index cc59154..0000000
--- a/scripts/staging/slicing/tests/regression/__init__.py
+++ /dev/null
@@ -1,20 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
diff --git a/scripts/staging/slicing/tests/regression/bd_spark_salary.py b/scripts/staging/slicing/tests/regression/bd_spark_salary.py
deleted file mode 100644
index 1889292..0000000
--- a/scripts/staging/slicing/tests/regression/bd_spark_salary.py
+++ /dev/null
@@ -1,130 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, join_data_parallel, union_data_parallel
-
-binner = udf(lambda arg: int(int(arg) // 5))
-
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- dataset = args[8]
- else:
- k = 10
- w = 0.5
- alpha = 6
- b_update = True
- debug = True
- loss_type = 0
- dataset = 'slicing/datasets/parallel_data/salaries/rows1000.csv'
- enumerator = "join"
-
- conf = SparkConf().setAppName("salary_test").setMaster('local[4]')
- num_partitions = 8
- model_type = "regression"
- label = 'salary'
- sparkContext = SparkContext(conf=conf)
- sqlContext = SQLContext(sparkContext)
- fileRDD = sparkContext.textFile(dataset, num_partitions)
- header = fileRDD.first()
- head_split = header.split(",")
- fileRDD = fileRDD.filter(lambda line: line != header)
- data = fileRDD.map(lambda row: row.split(","))
- dataset_df = sqlContext.createDataFrame(data, head_split)
-
- cat_features = ["rank", "discipline", "sincephd_bin", "service_bin", "sex"]
- # initializing stages of main transformation pipeline
- stages = []
- dataset_df = dataset_df.withColumn("id", sf.monotonically_increasing_id())
- # bining numeric features by local binner udf function (specified for current dataset if needed)
- dataset_df = dataset_df.withColumn('sincephd_bin', binner(dataset_df['sincephd']))
- dataset_df = dataset_df.withColumn('service_bin', binner(dataset_df['service']))
- dataset_df = dataset_df.withColumn('model_type', sf.lit(0))
- dataset_df = dataset_df.drop('sincephd', 'service')
- dataset_df = dataset_df.withColumn('target', dataset_df[label].cast("int"))
- # hot encoding categorical features
- for feature in cat_features:
- string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index")
- encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
- encoder.setDropLast(False)
- stages += [string_indexer, encoder]
- assembler_inputs = [feature + "_vec" for feature in cat_features]
- assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
- stages += [assembler]
- assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
- stages += [assembler_final]
-
- pipeline = Pipeline(stages=stages)
- pipeline_model = pipeline.fit(dataset_df)
- dataset_transformed = pipeline_model.transform(dataset_df)
- df_transform_fin = dataset_transformed.select('id', 'features', 'target', 'model_type').toPandas()
-
- cat = 0
- counter = 0
- decode_dict = {}
- for feature in cat_features:
- colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
- colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
- for item in colIdx:
- decode_dict[counter] = (cat, item, colIdx[item])
- counter = counter + 1
- cat = cat + 1
-
- train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
- train_df = sqlContext.createDataFrame(train)
- test_df = sqlContext.createDataFrame(test)
- lr = LinearRegression(featuresCol='features', labelCol='target', maxIter=10, regParam=0.3, elasticNetParam=0.8)
- lr_model = lr.fit(train_df)
- eval = lr_model.evaluate(test_df)
- f_l2 = eval.meanSquaredError
- pred = eval.predictions
- pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred['target'], pred['prediction'], pred['model_type']))
- predictions = pred_df_fin.select('id', 'features', 'error').repartition(num_partitions)
- converter = IndexToString(inputCol='features', outputCol='cats')
- all_features = list(decode_dict.keys())
- if enumerator == "join":
- join_data_parallel.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha,
- k=k, w=w, loss_type=loss_type)
- elif enumerator == "union":
- union_data_parallel.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha,
- k=k, w=w, loss_type=loss_type)
diff --git a/scripts/staging/slicing/tests/regression/spark_salary.py b/scripts/staging/slicing/tests/regression/spark_salary.py
deleted file mode 100644
index be9868f..0000000
--- a/scripts/staging/slicing/tests/regression/spark_salary.py
+++ /dev/null
@@ -1,157 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-from pyspark import SparkConf, SparkContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-from pyspark.sql import SQLContext
-import pyspark.sql.functions as sf
-from pyspark.sql.functions import udf
-from sklearn.model_selection import train_test_split
-
-from slicing.spark_modules import spark_utils, spark_slicer, spark_union_slicer
-
-
-binner = udf(lambda arg: int(arg / 5))
-
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- dataset = args[8]
- else:
- k = 10
- w = 0.5
- alpha = 10
- b_update = True
- debug = True
- loss_type = 0
- enumerator = "union"
- dataset = 'slicing/datasets/salaries.csv'
- # dataset = 'slicing/datasets/parallel_attr/salaries/attr3000.csv'
-
- conf = SparkConf().setAppName("salary_test").setMaster('local[4]')
- num_partitions = 4
- model_type = "regression"
- label = 'salary'
- sparkContext = SparkContext(conf=conf)
- sqlContext = SQLContext(sparkContext)
- dataset_df = sqlContext.read.csv(dataset, header='true',
- inferSchema='true')
- # initializing stages of main transformation pipeline
- stages = []
- # list of categorical features for further hot-encoding
- cat_features = ["rank", "discipline", "sincephd_bin", "service_bin", "sex"] # base
- # cat_features = ["rank", "discipline", "sex"] # 10
- # cat_features = ["rank", "discipline", "sincephd", "sex"] # 50
- # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "start", "dept", "card"] # 100
- # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
- # "uni", "dept", "race", "code", "job", "size", "previous", "stock"] # 1013
- # cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
- # "uni", "dept", "origin", "children", "bmi", "code", "job", "size", "previous", "stock", "market",
- # "freq", "smoker", "region", "WorkClass", "Education", "EducationNum", "MaritalStatus",
- # "Occupation", "Relationship", "color", "sepal_length", "sepal_width", "petal_length",
- # "petal_width", "variety", "symboling", "normalized-losses", "make", "fuel-type", "aspiration",
- # "num-of-doors", "body-style", "drive-wheels", "engine-location", "wheel-base", "length", "width",
- # "height", "curb-weight", "engine-type", "num-of-cylinders", "engine-size", "fuel-system", "bore",
- # "stroke", "compression-ratio", "horsepower", "peak-rpm", "city-mpg", "highway-mpg", "price",
- # "codeNumber", "clump", "cellsize", "cellshape", "adhesion", "epitel", "nuclei", "chromatin",
- # "nucleoli", "mitoses", "class", "elevation", "aspect", "slope", "distToHydr"]
-
-# cat_features = ["rank", "discipline", "sex", "company", "country", "city", "skills", "language", "tz", "card",
-# "uni", "dept", "origin", "children", "bmi", "code", "job", "size", "previous", "stock", "market",
-# "freq", "smoker", "region", "WorkClass", "Education", "EducationNum", "MaritalStatus",
-# "Occupation", "Relationship", "color", "sepal_length", "sepal_width", "petal_length",
-# "petal_width", "variety", "symboling", "normalized-losses", "make", "fuel-type", "aspiration",
-# "num-of-doors", "body-style", "drive-wheels", "engine-location", "wheel-base", "length", "width",
-# "height", "curb-weight", "engine-type", "num-of-cylinders", "engine-size", "fuel-system", "bore",
-# "stroke", "compression-ratio", "horsepower", "peak-rpm", "city-mpg", "highway-mpg", "price",
-# "codeNumber", "clump", "cellsize", "cellshape", "adhesion", "epitel", "nuclei", "chromatin",
-# "nucleoli", "mitoses", "class", "elevation", "aspect", "slope", "distToHydr", "distToRoad",
-# "hillshadeMorning", "hillSahdeNoon", "hillSadeAfternoon", "distToFire", "DSName", "T", "N", "p",
-# "k", "Bin", "Cost", "Sdratio", "correl", "cancor1", "cancor2", "fract1", "fract2", "skewness",
-# "kurtosis", "Hc", "Hx", "Mcx", "EnAttr", "NSRation", "Alg", "error"] # 3000
-
- # lines till 87 only for base case
- # removing column with ID field
- # dataset_df = dataset_df.drop('_c0')
- # bining numeric features by local binner udf function (specified for current dataset if needed)
- dataset_df = dataset_df.withColumn('sincephd_bin', binner(dataset_df['sincephd']))
- dataset_df = dataset_df.withColumn('service_bin', binner(dataset_df['service']))
- dataset_df = dataset_df.drop('sincephd', 'service')
- dataset_df = dataset_df.withColumn('model_type', sf.lit(0))
- # hot encoding categorical features
- for feature in cat_features:
- string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
- encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
- encoder.setDropLast(False)
- stages += [string_indexer, encoder]
- assembler_inputs = [feature + "_vec" for feature in cat_features]
- assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
- stages += [assembler]
- assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
- stages += [assembler_final]
- pipeline = Pipeline(stages=stages)
- pipeline_model = pipeline.fit(dataset_df)
- dataset_transformed = pipeline_model.transform(dataset_df)
- df_transform_fin = dataset_transformed.select('features', label, 'model_type').toPandas()
- train, test = train_test_split(df_transform_fin, test_size=0.3, random_state=0)
- train_df = sqlContext.createDataFrame(train)
- test_df = sqlContext.createDataFrame(test)
- decode_dict = {}
- counter = 0
- cat = 0
- for feature in cat_features:
- colIdx = dataset_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
- colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
- for item in colIdx:
- decode_dict[counter] = (cat, item, colIdx[item])
- counter = counter + 1
- cat = cat + 1
- lr = LinearRegression(featuresCol='features', labelCol=label, maxIter=10, regParam=0.3, elasticNetParam=0.8)
- lr_model = lr.fit(train_df)
- eval = lr_model.evaluate(test_df)
- f_l2 = eval.meanSquaredError
- pred = eval.predictions
- pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred[label], pred['prediction'], pred['model_type']))
- predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
- converter = IndexToString(inputCol='features', outputCol='cats')
- all_features = list(decode_dict.keys())
- predictions = predictions.collect()
- if enumerator == "join":
- spark_slicer.parallel_process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha, k=k, w=w,
- loss_type=loss_type, enumerator=enumerator)
- elif enumerator == "union":
- spark_union_slicer.process(all_features, predictions, f_l2, sparkContext, debug=debug, alpha=alpha, k=k, w=w,
- loss_type=loss_type, enumerator=enumerator)
diff --git a/scripts/staging/slicing/tests/regression/test_insurance.py b/scripts/staging/slicing/tests/regression/test_insurance.py
deleted file mode 100644
index 64b58c4..0000000
--- a/scripts/staging/slicing/tests/regression/test_insurance.py
+++ /dev/null
@@ -1,103 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import sys
-
-import pandas as pd
-from sklearn.linear_model import LinearRegression
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-
-from slicing.base import slicer as slicer, union_slicer
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- else:
- k = 10
- w = 0.5
- alpha = 4
- b_update = True
- debug = True
- loss_type = 0
- enumerator = "union"
- file_name = '/slicing/datasets/insurance.csv'
- dataset = pd.read_csv(file_name)
- attributes_amount = len(dataset.values[0])
- # for now working with regression datasets, assuming that target attribute is the last one
- # currently non-categorical features are not supported and should be binned
- y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- # starting with one not including id field
- x = dataset.iloc[:, 0:attributes_amount - 1].values
- # list of numerical columns
- non_categorical = [1, 3]
- for row in x:
- for attribute in non_categorical:
- # <attribute - 2> as we already excluded from x id column
- row[attribute - 1] = int(row[attribute - 1] / 5)
- # hot encoding of categorical features
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- all_features = enc.get_feature_names()
- # train model on a whole dataset
- x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
- for item in x_test:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- x_size = counter
- model = LinearRegression()
- model.fit(x_train, y_train)
- preds = (model.predict(x_test) - y_test) ** 2
- f_l2 = sum(preds)/x_size
- errors = []
- counter = 0
- for pred in preds:
- errors.append((counter, pred))
- counter = counter + 1
- # alpha is size significance coefficient
- # verbose option is for returning debug info while creating slices and printing it
- # k is number of top-slices we want
- # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
- # enumerator <union>/<join> indicates an approach of next level slices combination process:
- # in case of <join> in order to create new node of current level slicer
- # combines only nodes of previous layer with each other
- # <union> case implementation is based on DPSize algorithm
- if enumerator == "join":
- slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type, b_update=b_update)
- elif enumerator == "union":
- union_slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k,
- w=w, loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/test_salary.py b/scripts/staging/slicing/tests/regression/test_salary.py
deleted file mode 100644
index 92e0055..0000000
--- a/scripts/staging/slicing/tests/regression/test_salary.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn.linear_model import LinearRegression
-from sklearn.metrics import mean_squared_error
-from sklearn.model_selection import train_test_split
-from sklearn.preprocessing import OneHotEncoder
-import sys
-
-from slicing.base import slicer, union_slicer
-
-if __name__ == "__main__":
- args = sys.argv
- if len(args) > 1:
- k = int(args[1])
- w = float(args[2].replace(',', '.'))
- alpha = int(args[3])
- if args[4] == "True":
- b_update = True
- else:
- b_update = False
- debug = args[5]
- loss_type = int(args[6])
- enumerator = args[7]
- else:
- k = 10
- w = 0.5
- alpha = 4
- b_update = True
- debug = True
- loss_type = 0
- enumerator = "union"
- file_name = '/slicing/datasets/salaries.csv'
- dataset = pd.read_csv(file_name)
- attributes_amount = len(dataset.values[0])
- # for now working with regression datasets, assuming that target attribute is the last one
- # currently non-categorical features are not supported and should be binned
- y = dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- # starting with one not including id field
- x = dataset.iloc[:, 1:attributes_amount - 1].values
- # list of numerical columns
- non_categorical = [4, 5]
- for row in x:
- for attribute in non_categorical:
- # <attribute - 2> as we already excluded from x id column
- row[attribute - 2] = int(row[attribute - 2] / 5)
- # hot encoding of categorical features
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- all_features = enc.get_feature_names()
- # train model on a whole dataset
- x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=0)
- for item in x_test:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- x_size = counter
- model = LinearRegression()
- model.fit(x_train, y_train)
- predictions = model.predict(x_test)
- f_l2 = mean_squared_error(y_test, predictions)
- preds = (model.predict(x_test) - y_test) ** 2
- errors = []
- counter = 0
- for pred in preds:
- errors.append((counter, pred))
- counter = counter + 1
- # alpha is size significance coefficient
- # verbose option is for returning debug info while creating slices and printing it
- # k is number of top-slices we want
- # w is a weight of error function significance (1 - w) is a size significance propagated into optimization function
-
- # enumerator <union>/<join> indicates an approach of next level slices combination process:
- # in case of <join> in order to create new node of current level slicer
- # combines only nodes of previous layer with each other
- # <union> case implementation is based on DPSize algorithm
- if enumerator == "join":
- slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k, w=w,
- loss_type=loss_type, b_update=b_update)
- elif enumerator == "union":
- union_slicer.process(all_features, complete_x, f_l2, x_size, y_test, errors, debug=debug, alpha=alpha, k=k, w=w,
- loss_type=loss_type, b_update=b_update)
diff --git a/scripts/staging/slicing/tests/regression/toy_tests.py b/scripts/staging/slicing/tests/regression/toy_tests.py
deleted file mode 100644
index 876aa8b..0000000
--- a/scripts/staging/slicing/tests/regression/toy_tests.py
+++ /dev/null
@@ -1,428 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import unittest
-
-import pandas as pd
-from pyspark import SparkConf, SparkContext, SQLContext
-from pyspark.ml import Pipeline
-from pyspark.ml.feature import StringIndexer, OneHotEncoderEstimator, VectorAssembler, IndexToString
-from pyspark.ml.regression import LinearRegression
-import pyspark.sql.functions as sf
-from sklearn import linear_model
-from sklearn.metrics import mean_squared_error, r2_score
-from sklearn.preprocessing import OneHotEncoder
-
-import slicing.base.slicer as slicer
-from slicing.base import union_slicer
-from slicing.base.node import Node
-from slicing.base.top_k import Topk
-from slicing.spark_modules import spark_utils, spark_slicer, spark_union_slicer
-
-
-class SliceTests(unittest.TestCase):
- loss_type = 0
- # x, y = m.generate_dataset(10, 100)
- train_dataset = pd.read_csv("toy_train.csv")
- attributes_amount = len(train_dataset.values[0])
- model = linear_model.LinearRegression()
- y_train = train_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- x_train = train_dataset.iloc[:, 0:attributes_amount - 1].values
- model.fit(x_train, y_train)
- test_dataset = pd.read_csv("toy.csv")
- y_test = test_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- x_test = test_dataset.iloc[:, 0:attributes_amount - 1].values
- y_pred = model.predict(x_test)
- print("Mean squared error: %.2f"
- % mean_squared_error(y_test, y_pred))
- print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
- # Now that we have trained the model, we can print the coefficient of x that it has predicted
- print('Coefficients: \n', model.coef_)
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x_test).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- for item in x:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- all_features = enc.get_feature_names()
- loss = mean_squared_error(y_test, y_pred)
- devs = (y_pred - y_test) ** 2
- errors = []
- counter = 0
- for pred in devs:
- errors.append((counter, pred))
- counter = counter + 1
- k = 5
- w = 0.5
- alpha = 4
- top_k = Topk(k)
- debug = True
- b_update = True
- first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
- loss_type, top_k, alpha, w)
- first_level_nodes = first_level[0]
- slice_member = first_level_nodes[(7, 'x2_2')]
-
- def test_attr_spark(self):
- conf = SparkConf().setAppName("toy_test").setMaster('local[2]')
- num_partitions = 2
- enumerator = "join"
- model_type = "regression"
- label = 'target'
- sparkContext = SparkContext(conf=conf)
- sqlContext = SQLContext(sparkContext)
- train_df = sqlContext.read.csv("toy_train.csv", header='true',
- inferSchema='true')
- test_df = sqlContext.read.csv("toy.csv", header='true',
- inferSchema='true')
- # initializing stages of main transformation pipeline
- stages = []
- # list of categorical features for further hot-encoding
- cat_features = ['a', 'b', 'c']
- for feature in cat_features:
- string_indexer = StringIndexer(inputCol=feature, outputCol=feature + "_index").setHandleInvalid("skip")
- encoder = OneHotEncoderEstimator(inputCols=[string_indexer.getOutputCol()], outputCols=[feature + "_vec"])
- encoder.setDropLast(False)
- stages += [string_indexer, encoder]
- assembler_inputs = [feature + "_vec" for feature in cat_features]
- assembler = VectorAssembler(inputCols=assembler_inputs, outputCol="assembled_inputs")
- stages += [assembler]
- assembler_final = VectorAssembler(inputCols=["assembled_inputs"], outputCol="features")
- stages += [assembler_final]
- pipeline = Pipeline(stages=stages)
- train_pipeline_model = pipeline.fit(train_df)
- test_pipeline_model = pipeline.fit(test_df)
- train_df_transformed = train_pipeline_model.transform(train_df)
- test_df_transformed = test_pipeline_model.transform(test_df)
- train_df_transformed = train_df_transformed.withColumn('model_type', sf.lit(0))
- test_df_transformed = test_df_transformed.withColumn('model_type', sf.lit(0))
- decode_dict = {}
- counter = 0
- cat = 0
- for feature in cat_features:
- colIdx = test_df_transformed.select(feature, feature + "_index").distinct().rdd.collectAsMap()
- colIdx = {k: v for k, v in sorted(colIdx.items(), key=lambda item: item[1])}
- for item in colIdx:
- decode_dict[counter] = (cat, item, colIdx[item], counter)
- counter = counter + 1
- cat = cat + 1
- train_df_transform_fin = train_df_transformed.select('features', label, 'model_type')
- test_df_transform_fin = test_df_transformed.select('features', label, 'model_type')
- lr = LinearRegression(featuresCol='features', labelCol=label, maxIter=10, regParam=0.0, elasticNetParam=0.8)
- lr_model = lr.fit(train_df_transform_fin)
- eval = lr_model.evaluate(test_df_transform_fin)
- f_l2 = eval.meanSquaredError
- pred = eval.predictions
- pred_df_fin = pred.withColumn('error', spark_utils.calc_loss(pred[label], pred['prediction'], pred['model_type']))
- predictions = pred_df_fin.select('features', 'error').repartition(num_partitions)
- converter = IndexToString(inputCol='features', outputCol='cats')
- all_features = list(decode_dict)
- predictions = predictions.collect()
- spark_join = spark_slicer.parallel_process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha,
- k=self.k, w=self.w, loss_type=self.loss_type, enumerator="join")
- spark_union = spark_union_slicer.process(all_features, predictions, f_l2, sparkContext, debug=self.debug, alpha=self.alpha,
- k=self.k, w=self.w, loss_type=self.loss_type, enumerator="union")
- self.assertEqual(3, len(spark_join.slices))
- print("check1")
- self.assertEqual(spark_join.min_score, spark_union.min_score)
- print("check2")
- self.assertEqual(spark_join.keys, spark_union.keys)
- print("check3")
- self.assertEqual(len(spark_join.slices), len(spark_union.slices))
- print("check4")
- idx = -1
- for sliced in spark_join.slices:
- idx += 1
- self.assertEqual(sliced.score, spark_union.slices[idx].score)
- print("check5")
-
- def test_features_number(self):
- self.assertEqual(len(self.all_features), 9)
- print("check 1")
-
- def test_base_first_level(self):
- self.assertEqual(9, len(self.first_level_nodes))
- print("check 2")
-
- def test_parents_first(self):
- self.assertIn(('x2_2', 7), self.slice_member.parents)
- print("check 3")
-
- def test_name(self):
- self.assertEqual('x2_2', self.slice_member.make_name())
- print("check 4")
-
- def test_size(self):
- self.assertEqual(36, self.slice_member.size)
- print("check 5")
-
- def test_e_upper(self):
- self.assertEqual(81, self.slice_member.e_upper)
- print("check 6")
-
- def test_loss(self):
- self.assertEqual(22, int(self.slice_member.loss))
- print("check 7")
-
- def test_opt_fun(self):
- self.slice_member.score = slicer.opt_fun(self.slice_member.loss, self.slice_member.size, self.loss, len(self.x_test), self.w)
- print("check 8")
-
- def test_score(self):
- self.assertEqual(1.2673015873015872, self.slice_member.score)
- print("check 9")
-
- def test_base_join_enum(self):
- cur_lvl_nodes = {}
- all_nodes = {}
- b_update = True
- cur_lvl = 1
- slice_index = (2, 'x0_3')
- combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
- len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
- self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
- self.assertEqual(6, len(combined[0]))
- print("check1")
-
- def test_parents_second(self):
- cur_lvl_nodes = {}
- all_nodes = {}
- b_update = True
- cur_lvl = 1
- slice_index = (2, 'x0_3')
- combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
- len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
- self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
- parent1 = combined[0][('x0_3 && x1_3')]
- parent2 = combined[0][('x0_3 && x2_2')]
- new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
- new_node.parents = [parent1, parent2]
- parent1_attr = parent1.attributes
- parent2_attr = parent2.attributes
- new_node_attr = slicer.union(parent1_attr, parent2_attr)
- self.assertEqual(new_node_attr, [('x0_3', 2), ('x1_3', 5), ('x2_2', 7)])
- print("check2")
-
- def test_nonsense(self):
- cur_lvl_nodes = {}
- all_nodes = {}
- b_update = True
- cur_lvl = 1
- slice_index = (2, 'x0_3')
- combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
- len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
- self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
- parent1 = combined[0][('x0_3 && x1_3')]
- parent2 = combined[0][('x0_3 && x2_2')]
- new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
- new_node.parents = [parent1, parent2]
- parent1_attr = parent1.attributes
- parent2_attr = parent2.attributes
- new_node_attr = slicer.union(parent1_attr, parent2_attr)
- new_node.attributes = new_node_attr
- new_node.name = new_node.make_name()
- flagTrue = slicer.slice_name_nonsense(parent1, parent2, 2)
- self.assertEqual(True, flagTrue)
- print("check3")
-
- def test_non_nonsense(self):
- cur_lvl_nodes = {}
- all_nodes = {}
- b_update = True
- cur_lvl = 1
- slice_index = (2, 'x0_3')
- parent3 = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
- parent3.parents = [self.first_level_nodes[(4, 'x1_2')], self.first_level_nodes[(7, 'x2_2')]]
- parent3.attributes = [('x1_2', 4), ('x2_2', 7)]
- combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
- len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
- self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
- parent2 = combined[0]['x0_3 && x2_3']
- parent3.key = (8, 'x1_2 && x2_2')
- flag_nonsense = slicer.slice_name_nonsense(parent2, parent3, 2)
- self.assertEqual(True, flag_nonsense)
- print("check4")
-
- def test_uppers(self):
- cur_lvl_nodes = {}
- all_nodes = {}
- b_update = True
- cur_lvl = 1
- slice_index = (2, 'x0_3')
- parent3 = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
- parent3.parents = [self.first_level_nodes[(4, 'x1_2')], self.first_level_nodes[(7, 'x2_2')]]
- parent3.attributes = [('x1_2', 4), ('x2_2', 7)]
- combined = slicer.join_enum(slice_index, self.first_level_nodes, self.complete_x, self.loss,
- len(self.complete_x), self.y_test, self.errors, self.debug, self.alpha, self.w,
- self.loss_type, b_update, cur_lvl, all_nodes, self.top_k, cur_lvl_nodes)
- parent1 = combined[0]['x0_3 && x1_3']
- parent2 = combined[0]['x0_3 && x2_3']
- new_node = Node(self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors)
- new_node.parents = [parent1, parent2]
- new_node.calc_bounds(2, self.w)
- self.assertEqual(25, new_node.s_upper)
- print("check5")
- self.assertEqual(398, int(new_node.c_upper))
- print("check6")
-
- def test_topk_slicing(self):
- join_top_k = slicer.process(self.all_features, self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors,
- self.debug, self.alpha, self.k, self.w, self.loss_type, self.b_update)
- union_top_k = union_slicer.process(self.all_features, self.complete_x, self.loss, len(self.complete_x), self.y_test, self.errors,
- self.debug, self.alpha, self.k, self.w, self.loss_type, self.b_update)
- self.assertEqual(join_top_k.min_score, union_top_k.min_score)
- print("check1")
- self.assertEqual(join_top_k.keys, union_top_k.keys)
- print("check2")
- self.assertEqual(len(join_top_k.slices), len(union_top_k.slices))
- print("check3")
- idx = -1
- for sliced in join_top_k.slices:
- idx += 1
- self.assertEqual(sliced.score, union_top_k.slices[idx].score)
- print("check4")
-
- def test_extreme_target(self):
- test_dataset = pd.read_csv("/home/lana/diploma/project/slicing/datasets/toy_extreme_change.csv")
- y_test = test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
- x_test = test_dataset.iloc[:, 0:self.attributes_amount - 1].values
- y_pred = self.model.predict(x_test)
- print("Mean squared error: %.2f"
- % mean_squared_error(y_test, y_pred))
- print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
- # Now that we have trained the model, we can print the coefficient of x that it has predicted
- print('Coefficients: \n', self.model.coef_)
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x_test).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- for item in x:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- all_features = enc.get_feature_names()
- loss = mean_squared_error(y_test, y_pred)
- devs = (y_pred - y_test) ** 2
- errors = []
- counter = 0
- for pred in devs:
- errors.append((counter, pred))
- counter = counter + 1
- k = 5
- w = 0.5
- alpha = 4
- top_k = Topk(k)
- debug = True
- b_update = True
- first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
- self.loss_type, top_k, alpha, w)
- first_level_nodes = first_level[0]
- slice_member = first_level_nodes[(7, 'x2_2')]
- self.assertGreater(slice_member.loss, self.slice_member.loss)
- print("check 1")
- self.assertGreater(slice_member.score, self.slice_member.score)
- print("check 2")
-
- def test_error_significance(self):
- y_test = self.test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
- x_test = self.test_dataset.iloc[:, 0:self.attributes_amount - 1].values
- y_pred = self.model.predict(x_test)
- print("Mean squared error: %.2f"
- % mean_squared_error(y_test, y_pred))
- print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
- # Now that we have trained the model, we can print the coefficient of x that it has predicted
- print('Coefficients: \n', self.model.coef_)
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x_test).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- for item in x:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- all_features = enc.get_feature_names()
- loss = mean_squared_error(y_test, y_pred)
- devs = (y_pred - y_test) ** 2
- errors = []
- counter = 0
- for pred in devs:
- errors.append((counter, pred))
- counter = counter + 1
- k = 5
- # Maximized size significance
- w = 0
- alpha = 4
- top_k = Topk(k)
- debug = True
- b_update = True
- first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
- self.loss_type, top_k, alpha, w)
- first_level_nodes = first_level[0]
- slice_member = first_level_nodes[(7, 'x2_2')]
- self.assertGreater(self.slice_member.score, slice_member.score)
-
- def test_size_significance(self):
- y_test = self.test_dataset.iloc[:, self.attributes_amount - 1:self.attributes_amount].values
- x_test = self.test_dataset.iloc[:, 0:self.attributes_amount - 1].values
- y_pred = self.model.predict(x_test)
- print("Mean squared error: %.2f"
- % mean_squared_error(y_test, y_pred))
- print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
- # Now that we have trained the model, we can print the coefficient of x that it has predicted
- print('Coefficients: \n', self.model.coef_)
- enc = OneHotEncoder(handle_unknown='ignore')
- x = enc.fit_transform(x_test).toarray()
- complete_x = []
- complete_y = []
- counter = 0
- for item in x:
- complete_x.append((counter, item))
- complete_y.append((counter, y_test[counter]))
- counter = counter + 1
- all_features = enc.get_feature_names()
- loss = mean_squared_error(y_test, y_pred)
- devs = (y_pred - y_test) ** 2
- errors = []
- counter = 0
- for pred in devs:
- errors.append((counter, pred))
- counter = counter + 1
- k = 5
- # Maximized size significance
- w = 1
- alpha = 4
- top_k = Topk(k)
- debug = True
- b_update = True
- first_level = slicer.make_first_level(all_features, list(complete_x), loss, len(complete_x), y_test, errors,
- self.loss_type, top_k, alpha, w)
- first_level_nodes = first_level[0]
- slice_member = first_level_nodes[(7, 'x2_2')]
- self.assertGreater(slice_member.score, self.slice_member.score)
-
-
-if __name__ == '__main__':
- unittest.main()
diff --git a/scripts/staging/slicing/utils/model_generator.py b/scripts/staging/slicing/utils/model_generator.py
deleted file mode 100644
index 8f80b74..0000000
--- a/scripts/staging/slicing/utils/model_generator.py
+++ /dev/null
@@ -1,77 +0,0 @@
-#-------------------------------------------------------------
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#-------------------------------------------------------------
-
-import pandas as pd
-from sklearn import linear_model
-from sklearn.datasets import make_regression
-import numpy as np
-from sklearn.metrics import mean_squared_error, r2_score
-
-
-def generate_dataset(beta, n):
- # Generate x as an array of `n` samples which can take a value between 0 and 5
- x = np.random.random_integers(1, 3, (n, 3))
- # Calculate `y` according to the equation discussed
- coeff = [2, 3, 1]
- coeff_mat = np.array(coeff).reshape(len(coeff), 1)
- y = np.matmul(x, coeff_mat)
- dataset = np.append(x, y, axis=1)
- np.savetxt("toy_train.csv", dataset, delimiter=",", fmt='%s')
- return x, y
-
-
-if __name__ == '__main__':
- def main():
- x, y = generate_dataset(10, 100)
- x_train = x
- y_train = y
- dataset = np.append(x, y, axis=1)
- np.savetxt("toy_train.csv", dataset, delimiter=",", fmt='%s')
- model = linear_model.LinearRegression()
-
- # Train the model using the training data that we created
- model.fit(x_train, y_train)
- # Now that we have trained the model, we can print the coefficient of x that it has predicted
- print('Coefficients: \n', model.coef_)
-
- # We then use the model to make predictions based on the test values of x
- test_dataset = pd.read_csv("../datasets/toy.csv")
- attributes_amount = len(test_dataset.values[0])
- y_test = test_dataset.iloc[:, attributes_amount - 1:attributes_amount].values
- x_test = test_dataset.iloc[:, 0:attributes_amount - 1].values
- y_pred = model.predict(x_test)
-
- # Now, we can calculate the models accuracy metrics based on what the actual value of y was
- print("Mean squared error: %.2f"
- % mean_squared_error(y_test, y_pred))
- print('r_2 statistic: %.2f' % r2_score(y_test, y_pred))
- # Coefficients: [[2. 3. 1.]]
- # Mean squared error: 3.00
- # r_2 statistic: 0.72
-
- # dataset = np.append(x, y, axis=1)
- # np.savetxt("toy.csv", dataset, delimiter=",", fmt='%s')
-
- return model, x_test, y_test, y_pred
-
-
-
-
diff --git a/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml b/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
index c0624a1..9e3aa93 100644
--- a/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
+++ b/src/test/scripts/functions/codegenalg/Algorithm_AutoEncoder.dml
@@ -19,221 +19,18 @@
#
#-------------------------------------------------------------
-# Trains a 2-layer autoencoder with minibatch SGD, momentum and step-size decay.
-# If invoked with H1 > H2 then it becomes a 'bowtie' structured autoencoder
-# Weights are initialized using Glorot & Bengio (2010) AISTATS initialization.
-# The script standardizes the input before training (can be turned off).
-# Also, it randomly reshuffles rows before training.
-# Currently, tanh is set to be the activation function.
-# By re-implementing 'func' DML-bodied function, one can change the activation.
-
-# INPUT PARAMETERS:
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# X String --- Filename where the input is stored
-# H1 Int --- Number of neurons in the 1st hidden layer
-# H2 Int --- Number of neurons in the 2nd hidden layer
-# EPOCH Int --- Number of epochs to train for
-# fmt String 'text' Output format ("text", "csv", "binary" etc.)
-# OBJ Boolean FALSE If TRUE, Computes objective function value (squared-loss)
-# at the end of each epoch. Note that, computing the full
-# objective can take a lot of time.
-# BATCH Int 256 Mini-batch size (training parameter)
-# STEP Double 1e-5 Initial step size (training parameter)
-# DECAY Double 0.95 Decays step size after each epoch (training parameter)
-# MOMENTUM Double 0.9 Momentum parameter (training parameter)
-# ---------------------------------------------------------------------------------------------
-#
-# OUTPUT PARAMETERS (all filenames):
-# ---------------------------------------------------------------------------------------------
-# NAME TYPE DEFAULT MEANING
-# ---------------------------------------------------------------------------------------------
-# W1_out String --- File to store weights between input layer and 1st hidden layer
-# b1_out String --- File to store bias between input layer and 1st hidden layer
-# W2_out String --- File to store weights between 1st hidden layer and 2nd hidden layer
-# b2_out String --- File to store bias between 1st hidden layer and 2nd hidden layer
-# W3_out String --- File to store weights between 2nd hidden layer and 3rd hidden layer
-# b3_out String --- File to store bias between 2nd hidden layer and 3rd hidden layer
-# W4_out String --- File to store weights between 3rd hidden layer and output layer
-# b4_out String --- File to store bias between 3rd hidden layer and output layer
-# HIDDEN String " " File to store the hidden (2nd) layer representation if needed
-# ---------------------------------------------------------------------------------------------
-#
-# INVOCATION:
-# -f autoencoder_2layer.dml --nvargs X=<input file> H1=500 H2=2 EPOCH=1 fmt="csv"
-# W1_out=<weights from input to 1st hidden layer> b1_out=<bias from input to 1st hidden layer>
-# W2_out=<weights from 1st hidden layer to 2nd hidden layer> b2_out=<bias from 1st hidden layer to 2nd hidden layer>
-# W3_out=<weights from 2nd hidden layer to 3rd hidden layer> b3_out=<bias from 2nd hidden layer to 3rd hidden layer>
-# W4_out=<weights from 3rd hidden layer to output> b4_out=<bias from 3rd hidden layer to output>
-#
-
-# NOTE: This is a copy of the `scripts/staging/autoencoder-2layer.dml` with rand()
-# replaced with random matrices generated in the corresponding test class.
-
-#implements tanh
-#to use another activation fn, implement a DML-bodied function with
-#function name 'func' and comment out this one
-func = function(Matrix[Double] X) return(Matrix[Double] Y, Matrix[Double] Y_prime){
- Y = tanh(X)
- Y_prime = 1 - Y^2
-}
-
-feedForward = function(Matrix[Double] X, Matrix[Double] W1, Matrix[Double] b1,
- Matrix[Double] W2, Matrix[Double] b2, Matrix[Double] W3, Matrix[Double] b3,
- Matrix[Double] W4, Matrix[Double] b4, Matrix[Double] Y)
- return(Matrix[Double] H1, Matrix[Double] H1_prime, Matrix[Double] H2,
- Matrix[Double] H2_prime, Matrix[Double] H3, Matrix[Double] H3_prime,
- Matrix[Double] Yhat, Matrix[Double] Yhat_prime, Matrix[Double] E)
-{
- H1_in = t(W1 %*% t(X) + b1)
- [H1, H1_prime] = func(H1_in)
-
- H2_in = t(W2 %*% t(H1) + b2)
- [H2, H2_prime] = func(H2_in)
-
- H3_in = t(W3 %*% t(H2) + b3)
- [H3, H3_prime] = func(H3_in)
-
- Yhat_in = t(W4 %*% t(H3) + b4)
- [Yhat, Yhat_prime] = func(Yhat_in)
- E = Yhat - Y
-}
-
-grad = function(Matrix[Double] X, Matrix[Double] H1, Matrix[Double] H1_prime,
- Matrix[Double] H2, Matrix[Double] H2_prime, Matrix[Double] H3,
- Matrix[Double] H3_prime, Matrix[Double] Yhat_prime, Matrix[Double] E,
- Matrix[Double] W1, Matrix[Double] W2, Matrix[Double] W3, Matrix[Double] W4)
- return(Matrix[Double] W1_grad, Matrix[Double] b1_grad, Matrix[Double] W2_grad,
- Matrix[Double] b2_grad, Matrix[Double] W3_grad, Matrix[Double] b3_grad,
- Matrix[Double] W4_grad, Matrix[Double] b4_grad)
-{
- #backprop
- delta4 = E * Yhat_prime
- delta3 = H3_prime * (delta4 %*% W4)
- delta2 = H2_prime * (delta3 %*% W3)
- delta1 = H1_prime * (delta2 %*% W2)
-
- #compute gradients
- b4_grad = t(colSums(delta4))
- b3_grad = t(colSums(delta3))
- b2_grad = t(colSums(delta2))
- b1_grad = t(colSums(delta1))
-
- W4_grad = t(delta4) %*% H3
- W3_grad = t(delta3) %*% H2
- W2_grad = t(delta2) %*% H1
- W1_grad = t(delta1) %*% X
-}
-
-obj = function(Matrix[Double] E) return(Double val){
- val = 0.5 * sum(E^2)
-}
-
batch_size = ifdef($BATCH, 256)
mu = ifdef($MOMENTUM, 0.9)
step = ifdef($STEP, 1e-5)
decay = ifdef($DECAY, 0.95)
-hfile = ifdef($HIDDEN, " ")
-fmt = ifdef($fmt, "text")
full_obj = ifdef($OBJ, FALSE)
+fmt = ifdef($fmt, "text")
-X = read($X)
-num_hidden1 = $H1
-num_hidden2 = $H2
-max_epochs = $EPOCH
-W1_rand = read($W1_rand)
-W2_rand = read($W2_rand)
-W3_rand = read($W3_rand)
-W4_rand = read($W4_rand)
-order_rand = read($order_rand)
-
-n = nrow(X)
-m = ncol(X)
-
-#randomly reordering rows
-#permut = table(seq(1,n,1), order(target=Rand(rows=n, cols=1, min=0, max=1, pdf="uniform"), by=1, index.return=TRUE), n, n)
-permut = table(seq(1,n,1), order(target=order_rand, by=1, index.return=TRUE), n, n)
-X = permut %*% X
-
-#z-transform, whitening operator is better
-means = colSums(X)/n
-stds = sqrt((colSums(X^2)/n - means*means)*n/(n-1)) + 1e-17
-X = (X - means)/stds
-
-W1 = sqrt(6)/sqrt(m + num_hidden1) * W1_rand #Rand(rows=num_hidden1, cols=m, min=-1, max=1, pdf="uniform")
-b1 = matrix(0, rows=num_hidden1, cols=1)
-W2 = sqrt(6)/sqrt(num_hidden1 + num_hidden2) * W2_rand #Rand(rows=num_hidden2, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b2 = matrix(0, rows=num_hidden2, cols=1)
-W3 = sqrt(6)/sqrt(num_hidden2 + num_hidden1) * W3_rand #Rand(rows=num_hidden1, cols=num_hidden2, min=-1, max=1, pdf="uniform")
-b3 = matrix(0, rows=num_hidden1, cols=1)
-W4 = sqrt(6)/sqrt(num_hidden2 + m) * W4_rand #Rand(rows=m, cols=num_hidden1, min=-1, max=1, pdf="uniform")
-b4 = matrix(0, rows=m, cols=1)
-
-upd_W1 = matrix(0, rows=nrow(W1), cols=ncol(W1))
-upd_b1 = matrix(0, rows=nrow(b1), cols=ncol(b1))
-upd_W2 = matrix(0, rows=nrow(W2), cols=ncol(W2))
-upd_b2 = matrix(0, rows=nrow(b2), cols=ncol(b2))
-upd_W3 = matrix(0, rows=nrow(W3), cols=ncol(W3))
-upd_b3 = matrix(0, rows=nrow(b3), cols=ncol(b3))
-upd_W4 = matrix(0, rows=nrow(W4), cols=ncol(W4))
-upd_b4 = matrix(0, rows=nrow(b4), cols=ncol(b4))
-
-if( full_obj ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime,
- full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- full_o = obj(full_E)
- print("EPOCHS=" + 0 + " OBJ (FULL DATA): " + full_o)
-}
-
-iter = 0
-num_iters_per_epoch = ceil(n / batch_size)
-max_iterations = max_epochs * num_iters_per_epoch
-#print("num_iters_per_epoch=" + num_iters_per_epoch + " max_iterations=" + max_iterations)
-beg = 1
-while( iter < max_iterations ) {
- end = beg + batch_size - 1
- if(end > n) end = n
- X_batch = X[beg:end,]
-
- [H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat, Yhat_prime, E] = feedForward(X_batch, W1, b1, W2, b2, W3, b3, W4, b4, X_batch)
- [W1_grad, b1_grad, W2_grad, b2_grad, W3_grad, b3_grad, W4_grad, b4_grad] = grad(X_batch, H1, H1_prime, H2, H2_prime, H3, H3_prime, Yhat_prime, E, W1, W2, W3, W4)
-
- o = obj(E)
- print("epochs=%5.4f BATCH beg=%d end=%d obj=%f", (iter / num_iters_per_epoch), beg, end, o)
-
- #update
- local_step = step / nrow(X_batch)
- upd_W1 = mu * upd_W1 - local_step * W1_grad
- upd_b1 = mu * upd_b1 - local_step * b1
- upd_W2 = mu * upd_W2 - local_step * W2_grad
- upd_b2 = mu * upd_b2 - local_step * b2
- upd_W3 = mu * upd_W3 - local_step * W3_grad
- upd_b3 = mu * upd_b3 - local_step * b3
- upd_W4 = mu * upd_W4 - local_step * W4_grad
- upd_b4 = mu * upd_b4 - local_step * b4
- W1 = W1 + upd_W1
- b1 = b1 + upd_b1
- W2 = W2 + upd_W2
- b2 = b2 + upd_b2
- W3 = W3 + upd_W3
- b3 = b3 + upd_b3
- W4 = W4 + upd_W4
- b4 = b4 + upd_b4
-
- iter = iter + 1
- if(end == n) beg = 1
- else beg = end + 1
-
- if( iter %% num_iters_per_epoch == 0 ) step = step * decay
-
- if( full_obj & iter %% num_iters_per_epoch == 0 ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- full_o = obj(full_E)
- epochs = iter %/% num_iters_per_epoch
- print("EPOCHS=" + epochs + " iter=" + iter + " OBJ (FULL DATA)=" + full_o)
- }
-}
+[W1, b1, W2, b2, W3, b3, W4, b4, hidden] = autoencoder_2layer(
+ X = read($X), W1_rand = read($W1_rand), W2_rand = read($W2_rand),
+ W3_rand = read($W3_rand), W4_rand = read($W4_rand), order_rand = read($order_rand),
+ num_hidden1 = $H1, num_hidden2 = $H2, max_epochs = $EPOCH, full_obj = full_obj,
+ batch_size = batch_size, step = step, decay = decay, mu = mu)
write(W1, $W1_out, format=fmt)
write(b1, $b1_out, format=fmt)
@@ -243,9 +40,3 @@ write(W3, $W3_out, format=fmt)
write(b3, $b3_out, format=fmt)
write(W4, $W4_out, format=fmt)
write(b4, $b4_out, format=fmt)
-
-if( hfile != " " ){
- [full_H1, full_H1_prime, full_H2, full_H2_prime, full_H3, full_H3_prime, full_Yhat, full_Yhat_prime, full_E] = feedForward(X, W1, b1, W2, b2, W3, b3, W4, b4, X)
- reordered_H = t(permut) %*% full_H2
- write(reordered_H, hfile, format=fmt)
-}