You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2018/03/22 22:01:57 UTC
[4/4] madlib git commit: MLP: Add minibatch gradient descent solver
MLP: Add minibatch gradient descent solver
JIRA: MADLIB-1206
This commit adds support for mini-batch based gradient descent for MLP.
If the input table contains a 2D matrix for independent variable,
minibatch is automatically used as the solver. Two minibatch specific
optimizers are also introduced: batch_size and n_epochs.
- batch_size is defaulted to min(200, buffer_size), where buffer_size is
equal to the number of original input rows packed into a single row in
the matrix.
- n_epochs is the number of times all the batches in a buffer are
iterated over (default 1).
Other changes include:
- dependent variable in the minibatch solver is also a matrix now. It
was initially a vector.
- Randomize the order of processing a batch within an epoch.
- MLP minibatch currently doesn't support weights param, an error is
thrown now.
- Delete an unused type named mlp_step_result.
- Add unit tests for newly added functions in python file.
Closes #243
Co-authored-by: Rahul Iyer <ri...@apache.org>
Co-authored-by: Nikhil Kak <nk...@pivotal.io>
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/a595e221
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/a595e221
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/a595e221
Branch: refs/heads/master
Commit: a595e2218c95685d73dc51f28e9fc1f0193abce9
Parents: d2ede20
Author: Nandish Jayaram <nj...@apache.org>
Authored: Thu Mar 22 15:01:18 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu Mar 22 15:01:18 2018 -0700
----------------------------------------------------------------------
src/modules/convex/algo/igd.hpp | 42 +-
src/modules/convex/mlp_igd.cpp | 198 ++++-
src/modules/convex/mlp_igd.hpp | 5 +
src/modules/convex/task/mlp.hpp | 75 +-
src/modules/convex/type/state.hpp | 172 ++++-
src/modules/convex/type/tuple.hpp | 1 +
src/ports/postgres/modules/convex/mlp.sql_in | 91 ++-
src/ports/postgres/modules/convex/mlp_igd.py_in | 772 +++++++++++++------
.../postgres/modules/convex/test/mlp.sql_in | 503 ++++++------
.../convex/test/unit_tests/plpy_mock.py_in | 34 +
.../convex/test/unit_tests/test_mlp_igd.py_in | 164 ++++
.../modules/utilities/validate_args.py_in | 42 +-
12 files changed, 1502 insertions(+), 597 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/algo/igd.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/algo/igd.hpp b/src/modules/convex/algo/igd.hpp
index 3ae4c13..1d7fc0f 100644
--- a/src/modules/convex/algo/igd.hpp
+++ b/src/modules/convex/algo/igd.hpp
@@ -90,20 +90,34 @@ IGD<State, ConstState, Task>::transition(state_type &state,
for (int curr_epoch=0; curr_epoch < n_epochs; curr_epoch++) {
double loss = 0.0;
- for (int curr_batch=0, curr_batch_row_index=0; curr_batch < n_batches;
- curr_batch++, curr_batch_row_index += batch_size) {
- Matrix X_batch;
- ColumnVector y_batch;
- if (curr_batch == n_batches-1) {
- // last batch
- X_batch = tuple.indVar.bottomRows(n_rows-curr_batch_row_index);
- y_batch = tuple.depVar.tail(n_rows-curr_batch_row_index);
- } else {
- X_batch = tuple.indVar.block(curr_batch_row_index, 0, batch_size, n_ind_cols);
- y_batch = tuple.depVar.segment(curr_batch_row_index, batch_size);
- }
- loss += Task::getLossAndUpdateModel(
- state.task.model, X_batch, y_batch, state.task.stepsize);
+ /*
+ Randomizing the input data before every iteration is good for
+ minibatch gradient descent convergence. Since we don't do that,
+ we are randomizing the order in which every batch is visited in
+ a buffer. Note that this still does not randomize rows within
+ a batch.
+ */
+ int random_curr_batch[n_batches];
+ for(int i=0; i<n_batches; i++) {
+ random_curr_batch[i] = i;
+ }
+ int curr_batch_row_index = 0;
+ std::random_shuffle(&random_curr_batch[0], &random_curr_batch[n_batches]);
+ for (int i=0; i < n_batches; i++) {
+ int curr_batch = random_curr_batch[i];
+ int curr_batch_row_index = curr_batch * batch_size;
+ Matrix X_batch;
+ Matrix Y_batch;
+ if (curr_batch == n_batches-1) {
+ // last batch
+ X_batch = tuple.indVar.bottomRows(n_rows-curr_batch_row_index);
+ Y_batch = tuple.depVar.bottomRows(n_rows-curr_batch_row_index);
+ } else {
+ X_batch = tuple.indVar.block(curr_batch_row_index, 0, batch_size, n_ind_cols);
+ Y_batch = tuple.depVar.block(curr_batch_row_index, 0, batch_size, tuple.depVar.cols());
+ }
+ loss += Task::getLossAndUpdateModel(
+ state.task.model, X_batch, Y_batch, state.task.stepsize);
}
// The first epoch will most likely have the highest loss.
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/mlp_igd.cpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/mlp_igd.cpp b/src/modules/convex/mlp_igd.cpp
index 7a7bf1d..b8738d4 100644
--- a/src/modules/convex/mlp_igd.cpp
+++ b/src/modules/convex/mlp_igd.cpp
@@ -47,6 +47,9 @@ namespace convex {
typedef IGD<MLPIGDState<MutableArrayHandle<double> >, MLPIGDState<ArrayHandle<double> >,
MLP<MLPModel<MutableArrayHandle<double> >, MLPTuple > > MLPIGDAlgorithm;
+typedef IGD<MLPMiniBatchState<MutableArrayHandle<double> >, MLPMiniBatchState<ArrayHandle<double> >,
+ MLP<MLPModel<MutableArrayHandle<double> >, MiniBatchTuple > > MLPMiniBatchAlgorithm;
+
typedef Loss<MLPIGDState<MutableArrayHandle<double> >, MLPIGDState<ArrayHandle<double> >,
MLP<MLPModel<MutableArrayHandle<double> >, MLPTuple > > MLPLossAlgorithm;
@@ -66,7 +69,6 @@ mlp_igd_transition::run(AnyType &args) {
// For other tuples: args[0] holds the computation state until last tuple
MLPIGDState<MutableArrayHandle<double> > state = args[0];
-
// initilize the state if first tuple
if (state.algo.numRows == 0) {
if (!args[3].isNull()) {
@@ -76,39 +78,48 @@ mlp_igd_transition::run(AnyType &args) {
previousState.task.numbersOfUnits);
state = previousState;
} else {
- // configuration parameters
+ // configuration parameters and initialization
+ // this is run only once (first iteration, first tuple)
ArrayHandle<double> numbersOfUnits = args[4].getAs<ArrayHandle<double> >();
int numberOfStages = numbersOfUnits.size() - 1;
double stepsize = args[5].getAs<double>();
-
state.allocate(*this, numberOfStages,
reinterpret_cast<const double *>(numbersOfUnits.ptr()));
state.task.stepsize = stepsize;
-
const int activation = args[6].getAs<int>();
const int is_classification = args[7].getAs<int>();
-
+ // args[8] is for weighting the input row, which is populated later.
const bool warm_start = args[9].getAs<bool>();
const double lambda = args[11].getAs<double>();
state.task.lambda = lambda;
MLPTask::lambda = lambda;
+
double is_classification_double = (double) is_classification;
double activation_double = (double) activation;
MappedColumnVector coeff = args[10].getAs<MappedColumnVector>();
state.task.model.rebind(&is_classification_double,&activation_double,
&coeff.data()[0], numberOfStages,
&numbersOfUnits[0]);
+
+ // state.task.model.is_classification =
+ // static_cast<double>(is_classification);
+ // state.task.model.activation = static_cast<double>(activation);
+ // MappedColumnVector initial_coeff = args[10].getAs<MappedColumnVector>();
+ // // copy initial_coeff into the model
+ // Index fan_in, fan_out, layer_start = 0;
+ // for (size_t k = 0; k < numberOfStages; ++k){
+ // fan_in = numbersOfUnits[k];
+ // fan_out = numbersOfUnits[k+1];
+ // state.task.model.u[k] << initial_coeff.segment(layer_start, (fan_in+1)*fan_out);
+ // layer_start = (fan_in + 1) * fan_out;
+ // }
}
// resetting in either case
state.reset();
}
- // meta data
- const uint16_t N = state.task.numberOfStages;
- const double *n = state.task.numbersOfUnits;
-
// tuple
ColumnVector indVar;
MappedColumnVector depVar;
@@ -125,11 +136,107 @@ mlp_igd_transition::run(AnyType &args) {
tuple.weight = args[8].getAs<double>();
MLPIGDAlgorithm::transition(state, tuple);
+ // Use the model from the previous iteration to compute the loss (note that
+ // it is stored in Task's state, and the Algo's state holds the model from
+ // the current iteration.
MLPLossAlgorithm::transition(state, tuple);
state.algo.numRows ++;
return state;
}
+/**
+ * @brief Perform the multilayer perceptron minibatch transition step
+ *
+ * Called for each tuple.
+ */
+AnyType
+mlp_minibatch_transition::run(AnyType &args) {
+ // For the first tuple: args[0] is nothing more than a marker that
+ // indicates that we should do some initial operations.
+ // For other tuples: args[0] holds the computation state until last tuple
+ MLPMiniBatchState<MutableArrayHandle<double> > state = args[0];
+
+ // initilize the state if first tuple
+ if (state.algo.numRows == 0) {
+ if (!args[3].isNull()) {
+ MLPMiniBatchState<ArrayHandle<double> > previousState = args[3];
+ state.allocate(*this, previousState.task.numberOfStages,
+ previousState.task.numbersOfUnits);
+ state = previousState;
+ } else {
+ // configuration parameters
+ ArrayHandle<double> numbersOfUnits = args[4].getAs<ArrayHandle<double> >();
+ int numberOfStages = numbersOfUnits.size() - 1;
+
+ double stepsize = args[5].getAs<double>();
+
+ state.allocate(*this, numberOfStages,
+ reinterpret_cast<const double *>(numbersOfUnits.ptr()));
+ state.task.stepsize = stepsize;
+ const int activation = args[6].getAs<int>();
+ const int is_classification = args[7].getAs<int>();
+ // args[8] is for weighting the input row, which is populated later.
+ const bool warm_start = args[9].getAs<bool>();
+ const double lambda = args[11].getAs<double>();
+ state.algo.batchSize = args[12].getAs<int>();
+ state.algo.nEpochs = args[13].getAs<int>();
+ state.task.lambda = lambda;
+ MLPTask::lambda = lambda;
+
+ /* FIXME: The state is set back to zero for second row onwards if
+ initialized as in IGD. The following avoids that, but there is
+ some failure with debug build that must be fixed.
+ */
+ state.task.model.is_classification =
+ static_cast<double>(is_classification);
+ state.task.model.activation = static_cast<double>(activation);
+ MappedColumnVector initial_coeff = args[10].getAs<MappedColumnVector>();
+ // copy initial_coeff into the model
+ Index fan_in, fan_out, layer_start = 0;
+ for (size_t k = 0; k < numberOfStages; ++k){
+ fan_in = numbersOfUnits[k];
+ fan_out = numbersOfUnits[k+1];
+ state.task.model.u[k] << initial_coeff.segment(layer_start, (fan_in+1)*fan_out);
+ layer_start = (fan_in + 1) * fan_out;
+ }
+ }
+ // resetting in either case
+ state.reset();
+ }
+
+ // tuple
+ Matrix indVar;
+ Matrix depVar;
+ try {
+ // Ideally there should be no NULLs in the pre-processed input data,
+ // but keep it in a try block in case the user has modified the
+ // pre-processed data in any way.
+ indVar = args[1].getAs<MappedMatrix>();
+ depVar = args[2].getAs<MappedMatrix>();
+ } catch (const ArrayWithNullException &e) {
+ return args[0];
+ }
+ MiniBatchTuple tuple;
+ // The matrices are by default read as column-major. We will have to
+ // transpose it to get back the matrix like how it is in the database.
+ tuple.indVar = trans(indVar);
+ tuple.depVar = trans(depVar);
+ tuple.weight = args[8].getAs<double>();
+
+ /*
+ Note that the IGD version uses the model in Task (model from the
+ previous iteration) to compute the loss.
+ Minibatch uses the model from Algo (the model based on current
+ iteration) to compute the loss. The difference in loss based on one
+ iteration is not too much, hence doing so here. We therefore don't
+ need to maintain another copy of the model (from previous iteration)
+ in the state. The model for the current iteration, and the loss are
+ both computed in one function now.
+ */
+ MLPMiniBatchAlgorithm::transitionInMiniBatch(state, tuple);
+ state.algo.numRows += tuple.indVar.rows();
+ return state;
+}
/**
* @brief Perform the perliminary aggregation function: Merge transition states
@@ -145,9 +252,29 @@ mlp_igd_merge::run(AnyType &args) {
MLPIGDAlgorithm::merge(stateLeft, stateRight);
MLPLossAlgorithm::merge(stateLeft, stateRight);
+ // The following numRows update cannot be put above, because the model
+ // averaging depends on their original values
+ stateLeft.algo.numRows += stateRight.algo.numRows;
+
+ return stateLeft;
+}
+/**
+ * @brief Perform the perliminary aggregation function: Merge transition states
+ */
+AnyType
+mlp_minibatch_merge::run(AnyType &args) {
+ MLPMiniBatchState<MutableArrayHandle<double> > stateLeft = args[0];
+ MLPMiniBatchState<ArrayHandle<double> > stateRight = args[1];
+
+ if (stateLeft.algo.numRows == 0) { return stateRight; }
+ else if (stateRight.algo.numRows == 0) { return stateLeft; }
+
+ MLPMiniBatchAlgorithm::mergeInPlace(stateLeft, stateRight);
+
// The following numRows update, cannot be put above, because the model
// averaging depends on their original values
stateLeft.algo.numRows += stateRight.algo.numRows;
+ stateLeft.algo.loss += stateRight.algo.loss;
return stateLeft;
}
@@ -170,6 +297,24 @@ mlp_igd_final::run(AnyType &args) {
return state;
}
+
+/**
+ * @brief Perform the multilayer perceptron final step
+ */
+AnyType
+mlp_minibatch_final::run(AnyType &args) {
+ // We request a mutable object. Depending on the backend, this might perform
+ // a deep copy.
+ MLPMiniBatchState<MutableArrayHandle<double> > state = args[0];
+ // Aggregates that haven't seen any data just return Null.
+ if (state.algo.numRows == 0) { return Null(); }
+
+ L2<MLPModelType>::lambda = state.task.lambda;
+ state.algo.loss = state.algo.loss/static_cast<double>(state.algo.numRows);
+ state.algo.loss += L2<MLPModelType>::loss(state.task.model);
+ return state;
+}
+
/**
* @brief Return the difference in RMSE between two states
*/
@@ -180,18 +325,44 @@ internal_mlp_igd_distance::run(AnyType &args) {
return std::abs(stateLeft.algo.loss - stateRight.algo.loss);
}
+
+AnyType
+internal_mlp_minibatch_distance::run(AnyType &args) {
+ MLPMiniBatchState<ArrayHandle<double> > stateLeft = args[0];
+ MLPMiniBatchState<ArrayHandle<double> > stateRight = args[1];
+
+ return std::abs(stateLeft.algo.loss - stateRight.algo.loss);
+}
+
/**
* @brief Return the coefficients and diagnostic statistics of the state
*/
AnyType
internal_mlp_igd_result::run(AnyType &args) {
MLPIGDState<ArrayHandle<double> > state = args[0];
-
HandleTraits<ArrayHandle<double> >::ColumnVectorTransparentHandleMap
flattenU;
flattenU.rebind(&state.task.model.u[0](0, 0),
- state.task.model.arraySize(state.task.numberOfStages,
- state.task.numbersOfUnits));
+ state.task.model.arraySize(state.task.numberOfStages,
+ state.task.numbersOfUnits));
+ double loss = state.algo.loss;
+
+ AnyType tuple;
+ tuple << flattenU
+ << loss;
+ return tuple;
+}
+
+/**
+ * @brief Return the coefficients and diagnostic statistics of the state
+ */
+AnyType
+internal_mlp_minibatch_result::run(AnyType &args) {
+ MLPMiniBatchState<ArrayHandle<double> > state = args[0];
+ HandleTraits<ArrayHandle<double> >::ColumnVectorTransparentHandleMap flattenU;
+ flattenU.rebind(&state.task.model.u[0](0, 0),
+ state.task.model.arraySize(state.task.numberOfStages,
+ state.task.numbersOfUnits));
double loss = state.algo.loss;
AnyType tuple;
@@ -215,7 +386,8 @@ internal_predict_mlp::run(AnyType &args) {
double activation = args[3].getAs<double>();
bool get_class = is_classification && is_response;
- model.rebind(&is_classification,&activation,&coeff.data()[0],numberOfStages,&layerSizes.data()[0]);
+ model.rebind(&is_classification, &activation, &coeff.data()[0],
+ numberOfStages, &layerSizes.data()[0]);
try {
indVar = (args[1].getAs<MappedColumnVector>()-x_means).cwiseQuotient(x_stds);
} catch (const ArrayWithNullException &e) {
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/mlp_igd.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/mlp_igd.hpp b/src/modules/convex/mlp_igd.hpp
index c957d97..798efe3 100644
--- a/src/modules/convex/mlp_igd.hpp
+++ b/src/modules/convex/mlp_igd.hpp
@@ -25,28 +25,33 @@
* @brief Multilayer perceptron (incremental gradient): Transition function
*/
DECLARE_UDF(convex, mlp_igd_transition)
+DECLARE_UDF(convex, mlp_minibatch_transition)
/**
* @brief Multilayer perceptron (incremental gradient): State merge function
*/
DECLARE_UDF(convex, mlp_igd_merge)
+DECLARE_UDF(convex, mlp_minibatch_merge)
/**
* @brief Multilayer perceptron (incremental gradient): Final function
*/
DECLARE_UDF(convex, mlp_igd_final)
+DECLARE_UDF(convex, mlp_minibatch_final)
/**
* @brief Multilayer perceptron (incremental gradient): Difference in
* log-likelihood between two transition states
*/
DECLARE_UDF(convex, internal_mlp_igd_distance)
+DECLARE_UDF(convex, internal_mlp_minibatch_distance)
/**
* @brief Multilayer perceptron (incremental gradient): Convert
* transition state to result tuple
*/
DECLARE_UDF(convex, internal_mlp_igd_result)
+DECLARE_UDF(convex, internal_mlp_minibatch_result)
/**
* @brief Multilayer perceptron (incremental gradient): Predict
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/task/mlp.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/task/mlp.hpp b/src/modules/convex/task/mlp.hpp
index 0032b81..8a68aaa 100644
--- a/src/modules/convex/task/mlp.hpp
+++ b/src/modules/convex/task/mlp.hpp
@@ -52,6 +52,12 @@ public:
const dependent_variable_type &y,
const double &stepsize);
+ static double getLossAndUpdateModel(
+ model_type &model,
+ const Matrix &x,
+ const Matrix &y,
+ const double &stepsize);
+
static double loss(
const model_type &model,
const independent_variables_type &x,
@@ -112,6 +118,59 @@ template <class Model, class Tuple>
double MLP<Model, Tuple>::lambda = 0;
template <class Model, class Tuple>
+double
+MLP<Model, Tuple>::getLossAndUpdateModel(
+ model_type &model,
+ const Matrix &x_batch,
+ const Matrix &y_true_batch,
+ const double &stepsize) {
+
+ uint16_t num_layers = model.u.size(); // assuming nu. of layers >= 1
+ size_t num_rows_in_batch = x_batch.rows();
+ size_t i, k;
+ double total_loss = 0.;
+
+ // gradient added over the batch
+ std::vector<Matrix> total_gradient_per_layer(num_layers);
+ for (k=0; k < num_layers; ++k)
+ total_gradient_per_layer[k] = Matrix::Zero(model.u[k].rows(),
+ model.u[k].cols());
+
+ for (i=0; i < num_rows_in_batch; i++){
+ ColumnVector x = x_batch.row(i);
+ ColumnVector y_true = y_true_batch.row(i);
+
+ std::vector<ColumnVector> net, o, delta;
+ feedForward(model, x, net, o);
+ backPropogate(y_true, o.back(), net, model, delta);
+
+ for (k=0; k < num_layers; k++){
+ total_gradient_per_layer[k] += o[k] * delta[k].transpose();
+ }
+
+ // loss computation
+ ColumnVector y_estimated = o.back();
+ if(model.is_classification){
+ double clip = 1.e-10;
+ y_estimated = y_estimated.cwiseMax(clip).cwiseMin(1.-clip);
+ total_loss += - (y_true.array()*y_estimated.array().log()
+ + (-y_true.array()+1)*(-y_estimated.array()+1).log()).sum();
+ }
+ else{
+ total_loss += 0.5 * (y_estimated - y_true).squaredNorm();
+ }
+ }
+ for (k=0; k < num_layers; k++){
+ Matrix regularization = MLP<Model, Tuple>::lambda * model.u[k];
+ regularization.row(0).setZero(); // Do not update bias
+ model.u[k] -= stepsize * (total_gradient_per_layer[k] / \
+ num_rows_in_batch + \
+ regularization);
+ }
+ return total_loss;
+}
+
+template <class Model, class Tuple>
void
MLP<Model, Tuple>::gradientInPlace(
model_type &model,
@@ -151,7 +210,7 @@ MLP<Model, Tuple>::loss(
+ (-y_true.array()+1)*(-y_estimated.array()+1).log()).sum();
}
else{
- return 0.5 * (y_estimated-y_true).squaredNorm();
+ return 0.5 * (y_estimated - y_true).squaredNorm();
}
}
@@ -165,6 +224,7 @@ MLP<Model, Tuple>::predict(
feedForward(model, x, net, o);
ColumnVector output = o.back();
+
if(get_class){ // Return a length 1 array with the predicted index
int max_idx;
output.maxCoeff(&max_idx);
@@ -183,8 +243,14 @@ MLP<Model, Tuple>::feedForward(
std::vector<ColumnVector> &net,
std::vector<ColumnVector> &o){
uint16_t k, N;
+ /*
+ The network starts with the 0th layer (input), followed by n_layers
+ number of hidden layers, and then an output layer.
+ */
+ // Total number of coefficients in the model
N = model.u.size(); // assuming >= 1
net.resize(N + 1);
+ // o[k] is a vector of the output of the kth layer
o.resize(N + 1);
double (*activation)(const double&);
@@ -195,12 +261,15 @@ MLP<Model, Tuple>::feedForward(
else
activation = &tanh;
- o[0].resize(x.size()+1);
+ o[0].resize(x.size() + 1);
o[0] << 1.,x;
for (k = 1; k < N; k ++) {
+ // o_k = activation(sum(o_{k-1} * u_{k-1}))
+ // net_k just does the inner sum: input to the activation function
net[k] = model.u[k-1].transpose() * o[k-1];
- o[k] = ColumnVector(model.u[k-1].cols()+1);
+ o[k] = ColumnVector(model.u[k-1].cols() + 1);
+ // This applies the activation function to give the actual node output
o[k] << 1., net[k].unaryExpr(activation);
}
o[N] = model.u[N-1].transpose() * o[N-1];
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/type/state.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/type/state.hpp b/src/modules/convex/type/state.hpp
index f846e8f..a3b0d43 100644
--- a/src/modules/convex/type/state.hpp
+++ b/src/modules/convex/type/state.hpp
@@ -712,10 +712,9 @@ public:
+ 1 // is_classification
+ 1 // activation
+ sizeOfModel // model
-
+ + sizeOfModel // incrModel
+ 1 // numRows
- + 1 // loss
- + sizeOfModel; // incrModel
+ + 1; // loss
}
private:
@@ -728,14 +727,18 @@ private:
* - 1: numbersOfUnits (numbers of activation units, design doc: n_0,...,n_N)
* - N + 2: stepsize (step size of gradient steps)
* - N + 3: lambda (regularization term)
+// is_classification, activation, and coeff together form the model
* - N + 4: is_classification (do classification)
* - N + 5: activation (activation function)
* - N + 6: coeff (coefficients, design doc: u)
*
* Intra-iteration components (updated in transition step):
* sizeOfModel = # of entries in u + 2, (\sum_1^N n_{k-1} n_k)
+// incremental model
* - N + 6 + sizeOfModel: coeff (volatile model for incrementally update)
+// number of rows
* - N + 6 + 2*sizeOfModel: numRows (number of rows processed in this iteration)
+// loss
* - N + 7 + 2*sizeOfModel: loss (loss value, the sum of squared errors)
*/
void rebind() {
@@ -779,6 +782,169 @@ public:
};
+
+/**
+ * @brief Inter- (Task State) and intra-iteration (Algo State) state of
+ * incremental gradient descent for multilayer perceptron
+ *
+ * TransitionState encapsualtes the transition state during the
+ * aggregate function during an iteration. To the database, the state is
+ * exposed as a single DOUBLE PRECISION array, to the C++ code it is a proper
+ * object containing scalars and vectors.
+ *
+ * Note: We assume that the DOUBLE PRECISION array is initialized by the
+ * database with length at least 6, and at least first elemenet
+ * is 0 (exact values of other elements are ignored).
+ *
+ */
+template <class Handle>
+class MLPMiniBatchState {
+ template <class OtherHandle>
+ friend class MLPMiniBatchState;
+
+public:
+ MLPMiniBatchState(const AnyType &inArray) : mStorage(inArray.getAs<Handle>()) {
+ rebind();
+ }
+
+ /**
+ * @brief Reset the intra-iteration fields.
+ */
+ inline void reset() {
+ algo.numRows = 0;
+ algo.loss = 0.;
+ }
+
+ /**
+ * @brief Convert to backend representation
+ *
+ * We define this function so that we can use State in the
+ * argument list and as a return type.
+ */
+ inline operator AnyType() const {
+ return mStorage;
+ }
+
+ /**
+ * @brief Allocating the incremental gradient state.
+ */
+ inline void allocate(const Allocator &inAllocator,
+ const uint16_t &inNumberOfStages,
+ const double *inNumbersOfUnits) {
+ mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
+ dbal::DoZero, dbal::ThrowBadAlloc>(
+ arraySize(inNumberOfStages, inNumbersOfUnits));
+
+ // This rebind is for the following lines of code to take
+ // effect. I can also do something like "mStorage[0] = N",
+ // but I am not clear about the type binding/alignment
+ rebind();
+ task.numberOfStages = inNumberOfStages;
+ uint16_t N = inNumberOfStages;
+ uint16_t k;
+ for (k = 0; k <= N; k ++) {
+ task.numbersOfUnits[k] = inNumbersOfUnits[k];
+ }
+
+ // This time all the member fields are correctly binded
+ rebind();
+ }
+
+ /**
+ * @brief We need to support assigning the previous state
+ */
+ template <class OtherHandle>
+ MLPMiniBatchState &operator=(const MLPMiniBatchState<OtherHandle> &inOtherState) {
+ for (size_t i = 0; i < mStorage.size(); i++) {
+ mStorage[i] = inOtherState.mStorage[i];
+ }
+
+ return *this;
+ }
+
+ static inline uint32_t arraySize(const uint16_t &inNumberOfStages,
+ const double *inNumbersOfUnits) {
+ uint32_t sizeOfModel =
+ MLPModel<Handle>::arraySize(inNumberOfStages, inNumbersOfUnits);
+ return 1 // numberOfStages = N
+ + (inNumberOfStages + 1) // numbersOfUnits: size is (N + 1)
+ + 1 // stepsize
+ + 1 // lambda
+ + 1 // is_classification
+ + 1 // activation
+ + sizeOfModel // model
+ + 1 // numRows
+ + 1 // batchSize
+ + 1 // nEpochs
+ + 1; // loss
+
+ }
+
+ Handle mStorage;
+private:
+ /**
+ * @brief Rebind to a new storage array.
+ *
+ * Array layout (iteration refers to one aggregate-function call):
+ * Inter-iteration components (updated in final function):
+ * - 0: numberOfStages (number of stages (layers), design doc: N)
+ * - 1: numbersOfUnits (numbers of activation units, design doc: n_0,...,n_N)
+ * - N + 2: stepsize (step size of gradient steps)
+ * - N + 3: lambda (regularization term)
+// is_classification, activation, and coeff together form the model
+ * - N + 4: is_classification (do classification)
+ * - N + 5: activation (activation function)
+ * - N + 6: coeff (coefficients, design doc: u)
+// model is done, now bind numRows
+ * - N + 6 + sizeOfModel: numRows (number of rows processed in this iteration)
+ * - N + 7 + sizeOfModel: batchSize (number of rows processed in this iteration)
+ * - N + 8 + sizeOfModel: nEpochs (number of rows processed in this iteration)
+ * - N + 9 + sizeOfModel: loss (loss value, the sum of squared errors)
+ */
+ void rebind() {
+
+ task.numberOfStages.rebind(&mStorage[0]);
+ size_t N = task.numberOfStages;
+
+ task.numbersOfUnits =
+ reinterpret_cast<dimension_pointer_type>(&mStorage[1]);
+ task.stepsize.rebind(&mStorage[N + 2]);
+ task.lambda.rebind(&mStorage[N + 3]);
+ size_t sizeOfModel = task.model.rebind(&mStorage[N + 4],
+ &mStorage[N + 5],
+ &mStorage[N + 6],
+ task.numberOfStages,
+ task.numbersOfUnits);
+ algo.numRows.rebind(&mStorage[N + 6 + sizeOfModel]);
+ algo.batchSize.rebind(&mStorage[N + 7 + sizeOfModel]);
+ algo.nEpochs.rebind(&mStorage[N + 8 + sizeOfModel]);
+ algo.loss.rebind(&mStorage[N + 9 + sizeOfModel]);
+ }
+
+
+ typedef typename HandleTraits<Handle>::ReferenceToUInt16 dimension_type;
+ typedef typename HandleTraits<Handle>::DoublePtr dimension_pointer_type;
+ typedef typename HandleTraits<Handle>::ReferenceToUInt64 count_type;
+ typedef typename HandleTraits<Handle>::ReferenceToDouble numeric_type;
+
+public:
+ struct TaskState {
+ dimension_type numberOfStages;
+ dimension_pointer_type numbersOfUnits;
+ numeric_type stepsize;
+ numeric_type lambda;
+ MLPModel<Handle> model;
+ } task;
+
+ struct AlgoState {
+ count_type numRows;
+ dimension_type batchSize;
+ dimension_type nEpochs;
+ numeric_type loss;
+ } algo;
+};
+
+
} // namespace convex
} // namespace modules
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/type/tuple.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/type/tuple.hpp b/src/modules/convex/type/tuple.hpp
index ac070b6..492a1b0 100644
--- a/src/modules/convex/type/tuple.hpp
+++ b/src/modules/convex/type/tuple.hpp
@@ -68,6 +68,7 @@ typedef ExampleTuple<Matrix, ColumnVector> SVMMiniBatchTuple;
typedef ExampleTuple<MatrixIndex, double> LMFTuple;
typedef ExampleTuple<ColumnVector, MappedColumnVector> MLPTuple;
+typedef ExampleTuple<Matrix, Matrix> MiniBatchTuple;
} // namespace convex
http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/ports/postgres/modules/convex/mlp.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/convex/mlp.sql_in b/src/ports/postgres/modules/convex/mlp.sql_in
index e6e2716..739007e 100644
--- a/src/ports/postgres/modules/convex/mlp.sql_in
+++ b/src/ports/postgres/modules/convex/mlp.sql_in
@@ -316,7 +316,9 @@ the parameter is ignored.
n_iterations = <value>,
n_tries = <value>,
lambda = <value>,
- tolerance = <value>'
+ tolerance = <value>,
+ batch_size = <value>,
+ n_epochs = <value>'
</pre>
\b Optimizer \b Parameters
<DL class="arglist">
@@ -376,6 +378,22 @@ If you want to run the full number of iterations specified in \e n_interations,
set tolerance=0.0
</DD>
+<DT>tolerance</dt>
+<DD>Default: 1 for IGD, 20 for Minibatch
+If the source_table is detected to contain data
+that is supported by minibatch, then the solver
+uses mini-batch gradient descent, with the specified
+batch_size.
+</DD>
+
+<DT>tolerance</dt>
+<DD>Default: 1 for IGD, 20 for Minibatch
+If the source_table is detected to contain data
+that is supported by minibatch, then the solver
+uses mini-batch gradient descent. During gradient
+descent, n_epochs represents the number of times
+all batches in a buffer are iterated over.
+</DD>
</DL>
@anchor predict
@@ -1241,11 +1259,6 @@ File mlp.sql_in documenting the training function
*/
-CREATE TYPE MADLIB_SCHEMA.mlp_step_result AS (
- state DOUBLE PRECISION[],
- loss DOUBLE PRECISION
-);
-
CREATE TYPE MADLIB_SCHEMA.mlp_result AS (
coeff DOUBLE PRECISION[],
loss DOUBLE PRECISION
@@ -1272,6 +1285,26 @@ RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
LANGUAGE C IMMUTABLE;
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
+ state DOUBLE PRECISION[],
+ ind_var DOUBLE PRECISION[],
+ dep_var DOUBLE PRECISION[],
+ previous_state DOUBLE PRECISION[],
+ layer_sizes DOUBLE PRECISION[],
+ learning_rate_init DOUBLE PRECISION,
+ activation INTEGER,
+ is_classification INTEGER,
+ weight DOUBLE PRECISION,
+ warm_start BOOLEAN,
+ warm_start_coeff DOUBLE PRECISION[],
+ lambda DOUBLE PRECISION,
+ batch_size INTEGER,
+ n_epochs INTEGER
+ )
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE;
+
CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_merge(
state1 DOUBLE PRECISION[],
state2 DOUBLE PRECISION[])
@@ -1285,6 +1318,19 @@ RETURNS DOUBLE PRECISION[]
AS 'MODULE_PATHNAME'
LANGUAGE C IMMUTABLE STRICT;
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_merge(
+ state1 DOUBLE PRECISION[],
+ state2 DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE STRICT;
+
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_final(
+ state DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE STRICT;
+
/**
* @internal
* @brief Perform one iteration of backprop
@@ -1310,6 +1356,33 @@ CREATE AGGREGATE MADLIB_SCHEMA.mlp_igd_step(
);
-------------------------------------------------------------------------
+/**
+ * @internal
+ * @brief Perform one iteration of backprop
+ */
+CREATE AGGREGATE MADLIB_SCHEMA.mlp_minibatch_step(
+ /* ind_var */ DOUBLE PRECISION[],
+ /* dep_var */ DOUBLE PRECISION[],
+ /* previous_state */ DOUBLE PRECISION[],
+ /* layer_sizes */ DOUBLE PRECISION[],
+ /* learning_rate_init */ DOUBLE PRECISION,
+ /* activation */ INTEGER,
+ /* is_classification */ INTEGER,
+ /* weight */ DOUBLE PRECISION,
+ /* warm_start */ BOOLEAN,
+ /* warm_start_coeff */ DOUBLE PRECISION[],
+ /* lambda */ DOUBLE PRECISION,
+ /* batch_size */ INTEGER,
+ /* n_epochs */ INTEGER
+ )(
+ STYPE=DOUBLE PRECISION[],
+ SFUNC=MADLIB_SCHEMA.mlp_minibatch_transition,
+ m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.mlp_minibatch_merge,')
+ FINALFUNC=MADLIB_SCHEMA.mlp_minibatch_final,
+ INITCOND='{0,0,0,0,0,0,0,0,0,0,0,0}'
+);
+-------------------------------------------------------------------------
+
CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_distance(
/*+ state1 */ DOUBLE PRECISION[],
/*+ state2 */ DOUBLE PRECISION[])
@@ -1322,6 +1395,12 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_result(
RETURNS MADLIB_SCHEMA.mlp_result AS
'MODULE_PATHNAME'
LANGUAGE c IMMUTABLE STRICT;
+
+CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_minibatch_result(
+ /*+ state */ DOUBLE PRECISION[])
+RETURNS MADLIB_SCHEMA.mlp_result AS
+'MODULE_PATHNAME'
+LANGUAGE c IMMUTABLE STRICT;
-------------------------------------------------------------------------
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(