You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2018/03/22 22:01:57 UTC

[4/4] madlib git commit: MLP: Add minibatch gradient descent solver

MLP: Add minibatch gradient descent solver

JIRA: MADLIB-1206

This commit adds support for mini-batch based gradient descent for MLP.
If the input table contains a 2D matrix for independent variable,
minibatch is automatically used as the solver. Two minibatch specific
optimizers are also introduced: batch_size and n_epochs.
- batch_size is defaulted to min(200, buffer_size), where buffer_size is
equal to the number of original input rows packed into a single row in
the matrix.
- n_epochs is the number of times all the batches in a buffer are
iterated over (default 1).

Other changes include:
- dependent variable in the minibatch solver is also a matrix now. It
was initially a vector.
- Randomize the order of processing a batch within an epoch.
- MLP minibatch currently doesn't support weights param, an error is
thrown now.
- Delete an unused type named mlp_step_result.
- Add unit tests for newly added functions in python file.

Closes #243

Co-authored-by: Rahul Iyer <ri...@apache.org>
Co-authored-by: Nikhil Kak <nk...@pivotal.io>


Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/a595e221
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/a595e221
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/a595e221

Branch: refs/heads/master
Commit: a595e2218c95685d73dc51f28e9fc1f0193abce9
Parents: d2ede20
Author: Nandish Jayaram <nj...@apache.org>
Authored: Thu Mar 22 15:01:18 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu Mar 22 15:01:18 2018 -0700

----------------------------------------------------------------------
 src/modules/convex/algo/igd.hpp                 |  42 +-
 src/modules/convex/mlp_igd.cpp                  | 198 ++++-
 src/modules/convex/mlp_igd.hpp                  |   5 +
 src/modules/convex/task/mlp.hpp                 |  75 +-
 src/modules/convex/type/state.hpp               | 172 ++++-
 src/modules/convex/type/tuple.hpp               |   1 +
 src/ports/postgres/modules/convex/mlp.sql_in    |  91 ++-
 src/ports/postgres/modules/convex/mlp_igd.py_in | 772 +++++++++++++------
 .../postgres/modules/convex/test/mlp.sql_in     | 503 ++++++------
 .../convex/test/unit_tests/plpy_mock.py_in      |  34 +
 .../convex/test/unit_tests/test_mlp_igd.py_in   | 164 ++++
 .../modules/utilities/validate_args.py_in       |  42 +-
 12 files changed, 1502 insertions(+), 597 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/algo/igd.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/algo/igd.hpp b/src/modules/convex/algo/igd.hpp
index 3ae4c13..1d7fc0f 100644
--- a/src/modules/convex/algo/igd.hpp
+++ b/src/modules/convex/algo/igd.hpp
@@ -90,20 +90,34 @@ IGD<State, ConstState, Task>::transition(state_type &state,
 
     for (int curr_epoch=0; curr_epoch < n_epochs; curr_epoch++) {
         double loss = 0.0;
-        for (int curr_batch=0, curr_batch_row_index=0; curr_batch < n_batches;
-             curr_batch++, curr_batch_row_index += batch_size) {
-           Matrix X_batch;
-           ColumnVector y_batch;
-           if (curr_batch == n_batches-1) {
-              // last batch
-              X_batch = tuple.indVar.bottomRows(n_rows-curr_batch_row_index);
-              y_batch = tuple.depVar.tail(n_rows-curr_batch_row_index);
-           } else {
-               X_batch = tuple.indVar.block(curr_batch_row_index, 0, batch_size, n_ind_cols);
-               y_batch = tuple.depVar.segment(curr_batch_row_index, batch_size);
-           }
-           loss += Task::getLossAndUpdateModel(
-               state.task.model, X_batch, y_batch, state.task.stepsize);
+        /*
+            Randomizing the input data before every iteration is good for
+            minibatch gradient descent convergence. Since we don't do that,
+            we are randomizing the order in which every batch is visited in
+            a buffer. Note that this still does not randomize rows within
+            a batch.
+        */
+        int random_curr_batch[n_batches];
+        for(int i=0; i<n_batches; i++) {
+            random_curr_batch[i] = i;
+        }
+        int curr_batch_row_index = 0;
+        std::random_shuffle(&random_curr_batch[0], &random_curr_batch[n_batches]);
+        for (int i=0; i < n_batches; i++) {
+            int curr_batch = random_curr_batch[i];
+            int curr_batch_row_index = curr_batch * batch_size;
+            Matrix X_batch;
+            Matrix Y_batch;
+            if (curr_batch == n_batches-1) {
+               // last batch
+               X_batch = tuple.indVar.bottomRows(n_rows-curr_batch_row_index);
+               Y_batch = tuple.depVar.bottomRows(n_rows-curr_batch_row_index);
+            } else {
+                X_batch = tuple.indVar.block(curr_batch_row_index, 0, batch_size, n_ind_cols);
+                Y_batch = tuple.depVar.block(curr_batch_row_index, 0, batch_size, tuple.depVar.cols());
+            }
+            loss += Task::getLossAndUpdateModel(
+                state.task.model, X_batch, Y_batch, state.task.stepsize);
         }
 
         // The first epoch will most likely have the highest loss.

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/mlp_igd.cpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/mlp_igd.cpp b/src/modules/convex/mlp_igd.cpp
index 7a7bf1d..b8738d4 100644
--- a/src/modules/convex/mlp_igd.cpp
+++ b/src/modules/convex/mlp_igd.cpp
@@ -47,6 +47,9 @@ namespace convex {
 typedef IGD<MLPIGDState<MutableArrayHandle<double> >, MLPIGDState<ArrayHandle<double> >,
         MLP<MLPModel<MutableArrayHandle<double> >, MLPTuple > > MLPIGDAlgorithm;
 
+typedef IGD<MLPMiniBatchState<MutableArrayHandle<double> >, MLPMiniBatchState<ArrayHandle<double> >,
+        MLP<MLPModel<MutableArrayHandle<double> >, MiniBatchTuple > > MLPMiniBatchAlgorithm;
+
 typedef Loss<MLPIGDState<MutableArrayHandle<double> >, MLPIGDState<ArrayHandle<double> >,
         MLP<MLPModel<MutableArrayHandle<double> >, MLPTuple > > MLPLossAlgorithm;
 
@@ -66,7 +69,6 @@ mlp_igd_transition::run(AnyType &args) {
     // For other tuples: args[0] holds the computation state until last tuple
     MLPIGDState<MutableArrayHandle<double> > state = args[0];
 
-
     // initilize the state if first tuple
     if (state.algo.numRows == 0) {
         if (!args[3].isNull()) {
@@ -76,39 +78,48 @@ mlp_igd_transition::run(AnyType &args) {
                            previousState.task.numbersOfUnits);
             state = previousState;
         } else {
-            // configuration parameters
+            // configuration parameters and initialization
+            // this is run only once (first iteration, first tuple)
             ArrayHandle<double> numbersOfUnits = args[4].getAs<ArrayHandle<double> >();
             int numberOfStages = numbersOfUnits.size() - 1;
 
             double stepsize = args[5].getAs<double>();
-
             state.allocate(*this, numberOfStages,
                            reinterpret_cast<const double *>(numbersOfUnits.ptr()));
             state.task.stepsize = stepsize;
 
-
             const int activation = args[6].getAs<int>();
             const int is_classification = args[7].getAs<int>();
-
+            // args[8] is for weighting the input row, which is populated later.
             const bool warm_start = args[9].getAs<bool>();
             const double lambda = args[11].getAs<double>();
             state.task.lambda = lambda;
             MLPTask::lambda = lambda;
+
             double is_classification_double = (double) is_classification;
             double activation_double = (double) activation;
             MappedColumnVector coeff = args[10].getAs<MappedColumnVector>();
             state.task.model.rebind(&is_classification_double,&activation_double,
                                     &coeff.data()[0], numberOfStages,
                                     &numbersOfUnits[0]);
+
+            // state.task.model.is_classification =
+            //     static_cast<double>(is_classification);
+            // state.task.model.activation = static_cast<double>(activation);
+            // MappedColumnVector initial_coeff = args[10].getAs<MappedColumnVector>();
+            // // copy initial_coeff into the model
+            // Index fan_in, fan_out, layer_start = 0;
+            // for (size_t k = 0; k < numberOfStages; ++k){
+            //     fan_in = numbersOfUnits[k];
+            //     fan_out = numbersOfUnits[k+1];
+            //     state.task.model.u[k] << initial_coeff.segment(layer_start, (fan_in+1)*fan_out);
+            //     layer_start = (fan_in + 1) * fan_out;
+            // }
         }
         // resetting in either case
         state.reset();
     }
 
-    // meta data
-    const uint16_t N = state.task.numberOfStages;
-    const double *n = state.task.numbersOfUnits;
-
     // tuple
     ColumnVector indVar;
     MappedColumnVector depVar;
@@ -125,11 +136,107 @@ mlp_igd_transition::run(AnyType &args) {
     tuple.weight = args[8].getAs<double>();
 
     MLPIGDAlgorithm::transition(state, tuple);
+    // Use the model from the previous iteration to compute the loss (note that
+    // it is stored in Task's state, and the Algo's state holds the model from
+    // the current iteration.
     MLPLossAlgorithm::transition(state, tuple);
     state.algo.numRows ++;
 
     return state;
 }
+/**
+ * @brief Perform the multilayer perceptron minibatch transition step
+ *
+ * Called for each tuple.
+ */
+AnyType
+mlp_minibatch_transition::run(AnyType &args) {
+    // For the first tuple: args[0] is nothing more than a marker that
+    // indicates that we should do some initial operations.
+    // For other tuples: args[0] holds the computation state until last tuple
+    MLPMiniBatchState<MutableArrayHandle<double> > state = args[0];
+
+    // initilize the state if first tuple
+    if (state.algo.numRows == 0) {
+        if (!args[3].isNull()) {
+            MLPMiniBatchState<ArrayHandle<double> > previousState = args[3];
+            state.allocate(*this, previousState.task.numberOfStages,
+                           previousState.task.numbersOfUnits);
+            state = previousState;
+        } else {
+            // configuration parameters
+            ArrayHandle<double> numbersOfUnits = args[4].getAs<ArrayHandle<double> >();
+            int numberOfStages = numbersOfUnits.size() - 1;
+
+            double stepsize = args[5].getAs<double>();
+
+            state.allocate(*this, numberOfStages,
+                           reinterpret_cast<const double *>(numbersOfUnits.ptr()));
+            state.task.stepsize = stepsize;
+            const int activation = args[6].getAs<int>();
+            const int is_classification = args[7].getAs<int>();
+            // args[8] is for weighting the input row, which is populated later.
+            const bool warm_start = args[9].getAs<bool>();
+            const double lambda = args[11].getAs<double>();
+            state.algo.batchSize = args[12].getAs<int>();
+            state.algo.nEpochs = args[13].getAs<int>();
+            state.task.lambda = lambda;
+            MLPTask::lambda = lambda;
+
+            /* FIXME: The state is set back to zero for second row onwards if
+               initialized as in IGD. The following avoids that, but there is
+               some failure with debug build that must be fixed.
+            */
+            state.task.model.is_classification =
+                static_cast<double>(is_classification);
+            state.task.model.activation = static_cast<double>(activation);
+            MappedColumnVector initial_coeff = args[10].getAs<MappedColumnVector>();
+            // copy initial_coeff into the model
+            Index fan_in, fan_out, layer_start = 0;
+            for (size_t k = 0; k < numberOfStages; ++k){
+                fan_in = numbersOfUnits[k];
+                fan_out = numbersOfUnits[k+1];
+                state.task.model.u[k] << initial_coeff.segment(layer_start, (fan_in+1)*fan_out);
+                layer_start = (fan_in + 1) * fan_out;
+            }
+        }
+        // resetting in either case
+        state.reset();
+    }
+
+    // tuple
+    Matrix indVar;
+    Matrix depVar;
+    try {
+        // Ideally there should be no NULLs in the pre-processed input data,
+        // but keep it in a try block in case the user has modified the
+        // pre-processed data in any way.
+        indVar = args[1].getAs<MappedMatrix>();
+        depVar = args[2].getAs<MappedMatrix>();
+    } catch (const ArrayWithNullException &e) {
+        return args[0];
+    }
+    MiniBatchTuple tuple;
+    // The matrices are by default read as column-major. We will have to
+    // transpose it to get back the matrix like how it is in the database.
+    tuple.indVar = trans(indVar);
+    tuple.depVar = trans(depVar);
+    tuple.weight = args[8].getAs<double>();
+
+    /*
+        Note that the IGD version uses the model in Task (model from the
+        previous iteration) to compute the loss.
+        Minibatch uses the model from Algo (the model based on current
+        iteration) to compute the loss. The difference in loss based on one
+        iteration is not too much, hence doing so here. We therefore don't
+        need to maintain another copy of the model (from previous iteration)
+        in the state. The model for the current iteration, and the loss are
+        both computed in one function now.
+    */
+    MLPMiniBatchAlgorithm::transitionInMiniBatch(state, tuple);
+    state.algo.numRows += tuple.indVar.rows();
+    return state;
+}
 
 /**
  * @brief Perform the perliminary aggregation function: Merge transition states
@@ -145,9 +252,29 @@ mlp_igd_merge::run(AnyType &args) {
     MLPIGDAlgorithm::merge(stateLeft, stateRight);
     MLPLossAlgorithm::merge(stateLeft, stateRight);
 
+    // The following numRows update cannot be put above, because the model
+    // averaging depends on their original values
+    stateLeft.algo.numRows += stateRight.algo.numRows;
+
+    return stateLeft;
+}
+/**
+ * @brief Perform the perliminary aggregation function: Merge transition states
+ */
+AnyType
+mlp_minibatch_merge::run(AnyType &args) {
+    MLPMiniBatchState<MutableArrayHandle<double> > stateLeft = args[0];
+    MLPMiniBatchState<ArrayHandle<double> > stateRight = args[1];
+
+    if (stateLeft.algo.numRows == 0) { return stateRight; }
+    else if (stateRight.algo.numRows == 0) { return stateLeft; }
+
+    MLPMiniBatchAlgorithm::mergeInPlace(stateLeft, stateRight);
+
     // The following numRows update, cannot be put above, because the model
     // averaging depends on their original values
     stateLeft.algo.numRows += stateRight.algo.numRows;
+    stateLeft.algo.loss += stateRight.algo.loss;
 
     return stateLeft;
 }
@@ -170,6 +297,24 @@ mlp_igd_final::run(AnyType &args) {
     return state;
 }
 
+
+/**
+ * @brief Perform the multilayer perceptron final step
+ */
+AnyType
+mlp_minibatch_final::run(AnyType &args) {
+    // We request a mutable object. Depending on the backend, this might perform
+    // a deep copy.
+    MLPMiniBatchState<MutableArrayHandle<double> > state = args[0];
+    // Aggregates that haven't seen any data just return Null.
+    if (state.algo.numRows == 0) { return Null(); }
+
+    L2<MLPModelType>::lambda = state.task.lambda;
+    state.algo.loss = state.algo.loss/static_cast<double>(state.algo.numRows);
+    state.algo.loss += L2<MLPModelType>::loss(state.task.model);
+    return state;
+}
+
 /**
  * @brief Return the difference in RMSE between two states
  */
@@ -180,18 +325,44 @@ internal_mlp_igd_distance::run(AnyType &args) {
     return std::abs(stateLeft.algo.loss - stateRight.algo.loss);
 }
 
+
+AnyType
+internal_mlp_minibatch_distance::run(AnyType &args) {
+    MLPMiniBatchState<ArrayHandle<double> > stateLeft = args[0];
+    MLPMiniBatchState<ArrayHandle<double> > stateRight = args[1];
+
+    return std::abs(stateLeft.algo.loss - stateRight.algo.loss);
+}
+
 /**
  * @brief Return the coefficients and diagnostic statistics of the state
  */
 AnyType
 internal_mlp_igd_result::run(AnyType &args) {
     MLPIGDState<ArrayHandle<double> > state = args[0];
-
     HandleTraits<ArrayHandle<double> >::ColumnVectorTransparentHandleMap
         flattenU;
     flattenU.rebind(&state.task.model.u[0](0, 0),
-            state.task.model.arraySize(state.task.numberOfStages,
-                    state.task.numbersOfUnits));
+                    state.task.model.arraySize(state.task.numberOfStages,
+                                               state.task.numbersOfUnits));
+    double loss = state.algo.loss;
+
+    AnyType tuple;
+    tuple << flattenU
+          << loss;
+    return tuple;
+}
+
+/**
+ * @brief Return the coefficients and diagnostic statistics of the state
+ */
+AnyType
+internal_mlp_minibatch_result::run(AnyType &args) {
+    MLPMiniBatchState<ArrayHandle<double> > state = args[0];
+    HandleTraits<ArrayHandle<double> >::ColumnVectorTransparentHandleMap flattenU;
+    flattenU.rebind(&state.task.model.u[0](0, 0),
+                    state.task.model.arraySize(state.task.numberOfStages,
+                                               state.task.numbersOfUnits));
     double loss = state.algo.loss;
 
     AnyType tuple;
@@ -215,7 +386,8 @@ internal_predict_mlp::run(AnyType &args) {
     double activation = args[3].getAs<double>();
     bool get_class = is_classification && is_response;
 
-    model.rebind(&is_classification,&activation,&coeff.data()[0],numberOfStages,&layerSizes.data()[0]);
+    model.rebind(&is_classification, &activation, &coeff.data()[0],
+                 numberOfStages, &layerSizes.data()[0]);
     try {
         indVar = (args[1].getAs<MappedColumnVector>()-x_means).cwiseQuotient(x_stds);
     } catch (const ArrayWithNullException &e) {

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/mlp_igd.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/mlp_igd.hpp b/src/modules/convex/mlp_igd.hpp
index c957d97..798efe3 100644
--- a/src/modules/convex/mlp_igd.hpp
+++ b/src/modules/convex/mlp_igd.hpp
@@ -25,28 +25,33 @@
  * @brief Multilayer perceptron (incremental gradient): Transition function
  */
 DECLARE_UDF(convex, mlp_igd_transition)
+DECLARE_UDF(convex, mlp_minibatch_transition)
 
 /**
  * @brief Multilayer perceptron (incremental gradient): State merge function
  */
 DECLARE_UDF(convex, mlp_igd_merge)
+DECLARE_UDF(convex, mlp_minibatch_merge)
 
 /**
  * @brief Multilayer perceptron (incremental gradient): Final function
  */
 DECLARE_UDF(convex, mlp_igd_final)
+DECLARE_UDF(convex, mlp_minibatch_final)
 
 /**
  * @brief Multilayer perceptron (incremental gradient): Difference in
  *     log-likelihood between two transition states
  */
 DECLARE_UDF(convex, internal_mlp_igd_distance)
+DECLARE_UDF(convex, internal_mlp_minibatch_distance)
 
 /**
  * @brief Multilayer perceptron (incremental gradient): Convert
  *     transition state to result tuple
  */
 DECLARE_UDF(convex, internal_mlp_igd_result)
+DECLARE_UDF(convex, internal_mlp_minibatch_result)
 
 /**
  * @brief Multilayer perceptron (incremental gradient): Predict

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/task/mlp.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/task/mlp.hpp b/src/modules/convex/task/mlp.hpp
index 0032b81..8a68aaa 100644
--- a/src/modules/convex/task/mlp.hpp
+++ b/src/modules/convex/task/mlp.hpp
@@ -52,6 +52,12 @@ public:
             const dependent_variable_type       &y,
             const double                        &stepsize);
 
+    static double getLossAndUpdateModel(
+            model_type                          &model,
+            const Matrix                        &x,
+            const Matrix                        &y,
+            const double                        &stepsize);
+
     static double loss(
             const model_type                    &model,
             const independent_variables_type    &x,
@@ -112,6 +118,59 @@ template <class Model, class Tuple>
 double MLP<Model, Tuple>::lambda = 0;
 
 template <class Model, class Tuple>
+double
+MLP<Model, Tuple>::getLossAndUpdateModel(
+        model_type           &model,
+        const Matrix         &x_batch,
+        const Matrix         &y_true_batch,
+        const double         &stepsize) {
+
+    uint16_t num_layers = model.u.size(); // assuming nu. of layers >= 1
+    size_t num_rows_in_batch = x_batch.rows();
+    size_t i, k;
+    double total_loss = 0.;
+
+    // gradient added over the batch
+    std::vector<Matrix> total_gradient_per_layer(num_layers);
+    for (k=0; k < num_layers; ++k)
+        total_gradient_per_layer[k] = Matrix::Zero(model.u[k].rows(),
+                                                   model.u[k].cols());
+
+    for (i=0; i < num_rows_in_batch; i++){
+        ColumnVector x = x_batch.row(i);
+        ColumnVector y_true = y_true_batch.row(i);
+
+        std::vector<ColumnVector> net, o, delta;
+        feedForward(model, x, net, o);
+        backPropogate(y_true, o.back(), net, model, delta);
+
+        for (k=0; k < num_layers; k++){
+                total_gradient_per_layer[k] += o[k] * delta[k].transpose();
+        }
+
+        // loss computation
+        ColumnVector y_estimated = o.back();
+        if(model.is_classification){
+            double clip = 1.e-10;
+            y_estimated = y_estimated.cwiseMax(clip).cwiseMin(1.-clip);
+            total_loss += - (y_true.array()*y_estimated.array().log()
+                   + (-y_true.array()+1)*(-y_estimated.array()+1).log()).sum();
+        }
+        else{
+            total_loss += 0.5 * (y_estimated - y_true).squaredNorm();
+        }
+    }
+    for (k=0; k < num_layers; k++){
+        Matrix regularization = MLP<Model, Tuple>::lambda * model.u[k];
+        regularization.row(0).setZero(); // Do not update bias
+        model.u[k] -= stepsize * (total_gradient_per_layer[k] / \
+                                  num_rows_in_batch + \
+                                  regularization);
+    }
+    return total_loss;
+}
+
+template <class Model, class Tuple>
 void
 MLP<Model, Tuple>::gradientInPlace(
         model_type                          &model,
@@ -151,7 +210,7 @@ MLP<Model, Tuple>::loss(
                + (-y_true.array()+1)*(-y_estimated.array()+1).log()).sum();
     }
     else{
-        return 0.5 * (y_estimated-y_true).squaredNorm();
+        return 0.5 * (y_estimated - y_true).squaredNorm();
     }
 }
 
@@ -165,6 +224,7 @@ MLP<Model, Tuple>::predict(
 
     feedForward(model, x, net, o);
     ColumnVector output = o.back();
+
     if(get_class){ // Return a length 1 array with the predicted index
         int max_idx;
         output.maxCoeff(&max_idx);
@@ -183,8 +243,14 @@ MLP<Model, Tuple>::feedForward(
         std::vector<ColumnVector>           &net,
         std::vector<ColumnVector>           &o){
     uint16_t k, N;
+    /*
+        The network starts with the 0th layer (input), followed by n_layers
+        number of hidden layers, and then an output layer.
+    */
+    // Total number of coefficients in the model
     N = model.u.size(); // assuming >= 1
     net.resize(N + 1);
+    // o[k] is a vector of the output of the kth layer
     o.resize(N + 1);
 
     double (*activation)(const double&);
@@ -195,12 +261,15 @@ MLP<Model, Tuple>::feedForward(
     else
         activation = &tanh;
 
-    o[0].resize(x.size()+1);
+    o[0].resize(x.size() + 1);
     o[0] << 1.,x;
 
     for (k = 1; k < N; k ++) {
+        // o_k = activation(sum(o_{k-1} * u_{k-1}))
+        // net_k just does the inner sum: input to the activation function
         net[k] = model.u[k-1].transpose() * o[k-1];
-        o[k] = ColumnVector(model.u[k-1].cols()+1);
+        o[k] = ColumnVector(model.u[k-1].cols() + 1);
+        // This applies the activation function to give the actual node output
         o[k] << 1., net[k].unaryExpr(activation);
     }
     o[N] = model.u[N-1].transpose() * o[N-1];

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/type/state.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/type/state.hpp b/src/modules/convex/type/state.hpp
index f846e8f..a3b0d43 100644
--- a/src/modules/convex/type/state.hpp
+++ b/src/modules/convex/type/state.hpp
@@ -712,10 +712,9 @@ public:
             + 1                         // is_classification
             + 1                         // activation
             + sizeOfModel               // model
-
+            + sizeOfModel               // incrModel
             + 1                         // numRows
-            + 1                         // loss
-            + sizeOfModel;              // incrModel
+            + 1;                        // loss
     }
 
 private:
@@ -728,14 +727,18 @@ private:
      * - 1: numbersOfUnits (numbers of activation units, design doc: n_0,...,n_N)
      * - N + 2: stepsize (step size of gradient steps)
      * - N + 3: lambda (regularization term)
+// is_classification, activation, and coeff together form the model
      * - N + 4: is_classification (do classification)
      * - N + 5: activation (activation function)
      * - N + 6: coeff (coefficients, design doc: u)
      *
      * Intra-iteration components (updated in transition step):
      *   sizeOfModel = # of entries in u + 2, (\sum_1^N n_{k-1} n_k)
+// incremental model
      * - N + 6 + sizeOfModel: coeff (volatile model for incrementally update)
+// number of rows
      * - N + 6 + 2*sizeOfModel: numRows (number of rows processed in this iteration)
+// loss
      * - N + 7 + 2*sizeOfModel: loss (loss value, the sum of squared errors)
      */
     void rebind() {
@@ -779,6 +782,169 @@ public:
 };
 
 
+
+/**
+ * @brief Inter- (Task State) and intra-iteration (Algo State) state of
+ *        incremental gradient descent for multilayer perceptron
+ *
+ * TransitionState encapsualtes the transition state during the
+ * aggregate function during an iteration. To the database, the state is
+ * exposed as a single DOUBLE PRECISION array, to the C++ code it is a proper
+ * object containing scalars and vectors.
+ *
+ * Note: We assume that the DOUBLE PRECISION array is initialized by the
+ * database with length at least 6, and at least first elemenet
+ * is 0 (exact values of other elements are ignored).
+ *
+ */
+template <class Handle>
+class MLPMiniBatchState {
+    template <class OtherHandle>
+    friend class MLPMiniBatchState;
+
+public:
+    MLPMiniBatchState(const AnyType &inArray) : mStorage(inArray.getAs<Handle>()) {
+        rebind();
+    }
+
+    /**
+     * @brief Reset the intra-iteration fields.
+     */
+    inline void reset() {
+        algo.numRows = 0;
+        algo.loss = 0.;
+    }
+
+    /**
+     * @brief Convert to backend representation
+     *
+     * We define this function so that we can use State in the
+     * argument list and as a return type.
+     */
+    inline operator AnyType() const {
+        return mStorage;
+    }
+
+    /**
+     * @brief Allocating the incremental gradient state.
+     */
+    inline void allocate(const Allocator &inAllocator,
+                         const uint16_t &inNumberOfStages,
+                         const double *inNumbersOfUnits) {
+        mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
+                dbal::DoZero, dbal::ThrowBadAlloc>(
+                        arraySize(inNumberOfStages, inNumbersOfUnits));
+
+        // This rebind is for the following lines of code to take
+        // effect. I can also do something like "mStorage[0] = N",
+        // but I am not clear about the type binding/alignment
+        rebind();
+        task.numberOfStages = inNumberOfStages;
+        uint16_t N = inNumberOfStages;
+        uint16_t k;
+        for (k = 0; k <= N; k ++) {
+            task.numbersOfUnits[k] = inNumbersOfUnits[k];
+        }
+
+        // This time all the member fields are correctly binded
+        rebind();
+    }
+
+    /**
+     * @brief We need to support assigning the previous state
+     */
+    template <class OtherHandle>
+    MLPMiniBatchState &operator=(const MLPMiniBatchState<OtherHandle> &inOtherState) {
+        for (size_t i = 0; i < mStorage.size(); i++) {
+            mStorage[i] = inOtherState.mStorage[i];
+        }
+
+        return *this;
+    }
+
+    static inline uint32_t arraySize(const uint16_t &inNumberOfStages,
+                                     const double *inNumbersOfUnits) {
+        uint32_t sizeOfModel =
+            MLPModel<Handle>::arraySize(inNumberOfStages, inNumbersOfUnits);
+        return 1                        // numberOfStages = N
+            + (inNumberOfStages + 1)    // numbersOfUnits: size is (N + 1)
+            + 1                         // stepsize
+            + 1                         // lambda
+            + 1                         // is_classification
+            + 1                         // activation
+            + sizeOfModel               // model
+            + 1                         // numRows
+            + 1                         // batchSize
+            + 1                         // nEpochs
+            + 1;                        // loss
+
+    }
+
+    Handle mStorage;
+private:
+    /**
+     * @brief Rebind to a new storage array.
+     *
+     * Array layout (iteration refers to one aggregate-function call):
+     * Inter-iteration components (updated in final function):
+     * - 0: numberOfStages (number of stages (layers), design doc: N)
+     * - 1: numbersOfUnits (numbers of activation units, design doc: n_0,...,n_N)
+     * - N + 2: stepsize (step size of gradient steps)
+     * - N + 3: lambda (regularization term)
+// is_classification, activation, and coeff together form the model
+     * - N + 4: is_classification (do classification)
+     * - N + 5: activation (activation function)
+     * - N + 6: coeff (coefficients, design doc: u)
+// model is done, now bind numRows
+     * - N + 6 + sizeOfModel: numRows (number of rows processed in this iteration)
+     * - N + 7 + sizeOfModel: batchSize (number of rows processed in this iteration)
+     * - N + 8 + sizeOfModel: nEpochs (number of rows processed in this iteration)
+     * - N + 9 + sizeOfModel: loss (loss value, the sum of squared errors)
+     */
+    void rebind() {
+
+        task.numberOfStages.rebind(&mStorage[0]);
+        size_t N = task.numberOfStages;
+
+        task.numbersOfUnits =
+            reinterpret_cast<dimension_pointer_type>(&mStorage[1]);
+        task.stepsize.rebind(&mStorage[N + 2]);
+        task.lambda.rebind(&mStorage[N + 3]);
+        size_t sizeOfModel = task.model.rebind(&mStorage[N + 4],
+                                               &mStorage[N + 5],
+                                               &mStorage[N + 6],
+                                               task.numberOfStages,
+                                               task.numbersOfUnits);
+        algo.numRows.rebind(&mStorage[N + 6 + sizeOfModel]);
+        algo.batchSize.rebind(&mStorage[N + 7 + sizeOfModel]);
+        algo.nEpochs.rebind(&mStorage[N + 8 + sizeOfModel]);
+        algo.loss.rebind(&mStorage[N + 9 + sizeOfModel]);
+    }
+
+
+    typedef typename HandleTraits<Handle>::ReferenceToUInt16 dimension_type;
+    typedef typename HandleTraits<Handle>::DoublePtr dimension_pointer_type;
+    typedef typename HandleTraits<Handle>::ReferenceToUInt64 count_type;
+    typedef typename HandleTraits<Handle>::ReferenceToDouble numeric_type;
+
+public:
+    struct TaskState {
+        dimension_type numberOfStages;
+        dimension_pointer_type numbersOfUnits;
+        numeric_type stepsize;
+        numeric_type lambda;
+        MLPModel<Handle> model;
+    } task;
+
+    struct AlgoState {
+        count_type numRows;
+        dimension_type batchSize;
+        dimension_type nEpochs;
+        numeric_type loss;
+    } algo;
+};
+
+
 } // namespace convex
 
 } // namespace modules

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/modules/convex/type/tuple.hpp
----------------------------------------------------------------------
diff --git a/src/modules/convex/type/tuple.hpp b/src/modules/convex/type/tuple.hpp
index ac070b6..492a1b0 100644
--- a/src/modules/convex/type/tuple.hpp
+++ b/src/modules/convex/type/tuple.hpp
@@ -68,6 +68,7 @@ typedef ExampleTuple<Matrix, ColumnVector> SVMMiniBatchTuple;
 typedef ExampleTuple<MatrixIndex, double> LMFTuple;
 
 typedef ExampleTuple<ColumnVector, MappedColumnVector> MLPTuple;
+typedef ExampleTuple<Matrix, Matrix> MiniBatchTuple;
 
 } // namespace convex
 

http://git-wip-us.apache.org/repos/asf/madlib/blob/a595e221/src/ports/postgres/modules/convex/mlp.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/convex/mlp.sql_in b/src/ports/postgres/modules/convex/mlp.sql_in
index e6e2716..739007e 100644
--- a/src/ports/postgres/modules/convex/mlp.sql_in
+++ b/src/ports/postgres/modules/convex/mlp.sql_in
@@ -316,7 +316,9 @@ the parameter is ignored.
    n_iterations = &lt;value>,
    n_tries = &lt;value>,
    lambda = &lt;value>,
-   tolerance = &lt;value>'
+   tolerance = &lt;value>,
+   batch_size = &lt;value>,
+   n_epochs = &lt;value>'
 </pre>
 \b Optimizer \b Parameters
 <DL class="arglist">
@@ -376,6 +378,22 @@ If you want to run the full number of iterations specified in \e n_interations,
 set tolerance=0.0
 </DD>
 
+<DT>tolerance</dt>
+<DD>Default: 1 for IGD, 20 for Minibatch
+If the source_table is detected to contain data
+that is supported by minibatch, then the solver
+uses mini-batch gradient descent, with the specified
+batch_size.
+</DD>
+
+<DT>tolerance</dt>
+<DD>Default: 1 for IGD, 20 for Minibatch
+If the source_table is detected to contain data
+that is supported by minibatch, then the solver
+uses mini-batch gradient descent. During gradient
+descent, n_epochs represents the number of times
+all batches in a buffer are iterated over.
+</DD>
 </DL>
 
 @anchor predict
@@ -1241,11 +1259,6 @@ File mlp.sql_in documenting the training function
 
 */
 
-CREATE TYPE MADLIB_SCHEMA.mlp_step_result AS (
-        state    DOUBLE PRECISION[],
-        loss     DOUBLE PRECISION
-);
-
 CREATE TYPE MADLIB_SCHEMA.mlp_result AS (
         coeff    DOUBLE PRECISION[],
         loss     DOUBLE PRECISION
@@ -1272,6 +1285,26 @@ RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE;
 
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_transition(
+        state              DOUBLE PRECISION[],
+        ind_var            DOUBLE PRECISION[],
+        dep_var            DOUBLE PRECISION[],
+        previous_state     DOUBLE PRECISION[],
+        layer_sizes        DOUBLE PRECISION[],
+        learning_rate_init DOUBLE PRECISION,
+        activation         INTEGER,
+        is_classification  INTEGER,
+        weight             DOUBLE PRECISION,
+        warm_start         BOOLEAN,
+        warm_start_coeff   DOUBLE PRECISION[],
+        lambda             DOUBLE PRECISION,
+        batch_size         INTEGER,
+        n_epochs           INTEGER
+    )
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE;
+
 CREATE FUNCTION MADLIB_SCHEMA.mlp_igd_merge(
         state1 DOUBLE PRECISION[],
         state2 DOUBLE PRECISION[])
@@ -1285,6 +1318,19 @@ RETURNS DOUBLE PRECISION[]
 AS 'MODULE_PATHNAME'
 LANGUAGE C IMMUTABLE STRICT;
 
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_merge(
+        state1 DOUBLE PRECISION[],
+        state2 DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE STRICT;
+
+CREATE FUNCTION MADLIB_SCHEMA.mlp_minibatch_final(
+        state DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME'
+LANGUAGE C IMMUTABLE STRICT;
+
 /**
  * @internal
  * @brief Perform one iteration of backprop
@@ -1310,6 +1356,33 @@ CREATE AGGREGATE MADLIB_SCHEMA.mlp_igd_step(
 );
 -------------------------------------------------------------------------
 
+/**
+ * @internal
+ * @brief Perform one iteration of backprop
+ */
+CREATE AGGREGATE MADLIB_SCHEMA.mlp_minibatch_step(
+        /* ind_var */             DOUBLE PRECISION[],
+        /* dep_var */             DOUBLE PRECISION[],
+        /* previous_state */      DOUBLE PRECISION[],
+        /* layer_sizes */         DOUBLE PRECISION[],
+        /* learning_rate_init */  DOUBLE PRECISION,
+        /* activation */          INTEGER,
+        /* is_classification */   INTEGER,
+        /* weight */              DOUBLE PRECISION,
+        /* warm_start */          BOOLEAN,
+        /* warm_start_coeff */    DOUBLE PRECISION[],
+        /* lambda */              DOUBLE PRECISION,
+        /* batch_size */          INTEGER,
+        /* n_epochs */            INTEGER
+        )(
+    STYPE=DOUBLE PRECISION[],
+    SFUNC=MADLIB_SCHEMA.mlp_minibatch_transition,
+    m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.mlp_minibatch_merge,')
+    FINALFUNC=MADLIB_SCHEMA.mlp_minibatch_final,
+    INITCOND='{0,0,0,0,0,0,0,0,0,0,0,0}'
+);
+-------------------------------------------------------------------------
+
 CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_distance(
     /*+ state1 */ DOUBLE PRECISION[],
     /*+ state2 */ DOUBLE PRECISION[])
@@ -1322,6 +1395,12 @@ CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_igd_result(
 RETURNS MADLIB_SCHEMA.mlp_result AS
 'MODULE_PATHNAME'
 LANGUAGE c IMMUTABLE STRICT;
+
+CREATE FUNCTION MADLIB_SCHEMA.internal_mlp_minibatch_result(
+    /*+ state */ DOUBLE PRECISION[])
+RETURNS MADLIB_SCHEMA.mlp_result AS
+'MODULE_PATHNAME'
+LANGUAGE c IMMUTABLE STRICT;
 -------------------------------------------------------------------------
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.mlp_classification(