You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2015/11/04 22:45:39 UTC
[34/34] incubator-madlib git commit: Add hello-world examples
Add hello-world examples
Pivotal Tracker: #103042238
- added average and variance function as a hello-world example
- added simple logistic regression as an iterative example
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/a184a8ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/a184a8ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/a184a8ac
Branch: refs/heads/master
Commit: a184a8ac6f1d17cc364fecf5e5f8d55e229210b9
Parents: a32fc63
Author: Xiaocheng Tang <xt...@pivotal.io>
Authored: Fri Sep 18 11:49:28 2015 -0700
Committer: Rahul Iyer <ri...@pivotal.io>
Committed: Mon Nov 2 10:57:01 2015 -0800
----------------------------------------------------------------------
CMakeLists.txt | 2 +
.../hello_world/iterative/simple_logistic.cpp | 433 +++++++++++++++++++
.../hello_world/iterative/simple_logistic.hpp | 32 ++
.../hello_world/iterative/simple_logistic.py_in | 222 ++++++++++
.../iterative/simple_logistic.sql_in | 203 +++++++++
examples/hello_world/non-iterative/avg_var.cpp | 141 ++++++
examples/hello_world/non-iterative/avg_var.hpp | 20 +
.../hello_world/non-iterative/avg_var.sql_in | 53 +++
8 files changed, 1106 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index c5f40f4..c00b79f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -103,6 +103,8 @@ if(CMAKE_COMPILER_IS_GNUCXX)
if(APPLE)
set(CMAKE_INCLUDE_SYSTEM_FLAG_CXX "-isystem ")
endif(APPLE)
+elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
+ set(CMAKE_CXX_FLAGS "-stdlib=libstdc++")
endif(CMAKE_COMPILER_IS_GNUCXX)
# force a `m4_' prefix to all builtins
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/iterative/simple_logistic.cpp
----------------------------------------------------------------------
diff --git a/examples/hello_world/iterative/simple_logistic.cpp b/examples/hello_world/iterative/simple_logistic.cpp
new file mode 100644
index 0000000..15c4e96
--- /dev/null
+++ b/examples/hello_world/iterative/simple_logistic.cpp
@@ -0,0 +1,433 @@
+/* ------------------------------------------------------
+ *
+ * @file logistic.cpp
+ *
+ * @brief Logistic-Regression functions
+ *
+ * We implement the conjugate-gradient method and the iteratively-reweighted-
+ * least-squares method.
+ *
+ *//* ----------------------------------------------------------------------- */
+#include <limits>
+#include <dbconnector/dbconnector.hpp>
+#include <modules/shared/HandleTraits.hpp>
+#include <modules/prob/boost.hpp>
+#include <boost/math/distributions.hpp>
+#include <modules/prob/student.hpp>
+#include "simple_logistic.hpp"
+
+namespace madlib {
+
+// Use Eigen
+using namespace dbal::eigen_integration;
+
+namespace modules {
+
+// Import names from other MADlib modules
+using dbal::NoSolutionFoundException;
+
+namespace hello_world {
+
+// FIXME this enum should be accessed by all modules that may need grouping
+// valid status values
+enum { IN_PROCESS, COMPLETED, TERMINATED, NULL_EMPTY };
+
+// Internal functions
+AnyType stateToResult(const Allocator &inAllocator,
+ const HandleMap<const ColumnVector, TransparentHandle<double> >& inCoef,
+ const Matrix & hessian,
+ const double &logLikelihood,
+ int status,
+ const uint64_t &numRows);
+
+
+// ---------------------------------------------------------------------------
+// Logistic Regression States
+// ---------------------------------------------------------------------------
+
+/**
+ * @brief Inter- and intra-iteration state for conjugate-gradient method for
+ * logistic regression
+ *
+ * TransitionState encapsualtes the transition state during the
+ * logistic-regression aggregate function. To the database, the state is
+ * exposed as a single DOUBLE PRECISION array, to the C++ code it is a proper
+ * object containing scalars and vectors.
+ *
+ * Note: We assume that the DOUBLE PRECISION array is initialized by the
+ * database with length at least 5, and all elemenets are 0.
+ *
+ */
+template <class Handle>
+class LogRegrSPTransitionState {
+ template <class OtherHandle>
+ friend class LogRegrSPTransitionState;
+
+ public:
+ LogRegrSPTransitionState(const AnyType &inArray)
+ : mStorage(inArray.getAs<Handle>()) {
+
+ rebind(static_cast<uint16_t>(mStorage[1]));
+ }
+
+ /**
+ * @brief Convert to backend representation
+ *
+ * We define this function so that we can use State in the
+ * argument list and as a return type.
+ */
+ inline operator AnyType() const {
+ return mStorage;
+ }
+
+ /**
+ * @brief Initialize the conjugate-gradient state.
+ *
+ * This function is only called for the first iteration, for the first row.
+ */
+ inline void initialize(const Allocator &inAllocator, uint16_t inWidthOfX) {
+ mStorage = inAllocator.allocateArray<double, dbal::AggregateContext,
+ dbal::DoZero, dbal::ThrowBadAlloc>(arraySize(inWidthOfX));
+ rebind(inWidthOfX);
+ widthOfX = inWidthOfX;
+ }
+
+ /**
+ * @brief We need to support assigning the previous state
+ */
+ template <class OtherHandle>
+ LogRegrSPTransitionState &operator=(
+ const LogRegrSPTransitionState<OtherHandle> &inOtherState) {
+
+ for (size_t i = 0; i < mStorage.size(); i++)
+ mStorage[i] = inOtherState.mStorage[i];
+ return *this;
+ }
+
+ /**
+ * @brief Merge with another State object by copying the intra-iteration
+ * fields
+ */
+ template <class OtherHandle>
+ LogRegrSPTransitionState &operator+=(
+ const LogRegrSPTransitionState<OtherHandle> &inOtherState) {
+
+ if (mStorage.size() != inOtherState.mStorage.size() ||
+ widthOfX != inOtherState.widthOfX)
+ throw std::logic_error("Internal error: Incompatible transition "
+ "states");
+
+ numRows += inOtherState.numRows;
+ gradNew += inOtherState.gradNew;
+ X_transp_AX += inOtherState.X_transp_AX;
+ logLikelihood += inOtherState.logLikelihood;
+ // merged state should have the higher status
+ // (see top of file for more on 'status' )
+ status = (inOtherState.status > status) ? inOtherState.status : status;
+ return *this;
+ }
+
+ /**
+ * @brief Reset the inter-iteration fields.
+ */
+ inline void reset() {
+ numRows = 0;
+ X_transp_AX.fill(0);
+ gradNew.fill(0);
+ logLikelihood = 0;
+ status = IN_PROCESS;
+ }
+
+ private:
+ static inline size_t arraySize(const uint16_t inWidthOfX) {
+ return 6 + inWidthOfX * inWidthOfX + 4 * inWidthOfX;
+ }
+
+ /**
+ * @brief Rebind to a new storage array
+ *
+ * @param inWidthOfX The number of independent variables.
+ *
+ * Array layout (iteration refers to one aggregate-function call):
+ * Inter-iteration components (updated in final function):
+ * - 0: iteration (current iteration)
+ * - 1: widthOfX (number of coefficients)
+ * - 2: coef (vector of coefficients)
+ * - 2 + widthOfX: dir (direction)
+ * - 2 + 2 * widthOfX: grad (gradient)
+ * - 2 + 3 * widthOfX: beta (scale factor)
+ *
+ * Intra-iteration components (updated in transition step):
+ * - 3 + 3 * widthOfX: numRows (number of rows already processed in this iteration)
+ * - 4 + 3 * widthOfX: gradNew (intermediate value for gradient)
+ * - 4 + 4 * widthOfX: X_transp_AX (X^T A X)
+ * - 4 + widthOfX * widthOfX + 4 * widthOfX: logLikelihood ( ln(l(c)) )
+ */
+ void rebind(uint16_t inWidthOfX) {
+ iteration.rebind(&mStorage[0]);
+ widthOfX.rebind(&mStorage[1]);
+ coef.rebind(&mStorage[2], inWidthOfX);
+ dir.rebind(&mStorage[2 + inWidthOfX], inWidthOfX);
+ grad.rebind(&mStorage[2 + 2 * inWidthOfX], inWidthOfX);
+ beta.rebind(&mStorage[2 + 3 * inWidthOfX]);
+ numRows.rebind(&mStorage[3 + 3 * inWidthOfX]);
+ gradNew.rebind(&mStorage[4 + 3 * inWidthOfX], inWidthOfX);
+ X_transp_AX.rebind(&mStorage[4 + 4 * inWidthOfX], inWidthOfX, inWidthOfX);
+ logLikelihood.rebind(&mStorage[4 + inWidthOfX * inWidthOfX + 4 * inWidthOfX]);
+ status.rebind(&mStorage[5 + inWidthOfX * inWidthOfX + 4 * inWidthOfX]);
+ }
+
+ Handle mStorage;
+
+ public:
+ typename HandleTraits<Handle>::ReferenceToUInt32 iteration;
+ typename HandleTraits<Handle>::ReferenceToUInt16 widthOfX;
+ typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap coef;
+ typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap dir;
+ typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap grad;
+ typename HandleTraits<Handle>::ReferenceToDouble beta;
+
+ typename HandleTraits<Handle>::ReferenceToUInt64 numRows;
+ typename HandleTraits<Handle>::ColumnVectorTransparentHandleMap gradNew;
+ typename HandleTraits<Handle>::MatrixTransparentHandleMap X_transp_AX;
+ typename HandleTraits<Handle>::ReferenceToDouble logLikelihood;
+ typename HandleTraits<Handle>::ReferenceToUInt16 status;
+};
+
+
+
+
+/**
+ * @brief Logistic function
+ */
+inline double sigma(double x) {
+ return 1. / (1. + std::exp(-x));
+}
+
+/**
+ * @brief Perform the logistic-regression transition step
+ */
+AnyType
+logregr_simple_step_transition::run(AnyType &args) {
+ LogRegrSPTransitionState<MutableArrayHandle<double> > state = args[0];
+ if (args[1].isNull() || args[2].isNull()) { return args[0]; }
+ double y = args[1].getAs<bool>() ? 1. : -1.;
+ MappedColumnVector x;
+ try {
+ // an exception is raised in the backend if args[2] contains nulls
+ MappedColumnVector xx = args[2].getAs<MappedColumnVector>();
+ // x is a const reference, we can only rebind to change its pointer
+ x.rebind(xx.memoryHandle(), xx.size());
+ } catch (const ArrayWithNullException &e) {
+ return args[0];
+ }
+
+ // The following check was added with MADLIB-138.
+ if (!dbal::eigen_integration::isfinite(x)) {
+ warning("Design matrix is not finite.");
+ state.status = TERMINATED;
+ return state;
+ }
+
+ if (state.numRows == 0) {
+ if (x.size() > std::numeric_limits<uint16_t>::max()){
+ warning("Number of independent variables cannot be larger than 65535.");
+ state.status = TERMINATED;
+ return state;
+ }
+
+
+ state.initialize(*this, static_cast<uint16_t>(x.size()));
+ if (!args[3].isNull()) {
+ LogRegrSPTransitionState<ArrayHandle<double> > previousState = args[3];
+
+ state = previousState;
+ state.reset();
+ }
+ }
+ // Now do the transition step
+ state.numRows++;
+ double xc = dot(x, state.coef);
+ state.gradNew.noalias() += sigma(-y * xc) * y * trans(x);
+
+ // Note: sigma(-x) = 1 - sigma(x).
+ double a = sigma(xc) * sigma(-xc);
+ state.X_transp_AX += x * trans(x) * a;
+
+ state.logLikelihood -= std::log( 1. + std::exp(-y * xc) );
+
+ return state;
+}
+
+
+/**
+ * @brief Perform the perliminary aggregation function: Merge transition states
+ */
+AnyType
+logregr_simple_step_merge_states::run(AnyType &args) {
+ LogRegrSPTransitionState<MutableArrayHandle<double> > stateLeft = args[0];
+ LogRegrSPTransitionState<ArrayHandle<double> > stateRight = args[1];
+
+ // We first handle the trivial case where this function is called with one
+ // of the states being the initial state
+ if (stateLeft.numRows == 0)
+ return stateRight;
+ else if (stateRight.numRows == 0)
+ return stateLeft;
+
+ // Merge states together and return
+ stateLeft += stateRight;
+ return stateLeft;
+}
+
+
+
+/**
+ * @brief Perform the logistic-regression final step
+ */
+AnyType
+logregr_simple_step_final::run(AnyType &args) {
+ // We request a mutable object. Depending on the backend, this might perform
+ // a deep copy.
+ LogRegrSPTransitionState<MutableArrayHandle<double> > state = args[0];
+
+ // Aggregates that haven't seen any data just return Null.
+ if (state.numRows == 0){
+ state.status = NULL_EMPTY;
+ return state;
+ }
+
+ // Note: k = state.iteration
+ if (state.iteration == 0) {
+ // Iteration computes the gradient
+
+ state.dir = state.gradNew;
+ state.grad = state.gradNew;
+ } else {
+ // We use the Hestenes-Stiefel update formula:
+ //
+ // g_k^T (g_k - g_{k-1})
+ // beta_k = -------------------------
+ // d_{k-1}^T (g_k - g_{k-1})
+ ColumnVector gradNewMinusGrad = state.gradNew - state.grad;
+ state.beta
+ = dot(state.gradNew, gradNewMinusGrad)
+ / dot(state.dir, gradNewMinusGrad);
+
+ // Do a direction restart (Powell restart)
+ // Note: This is testing whether state.beta < 0 if state.beta were
+ // assigned according to Polak-Ribière
+ if (dot(state.gradNew, gradNewMinusGrad)
+ / dot(state.grad, state.grad) <= std::numeric_limits<double>::denorm_min()) state.beta = 0;
+
+ // d_k = g_k - beta_k * d_{k-1}
+ state.dir = state.gradNew - state.beta * state.dir;
+ state.grad = state.gradNew;
+ }
+
+ // H_k = - X^T A_k X
+ // where A_k = diag(a_1, ..., a_n) and a_i = sigma(x_i c_{k-1}) sigma(-x_i c_{k-1})
+ //
+ // g_k^T d_k
+ // alpha_k = -------------
+ // d_k^T H_k d_k
+ //
+ // c_k = c_{k-1} - alpha_k * d_k
+ state.coef += dot(state.grad, state.dir) /
+ as_scalar(trans(state.dir) * state.X_transp_AX * state.dir)
+ * state.dir;
+
+ if(!state.coef.is_finite()){
+ // we don't throw an error - instead set the state status to allow
+ // other states (possibly trained as part of group by) to continue
+ // training
+ warning("Over- or underflow in conjugate-gradient step, while updating "
+ "coefficients. Input data is likely of poor numerical condition.");
+ state.status = TERMINATED;
+ return state;
+ }
+
+ state.iteration++;
+ return state;
+}
+
+/**
+ * @brief Return the difference in log-likelihood between two states
+ */
+AnyType
+internal_logregr_simple_step_distance::run(AnyType &args) {
+ LogRegrSPTransitionState<ArrayHandle<double> > stateLeft = args[0];
+ LogRegrSPTransitionState<ArrayHandle<double> > stateRight = args[1];
+
+ if(stateLeft.status == NULL_EMPTY || stateRight.status == NULL_EMPTY){
+ return 0.0;
+ }
+ return std::abs(stateLeft.logLikelihood - stateRight.logLikelihood);
+}
+
+/**
+ * @brief Return the coefficients and diagnostic statistics of the state
+ */
+AnyType
+internal_logregr_simple_result::run(AnyType &args) {
+ LogRegrSPTransitionState<ArrayHandle<double> > state = args[0];
+ if (state.status == NULL_EMPTY)
+ return Null();
+
+ SymmetricPositiveDefiniteEigenDecomposition<Matrix> decomposition(
+ state.X_transp_AX, EigenvaluesOnly, ComputePseudoInverse);
+
+ return stateToResult(*this, state.coef,
+ state.X_transp_AX, state.logLikelihood,
+ state.status, state.numRows);
+}
+
+
+/**
+ * @brief Compute the diagnostic statistics
+ *
+ * This function wraps the common parts of computing the results for both the
+ * CG and the IRLS method.
+ */
+AnyType stateToResult(
+ const Allocator &inAllocator,
+ const HandleMap<const ColumnVector, TransparentHandle<double> > &inCoef,
+ const Matrix & hessian,
+ const double &logLikelihood,
+ int status,
+ const uint64_t &numRows) {
+
+ SymmetricPositiveDefiniteEigenDecomposition<Matrix> decomposition(
+ hessian, EigenvaluesOnly, ComputePseudoInverse);
+
+ const Matrix &inverse_of_X_transp_AX = decomposition.pseudoInverse();
+ const ColumnVector &diagonal_of_X_transp_AX = inverse_of_X_transp_AX.diagonal();
+
+ MutableNativeColumnVector stdErr(
+ inAllocator.allocateArray<double>(inCoef.size()));
+ MutableNativeColumnVector waldZStats(
+ inAllocator.allocateArray<double>(inCoef.size()));
+ MutableNativeColumnVector waldPValues(
+ inAllocator.allocateArray<double>(inCoef.size()));
+ MutableNativeColumnVector oddsRatios(
+ inAllocator.allocateArray<double>(inCoef.size()));
+
+ for (Index i = 0; i < inCoef.size(); ++i) {
+ stdErr(i) = std::sqrt(diagonal_of_X_transp_AX(i));
+ waldZStats(i) = inCoef(i) / stdErr(i);
+ waldPValues(i) = 2. * prob::cdf( prob::normal(),
+ -std::abs(waldZStats(i)));
+ oddsRatios(i) = std::exp( inCoef(i) );
+ }
+
+ // Return all coefficients, standard errors, etc. in a tuple
+ AnyType tuple;
+ tuple << inCoef << logLikelihood << stdErr << waldZStats << waldPValues
+ << oddsRatios << inverse_of_X_transp_AX
+ << sqrt(decomposition.conditionNo()) << status << numRows;
+ return tuple;
+}
+
+} // namespace hello_world
+} // namespace modules
+} // namespace madlib
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/iterative/simple_logistic.hpp
----------------------------------------------------------------------
diff --git a/examples/hello_world/iterative/simple_logistic.hpp b/examples/hello_world/iterative/simple_logistic.hpp
new file mode 100644
index 0000000..8fc38d2
--- /dev/null
+++ b/examples/hello_world/iterative/simple_logistic.hpp
@@ -0,0 +1,32 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * @file logistic.hpp
+ *
+ *//* ----------------------------------------------------------------------- */
+
+/**
+ * @brief Logistic regression (conjugate-gradient step): Transition function
+ */
+DECLARE_UDF(hello_world, logregr_simple_step_transition)
+
+/**
+ * @brief Logistic regression (conjugate-gradient step): State merge function
+ */
+DECLARE_UDF(hello_world, logregr_simple_step_merge_states)
+
+/**
+ * @brief Logistic regression (conjugate-gradient step): Final function
+ */
+DECLARE_UDF(hello_world, logregr_simple_step_final)
+
+/**
+ * @brief Logistic regression (conjugate-gradient): Difference in log-likelihood
+ * between two transition states
+ */
+DECLARE_UDF(hello_world, internal_logregr_simple_step_distance)
+
+/**
+ * @brief Logistic regression (conjugate-gradient): Convert transition state to
+ * result tuple
+ */
+DECLARE_UDF(hello_world, internal_logregr_simple_result)
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/iterative/simple_logistic.py_in
----------------------------------------------------------------------
diff --git a/examples/hello_world/iterative/simple_logistic.py_in b/examples/hello_world/iterative/simple_logistic.py_in
new file mode 100644
index 0000000..cd03be5
--- /dev/null
+++ b/examples/hello_world/iterative/simple_logistic.py_in
@@ -0,0 +1,222 @@
+# coding=utf-8
+
+"""
+@file simple_logistic.py_in
+
+@brief Logistic Regression: Driver functions
+
+@namespace simple_logistic
+
+@brief Logistic Regression: Driver functions
+"""
+import plpy
+from utilities.validate_args import table_exists
+from utilities.validate_args import table_is_empty
+# ------------------------------------------------------------------------
+
+
+def logregr_simple_train(
+ schema_madlib, source_table, out_table, dependent_varname,
+ independent_varname, max_iter=None,
+ tolerance=None, verbose=None, **kwargs):
+ """
+ Train logistic model
+
+ @param schema_madlib Name of the MADlib schema, properly escaped/quoted
+ @param source_table Name of relation containing the training data
+ @param out_table Name of relation where model will be outputted
+ @param dependent_varname Name of dependent column in training data (of type BOOLEAN)
+ @param independent_varname Name of independent column in training data (of type
+ DOUBLE PRECISION[])
+ @param max_iter The maximum number of iterations that are allowed.
+ @param tolerance The precision that the results should have
+ @param kwargs We allow the caller to specify additional arguments (all of
+ which will be ignored though). The purpose of this is to allow the
+ caller to unpack a dictionary whose element set is a superset of
+ the required arguments by this function.
+
+ @return A composite value which is __logregr_simple_result defined in simple_logistic.sql_in
+ """
+
+ return __logregr_train_compute(
+ schema_madlib, source_table, out_table, dependent_varname,
+ independent_varname, max_iter, tolerance,
+ verbose, **kwargs)
+
+# ========================================================================
+
+
+def __logregr_train_compute(schema_madlib, tbl_source, tbl_output, dep_col,
+ ind_col, max_iter, tolerance, verbose, **kwargs):
+ """
+ Validate the arguments
+ """
+ if tbl_source is None or tbl_source.strip().lower() in ('null', ''):
+ plpy.error("Logregr error: Invalid data table name!")
+ if not table_exists(tbl_source):
+ plpy.error("Logregr error: Data table does not exist!")
+ if table_is_empty(tbl_source):
+ plpy.error("Logregr error: Data table is empty!")
+
+ if tbl_output is None or tbl_output.strip().lower() in ('null', ''):
+ plpy.error("Logregr error: Invalid output table name!")
+
+ if not dep_col or dep_col.strip().lower() in ('null', ''):
+ plpy.error("Logregr error: Invalid dependent column name!")
+
+ # if not columns_exist_in_table(tbl_source, [dep_col]):
+ # plpy.error("Logregr error: Dependent column does not exist!")
+
+ if not ind_col or ind_col.lower() in ('null', ''):
+ plpy.error("Logregr error: Invalid independent column name!")
+
+ if max_iter <= 0:
+ plpy.error("Logregr error: Maximum number of iterations must be positive!")
+
+ if tolerance < 0:
+ plpy.error("Logregr error: The tolerance cannot be negative!")
+
+ update_plan = plpy.prepare(
+ """
+ SELECT
+ {schema_madlib}.__logregr_simple_step(
+ ({dep_col})::boolean,
+ ({ind_col})::double precision[],
+ ($1))
+ FROM {tbl_source}
+ """.format(
+ tbl_output=tbl_output,
+ schema_madlib=schema_madlib,
+ dep_col=dep_col,
+ ind_col=ind_col,
+ tbl_source=tbl_source), ["double precision[]"])
+
+ state = None
+ for it in range(0, max_iter):
+ res_tuple = plpy.execute(update_plan, [state])
+ state = res_tuple[0].values()[0]
+
+ output_table = plpy.prepare(
+ """
+ drop table if exists {tbl_output};
+ create table {tbl_output} as
+ select
+ (result).coef as coef,
+ (result).log_likelihood as log_likelihood
+ from
+ (
+ select * from
+ {schema_madlib}.__logregr_simple_finalizer($1)
+ ) result
+ """.format(schema_madlib=schema_madlib,
+ tbl_output=tbl_output), ["double precision[]"])
+
+ plpy.execute(output_table, [state])
+ return None
+# --------------------------------------------------------------------
+
+
+def logregr_simple_help_msg(schema_madlib, message, **kwargs):
+ """ Help message for logistic regression
+
+ @param message A string, the help message indicator
+
+ Returns:
+ A string, contains the help message
+ """
+ if not message:
+
+ help_string = """
+----------------------------------------------------------------
+ SUMMARY
+----------------------------------------------------------------
+Binomial logistic regression models the relationship between a
+dichotomous dependent variable and one or more predictor variables.
+
+The dependent variable may be a Boolean value or a categorical variable
+that can be represented with a Boolean expression.
+
+For more details on function usage:
+ SELECT {schema_madlib}.logregr_train('usage')
+
+For a small example on using the function:
+ SELECT {schema_madlib}.logregr_train('example')
+ """
+ elif message in ['usage', 'help', '?']:
+
+ help_string = """
+------------------------------------------------------------------
+ USAGE
+------------------------------------------------------------------
+SELECT {schema_madlib}.logregr_train(
+ source_table, -- name of input table
+ out_table, -- name of output table
+ dependent_varname, -- name of dependent variable
+ independent_varname, -- names of independent variables
+ max_iter, -- optional, default 20, maximum iteration number
+ tolerance, -- optional, default 0.0001, the stopping threshold
+ verbose -- optional, default FALSE, whether to print useful info
+);
+
+------------------------------------------------------------------
+ OUTPUT
+------------------------------------------------------------------
+The output table ('out_table' above) has the following columns:
+ <...>, -- Grouping column values used during training
+ 'coef', double precision[], -- vector of fitting coefficients
+ 'log_likelihood', double precision, -- log likelihood
+ 'std_err', double precision[], -- vector of standard errors of the fitting coefficients
+ 'z_stats', double precision[], -- vector of the z-statistics of the coefficients
+ 'p_values', double precision[], -- vector of the p values
+ 'odds_ratios', double precision[], -- vector of odds ratios, exp(coefficients)
+ 'condition_no', double precision, -- the condition number
+ 'num_rows_processed', integer, -- how many rows are actually used in the computation
+ 'num_missing_rows_skipped', integer, -- number of rows that contain NULL and were skipped per group
+ 'num_iterations' double precision -- how many iterations are used in the computation per group
+ """
+ elif message in ['example', 'examples']:
+
+ help_string = """
+CREATE TABLE patients( id INTEGER NOT NULL,
+ second_attack BOOLEAN,
+ treatment INTEGER,
+ trait_anxiety INTEGER);
+COPY patients FROM STDIN WITH DELIMITER '|';
+ 1 | True | 1 | 70
+ 3 | True | 1 | 50
+ 5 | True | 0 | 40
+ 7 | True | 0 | 75
+ 9 | True | 0 | 70
+ 11 | False | 1 | 65
+ 13 | False | 1 | 45
+ 15 | False | 1 | 40
+ 17 | False | 0 | 55
+ 19 | False | 0 | 50
+ 2 | True | 1 | 80
+ 4 | True | 0 | 60
+ 6 | True | 0 | 65
+ 8 | True | 0 | 80
+ 10 | True | 0 | 60
+ 12 | False | 1 | 50
+ 14 | False | 1 | 35
+ 16 | False | 1 | 50
+ 18 | False | 0 | 45
+ 20 | False | 0 | 60
+\.
+
+-- Drop output tables before calling the function
+DROP TABLE IF EXISTS patients_logregr;
+DROP TABLE IF EXISTS patients_logregr_summary;
+
+SELECT madlib.logregr_simple_train( 'patients',
+ 'patients_logregr',
+ 'second_attack',
+ 'ARRAY[1, treatment, trait_anxiety]');
+
+SELECT * from patients_logregr;
+ """
+ else:
+ help_string = "No such option. Use {schema_madlib}.logregr_simple_train('help')"
+
+ return help_string.format(schema_madlib=schema_madlib)
+# ------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/iterative/simple_logistic.sql_in
----------------------------------------------------------------------
diff --git a/examples/hello_world/iterative/simple_logistic.sql_in b/examples/hello_world/iterative/simple_logistic.sql_in
new file mode 100644
index 0000000..416831d
--- /dev/null
+++ b/examples/hello_world/iterative/simple_logistic.sql_in
@@ -0,0 +1,203 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * @file logistic.sql_in
+ *
+ * @brief SQL functions for logistic regression
+ * @date January 2011
+ *
+ * @sa For a brief introduction to logistic regression, see the
+ * module description \ref grp_logreg.
+ *
+ *//* ----------------------------------------------------------------------- */
+
+m4_include(`SQLCommon.m4')
+
+DROP TYPE IF EXISTS MADLIB_SCHEMA.__logregr_simple_result CASCADE;
+CREATE TYPE MADLIB_SCHEMA.__logregr_simple_result AS (
+ coef DOUBLE PRECISION[],
+ log_likelihood DOUBLE PRECISION,
+ std_err DOUBLE PRECISION[],
+ z_stats DOUBLE PRECISION[],
+ p_values DOUBLE PRECISION[],
+ odds_ratios DOUBLE PRECISION[],
+ vcov DOUBLE PRECISION[],
+ condition_no DOUBLE PRECISION,
+ status INTEGER,
+ num_processed BIGINT,
+ num_iterations INTEGER
+);
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_simple_step_transition(
+ DOUBLE PRECISION[],
+ BOOLEAN, /*+ y */
+ DOUBLE PRECISION[], /*+ x */
+ DOUBLE PRECISION[]) /*+ previous state */
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME', 'logregr_simple_step_transition'
+LANGUAGE C IMMUTABLE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_simple_step_merge_states(
+ state1 DOUBLE PRECISION[],
+ state2 DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME', 'logregr_simple_step_merge_states'
+LANGUAGE C IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_simple_step_final(
+ state DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION[]
+AS 'MODULE_PATHNAME', 'logregr_simple_step_final'
+LANGUAGE C IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+------------------------------------------------------------------------
+
+/**
+ * @internal
+ * @brief Perform one iteration of the conjugate-gradient method for computing
+ * logistic regression
+ */
+DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.__logregr_simple_step(
+ BOOLEAN, DOUBLE PRECISION[], DOUBLE PRECISION[]);
+CREATE AGGREGATE MADLIB_SCHEMA.__logregr_simple_step(
+ /*+ y */ BOOLEAN,
+ /*+ x */ DOUBLE PRECISION[],
+ /*+ previous_state */ DOUBLE PRECISION[]) (
+
+ STYPE=DOUBLE PRECISION[],
+ SFUNC=MADLIB_SCHEMA.__logregr_simple_step_transition,
+ m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.__logregr_simple_step_merge_states,')
+ FINALFUNC=MADLIB_SCHEMA.__logregr_simple_step_final,
+ INITCOND='{0,0,0,0,0,0}'
+);
+
+------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_simple_step_distance(
+ /*+ state1 */ DOUBLE PRECISION[],
+ /*+ state2 */ DOUBLE PRECISION[])
+RETURNS DOUBLE PRECISION AS
+'MODULE_PATHNAME', 'internal_logregr_simple_step_distance'
+LANGUAGE c IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.__logregr_simple_finalizer(
+ /*+ state */ DOUBLE PRECISION[])
+RETURNS MADLIB_SCHEMA.__logregr_simple_result
+AS 'MODULE_PATHNAME', 'internal_logregr_simple_result'
+LANGUAGE c IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+------------------------------------------------------------------------
+
+
+/**
+ * @brief Compute logistic-regression coefficients and diagnostic statistics
+ *
+ * To include an intercept in the model, set one coordinate in the
+ * <tt>independentVariables</tt> array to 1.
+ *
+ * @param source_table Name of the source relation containing the training data
+ * @param out_table Name of the output relation to store the model results
+ *
+ * Columns of the output relation are as follows:
+ * - <tt>coef FLOAT8[]</tt> - Array of coefficients, \f$ \boldsymbol c \f$
+ * - <tt>log_likelihood FLOAT8</tt> - Log-likelihood \f$ l(\boldsymbol c) \f$
+ * - <tt>std_err FLOAT8[]</tt> - Array of standard errors,
+ * \f$ \mathit{se}(c_1), \dots, \mathit{se}(c_k) \f$
+ * - <tt>z_stats FLOAT8[]</tt> - Array of Wald z-statistics, \f$ \boldsymbol z \f$
+ * - <tt>p_values FLOAT8[]</tt> - Array of Wald p-values, \f$ \boldsymbol p \f$
+ * - <tt>odds_ratios FLOAT8[]</tt>: Array of odds ratios,
+ * \f$ \mathit{odds}(c_1), \dots, \mathit{odds}(c_k) \f$
+ * - <tt>condition_no FLOAT8</tt> - The condition number of
+ * matrix \f$ X^T A X \f$ during the iteration
+ * immediately <em>preceding</em> convergence
+ * (i.e., \f$ A \f$ is computed using the coefficients
+ * of the previous iteration)
+ * @param dependent_varname Name of the dependent column (of type BOOLEAN)
+ * @param independent_varname Name of the independent column (of type DOUBLE
+ * PRECISION[])
+ * @param max_iter The maximum number of iterations
+ * @param tolerance The difference between log-likelihood values in successive
+ * iterations that should indicate convergence. This value should be
+ * non-negative and a zero value here disables the convergence criterion,
+ * and execution will only stop after \c maxNumIterations iterations.
+ * @param verbose If true, any error or warning message will be printed to the
+ * console (irrespective of the 'client_min_messages' set by server).
+ * If false, no error/warning message is printed to console.
+ *
+ *
+ * @usage
+ * - Get vector of coefficients \f$ \boldsymbol c \f$ and all diagnostic
+ * statistics:\n
+ * <pre>SELECT logregr_train('<em>sourceName</em>', '<em>outName</em>'
+ * '<em>dependentVariable</em>', '<em>independentVariables</em>');
+ * SELECT * from outName;
+ * </pre>
+ * - Get vector of coefficients \f$ \boldsymbol c \f$:\n
+ * <pre>SELECT coef from outName;</pre>
+ * - Get a subset of the output columns, e.g., only the array of coefficients
+ * \f$ \boldsymbol c \f$, the log-likelihood of determination
+ * \f$ l(\boldsymbol c) \f$, and the array of p-values \f$ \boldsymbol p \f$:
+ * <pre>SELECT coef, log_likelihood, p_values FROM outName;</pre>
+ *
+ * @note This function starts an iterative algorithm. It is not an aggregate
+ * function. Source, output, and column names have to be passed as strings
+ * (due to limitations of the SQL syntax).
+ *
+ * @internal
+ * @sa This function is a wrapper for logistic::compute_logregr(), which
+ * sets the default values.
+ */
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_simple_train (
+ source_table VARCHAR,
+ out_table VARCHAR,
+ dependent_varname VARCHAR,
+ independent_varname VARCHAR,
+ max_iter INTEGER,
+ tolerance DOUBLE PRECISION,
+ verbose BOOLEAN
+) RETURNS VOID AS $$
+PythonFunction(hello_world, simple_logistic, logregr_simple_train)
+$$ LANGUAGE plpythonu
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+
+------------------------------------------------------------------------
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_simple_train (
+ source_table VARCHAR,
+ out_table VARCHAR,
+ dependent_varname VARCHAR,
+ independent_varname VARCHAR)
+RETURNS VOID AS $$
+ SELECT MADLIB_SCHEMA.logregr_simple_train($1, $2, $3, $4, 20, 0.0001, False);
+$$ LANGUAGE sql VOLATILE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
+
+------------------------------------------------------------------------
+
+-- Help messages -------------------------------------------------------
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_simple_train(
+ message TEXT
+) RETURNS TEXT AS $$
+ PythonFunction(hello_world, simple_logistic, logregr_simple_help_msg)
+$$ LANGUAGE plpythonu IMMUTABLE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.logregr_simple_train()
+RETURNS TEXT
+AS $$
+ SELECT MADLIB_SCHEMA.logregr_simple_train(NULL::TEXT);
+$$ LANGUAGE SQL IMMUTABLE
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `CONTAINS SQL', `');
+
+-------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/non-iterative/avg_var.cpp
----------------------------------------------------------------------
diff --git a/examples/hello_world/non-iterative/avg_var.cpp b/examples/hello_world/non-iterative/avg_var.cpp
new file mode 100644
index 0000000..cbb5e19
--- /dev/null
+++ b/examples/hello_world/non-iterative/avg_var.cpp
@@ -0,0 +1,141 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * @file avg_var.cpp
+ *
+ * @brief average population variance functions
+ *
+ *//* ----------------------------------------------------------------------- */
+
+#include <dbconnector/dbconnector.hpp>
+#include <modules/shared/HandleTraits.hpp>
+#include "avg_var.hpp"
+
+namespace madlib {
+
+namespace modules {
+
+namespace hello_world {
+
+template <class Handle>
+class AvgVarTransitionState {
+ template <class OtherHandle>
+ friend class AvgVarTransitionState;
+
+ public:
+ AvgVarTransitionState(const AnyType &inArray)
+ : mStorage(inArray.getAs<Handle>()) {
+
+ rebind();
+ }
+
+ /**
+ * @brief Convert to backend representation
+ *
+ * We define this function so that we can use State in the
+ * argument list and as a return type.
+ */
+ inline operator AnyType() const {
+ return mStorage;
+ }
+
+ /**
+ * @brief Update state with a new data point
+ */
+ template <class OtherHandle>
+ AvgVarTransitionState &operator+=(const double x){
+ double diff = (x - avg);
+ double normalizer = static_cast<double>(numRows + 1);
+ // online update mean
+ this.avg += diff / normalizer;
+
+ // online update variance
+ double new_diff = (x - avg);
+ double a = static_cast<double>(state.numRows) / normalizer;
+ this.var = (var * a) + (diff * new_diff) / normalizer;
+ }
+
+ /**
+ * @brief Merge with another state object
+ *
+ * We update mean and variance in a online fashion
+ * to avoid intermediate large sum.
+ */
+ template <class OtherHandle>
+ AvgVarTransitionState &operator+=(
+ const AvgVarTransitionState<OtherHandle> &inOtherState) {
+
+ if (mStorage.size() != inOtherState.mStorage.size())
+ throw std::logic_error("Internal error: Incompatible transition "
+ "states");
+ double avg_ = inOtherState.avg;
+ double var_ = inOtherState.var;
+ uint16_t numRows_ = static_cast<uint16_t>(inOtherState.numRows);
+ double totalNumRows = static_cast<double>(numRows + numRows_);
+
+ // we perform a weighted average between states
+ double w = static_cast<double>(numRows) / totalNumRows;
+ double w_ = static_cast<double>(numRows_) / totalNumRows;
+ double totalAvg = avg * w + avg_ * w_;
+ double a = avg - totalAvg;
+ double a_ = avg_ - totalAvg;
+
+ numRows += numRows_;
+ this.var = (w * var) + (w_ * var_) + (w * a * a) + (w_ * a_ * a_);
+ this.avg = totalAvg;
+ return *this;
+ }
+
+ private:
+ void rebind() {
+ avg.rebind(&mStorage[0]);
+ var.rebind(&mStorage[1]);
+ numRows.rebind(&mStorage[2]);
+ }
+
+ Handle mStorage;
+
+ public:
+ typename HandleTraits<Handle>::ReferenceToDouble avg;
+ typename HandleTraits<Handle>::ReferenceToDouble var;
+ typename HandleTraits<Handle>::ReferenceToUInt64 numRows;
+};
+
+
+AnyType
+avg_var_transition::run(AnyType& args) {
+ // get current state value
+ AvgVarTransitionState<MutableArrayHandle<double> > state = args[0];
+ // update state with current row value
+ double x = args[1].getAs<double>();
+ state += x;
+ state.numRows ++;
+ return state;
+}
+
+AnyType
+avg_var_merge_states::run(AnyType& args) {
+ AvgVarTransitionState<MutableArrayHandle<double> > stateLeft = args[0];
+ AvgVarTransitionState<ArrayHandle<double> > stateRight = args[1];
+
+ // Merge states together and return
+ stateLeft += stateRight;
+ return stateLeft;
+}
+
+AnyType
+avg_var_final::run(AnyType& args) {
+ AvgVarTransitionState<MutableArrayHandle<double> > state = args[0];
+
+ // If we haven't seen any data, just return Null. This is the standard
+ // behavior of aggregate function on empty data sets (compare, e.g.,
+ // how PostgreSQL handles sum or avg on empty inputs)
+ if (state.numRows == 0)
+ return Null();
+
+ return state;
+}
+// -----------------------------------------------------------------------
+
+} // namespace hello_world
+} // namespace modules
+} // namespace madlib
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/non-iterative/avg_var.hpp
----------------------------------------------------------------------
diff --git a/examples/hello_world/non-iterative/avg_var.hpp b/examples/hello_world/non-iterative/avg_var.hpp
new file mode 100644
index 0000000..7fb4a93
--- /dev/null
+++ b/examples/hello_world/non-iterative/avg_var.hpp
@@ -0,0 +1,20 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * @file avg_var.hpp
+ *
+ *//* ----------------------------------------------------------------------- */
+
+/**
+ * @brief average and variance: Transition function
+ */
+DECLARE_UDF(hello_world, avg_var_transition)
+
+/**
+ * @brief average and variance: State merge function
+ */
+DECLARE_UDF(hello_world, avg_var_merge_states)
+
+/**
+ * @brief average and variance: Final function
+ */
+DECLARE_UDF(hello_world, avg_var_final)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/a184a8ac/examples/hello_world/non-iterative/avg_var.sql_in
----------------------------------------------------------------------
diff --git a/examples/hello_world/non-iterative/avg_var.sql_in b/examples/hello_world/non-iterative/avg_var.sql_in
new file mode 100644
index 0000000..bfbfe5a
--- /dev/null
+++ b/examples/hello_world/non-iterative/avg_var.sql_in
@@ -0,0 +1,53 @@
+/* ----------------------------------------------------------------------- *//**
+ *
+ * @file avg_var.sql_in
+ *
+ * @brief SQL functions for average and variance
+ * @date September 2015
+ *
+ *
+ *//* ----------------------------------------------------------------------- */
+
+m4_include(`SQLCommon.m4')
+
+---------------------------------------------------------------------------
+-- Functions for user-defined aggregates
+---------------------------------------------------------------------------
+-- normal
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_var_merge_states(
+ state1 double precision[],
+ state2 double precision[])
+RETURNS double precision[] AS 'MODULE_PATHNAME', 'avg_var_merge_states'
+LANGUAGE C
+IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_var_final(
+ state double precision[])
+RETURNS double precision[] AS 'MODULE_PATHNAME'
+LANGUAGE C
+IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+
+CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.avg_var_transition(
+ state double precision[],
+ x double precision)
+RETURNS double precision[] AS 'MODULE_PATHNAME'
+LANGUAGE C
+IMMUTABLE STRICT
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `NO SQL', `');
+---------------------------------------------------------------------------
+
+---------------------------------------------------------------------------
+-- User-defined aggregates
+---------------------------------------------------------------------------
+DROP AGGREGATE IF EXISTS MADLIB_SCHEMA.avg_var(DOUBLE PRECISION);
+CREATE AGGREGATE MADLIB_SCHEMA.avg_var(
+ DOUBLE PRECISION) (
+
+ SFUNC=MADLIB_SCHEMA.avg_var_transition,
+ STYPE=double precision[],
+ FINALFUNC=MADLIB_SCHEMA.avg_var_final,
+ m4_ifdef(`__POSTGRESQL__', `', `prefunc=MADLIB_SCHEMA.avg_var_merge_states,')
+ INITCOND='{0, 0, 0}'
+);
\ No newline at end of file