You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2017/01/26 22:47:53 UTC
incubator-madlib git commit: Documentation: Fix misc errors
Repository: incubator-madlib
Updated Branches:
refs/heads/master faec6bee3 -> 29acc5386
Documentation: Fix misc errors
Closes #92
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/29acc538
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/29acc538
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/29acc538
Branch: refs/heads/master
Commit: 29acc53868a2ea8d007b07bc3464d5481376cb65
Parents: faec6be
Author: Frank McQuillan <fm...@pivotal.io>
Authored: Tue Jan 24 16:04:22 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Thu Jan 26 14:46:47 2017 -0800
----------------------------------------------------------------------
src/ports/postgres/modules/lda/lda.sql_in | 19 +-
src/ports/postgres/modules/linalg/svd.sql_in | 4 +-
.../recursive_partitioning/decision_tree.sql_in | 328 ++++++++++---------
.../recursive_partitioning/random_forest.sql_in | 210 +++++++-----
.../postgres/modules/summary/summary.sql_in | 2 +-
.../postgres/modules/utilities/path.sql_in | 3 +-
.../postgres/modules/utilities/pivot.sql_in | 3 +
7 files changed, 325 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/lda/lda.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/lda/lda.sql_in b/src/ports/postgres/modules/lda/lda.sql_in
index c9c7323..a26deab 100644
--- a/src/ports/postgres/modules/lda/lda.sql_in
+++ b/src/ports/postgres/modules/lda/lda.sql_in
@@ -185,10 +185,14 @@ lda_train( data_table,
<dd>TEXT. The name of the table storing the training dataset. Each row is
in the form <tt><docid, wordid, count></tt> where \c docid, \c wordid, and \c count
are non-negative integers.
-
+
The \c docid column refers to the document ID, the \c wordid column is the
word ID (the index of a word in the vocabulary), and \c count is the
- number of occurrences of the word in the document. </dd>
+ number of occurrences of the word in the document.
+
+ Please note that column names for \c docid, \c wordid, and \c count are currently fixed, so you must use these
+ exact names in the data_table.</dd>
+
<dt>model_table</dt>
<dd>TEXT. The name of the table storing the learned models. This table has one row and the following columns.
<table class="output">
@@ -312,12 +316,13 @@ lda_get_perplexity( model_table,
-# Prepare a training dataset for LDA. The examples below are small strings extracted from various Wikipedia documents .
<pre class="example">
+DROP TABLE IF EXISTS documents;
CREATE TABLE documents(docid INT4, contents TEXT);
INSERT INTO documents VALUES
(0, 'Statistical topic models are a class of Bayesian latent variable models, originally developed for analyzing the semantic content of large document corpora.'),
(1, 'By the late 1960s, the balance between pitching and hitting had swung in favor of the pitchers. In 1968 Carl Yastrzemski won the American League batting title with an average of just .301, the lowest in history.'),
(2, 'Machine learning is closely related to and often overlaps with computational statistics; a discipline that also specializes in prediction-making. It has strong ties to mathematical optimization, which deliver methods, theory and application domains to the field.'),
-(3, 'California''s diverse geography ranges from the Sierra Nevada in the east to the Pacific Coast in the west, from the Redwood\u2013Douglas fir forests of the northwest, to the Mojave Desert areas in the southeast. The center of the state is dominated by the Central Valley, a major agricultural area. ')
+(3, 'California''s diverse geography ranges from the Sierra Nevada in the east to the Pacific Coast in the west, from the Redwood\u2013Douglas fir forests of the northwest, to the Mojave Desert areas in the southeast. The center of the state is dominated by the Central Valley, a major agricultural area. ');
</pre>
-# Build a word count table by extracting the words and building a histogram for
@@ -328,7 +333,7 @@ ALTER TABLE documents ADD COLUMN words TEXT[];
UPDATE documents SET words = regexp_split_to_array(lower(contents), E'[\\\\s+\\\\.\\\\,]');
\nbsp
-- Create the term frequency table
-DROP TABLE IF EXISTS my_training;
+DROP TABLE IF EXISTS my_training, my_training_vocabulary;
SELECT madlib.term_frequency('documents', 'docid', 'words', 'my_training', TRUE);
SELECT * FROM my_training order by docid limit 20;
</pre>
@@ -388,6 +393,7 @@ SELECT * FROM my_training_vocabulary order by wordid limit 20;
-# Create an LDA model using the \c lda_train() function.
<pre class="example">
+DROP TABLE IF EXISTS my_model, my_outdata;
SELECT madlib.lda_train( 'my_training',
'my_model',
'my_outdata',
@@ -398,12 +404,15 @@ SELECT madlib.lda_train( 'my_training',
0.01
);
</pre>
+Reminder that column names for \c docid, \c wordid, and \c count are currently fixed,
+so you must use these exact names in the input table.
After a successful run of the lda_train() function two tables are generated,
one for storing the learned model and the other for storing the output data table.
-# To get the detailed information about the learned model, run these commands:
<pre class="example">
-- The topic description by top-k words
+DROP TABLE IF EXISTS my_topic_desc;
SELECT madlib.lda_get_topic_desc( 'my_model',
'my_training_vocabulary',
'my_topic_desc',
@@ -433,8 +442,10 @@ select * from my_topic_desc order by topicid, prob DESC;
\nbsp
<pre class="example">
-- The per-word topic counts (sorted by topic id)
+DROP TABLE IF EXISTS my_word_topic_count;
SELECT madlib.lda_get_word_topic_count( 'my_model',
'my_word_topic_count');
+SELECT * FROM my_word_topic_count ORDER BY wordid;
</pre>
\nbsp
<pre class="result">
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/linalg/svd.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/linalg/svd.sql_in b/src/ports/postgres/modules/linalg/svd.sql_in
index 799e9cb..b6d763b 100644
--- a/src/ports/postgres/modules/linalg/svd.sql_in
+++ b/src/ports/postgres/modules/linalg/svd.sql_in
@@ -416,8 +416,8 @@ called the <em>singular values</em>.
It is possible to formulate the problem of computing the singular triplets
(\f$\sigma_i, u_i, v_i\f$) of \f$A\f$ as an eigenvalue problem involving a Hermitian
matrix related to \f$A\f$. There are two possible ways of achieving this:
--# With the cross product matrix, \f$A^TA\f$ and \f$AA^T\f$
--# With the cyclic matrix
+- With the cross product matrix, \f$A^TA\f$ and \f$AA^T\f$
+- With the cyclic matrix
\f[
H(A) =
\begin{bmatrix}
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index d6b7b5a..b5ed4a2 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -24,17 +24,15 @@ m4_include(`SQLCommon.m4')
</ul></div>
@brief Decision Trees.
-Decision trees use a tree-based predictive model to
-predict the value of a target variable based on several input variables.
-
-Decision trees are a supervised learning method that uses a predictive model to
-predict the value of a target variable, based on several input variables. They
-use a tree-based representation of the model such that, the interior nodes of
-the tree correspond to the input variables, the edges of the nodes correspond to
-values that the input variables can take, and leaf nodes represent values of the
-target variable, given the values of the input variables, represented by the
-path from the root to the leaf nodes.
+Decision trees use a tree-based model to predict the value of a
+target variable based on several input variables.
+A decision tree is a supervised learning method that can be used for
+classification and regression. It consists of a structure in which
+internal nodes represent tests on attributes, and the branches from
+nodes represent the result of those tests. Each leaf node is a class
+label and the paths from root to leaf nodes define the set of classification
+or regression rules.
@anchor train
@par Training Function
@@ -63,18 +61,18 @@ tree_train(
\b Arguments
<dl class="arglist">
<dt>training_table_name</dt>
- <dd>TEXT. The name of the table containing the training data</dd>
+ <dd>TEXT. The name of the table containing the training data.</dd>
<dt>output_table_name</dt>
<dd>TEXT. The name of the generated table containing the model. If a table
with the same name already exists, then the function will return an error.</dd>
- The model table produced by the train function contains the following columns:
+ The model table produced by the training function contains the following columns:
<table class="output">
<tr>
<th><...></th>
- <td>Grouping columns, if provided in input, same types as in the training table.
+ <td>Grouping columns, if provided as input, in the same types as the training table.
This could be multiple columns depending on the \c grouping_cols input.</td>
</tr>
<tr>
@@ -83,22 +81,22 @@ tree_train(
</tr>
<tr>
<th>cat_levels_in_text</th>
- <td>TEXT[]. Ordered levels of categorical variables</td>
+ <td>TEXT[]. Ordered levels of categorical variables.</td>
</tr>
<tr>
<th>cat_n_levels</th>
- <td>INTEGER[]. Number of levels for each categorical variable</td>
+ <td>INTEGER[]. Number of levels for each categorical variable.</td>
</tr>
<tr>
<th>tree_depth</th>
- <td>INTEGER. The maximum depth the tree obtained after training (root has depth 0)</td>
+ <td>INTEGER. The maximum depth the tree obtained after training (root has depth 0).</td>
</tr>
<tr>
<th>pruning_cp</th>
<td>DOUBLE PRECISION. The cost-complexity parameter used for pruning
- the trained tree(s). This would be different from the input cp value if
+ the trained tree(s). This would be different from the 'input_cp' value if
cross-validation is used.
</td>
</tr>
@@ -116,89 +114,90 @@ tree_train(
<tr>
<th>is_classification</th>
- <td>BOOLEAN. TRUE if the decision trees are for classification, FALSE if regression</td>
+ <td>BOOLEAN. TRUE if the decision trees are for classification, FALSE if for regression.</td>
</tr>
<tr>
<th>source_table</th>
- <td>TEXT. The data source table name</td>
+ <td>TEXT. The data source table name.</td>
</tr>
<tr>
<th>model_table</th>
- <td>TEXT. The model table name</td>
+ <td>TEXT. The model table name.</td>
</tr>
<tr>
<th>id_col_name</th>
- <td>TEXT. The ID column name</td>
+ <td>TEXT. The ID column name.</td>
</tr>
<tr>
<th>dependent_varname</th>
- <td>TEXT. The dependent variable</td>
+ <td>TEXT. The dependent variable.</td>
</tr>
<tr>
<th>independent_varname</th>
- <td>TEXT. The independent variables</td>
+ <td>TEXT. The independent variables.</td>
</tr>
<tr>
<th>cat_features</th>
- <td>TEXT. The list of categorical feature names as a comma-separated string</td>
+ <td>TEXT. The list of categorical feature names as a comma-separated string.</td>
</tr>
+
<tr>
<th>con_features</th>
- <td>TEXT. The list of continuous feature names as a comma-separated string</td>
+ <td>TEXT. The list of continuous feature names as a comma-separated string.</td>
</tr>
<tr>
<th>grouping_col</th>
- <td>TEXT. Names of grouping columns</td>
+ <td>TEXT. Names of grouping columns.</td>
</tr>
<tr>
<th>num_all_groups</th>
- <td>INTEGER. Number of groups in decision tree training</td>
+ <td>INTEGER. Number of groups in decision tree training.</td>
</tr>
<tr>
<th>num_failed_groups</th>
- <td>INTEGER. Number of failed groups in decision tree training</td>
+ <td>INTEGER. Number of failed groups in decision tree training.</td>
</tr>
<tr>
<th>total_rows_processed</th>
- <td>BIGINT. Total numbers of rows processed in all groups</td>
+ <td>BIGINT. Total numbers of rows processed in all groups.</td>
</tr>
<tr>
<th>total_rows_skipped</th>
<td>BIGINT. Total numbers of rows skipped in all groups due to missing
- values or failures</td>
+ values or failures.</td>
</tr>
<tr>
<th>dependent_var_levels</th>
- <td>TEXT. For classification, the distinct levels of the dependent variable</td>
+ <td>TEXT. For classification, the distinct levels of the dependent variable.</td>
</tr>
<tr>
<th>dependent_var_type</th>
- <td>TEXT. The type of dependent variable</td>
+ <td>TEXT. The type of dependent variable.</td>
</tr>
<tr>
<th>input_cp</th>
<td>DOUBLE PRECISION. The complexity parameter (cp) used for pruning the trained tree(s)
- (before cross-validation is run). This is same as the cp value inputed
- through the <em>pruning_params</em></td>
+ before cross-validation is run. This is same as the cp value input
+ using the <em>pruning_params</em>.</td>
</tr>
<tr>
<th>independent_var_types</th>
- <td>TEXT. A comma separated string, the types of independent variables</td>
+ <td>TEXT. A comma separated string for the types of independent variables.</td>
</tr>
</table>
@@ -207,7 +206,7 @@ tree_train(
<DT>id_col_name</DT>
<DD>TEXT. Name of the column containing id information in the training data.
This is a mandatory argument and is used for prediction and cross-validation.
- The values are expected to be unique for each row
+ The values are expected to be unique for each row.
</DD>
<DT>dependent_variable</DT>
@@ -215,7 +214,7 @@ tree_train(
training. Boolean, integer and text types are considered classification
outputs, while double precision values are considered regression outputs.
The response variable for a classification tree can be multinomial, but the
- time and space complexity of train function increases linearly as the
+ time and space complexity of the training function increases linearly as the
number of response classes increases.</DD>
<DT>list_of_features</DT>
@@ -226,11 +225,11 @@ tree_train(
double precision columns are considered continuous. The categorical variables
are not encoded and used as is for the training.
- There are no limitations to the number of levels in a categorical variable.
+ There are no limitations on the number of levels in a categorical variable.
It is, however, important to note that we don't test for every combination of
- levels of a categorical variable for evaluating a split. We order the levels
- of the variable by the entropy of the varible in predicting the response. The
- splits at each node is evaluated between these ordered levels
+ levels of a categorical variable when evaluating a split. We order the levels
+ of the variable by the entropy of the variable in predicting the response. The
+ split at each node is evaluated between these ordered levels.
</DD>
<DT>list_of_features_to_exclude</DT>
@@ -240,44 +239,44 @@ tree_train(
<em>dependent_variable</em> expression,
otherwise those columns will be included in the features.
The names in this parameter should be identical to the names used in the table and
- quoted appropriately</DD>
+ quoted appropriately.</DD>
<DT>split_criterion</DT>
<DD>TEXT, default = 'gini' for classification, 'mse' for regression.
Impurity function to compute the feature to use for the split.
Supported criteria are 'gini', 'entropy', 'misclassification' for
classification trees. For regression trees, split_criterion of 'mse'
- is always used (irrespective of the input for this argument)
+ is always used (irrespective of the input for this argument).
</DD>
<DT>grouping_cols (optional)</DT>
<DD>TEXT, default: NULL. Comma-separated list of column names to group the
- data by. This will lead to creating multiple decision trees, one for
- each group</DD>
+ data by. This will result in multiple decision trees, one for
+ each group.</DD>
<DT>weights (optional)</DT>
- <DD>TEXT. Column name containing weights for each observation</DD>
+ <DD>TEXT. Column name containing weights for each observation.</DD>
<DT>max_depth (optional)</DT>
<DD>INTEGER, default: 10. Maximum depth of any node of the final tree,
- with the root node counted as depth 0</DD>
+ with the root node counted as depth 0.</DD>
<DT>min_split (optional)</DT>
<DD>INTEGER, default: 20. Minimum number of observations that must exist
in a node for a split to be attempted. The best value for this parameter
- depends on the number of tuples in the dataset</DD>
+ depends on the number of tuples in the dataset.</DD>
<DT>min_bucket (optional)</DT>
<DD>INTEGER, default: min_split/3. Minimum number of observations in any terminal
node. If only one of min_bucket or min_split is specified, min_split is
- set to min_bucket*3 or min_bucket to min_split/3, as appropriate</DD>
+ set to min_bucket*3 or min_bucket to min_split/3, as appropriate.</DD>
<DT>num_splits (optional)</DT>
<DD>INTEGER, default: 100. Continuous-valued features are binned into
discrete quantiles to compute split boundaries. This global parameter
is used to compute the resolution of splits for continuous features.
Higher number of bins will lead to better prediction,
- but will also result in higher processing time</DD>
+ but will also result in longer processing.</DD>
<DT>pruning_params (optional)</DT>
<DD>TEXT. Comma-separated string of key-value pairs giving
@@ -296,7 +295,7 @@ tree_train(
<tr>
<th>n_folds</th>
<td>
- Default: 0 (i.e. No cross-validation).
+ Default: 0 (i.e. no cross-validation).
Number of cross-validation folds to use to compute the best value of
<em>cp</em>. To perform cross-validation, a positive value of
<em>n_folds</em> (greater than 2) should be given. An additional output
@@ -306,9 +305,9 @@ tree_train(
cross-validation error (we pick the maximum <em>cp</em> if multiple
values have same error).
- The list of <em>cp</em> values are automatically computed by parsing
+ The list of <em>cp</em> values is automatically computed by parsing
through the tree initially trained on the complete dataset. The tree
- outputted is a subset of this initial tree corresponding to the best
+ output is a subset of this initial tree corresponding to the best
computed <em>cp</em>.
</td>
@@ -321,33 +320,33 @@ tree_train(
of surrogate splits for each node. A surrogate variable is another predictor
variable that is associated (correlated) with the primary predictor variable
for a split. The surrogate variable comes into use when the primary predictior
- value is NULL. This parameter currently accepts the below argument:
+ value is NULL. This parameter currently accepts one argument:
<table class='output'>
<tr>
<th>max_surrogates</th>
- <td>Default: 0. Number of surrogates to store for each node</td>
+ <td>Default: 0. Number of surrogates to store for each node.</td>
</tr>
</table>
</DD>
<DT>verbosity (optional)</DT>
- <DD>BOOLEAN, default: FALSE. Provides verbose output of the results of training</DD>
+ <DD>BOOLEAN, default: FALSE. Provides verbose output of the training result.</DD>
</DL>
@note
- Many of the parameters are designed to be similar to the popular R package 'rpart'.
-An important distinction between rpart and the above MADlib function is that
+An important distinction between rpart and the MADlib function is that
for both response and feature variables, MADlib considers integer values as
categorical values, while rpart considers them as continuous.
-- When using no surrogates (<em>max_surrogates</em>=0), all rows containing NULL value
+- When using no surrogates (<em>max_surrogates</em>=0), all rows containing NULL values
for any of the features used for training will be ignored from training and prediction.
-- When cross-validation is not used (<em>n_folds</em>=0), each tree outputed
-is pruned by inputed cost-complextity (<em>cp</em>). With cross-validation,
-inputed <em>cp</em> is the minimum value of all the explored values of 'cp'.
+- When cross-validation is not used (<em>n_folds</em>=0), each tree output
+is pruned by the input cost-complextity (<em>cp</em>). With cross-validation,
+the input <em>cp</em> is the minimum value of all the explored values of 'cp'.
During cross-validation, we train an initial tree using the
-provided <em>cp</em> and explore all possible sub-trees (upto a single-node tree)
+provided <em>cp</em> and explore all possible sub-trees (up to a single-node tree)
to compute the optimal sub-tree. The optimal sub-tree and the 'cp' corresponding
-to this optimal sub-tree is placed in the <em>output_table</em>, with their
+to this optimal sub-tree is placed in the <em>output_table</em>, with the
columns named as <em>tree</em> and <em>pruning_cp</em> respectively.
- The main parameters that affect memory usage are: depth of tree, number
of features, and number of values per feature. If you are hitting VMEM limits,
@@ -355,7 +354,7 @@ consider reducing one or more of these parameters.
@anchor predict
@par Prediction Function
-The prediction function is provided to estimate the conditional mean given a new
+The prediction function estimates the conditional mean given a new
predictor. It has the following syntax:
<pre class="syntax">
tree_predict(tree_model,
@@ -369,16 +368,16 @@ tree_predict(tree_model,
<DL class="arglist">
<DT>tree_model</DT>
<DD>TEXT. Name of the table containing the decision tree model. This should
- be the output table returned from <em>tree_train</em></DD>
+ be the output table returned from <em>tree_train.</em></DD>
<DT>new_data_table</DT>
<DD>TEXT. Name of the table containing prediction data. This table is
expected to contain the same features that were used during training. The table
- should also contain <em>id_col_name</em> used for identifying each row</DD>
+ should also contain <em>id_col_name</em> used for identifying each row.</DD>
<DT>output_table</DT>
- <DD>TEXT. Name of the table to output prediction results to. If this table
- already exists then an error is returned.
+ <DD>TEXT. Name of the table to output prediction results. If this table
+ already exists, an error is returned.
The table contains the <em>id_col_name</em> column giving
the 'id' for each prediction and the prediction columns for the dependent variable.
@@ -389,7 +388,7 @@ tree_predict(tree_model,
If <em>type</em> = 'prob', then the table has multiple additional columns, one for each
possible value of the response variable. The columns are labeled as
'estimated_prob_<em>dep_value</em>', where <em>dep_value</em> represents each value
- of the response</DD>
+ of the response variable.</DD>
<DT>type</DT>
<DD>TEXT, optional, default: 'response'. For regression trees, the output is
@@ -397,26 +396,26 @@ tree_predict(tree_model,
trees, the <em>type</em> variable can be 'response', giving the
classification prediction as output, or 'prob', giving the class
probabilities as output. For each value of the dependent variable, a
- column with the probabilities is added to the output table
+ column with the probabilities is added to the output table.
</DD>
</DL>
@note If the <em>new_data_table</em> contains categories of categorical variables
-not seen in the training data then the prediction for that row will be NULL.
+not seen in the training data, the prediction for that row will be NULL.
@anchor display
@par Display Function
-The display function is provided to output a graph representation of the
+The display function outputs a graph representation of the
decision tree. The output can either be in the popular 'dot' format that can
be visualized using various programs including those in the GraphViz package, or
-in a simple text format. The details of the text format is outputted with the
+in a simple text format. The details of the text format are output with the
tree.
<pre class="syntax">
tree_display(tree_model, dot_format, verbosity)
</pre>
An additional display function is provided to output the surrogate splits chosen
-for each internal node.
+for each internal node:
<pre class="syntax">
tree_surr_display(tree_model)
</pre>
@@ -426,7 +425,7 @@ nodes are sorted in ascending order by id. This is equivalent to viewing the
tree in a breadth-first manner. For each surrogate, we output the surrogate
split (variable and threshold) and also give the number of rows that were common
between the primary split and the surrogate split. Finally, the number of rows
-present in the majority branch of the primary split is also presented. Only
+present in the majority branch of the primary split is also shown. Only
surrogates that perform better than this majority branch are included in the
surrogate list. When the primary variable has a NULL value the surrogate variables
are used in order to compute the split for that node. If all surrogates variables
@@ -434,23 +433,25 @@ are NULL, then the majority branch is used to compute the split for a tuple.
\b Arguments
<DL class="arglist">
- <DT>tree_model_name</DT>
- <DD>TEXT. Name of the table containing the decision tree model</DD>
+ <DT>tree_model</DT>
+ <DD>TEXT. Name of the table containing the decision tree model.</DD>
<DT>dot_format</DT>
<DD>BOOLEAN, default = TRUE. Output can either be in a dot format or a text
- format. If TRUE, the result is in the dot format, else output is in text format</DD>
+ format. If TRUE, the result is in the dot format, else output is in text format.</DD>
<DT>verbosity</DT>
- <DD>BOOLEAN, default = FALSE. If true, the dot format output will contain
- additional information (impurity, sample size, etc.)</DD>
+ <DD>BOOLEAN, default = FALSE. If set to TRUE, the dot format output will contain
+ additional information (impurity, sample size, number of weighted rows
+ for each response variable, classification or prediction if the tree
+ was pruned at this level)</DD>
</DL>
The output is always returned as a 'TEXT'. For the dot format, the output can be
redirected to a file on the client side and then rendered using visualization
programs.
-If the user wants to export the dot format result to an external file,
-he can use the following method (Note: the user needs to use unaligned
-table output mode for psql with '-A' flag. And inside psql client,
+To export the dot format result to an external file,
+use the method below. Use unaligned
+table output mode for psql with '-A' flag. And inside the psql client,
both '\\t' and '\\o' should be used):
<pre class="example">
@@ -464,19 +465,22 @@ both '\\t' and '\\o' should be used):
\# \\t
</pre>
-After the desired dot file has been generated, one can then use third-party
-plotting software to plot the trees in a nice figure:
+After the dot file has been generated, use third-party
+plotting software to plot the trees in a nice format:
<pre class="example">
\> \# under bash, convert the dot file into a PDF file
\> dot -Tpdf test.dot \> test.pdf
\> xpdf test.pdf\&
</pre>
+Please see the examples below for more details on the contents
+of the tree output formats.
+
@anchor examples
@examp
-*Decision tree classification example*
+<h4>Decision Tree Classification Example</h4>
--# Prepare input data.
+-# Prepare input data:
<pre class="example">
DROP TABLE IF EXISTS dt_golf;
CREATE TABLE dt_golf (
@@ -486,7 +490,7 @@ CREATE TABLE dt_golf (
humidity double precision,
windy text,
class text
-) ;
+);
</pre>
<pre class="example">
COPY dt_golf (id,"OUTLOOK",temperature,humidity,windy,class) FROM stdin WITH DELIMITER '|';
@@ -507,8 +511,9 @@ COPY dt_golf (id,"OUTLOOK",temperature,humidity,windy,class) FROM stdin WITH DEL
\\.
</pre>
--# Run Decision tree train function.
+-# Run the decision tree training function:
<pre class="example">
+DROP TABLE IF EXISTS train_output, train_output_summary;
SELECT madlib.tree_train('dt_golf', -- source table
'train_output', -- output model table
'id', -- id column
@@ -525,17 +530,19 @@ SELECT madlib.tree_train('dt_golf', -- source table
);
</pre>
--# Predict output categories for the same data as was used for input.
+-# Predict output categories for the same data that was used for input:
<pre class="example">
-SELECT madlib.tree_predict('train_output',
- 'dt_golf',
- 'prediction_results',
- 'response');
-SELECT * FROM prediction_results;
+DROP TABLE IF EXISTS prediction_results;
+SELECT madlib.tree_predict('train_output', -- tree model
+ 'dt_golf', -- new data table
+ 'prediction_results', -- output table
+ 'response'); -- show prediction
+SELECT * FROM prediction_results ORDER BY id;
</pre>
Result:
<pre class="result">
- ----+-----------------
+ id | estimated_class
+----+-----------------
1 | 'Don't Play'
2 | 'Don't Play'
3 | 'Play'
@@ -553,7 +560,51 @@ Result:
(14 rows)
</pre>
--# Obtain a dot format display of the tree
+-# Create a text display of the tree:
+<pre class="example">
+SELECT madlib.tree_display('train_output', FALSE);
+</pre>
+Result:
+<pre class="result">
+ -------------------------------------
+ - Each node represented by 'id' inside ().
+ - Leaf nodes have a * while internal nodes have the split condition at the end.
+ - For each internal node (i), it's children will be at (2i+1) and (2i+2).
+ - For each split the first indented child (2i+1) is the 'True' node and
+second indented child (2i+2) is the 'False' node.
+ - Number of (weighted) rows for each response variable inside [].
+ - Order of values = ['"Don\'t Play"', '"Play"']
+ -------------------------------------
+(0)[ 5 9] "OUTLOOK"<={overcast}
+ (1)[ 0 4] *
+ (2)[ 5 5] temperature<=75
+ (5)[ 3 5] temperature<=65
+ (11)[ 1 0] *
+ (12)[ 2 5] temperature<=70
+ (25)[ 0 3] *
+ (26)[ 2 2] temperature<=72
+ (53)[ 2 0] *
+ (54)[ 0 2] *
+ (6)[ 2 0] *
+ -------------------------------------
+</pre>
+Here are some more details on how to interpret the tree display above...
+Node numbering starts at 0 for the root node and would be
+contiguous 1,2...n if the tree was completely full (no pruning).
+Since the tree has been pruned, the node numbering is not
+contiguous.
+The order of values [x y] indicates the number of weighted
+rows that correspond to ["Don't play" "Play"] <em>before</em> the node test.
+For example, at the root node 0, there are 5 rows that "Don't play"
+and 9 rows that "Play" in the raw data.
+If we apply the test
+of "OUTLOOK" being overcast, then the True result is
+leaf node 1 which is "Play". There are 0 "Don't play" rows
+and 4 "Play" rows that correspond to this case (overcast).
+The remaining 5 "Don't play" rows and 5 "Play rows" are then
+tested at node 2 on temperature<=75. And so on down the tree.
+
+-# Create a dot format display of the tree:
<pre class="example">
SELECT madlib.tree_display('train_output', TRUE);
</pre>
@@ -587,7 +638,7 @@ digraph "Classification tree for dt_golf" {
} //---end of digraph---------
</pre>
--# Obtain a dot format display of the tree with additional info
+-# Now create a dot format display of the tree with additional information:
<pre class="example">
SELECT madlib.tree_display('train_output', TRUE, TRUE);
</pre>
@@ -620,41 +671,13 @@ digraph "Classification tree for dt_golf" {
} //--- end of subgraph------------
} //---end of digraph---------
</pre>
+The additional information in each node is: impurity, sample size, number of weighted rows for each response variable, and classification if the tree was pruned at this level.
--# Obtain a text display of the tree
-<pre class="example">
-SELECT madlib.tree_display('train_output', FALSE);
-</pre>
-Result:
-<pre class="result">
- -------------------------------------
- - Each node represented by 'id' inside ().
- - Leaf nodes have a * while internal nodes have the split condition at the end.
- - For each internal node (i), it's children will be at (2i+1) and (2i+2).
- - For each split the first indented child (2i+1) is the 'True' node and
-second indented child (2i+2) is the 'False' node.
- - Number of (weighted) rows for each response variable inside [].
- - Order of values = ['"Don\'t Play"', '"Play"']
- -------------------------------------
-(0)[ 5 9] "OUTLOOK"<={overcast}
- (1)[ 0 4] *
- (2)[ 5 5] temperature<=75
- (5)[ 3 5] temperature<=65
- (11)[ 1 0] *
- (12)[ 2 5] temperature<=70
- (25)[ 0 3] *
- (26)[ 2 2] temperature<=72
- (53)[ 2 0] *
- (54)[ 0 2] *
- (6)[ 2 0] *
- -------------------------------------
-</pre>
-
-
-*Decision tree regression example*
+<h4>Decision Tree Regression Example</h4>
-# Prepare input data.
<pre class="example">
+DROP TABLE IF EXISTS mt_cars;
CREATE TABLE mt_cars (
id integer NOT NULL,
mpg double precision,
@@ -668,10 +691,10 @@ CREATE TABLE mt_cars (
am integer,
gear integer,
carb integer
-) ;
+);
</pre>
<pre class="example">
-COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH DELIMITER '|' NULL '\\null';
+COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH DELIMITER '|' NULL 'null';
1|18.7|8|360|175|3.15|3.44|17.02|0|0|3|2
2|21|6|160|110|3.9|2.62|16.46|0|1|4|4
3|24.4|4|146.7|62|3.69|3.19|20|1|0|4|2
@@ -680,7 +703,7 @@ COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH D
6|16.4|8|275.8|180|3.078|4.07|17.4|0|0|3|3
7|22.8|4|108|93|3.85|2.32|18.61|1|1|4|1
8|17.3|8|275.8|180|3.078|3.73|17.6|0|0|3|3
-9|21.4|\\null|258|110|3.08|3.215|19.44|1|0|3|1
+9|21.4|null|258|110|3.08|3.215|19.44|1|0|3|1
10|15.2|8|275.8|180|3.078|3.78|18|0|0|3|3
11|18.1|6|225|105|2.768|3.46|20.22|1|0|3|1
12|32.4|4|78.7|66|4.08|2.20|19.47|1|1|4|1
@@ -689,7 +712,7 @@ COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH D
15|30.4|4|75.7|52|4.93|1.615|18.52|1|1|4|2
16|19.2|6|167.6|123|3.92|3.44|18.3|1|0|4|4
17|33.9|4|71.14|65|4.22|1.835|19.9|1|1|4|1
-18|15.2|\\null|304|150|3.15|3.435|17.3|0|0|3|2
+18|15.2|null|304|150|3.15|3.435|17.3|0|0|3|2
19|10.4|8|472|205|2.93|5.25|17.98|0|0|3|4
20|27.3|4|79|66|4.08|1.935|18.9|1|1|4|1
21|10.4|8|460|215|3|5.424|17.82|0|0|3|4
@@ -707,28 +730,28 @@ COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH D
\\.
</pre>
--# Run Decision Tree train function.
+-# Run the decision tree training function:
<pre class="example">
DROP TABLE IF EXISTS train_output, train_output_summary;
-SELECT madlib.tree_train('mt_cars',
- 'train_output',
- 'id',
- 'mpg',
- '*',
+SELECT madlib.tree_train('mt_cars', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'mpg', -- dependent variable
+ '*', -- features
'id, hp, drat, am, gear, carb', -- exclude columns
- 'mse',
- NULL::text,
- NULL::text,
- 10,
- 8,
- 3,
- 10,
- NULL,
- 'max_surrogates=2'
+ 'mse', -- split criterion
+ NULL::text, -- no grouping
+ NULL::text, -- no weights
+ 10, -- max depth
+ 8, -- min split
+ 3, -- number of bins per continuous variable
+ 10, -- number of splits
+ NULL, -- pruning parameters
+ 'max_surrogates=2' -- number of surrogates
);
</pre>
--# Display the decision tree in basic text format.
+-# Display the decision tree in basic text format:
<pre class="example">
SELECT madlib.tree_display('train_output', FALSE);
</pre>
@@ -757,7 +780,7 @@ Result:
(1 row)
</pre>
--# Display the surrogates in the decision tree.
+-# Display the surrogates in the decision tree:
<pre class="example">
SELECT madlib.tree_surr_display('train_output');
</pre>
@@ -789,7 +812,6 @@ Result:
-------------------------------------
(1 row)
</pre>
-
@note The 'cyl' parameter above has two tuples with null values.
In the prediction example below, the surrogate splits for the
<em>cyl in {8, 6}</em> split are used to predict those
@@ -799,14 +821,14 @@ the two tuples have non-NULL values for <em>disp</em>, hence the <em>disp > 146.
split is used to make the prediction. If all the surrogate variables had been
NULL then the majority branch would have been followed.
--# Predict regression output for the same data and compare with original.
+-# Predict regression output for the same data and compare with original:
<pre class="example">
DROP TABLE IF EXISTS prediction_results;
SELECT madlib.tree_predict('train_output',
'mt_cars',
'prediction_results',
'response');
-SELECT s.id, mpg, estimated_mpg FROM prediction_results p, mt_cars s where s.id = p.id;
+SELECT s.id, mpg, estimated_mpg FROM prediction_results p, mt_cars s where s.id = p.id ORDER BY id;
</pre>
Result:
<pre class="result">
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
index 0cc6534..3d4da87 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
@@ -24,17 +24,15 @@ m4_include(`SQLCommon.m4')
</ul></div>
@brief
-Random forests are an ensemble learning method for classification (and
-regression) that operate by constructing a multitude of decision trees at
-training time and outputting the class that is the mode of the classes output by
+Random forests are an ensemble learning method for classification and
+regression that construct a multitude of decision trees at
+training time, then produce the class that is the mode of the classes of the
individual trees.
Random forests build an ensemble of classifiers, each of which is a tree model
constructed using bootstrapped samples from the input data. The results of these
-models are then combined to yield a single prediction, which, although at the
+models are then combined to yield a single prediction, which, at the
expense of some loss in interpretation, have been found to be highly accurate.
-Such methods of using multiple Random Forests to make predictions are called
-random forest methods.
@anchor train
@par Training Function
@@ -64,12 +62,12 @@ forest_train(training_table_name,
\b Arguments
<dl class="arglist">
<dt>training_table_name</dt>
- <dd>text. the name of the table containing the training data.</dd>
+ <dd>text. Name of the table containing the training data.</dd>
<dt>output_table_name</dt>
- <dd>text. the name of the generated table containing the model.</dd>
+ <dd>text. Name of the generated table containing the model.</dd>
- The model table produced by the train function contains the following columns:
+ The model table produced by the training function contains the following columns:
<table class="output">
<tr>
@@ -78,16 +76,16 @@ forest_train(training_table_name,
</tr>
<tr>
<th>sample_id</th>
- <td>integer. id of the bootstrap sample that this tree is a part of.</td>
+ <td>integer. The id of the bootstrap sample that this tree is a part of.</td>
</tr>
<tr>
<th>tree</th>
- <td>bytea8. trained tree model stored in binary format.</td>
+ <td>bytea8. Trained tree model stored in binary format.</td>
</tr>
</table>
A summary table named <em>\<model_table\>_summary</em> is also created at
- the same time, which has the following columns:
+ the same time, which contains the following columns:
<table class="output">
<tr>
<th>method</th>
@@ -101,12 +99,12 @@ forest_train(training_table_name,
<tr>
<th>source_table</th>
- <td>text. The data source table name.</td>
+ <td>text. Data source table name.</td>
</tr>
<tr>
<th>model_table</th>
- <td>text. The model table name.</td>
+ <td>text. Model table name.</td>
</tr>
<tr>
@@ -116,22 +114,22 @@ forest_train(training_table_name,
<tr>
<th>dependent_varname</th>
- <td>text. The dependent variable.</td>
+ <td>text. Dependent variable.</td>
</tr>
<tr>
<th>independent_varname</th>
- <td>text. The independent variables</td>
+ <td>text. Independent variables</td>
</tr>
<tr>
<th>cat_features</th>
- <td>text. categorical feature names.</td>
+ <td>text. Categorical feature names.</td>
</tr>
<tr>
<th>con_features</th>
- <td>text. continuous feature names.</td>
+ <td>text. Continuous feature names.</td>
</tr>
<tr>
@@ -161,27 +159,27 @@ forest_train(training_table_name,
<tr>
<th>min_bucket</th>
- <td>int. minimum number of observations in any terminal node.</td>
+ <td>int. Minimum number of observations in any terminal node.</td>
</tr>
<tr>
<th>num_splits</th>
- <td>int. number of buckets for continuous variables.</td>
+ <td>int. Number of buckets for continuous variables.</td>
</tr>
<tr>
<th>verbose</th>
- <td>boolean. whether or not to display debug info.</td>
+ <td>boolean. Whether or not to display debug info.</td>
</tr>
<tr>
<th>importance</th>
- <td>boolean. whether or not to calculate variable importance.</td>
+ <td>boolean. Whether or not to calculate variable importance.</td>
</tr>
<tr>
<th>num_permutations</th>
- <td>int. number of times feature values are permuted while calculating
+ <td>int. Number of times feature values are permuted while calculating
variable importance. The default value is 1.</td>
</tr>
@@ -221,7 +219,7 @@ forest_train(training_table_name,
<tr>
<th>gid</th>
- <td>integer. group id that uniquely identifies a set of grouping column values.</td>
+ <td>integer. Group id that uniquely identifies a set of grouping column values.</td>
</tr>
<tr>
@@ -303,13 +301,16 @@ forest_train(training_table_name,
features to randomly select at each split.</DD>
<DT>importance (optional)</DT>
- <DD>boolean, default: true. Whether or not to calculate variable importance.</DD>
+ <DD>boolean, default: true. Whether or not to calculate variable importance.
+ If set to true, variable importance for categorical and continuous features
+ will be output in the group table <em>\<model_table\>_group</em> described
+ above. Will increase run time when variable importance is turned on. </DD>
<DT>num_permutations (optional)</DT>
<DD>integer, default: 1. Number of times to permute each feature value while
calculating variable importance.
- Variable importance for a feature is computed by permuting the variable with
+ @note Variable importance for a feature is computed by permuting the variable with
random values and computing the drop in predictive accuracy (using OOB samples).
Setting this greater than 1 performs an average over multiple importance
calculation. This increases the total run time and in most cases
@@ -356,13 +357,12 @@ forest_train(training_table_name,
table is expected to be used for training each tree in the forest. A ratio that
is close to 0 may result in trees with only the root node.
This allows users to experiment with the function in a speedy fashion.</DD>
-
+</DL>
@note The main parameters that affect memory usage are: depth of tree, number
- of features, and number of values per feature. If you are hitting VMEM limits,
+ of features, and number of values per feature (controlled by num_splits).
+ If you are hitting VMEM limits,
consider reducing one or more of these parameters.
-</DL>
-
@anchor predict
@par Prediction Function
The prediction function is provided to estimate the conditional mean given a new
@@ -397,8 +397,8 @@ forest_predict(random_forest_model,
@anchor get_tree
@par Display Function
-The get_tree function is provided to output a graph representation of a
-single tree of the Random Forest. The output can either be in the popular
+The 'get_tree' function is provided to output a graph representation of a
+single tree of the random forest. The output can either be in the popular
'dot' format that can be visualized using various programs including those
in the GraphViz package, or in a simple text format. The details of the
text format is outputted with the tree.
@@ -446,7 +446,9 @@ are NULL, then the majority branch is used to compute the split for a tuple.
<DT>verbose (optional)</DT>
<DD>boolean, default = FALSE. If true, the dot format output will contain
- additional information (impurity, sample size, etc.)</DD>
+ additional information (impurity, sample size, number of weighted rows for
+ each response variable, classification or prediction if the tree was
+ pruned at this level)</DD>
</DL>
The output is always returned as a 'TEXT'. For the dot format, the output can be
@@ -459,7 +461,7 @@ programs.
<b>Random Forest Classification Example</b>
--# Prepare input data.
+-# Prepare input data:
<pre class="example">
DROP TABLE IF EXISTS dt_golf;
CREATE TABLE dt_golf (
@@ -469,7 +471,7 @@ CREATE TABLE dt_golf (
humidity double precision,
windy text,
class text
-) ;
+);
</pre>
<pre class="example">
INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,windy,class) VALUES
@@ -489,7 +491,7 @@ INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,windy,class) VALUES
(14, 'rain', 71, 80, 'true', 'Don''t Play');
</pre>
--# Run Random Forest train function.
+-# Run the random forest training function and view summary output:
<pre class="example">
DROP TABLE IF EXISTS train_output, train_output_group, train_output_summary;
SELECT madlib.forest_train('dt_golf', -- source table
@@ -510,35 +512,79 @@ SELECT madlib.forest_train('dt_golf', -- source table
);
\\x on
SELECT * FROM train_output_summary;
+</pre>
+Result:
+<pre class="result">
+-[ RECORD 1 ]---------+-----------------------------------------------
+method | forest_train
+is_classification | t
+source_table | dt_golf
+model_table | train_output
+id_col_name | id
+dependent_varname | class
+independent_varnames | "OUTLOOK",windy,temperature,humidity
+cat_features | "OUTLOOK",windy
+con_features | temperature,humidity
+grouping_cols |
+num_trees | 20
+num_random_features | 2
+max_tree_depth | 8
+min_split | 3
+min_bucket | 1
+num_splits | 10
+verbose | f
+importance | t
+num_permutations | 1
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 14
+total_rows_skipped | 0
+dependent_var_levels | "Don't Play","Play"
+dependent_var_type | text
+independent_var_types | text, text, double precision, double precision
+</pre>
+View the group table output:
+<pre class="example">
SELECT * FROM train_output_group;
-\\x off
+</pre>
+Result:
+<pre class="result">
+-[ RECORD 1 ]------+----------------------------------------
+gid | 1
+success | t
+cat_n_levels | {3,2}
+cat_levels_in_text | {overcast,rain,sunny,false,true}
+oob_error | 0.50000000000000000000
+cat_var_importance | {-0.206309523809524,-0.234345238095238}
+con_var_importance | {-0.308690476190476,-0.272678571428571}
</pre>
-# Obtain a dot format display of a single tree
-within the forest.
+within the forest:
<pre class="example">
+\\x off
SELECT madlib.get_tree('train_output',1,2);
</pre>
Result:
<pre class="result">
-digraph "Classification tree for dt_golf" {
-"0" [label="temperature<=70", shape=ellipse];
-"0" -> "1"[label="yes"];
-"1" [label="\"'Play'\"",shape=box];
-"0" -> "2"[label="no"];
-"2" [label="\"OUTLOOK\"<={overcast}", shape=ellipse];
-"2" -> "5"[label="yes"];
-"5" [label="\"'Play'\"",shape=box];
-"2" -> "6"[label="no"];
-"6" [label="humidity<=70", shape=ellipse];
-"6" -> "13"[label="yes"];
-"13" [label="\"'Play'\"",shape=box];
-"6" -> "14"[label="no"];
-"14" [label="\"'Don''t Play'\"",shape=box];
-} //---end of digraph---------
+ digraph "Classification tree for dt_golf" {
+ "0" [label="humidity <= 75", shape=ellipse];
+ "0" -> "1"[label="yes"];
+ "1" [label="\"Play\"",shape=box];
+ "0" -> "2"[label="no"];
+ "2" [label="humidity <= 80", shape=ellipse];
+ "2" -> "5"[label="yes"];
+ "5" [label="\"Don't Play\"",shape=box];
+ "2" -> "6"[label="no"];
+ "6" [label="\"OUTLOOK\" in {overcast,rain}", shape=ellipse];
+ "6" -> "13"[label="yes"];
+ "13" [label="\"Play\"",shape=box];
+ "6" -> "14"[label="no"];
+ "14" [label="\"Don't Play\"",shape=box];
+ } //---end of digraph---------
</pre>
--# Obtain a text display of the tree
+-# Obtain a text display of the tree:
<pre class="example">
SELECT madlib.get_tree('train_output',1,2,FALSE);
</pre>
@@ -553,17 +599,17 @@ second indented child (2i+2) is the 'False' node.
- Number of (weighted) rows for each response variable inside [].
- Order of values = ['"Don\'t Play"', '"Play"']
-------------------------------------
-(0)[ 3 11] temperature<=70
- (1)[ 0 7] * --> "'Play'"
- (2)[ 3 4] "OUTLOOK"<={overcast}
- (5)[ 0 3] * --> "'Play'"
- (6)[ 3 1] humidity<=70
- (13)[ 0 1] * --> "'Play'"
- (14)[ 3 0] * --> "'Don''t Play'"
+ (0)[ 4 10] humidity <= 75
+ (1)[0 7] * --> "Play"
+ (2)[4 3] humidity <= 80
+ (5)[3 1] * --> "Don't Play"
+ (6)[1 2] "OUTLOOK" in {overcast,rain}
+ (13)[0 2] * --> "Play"
+ (14)[1 0] * --> "Don't Play"
-------------------------------------
</pre>
--# Predict output categories for the same data as was used for input.
+-# Predict output categories for the same data as was used for input:
<pre class="example">
DROP TABLE IF EXISTS prediction_results;
SELECT madlib.forest_predict('train_output',
@@ -577,7 +623,7 @@ ORDER BY id;
</pre>
Result:
<pre class="result">
- id | estimated_class | class
+ id | estimated_class | class
----+-----------------+------------
1 | Don't Play | Don't Play
2 | Don't Play | Don't Play
@@ -596,7 +642,7 @@ Result:
(14 rows)
</pre>
--# Predict probablities of output categories for the same data.
+-# Predict probablities of output categories for the same data:
<pre class="example">
DROP TABLE IF EXISTS prediction_prob;
SELECT madlib.forest_predict('train_output',
@@ -610,29 +656,29 @@ ORDER BY id;
</pre>
Result:
<pre class="result">
- id | estimated_prob_Play | class
+ id | estimated_prob_Play | class
----+---------------------+------------
- 1 | 0.15 | Don't Play
- 2 | 0.1 | Don't Play
+ 1 | 0.05 | Don't Play
+ 2 | 0.15 | Don't Play
3 | 0.95 | Play
- 4 | 0.7 | Play
- 5 | 0.85 | Play
- 6 | 0.25 | Don't Play
- 7 | 0.75 | Play
+ 4 | 0.65 | Play
+ 5 | 0.75 | Play
+ 6 | 0.4 | Don't Play
+ 7 | 0.7 | Play
8 | 0.1 | Don't Play
- 9 | 0.85 | Play
- 10 | 0.7 | Play
- 11 | 0.35 | Play
- 12 | 0.75 | Play
- 13 | 0.95 | Play
- 14 | 0.15 | Don't Play
+ 9 | 0.9 | Play
+ 10 | 0.85 | Play
+ 11 | 0.8 | Play
+ 12 | 0.7 | Play
+ 13 | 1 | Play
+ 14 | 0.4 | Don't Play
(14 rows)
</pre>
<b>Random Forest Regression Example</b>
--# Prepare input data.
+-# Prepare input data:
<pre class="example">
DROP TABLE IF EXISTS mt_cars;
CREATE TABLE mt_cars (
@@ -648,7 +694,7 @@ CREATE TABLE mt_cars (
am integer,
gear integer,
carb integer
-) ;
+);
</pre>
<pre class="example">
INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
@@ -686,7 +732,7 @@ INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
(32,21.4,4,121,109,4.11,2.78,18.6,1,1,4,2);
</pre>
--# Run Random Forest train function.
+-# Run the random forest training function:
<pre class="example">
DROP TABLE IF EXISTS mt_cars_output, mt_cars_output_group, mt_cars_output_summary;
SELECT madlib.forest_train('mt_cars',
@@ -711,7 +757,7 @@ SELECT * FROM mt_cars_output_group;
\\x off
</pre>
--# Display a single tree of the Random Forest in dot format.
+-# Display a single tree of the random forest in dot format:
<pre class="example">
SELECT madlib.get_tree('mt_cars_output',1,1);
</pre>
@@ -722,7 +768,7 @@ digraph "Regression tree for mt_cars" {
} //---end of digraph---------
</pre>
--# Predict regression output for the same data and compare with original.
+-# Predict regression output for the same data and compare with original:
<pre class="example">
DROP TABLE IF EXISTS prediction_results;
SELECT madlib.forest_predict('mt_cars_output',
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/summary/summary.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/summary/summary.sql_in b/src/ports/postgres/modules/summary/summary.sql_in
index 5457463..9f56b4d 100644
--- a/src/ports/postgres/modules/summary/summary.sql_in
+++ b/src/ports/postgres/modules/summary/summary.sql_in
@@ -186,7 +186,7 @@ provide an array of quantile values for the parameter 'ntile_array'.
<dt>how_many_mfv (optional)</dt>
<dd>INTEGER, default: 10. The number of most-frequent-values to compute.</dd>
<dt>get_estimates (optional)</dt>
-<dd>BOOLEAN, default TRUE. If TRUE, estimated values are produced. If FALSE, exact values are calculated.</dd>
+<dd>BOOLEAN, default TRUE. If TRUE, estimated values are produced for distinct values and most frequent values. If FALSE, exact values are calculated (may take longer to run depending on data size).</dd>
</DL>
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/utilities/path.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/path.sql_in b/src/ports/postgres/modules/utilities/path.sql_in
index 9f58536..f97c6a0 100644
--- a/src/ports/postgres/modules/utilities/path.sql_in
+++ b/src/ports/postgres/modules/utilities/path.sql_in
@@ -185,8 +185,7 @@ wine selection page, and checkout. Other pages on the site like help
pages show up in the logs as well. Let\u2019s assume that the log has been
sessionized.
-Create the date table:
-
+-# Create the date table:
<pre class="example">
DROP TABLE IF EXISTS eventlog;
CREATE TABLE eventlog (event_timestamp TIMESTAMP,
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/29acc538/src/ports/postgres/modules/utilities/pivot.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/pivot.sql_in b/src/ports/postgres/modules/utilities/pivot.sql_in
index 1c36cef..cb2c223 100644
--- a/src/ports/postgres/modules/utilities/pivot.sql_in
+++ b/src/ports/postgres/modules/utilities/pivot.sql_in
@@ -138,6 +138,9 @@ allowed so NULLs are ignored.
- It is not allowed to set the fill_value parameter without setting the
aggregate_func parameter due to possible ambiguity. Set
aggregate_func to NULL for the default behavior and use fill_value as desired.
+Please note that full_value must be of the same type as the output of the
+aggregate_func (or capable of being cast to the same type by PostgreSQL),
+or else an error will result.
- It is not allowed to set the output_col_dictionary parameter without setting
the keep_null parameter due to possible ambiguity. Set
keep_null to NULL for the default behavior and use output_col_dictionary as