You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2018/03/27 22:46:13 UTC
[1/5] madlib git commit: DT: Ensure proper quoting in grouping
coalesce
Repository: madlib
Updated Branches:
refs/heads/master 3810ca4e5 -> 7f3aae92f
DT: Ensure proper quoting in grouping coalesce
JIRA: MADLIB-1217
Grouping column value is coalesced with null_proxy to get the right null
identifier when null_as_category is True. The null_proxy needs to be
quoted appropriately as a string.
Closes #248
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/c1ff8c78
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/c1ff8c78
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/c1ff8c78
Branch: refs/heads/master
Commit: c1ff8c78765dee354045940f51bae4f492e09183
Parents: 3810ca4
Author: Rahul Iyer <ri...@apache.org>
Authored: Thu Mar 22 16:48:53 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Mar 27 15:28:22 2018 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 4 +-
.../test/decision_tree.sql_in | 50 +++++++++++---------
.../test/random_forest.sql_in | 6 +--
3 files changed, 32 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/c1ff8c78/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 133ce1d..f8189e4 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -949,7 +949,7 @@ def _get_bins_grps(
" THEN 'True' ELSE 'False' END)")
else:
if null_proxy is not None:
- all_col_expressions[col] = ("COALESCE({0}, {1})".
+ all_col_expressions[col] = ("COALESCE({0}::TEXT, '{1}')".
format(col, null_proxy))
else:
all_col_expressions[col] = col
@@ -1033,7 +1033,7 @@ def get_feature_str(schema_madlib, boolean_cats,
cat_features_str = "NULL"
if len(con_features) > 0:
- con_features_list = ["coalesce(" + str(con) + ", 'nan'::double precision)"
+ con_features_list = ["COALESCE(" + str(con) + ", 'nan'::double precision)"
for con in con_features]
con_features_str = "ARRAY[" + ", ".join(con_features_list) + "]"
else:
http://git-wip-us.apache.org/repos/asf/madlib/blob/c1ff8c78/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
index 53e8362..ba0c75e 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/decision_tree.sql_in
@@ -236,6 +236,7 @@ FROM (
DROP TABLE IF EXISTS dt_golf CASCADE;
CREATE TABLE dt_golf (
id integer NOT NULL,
+ id_2 integer,
"OUTLOOK" text,
temperature double precision,
humidity double precision,
@@ -263,8 +264,11 @@ INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,"Cont_features",cat_featu
(16, 'overcast', NULL, 75, ARRAY[81, 75], ARRAY['a', 'd'], false, 'Play'),
(14, 'rain', 71, 80, ARRAY[71, 80], ARRAY['c', 'b'], true, 'Don''t Play');
--- no grouping
-DROP TABLE IF EXISTS train_output, train_output_summary;
+update dt_golf set id_2 = id % 2;
+-------------------------------------------------------------------------
+
+-- no grouping, with cross_validation
+DROP TABLE IF EXISTS train_output, train_output_summary, train_output_cv;
SELECT tree_train('dt_golf'::text, -- source table
'train_output'::text, -- output model table
'id'::text, -- id column
@@ -277,12 +281,13 @@ SELECT tree_train('dt_golf'::text, -- source table
10::integer, -- max depth
6::integer, -- min split
2::integer, -- min bucket
- 8::integer, -- number of bins per continuous variable
- 'cp=0.01' -- cost-complexity pruning parameter
+ 3::integer, -- number of bins per continuous variable
+ 'cp=0.01, n_folds=2' -- cost-complexity pruning parameter
);
SELECT _print_decision_tree(tree) from train_output;
SELECT tree_display('train_output', False);
+SELECT * FROM train_output_cv;
-------------------------------------------------------------------------
-- grouping
@@ -311,7 +316,7 @@ SELECT tree_display('train_output', False);
CREATE TABLE dt_golf2 as
SELECT * FROM dt_golf
UNION
-SELECT 15 as id, 'humid' as "OUTLOOK", 71 as temperature, 80 as humidity,
+SELECT 15 as id, 1 as id_2, 'humid' as "OUTLOOK", 71 as temperature, 80 as humidity,
ARRAY[90, 90] as "Cont_features", ARRAY['b', 'c'] as cat_features,
true as windy, 'Don''t Play' as class;
\x off
@@ -328,24 +333,24 @@ select * from train_output;
select * from train_output_summary;
-------------------------------------------------------------------------
--- no grouping, cross-validation
-DROP TABLE IF EXISTS train_output, train_output_summary, train_output_cv, predict_output;
+-- grouping and null_as_category=True
+DROP TABLE IF EXISTS train_output, train_output_summary, predict_output;
SELECT tree_train('dt_golf'::text, -- source table
- 'train_output'::text, -- output model table
- 'id'::text, -- id column
- 'temperature::double precision'::text, -- response
- '"OUTLOOK", cat_features, "Cont_features"'::text, -- features
- NULL::text, -- exclude columns
- 'mse'::text, -- split criterion
- NULL::text, -- no grouping
- NULL::text, -- no weights
- NULL::integer, -- max depth
- 6::integer, -- min split
- 2::integer, -- min bucket
- 8::integer, -- number of bins per continuous variable
- 'cp=0.01, n_folds=3',
- 'max_surrogates=2, null_as_category=True'
- );
+ 'train_output'::text, -- output model table
+ 'id'::text, -- id column
+ 'temperature::double precision'::text, -- response
+ '"OUTLOOK", humidity'::text, -- features
+ NULL::text, -- exclude columns
+ 'mse'::text, -- split criterion
+ 'id_2'::text, -- grouping col
+ NULL::text, -- no weights
+ NULL::integer, -- max depth
+ 6::integer, -- min split
+ 2::integer, -- min bucket
+ 3::integer, -- number of bins per continuous variable
+ 'cp=0.01',
+ 'max_surrogates=2, null_as_category=True'
+ );
SELECT _print_decision_tree(tree) from train_output;
SELECT tree_display('train_output', False);
@@ -360,7 +365,6 @@ USING (id);
\x on
select * from train_output;
select * from train_output_summary;
-select * from train_output_cv;
-------------------------------------------------------------------------
drop table if exists group_cp;
http://git-wip-us.apache.org/repos/asf/madlib/blob/c1ff8c78/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
index cabca6a..8fcb75b 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
@@ -134,7 +134,7 @@ SELECT forest_train(
10, -- num of trees
1, -- num of random features
TRUE, -- importance
- 1, -- num_permutations
+ 3, -- num_permutations
10, -- max depth
1, -- min split
1, -- min bucket
@@ -148,7 +148,7 @@ SELECT * from train_output_summary;
SELECT * from train_output_group;
SELECT
assert(cat_var_importance[1] > con_var_importance[1], 'class should be important!'),
- assert(cat_var_importance[1] > con_var_importance[2], 'class should be important!')
+ assert(cat_var_importance[1] > cat_var_importance[2], 'class should be important!')
FROM train_output_group;
-------------------------------------------------------------------------
@@ -165,7 +165,7 @@ SELECT forest_train(
10, -- num of trees
1, -- num of random features
TRUE, -- importance
- 1, -- num_permutations
+ 3, -- num_permutations
10, -- max depth
1, -- min split
1, -- min bucket
[2/5] madlib git commit: DT: Fix the group value sorting in cat levels
Posted by ri...@apache.org.
DT: Fix the group value sorting in cat levels
JIRA: MADLIB-1218
In decision tree, categorical levels in the data is computed and then
sorted into multiple variables, presumably to avoid passing a dictionary
from Python to C++. This sorting was broken for grouping due to an
incorrect ordering of for-loops. This commit fixes that error and adds
multiple comments to elucidate that process.
Closes: #248
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/c35107d0
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/c35107d0
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/c35107d0
Branch: refs/heads/master
Commit: c35107d03a033dac82f19a8af03f4c127e985611
Parents: c1ff8c7
Author: Rahul Iyer <ri...@apache.org>
Authored: Fri Mar 23 00:40:13 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Mar 27 15:29:14 2018 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 35 +++++++++++++++-----
.../recursive_partitioning/decision_tree.sql_in | 4 +--
2 files changed, 29 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/c35107d0/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index f8189e4..6dca0f2 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -970,16 +970,35 @@ def _get_bins_grps(
"one value are dropped from the tree model.")
cat_features = [feature for feature in cat_features if feature in use_cat_features]
- grp_to_col_to_row = dict((grp_key, dict(
- (row['colname'], row['levels']) for row in items))
- for grp_key, items in groupby(all_levels, key=itemgetter('grp_key')))
-
+ # grp_col_to_levels is a list of tuples (pairs) with
+ # first value = group value,
+ # second value = a dict mapping a categorical column to its levels in data
+ # (these levels are specific to the group and can be different
+ # for different groups)
+ # The list of tuples can be converted to a dict, but the ordering
+ # will be lost.
+ # eg. grp_col_to_levels =
+ # [
+ # ('3', {'vs': [0, 1], 'cyl': [4,6,8]}),
+ # ('4', {'vs': [0, 1], 'cyl': [4,6]}),
+ # ('5', {'vs': [0, 1], 'cyl': [4,6,8]})
+ # ]
+ grp_to_col_to_levels = [
+ (grp_key, dict((row['colname'], row['levels']) for row in items))
+ for grp_key, items in groupby(all_levels, key=itemgetter('grp_key'))]
if cat_features:
- cat_items_list = [rows[col] for col in cat_features
- for grp_key, rows in grp_to_col_to_row.items() if col in rows]
+ # Below statements collect the grp_to_col_to_levels into multiple variables
+ # From above eg.
+ # cat_items_list = [[0,1], [4,6,8], [0,1], [4,6], [0,1], [4,6,8]]
+ # cat_n = [2, 3, 2, 2, 2, 3]
+ # cat_origin = [0, 1, 4, 6, 8, 0, 1, 4, 6, 0, 1, 4, 6, 8]
+ # grp_key_cat = ['3', '4', '5']
+ cat_items_list = [rows[col]
+ for grp_key, rows in grp_to_col_to_levels
+ for col in cat_features if col in rows]
cat_n = [len(i) for i in cat_items_list]
- cat_origin = [item for subl in cat_items_list for item in subl]
- grp_key_cat=[grp_key for grp_key in grp_to_col_to_row]
+ cat_origin = [item for sublist in cat_items_list for item in sublist]
+ grp_key_cat=[item[0] for item in grp_to_col_to_levels]
else:
cat_n = []
cat_origin = []
http://git-wip-us.apache.org/repos/asf/madlib/blob/c35107d0/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index e21f3f2..cb6a8ea 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -363,9 +363,9 @@ tree_train(
<tr>
<th>null_proxy</th>
- <td>TEXT. Describes how NULLs are handled. If NULL is not
+ <td>TEXT. Describes how NULLs are handled. If NULL is not
treated as a separate categorical variable, this will be NULL.
- If NULL is treated as a separate categorical value, this will be
+ If NULL is treated as a separate categorical value, this will be
set to "__NULL__"</td>
</tr>
</table>
[4/5] madlib git commit: Doc: Update DT and RF doc
Posted by ri...@apache.org.
http://git-wip-us.apache.org/repos/asf/madlib/blob/7f3aae92/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
index cc228ac..20fa18e 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.sql_in
@@ -18,28 +18,34 @@ m4_include(`SQLCommon.m4')
<div class="toc"><b>Contents</b><ul>
<li class="level1"><a href="#train">Training Function</a></li>
<li class="level1"><a href="#predict">Prediction Function</a></li>
-<li class="level1"><a href="#get_tree">Display Function</a></li>
+<li class="level1"><a href="#get_tree">Tree Display</a></li>
<li class="level1"><a href="#examples">Examples</a></li>
+<li class="level1"><a href="#literature">Literature</a></li>
<li class="level1"><a href="#related">Related Topics</a></li>
</ul></div>
@brief
-Random forests are an ensemble learning method for classification and
+Random forest is an ensemble learning method for classification and
regression that construct a multitude of decision trees at
-training time, then produce the class that is the mode of the classes of the
+training time, then produces the class that is the mean (regression)
+or mode (classification) of the prediction produced by the
individual trees.
-Random forests build an ensemble of classifiers, each of which is a tree model
+Random forest builds an ensemble of classifiers, each of which is a tree model
constructed using bootstrapped samples from the input data. The results of these
models are then combined to yield a single prediction, which, at the
-expense of some loss in interpretation, have been found to be highly accurate.
+expense of some loss in interpretation, can be highly accurate.
+Refer to Breiman and Cutler [1][2] for details on the implementation
+used here.
-Please also refer to the decision tree user documentation for
-information relevant to the implementation of random forests in MADlib.
+Also refer to
+the <a href="group__grp__decision__tree.html">decision tree user documentation</a>
+since many parameters and examples are similar to
+random forest.
@anchor train
@par Training Function
-Random Forest training function has the following format:
+Random forest training function has the following format:
<pre class="syntax">
forest_train(training_table_name,
output_table_name,
@@ -68,22 +74,183 @@ forest_train(training_table_name,
<dd>text. Name of the table containing the training data.</dd>
<dt>output_table_name</dt>
- <dd>text. Name of the generated table containing the model.</dd>
+ <dd>TEXT. Name of the generated table containing the model.
+ If a table with the same name already exists, an
+ error will be returned. A summary table
+ named <em>\<output_table_name\>_summary</em> and a grouping
+ table named <em>\<output_table_name\>_group</em>
+ are also created. These are described later on this page.
+ </DD>
+
+ <DT>id_col_name</DT>
+ <DD>TEXT. Name of the column containing id information
+ in the training data. This is a mandatory argument
+ and is used for prediction and other purposes. The values
+ are expected to be unique for each row.</DD>
+
+ <DT>dependent_variable</DT>
+ <DD>TEXT. Name of the column that contains the output (response) for
+ training. Boolean, integer and text types are considered to be classification
+ outputs, while double precision values are considered to be regression outputs.
+ The response variable for a classification tree can be multinomial, but the
+ time and space complexity of the training function increases linearly as the
+ number of response classes increases.</DD>
+
+ <DT>list_of_features</DT>
+ <DD>TEXT. Comma-separated string of column names or expressions to use as predictors.
+ Can also be a '*' implying all columns are to be used as predictors (except for the
+ ones included in the next argument that lists exclusions).
+ The types of the features can be mixed: boolean, integer, and text columns
+ are considered categorical and
+ double precision columns are considered continuous. Categorical variables
+ are not encoded and used as is in the training.
+
+ Array columns can also be included in the list, where the array is expanded
+ to treat each element of the array as a feature.
+
+ Note that not every combination of the levels of a
+ categorical variable is checked when evaluating a split. The levels of the
+ non-integer categorical variable are ordered by the entropy of the variable in
+ predicting the response. The split at each node is evaluated between these
+ ordered levels. Integer categorical variables, however, are simply ordered
+ by their value.
+ </DD>
+
+ <DT>list_of_features_to_exclude</DT>
+ <DD>TEXT. Comma-separated string of column names to exclude from the predictors
+ list. If the <em>dependent_variable</em> is an expression (including cast of a column name),
+ then this list should include the columns present in the
+ <em>dependent_variable</em> expression,
+ otherwise those columns will be included in the
+ features (resulting in meaningless trees).
+ The names in this parameter should be identical to the names used in the table and
+ quoted appropriately. </DD>
+
+ <DT>grouping_cols (optional)</DT>
+ <DD>TEXT, default: NULL. Comma-separated list of column names to group the
+ data by. This will produce multiple random forests, one for
+ each group.</DD>
+
+ <DT>num_trees (optional)</DT>
+ <DD>INTEGER, default: 100. Maximum number of trees to grow in the random
+ forest model. Actual number of trees grown may be different, depending
+ on the data.</DD>
+
+ <DT>num_random_features (optional)</DT>
+ <DD>INTEGER, default: sqrt(n) for classification, n/3
+ for regression. This is the number of features to randomly
+ select at each split.</DD>
+
+ <DT>importance (optional)</DT>
+ <DD>BOOLEAN, default: true. Whether or not to calculate variable importance.
+ If set to true, variable importance for categorical and continuous features
+ will be output in the group table <em>\<model_table\>_group</em>.
+ Note that total runtime will increase when variable
+ importance is turned on. </DD>
+
+ <DT>num_permutations (optional)</DT>
+ <DD>INTEGER, default: 1. Number of times to permute each feature value while
+ calculating variable importance.
+
+ @note Variable importance for a feature is determined by permuting the variable
+ and computing the drop in predictive accuracy using out-of-bag samples [1].
+ Setting this greater than 1 performs an average over multiple
+ importance calculations, but increases total run time. In most cases,
+ the default value of 1 is sufficient to compute the importance.
+ Due to nature of permutation, the importance value can end up being
+ negative if the number of levels for a categorical variable is small and is
+ unbalanced. In such a scenario, the importance values are shifted to ensure
+ that the lowest importance value is 0.
+
+ </DD>
+
+ <DT>max_tree_depth (optional)</DT>
+ <DD>INTEGER, default: 7. Maximum depth of any node of a tree,
+ with the root node counted as depth 0. A deeper tree can
+ lead to better prediction but will also result in
+ longer processing time and higher memory usage.</DD>
+
+ <DT>min_split (optional)</DT>
+ <DD>INTEGER, default: 20. Minimum number of observations that must exist
+ in a node for a split to be attempted.</DD>
+
+ <DT>min_bucket (optional)</DT>
+ <DD>INTEGER, default: min_split/3. Minimum number of observations in any terminal
+ node. If only one of min_bucket or min_split is specified, min_split is
+ set to min_bucket*3 or min_bucket to min_split/3, as appropriate.</DD>
+
+ <DT>num_splits (optional)</DT>
+ <DD>INTEGER, default: 20. Continuous-valued features are binned into
+ discrete quantiles to compute split boundaries. This global parameter
+ is used to compute the resolution of splits for continuous features.
+ Higher number of bins will lead to better prediction,
+ but will also result in longer processing time and higher memory usage.</DD>
+
+ <DT>null_handling_params (optional)</DT>
+ <DD>TEXT. Comma-separated string of key-value pairs controlling the behavior
+ of various features handling missing values. One of the following can
+ be used if desired (not both):
+ <table class='output'>
+ <tr>
+ <th>max_surrogates</th>
+ <td>Default: 0. Number of surrogates to store for each node.</td>
+ One approach to handling NULLs is to use surrogate splits for each
+ node. A surrogate variable enables you to make better use of
+ the data by using another predictor variable that is associated
+ (correlated) with the primary split variable. The surrogate
+ variable comes into use when the primary predictior value is NULL.
+ Surrogate rules implemented here are based on reference [1].
+ </tr>
+ <tr>
+ <th>null_as_category</th>
+ <td>Default: FALSE. Whether to treat NULL as a valid level
+ for categorical features. FALSE means that NULL is not a
+ valid level, which is probably the most common sitation.
+
+ If set to TRUE, NULL values are considered a categorical value and
+ placed at the end of the ordering of categorical levels. Placing at the
+ end ensures that NULL is never used as a value to split a node on.
+ One reason to make NULL a category is that it allows you to
+ predict on categorical levels that were not in the training
+ data by lumping them into an "other bucket."
+
+ This parameter is ignored for continuous-valued features.
+ </td>
+ </tr>
+ </table>
+ </DD>
+ <DT>verbose (optional)</DT>
+ <DD>BOOLEAN, default: FALSE. Provides verbose output of the results of training.</DD>
+
+ <DT>sample_ratio (optional)</DT>
+ <DD>DOUBLE PRECISION, in the range of (0, 1], default: 1.
+ If 'sample_ratio' is less than 1, a bootstrap sample size smaller than the data
+ table is used for training each tree in the forest. A ratio that
+ is close to 0 may result in trees with only the root node.
+ This sample parameter allows users to quickly experiment with the
+ random forest function since it reduces run time by
+ using only some of the data.</DD>
+</DL>
+
+\b Output
+<dl class="arglist">
+<DD>
The model table produced by the training function contains the following columns:
<table class="output">
<tr>
<th>gid</th>
- <td>integer. group id that uniquely identifies a set of grouping column values.</td>
+ <td>INTEGER. Group id that uniquely identifies a set of grouping column values.</td>
</tr>
<tr>
<th>sample_id</th>
- <td>integer. The id of the bootstrap sample that this tree is a part of.</td>
+ <td>INTEGER. The id of the bootstrap sample that this tree is a part of.</td>
</tr>
<tr>
<th>tree</th>
- <td>bytea8. Trained tree model stored in binary format.</td>
+ <td>BYTEA8. Trained tree model stored in binary
+ format (not human readable).</td>
</tr>
</table>
@@ -97,327 +264,220 @@ forest_train(training_table_name,
<tr>
<th>is_classification</th>
- <td>boolean. True if it is a classification model.</td>
+ <td>BOOLEAN. True if it is a classification model, false
+ if for regression.</td>
</tr>
<tr>
<th>source_table</th>
- <td>text. Data source table name.</td>
+ <td>TEXT. Data source table name.</td>
</tr>
<tr>
<th>model_table</th>
- <td>text. Model table name.</td>
+ <td>TEXT. Model table name.</td>
</tr>
<tr>
<th>id_col_name</th>
- <td>text. The ID column name.</td>
+ <td>TEXT. The ID column name.</td>
</tr>
<tr>
<th>dependent_varname</th>
- <td>text. Dependent variable.</td>
+ <td>TEXT. Dependent variable.</td>
</tr>
<tr>
- <th>independent_varname</th>
- <td>text. Independent variables</td>
+ <th>independent_varnames</th>
+ <td>TEXT. Independent variables</td>
</tr>
<tr>
<th>cat_features</th>
- <td>text. Categorical feature names.</td>
+ <td>TEXT. List of categorical features
+ as a comma-separated string.</td>
</tr>
<tr>
<th>con_features</th>
- <td>text. Continuous feature names.</td>
+ <td>TEXT. List of continuous feature
+ as a comma-separated string.</td>
</tr>
<tr>
- <th>grouping_col</th>
- <td>int. Names of grouping columns.</td>
+ <th>grouping_cols</th>
+ <td>INTEGER. Names of grouping columns.</td>
</tr>
<tr>
<th>num_trees</th>
- <td>int. Number of trees grown by the model.</td>
+ <td>INTEGER. Number of trees grown by the model.</td>
</tr>
<tr>
<th>num_random_features</th>
- <td>int. Number of features randomly selected for each split.</td>
+ <td>INTEGER. Number of features randomly selected for each split.</td>
</tr>
<tr>
<th>max_tree_depth</th>
- <td>int. Maximum depth of any tree in the random forest model_table.</td>
+ <td>INTEGER. Maximum depth of any tree in the random forest model_table.</td>
</tr>
<tr>
<th>min_split</th>
- <td>int. Minimum number of observations in a node for it to be split.</td>
+ <td>INTEGER. Minimum number of observations in a node for it to be split.</td>
</tr>
<tr>
<th>min_bucket</th>
- <td>int. Minimum number of observations in any terminal node.</td>
+ <td>INTEGER. Minimum number of observations in any terminal node.</td>
</tr>
<tr>
<th>num_splits</th>
- <td>int. Number of buckets for continuous variables.</td>
+ <td>INTEGER. Number of buckets for continuous variables.</td>
</tr>
<tr>
<th>verbose</th>
- <td>boolean. Whether or not to display debug info.</td>
+ <td>BOOLEAN. Whether or not to display debug info.</td>
</tr>
<tr>
<th>importance</th>
- <td>boolean. Whether or not to calculate variable importance.</td>
+ <td>BOOLEAN. Whether or not to calculate variable importance.</td>
</tr>
<tr>
<th>num_permutations</th>
- <td>int. Number of times feature values are permuted while calculating
- variable importance. The default value is 1.</td>
+ <td>INTEGER. Number of times feature values are permuted while calculating
+ variable importance.</td>
</tr>
<tr>
<th>num_all_groups</th>
- <td>int. Number of groups during forest training.</td>
+ <td>INTEGER. Number of groups during forest training.</td>
</tr>
<tr>
<th>num_failed_groups</th>
- <td>int. Number of failed groups during forest training.</td>
+ <td>INTEGER. Number of failed groups during forest training.</td>
</tr>
<tr>
<th>total_rows_processed</th>
- <td>bigint. Total numbers of rows processed in all groups.</td>
+ <td>BIGINT. Total numbers of rows processed in all groups.</td>
</tr>
<tr>
<th>total_rows_skipped</th>
- <td>bigint. Total numbers of rows skipped in all groups due to missing values or failures.</td>
+ <td>BIGINT. Total numbers of rows skipped in all
+ groups due to missing values or failures.</td>
</tr>
<tr>
<th>dependent_var_levels</th>
- <td>text. For classification, the distinct levels of the dependent variable.</td>
+ <td>TEXT. For classification, the distinct levels of the dependent variable.</td>
</tr>
<tr>
<th>dependent_var_type</th>
- <td>text. The type of dependent variable.</td>
+ <td>TEXT. The type of dependent variable.</td>
</tr>
<tr>
<th>independent_var_types</th>
- <td>text. A comma separated string for the types of independent variables.</td>
+ <td>TEXT. A comma separated string for the types of independent variables.</td>
</tr>
<tr>
<th>null_proxy</th>
- <td>text. Describes how NULLs are handled. If NULL is not
+ <td>TEXT. Describes how NULLs are handled. If NULL is not
treated as a separate categorical variable, this will be NULL.
If NULL is treated as a separate categorical value, this will be
set to "__NULL__"</td>
</tr>
</table>
- A group table named <em> \<model_table\>_group</em> is created, which has the following columns:
+ A table named <em>\<model_table\>_group</em> is also created at
+ the same time, even if no grouping is specified.
+ It contains the following columns:
<table class="output">
<tr>
<th>gid</th>
- <td>integer. Group id that uniquely identifies a set of grouping column values.</td>
+ <td>integer. Group id that uniquely identifies
+ a set of grouping column values. If grouping is not
+ used, this will always be 1.</td>
</tr>
<tr>
<th><...></th>
- <td>Same type as in the training data table. Grouping columns, if provided in input.
- This could be multiple columns depending on the \c grouping_cols input.</td>
+ <td>Same type as in the training data table 'grouping_cols'.
+ This could be multiple columns depending on
+ the 'grouping_cols' input.</td>
</tr>
<tr>
<th>success</th>
- <td>boolean. Indicator of the success of the group.</td>
+ <td>BOOLEAN. Indicator of the success of the group.</td>
</tr>
<tr>
<th>cat_levels_in_text</th>
- <td>text[]. Ordered levels of categorical variables.</td>
+ <td>TEXT[]. Ordered levels (values) of categorical variables
+ corresponding to the categorical features in
+ the 'list_of_features' argument above. Used to help
+ interpret the trained tree. For example, if the
+ categorical features specified are <em>weather_outlook</em>
+ and <em>windy</em> in that order, then 'cat_levels_in_text'
+ might be <em>[overcast, rain, sunny, False, True]</em>.</td>
</tr>
<tr>
<th>cat_n_levels</th>
- <td>integer[]. Number of levels for each categorical variable.</td>
+ <td>INTEGER[]. Number of levels for each categorical variable.
+ Used to help interpret the trained tree. In the example
+ from above, 'cat_n_levels' would
+ be <em>[3, 2]</em> since there are 3 levels
+ for <em>weather_outlook</em> and 2 levels
+ <em>windy</em>.</td>
</tr>
<tr>
<th>oob_error</th>
- <td>double precision. Out-of-bag error for the random forest model.</td>
+ <td>DOUBLE PRECISION. Out-of-bag error for the random forest model.</td>
</tr>
<tr>
<th>cat_var_importance</th>
- <td>double precision[]. Variable importance for categorical features.
+ <td>DOUBLE PRECISION[]. Variable importance for categorical features.
The order corresponds to the order of the variables as found in
- cat_features in <em> \<model_table\>_summary</em>.</td>
+ 'cat_features' in <em> \<model_table\>_summary</em>.</td>
</tr>
<tr>
<th>con_var_importance</th>
- <td>double precision[]. Variable importance for continuous features.
+ <td>DOUBLE PRECISION[]. Variable importance for continuous features.
The order corresponds to the order of the variables as found in
- con_features in <em> \<model_table\>_summary</em>.</td>
+ 'con_features' in <em> \<model_table\>_summary</em>.</td>
</tr>
</table>
</DD>
-
- <DT>id_col_name</DT>
- <DD>text. Name of the column containing id information in the training data.</DD>
-
- <DT>dependent_variable</DT>
- <DD>text. Name of the column that contains the output for
- training. Boolean, integer and text are considered classification outputs,
- while float values are considered regression outputs.</DD>
-
- <DT>list_of_features</DT>
- <DD>TEXT. Comma-separated string of column names or expressions to use as predictors.
- Can also be a '*' implying all columns are to be used as predictors (except for the
- ones included in the next argument that lists exclusions).
- The types of the features can be mixed - boolean, integer, and text columns
- are considered categorical and
- double precision columns are considered continuous. Categorical variables
- are not encoded and used as is for the training.
-
- Array columns can also be included in the list, where the array is expanded
- to treat each element of the array as a feature.
-
- It is important to note that not every combination of the levels of a
- categorical variable is checked when evaluating a split. The levels of the
- non-integer categorical variable are ordered by the entropy of the variable in
- predicting the response. The split at each node is evaluated between these
- ordered levels. Integer categorical variables, however, are simply ordered
- by their value.
- </DD>
-
- <DT>list_of_features_to_exclude</DT>
- <DD>text. Comma-separated string of column names to exclude from the predictors
- list. If the <em>dependent_variable</em> argument is an expression
- (including cast of a column name), then this list should include the
- columns that are included in the <em>dependent_variable</em> expression,
- otherwise those columns will be included in the features
- (resulting in meaningless trees).</DD>
-
- <DT>grouping_cols (optional)</DT>
- <DD>text, default: NULL. Comma-separated list of column names to group the
- data by. This will lead to creating multiple random forests, one for
- each group.</DD>
-
- <DT>num_trees (optional)</DT>
- <DD>integer, default: 100. Maximum number of trees to grow in the Random
- Forest model. Actual number of trees grown may be slighlty different.</DD>
-
- <DT>num_random_features (optional)</DT>
- <DD>integer, default: sqrt(n) if classification tree, otherwise n/3. Number of
- features to randomly select at each split.</DD>
-
- <DT>importance (optional)</DT>
- <DD>boolean, default: true. Whether or not to calculate variable importance.
- If set to true, variable importance for categorical and continuous features
- will be output in the group table <em>\<model_table\>_group</em> described
- above. Total runtime will increase when variable importance is turned on. </DD>
-
- <DT>num_permutations (optional)</DT>
- <DD>integer, default: 1. Number of times to permute each feature value while
- calculating variable importance.
-
- @note Variable importance for a feature is computed by permuting the variable
- and computing the drop in predictive accuracy (using OOB samples).
- Setting this greater than 1 performs an average over multiple
- importance calculations. This increases the total run time; in most cases
- the default value of 1 is sufficient to compute the importance.
- Due to nature of permutation, the importance value can end up being
- negative if the number of levels for a categorical variable is small and is
- unbalanced. In such a scenario the importance values are shifted to ensure
- that the lowest importance value is 0.
-
- </DD>
-
- <DT>max_tree_depth (optional)</DT>
- <DD>integer, default: 7. Maximum depth of any node of a tree,
- with the root node counted as depth 0. A deeper tree can
- lead to better prediction but will also result in
- longer processing time and higher memory usage.</DD>
-
- <DT>min_split (optional)</DT>
- <DD>integer, default: 20. Minimum number of observations that must exist
- in a node for a split to be attempted.</DD>
-
- <DT>min_bucket (optional)</DT>
- <DD>integer, default: min_split/3. Minimum number of observations in any terminal
- node. If only one of min_bucket or min_split is specified, min_split is
- set to min_bucket*3 or min_bucket to min_split/3, as appropriate.</DD>
-
- <DT>num_splits (optional)</DT>
- <DD>integer, default: 20. Continuous-valued features are binned into
- discrete quantiles to compute split boundaries. This global parameter
- is used to compute the resolution of splits for continuous features.
- Higher number of bins will lead to better prediction,
- but will also result in longer processing time and higher memory usage.</DD>
-
- <DT>null_handling_params</DT>
- <DD>TEXT. Comma-separated string of key-value pairs controlling the behavior
- of various features handling missing values.
- <table class='output'>
- <tr>
- <th>max_surrogates</th>
- <td>Default: 0. Number of surrogates to store for each node.</td>
- One of the approaches of handling NULLs is to use surrogate splits for each
- node. A surrogate variable is another predictor variable that is associated
- (correlated) with the primary split variable. The surrogate
- variable comes into use when the primary predictior value is NULL.
- </tr>
- <tr>
- <th>null_as_category</th>
- <td>Default: FALSE. Whether to treat NULL as a special categorical value.
-
- If this is set to TRUE, NULL values are considered a categorical
- value and placed at the end of the ordering of categorical levels. Placing it
- at the end ensures that NULL is never used as a value to split a node on.
- This parameter is ignored for continuous-valued features.
- </td>
- </tr>
- </table>
- </DD>
-
- <DT>verbose (optional)</DT>
- <DD>boolean, default: FALSE. Provides verbose output of the results of training.</DD>
-
- <DT>sample_ratio (optional)</DT>
- <DD>double precision, in the range of (0, 1], default: 1.
- If sample_ratio is less than 1, a bootstrap sample size smaller than the data
- table is expected to be used for training each tree in the forest. A ratio that
- is close to 0 may result in trees with only the root node.
- This allows users to experiment with the function in a speedy fashion.</DD>
</DL>
- @note The main parameters that affect memory usage are: depth of
- tree (‘max_tree_depth’), number of features, number of values per
- categorical feature, and number of bins for continuous features (‘num_splits’).
- If you are hitting memory limits, consider reducing one or
- more of these parameters.
+
+@note The main parameters that affect memory usage are: depth of
+tree (‘max_tree_depth’), number of features, number of values per
+categorical feature, and number of bins for continuous features (‘num_splits’).
+If you are hitting memory limits, consider reducing one or
+more of these parameters.
@anchor predict
@par Prediction Function
-The prediction function is provided to estimate the conditional mean given a new
+The prediction function estimates the conditional mean given a new
predictor. It has the following syntax:
<pre class="syntax">
forest_predict(random_forest_model,
@@ -429,18 +489,33 @@ forest_predict(random_forest_model,
\b Arguments
<DL class="arglist">
<DT>forest_model</DT>
- <DD>text. Name of the table containing the Random Forest model.</DD>
+ <DD>text. Name of the table containing the random forest model
+ from training.</DD>
<DT>new_data_table</DT>
- <DD>text. Name of the table containing prediction data.</DD>
+ <DD>TEXT. Name of the table containing prediction data. This table is
+ expected to contain the same features that were used during training. The table
+ should also contain <em>id_col_name</em> used for identifying each row.</DD>
<DT>output_table</DT>
- <DD>text. Name of the table to output prediction results to.</DD>
+ <DD>TEXT. Name of the table to output prediction results. If this table
+ already exists, an error is returned.
+ The table contains the <em>id_col_name</em> column giving
+ the 'id' for each prediction and the prediction columns for the dependent variable.
+
+ If <em>type</em> = 'response', then the table has a single additional column
+ with the prediction value of the response. The type of this column depends on
+ the type of the response variable used during training.
+
+ If <em>type</em> = 'prob', then the table has multiple additional columns, one
+ for each possible value of the response variable. The columns are labeled as
+ 'estimated_prob_<em>dep_value</em>', where <em>dep_value</em> represents each
+ value of the response variable.</DD>
<DT>type</DT>
- <DD>text, optional, default: 'response'. For regression models, the output is
+ <DD>TEXT, optional, default: 'response'. For regression trees, the output is
always the predicted value of the dependent variable. For classification
- models, the <em>type</em> variable can be 'response', giving the
+ trees, the <em>type</em> variable can be 'response', giving the
classification prediction as output, or 'prob', giving the class
probabilities as output. For each value of the dependent variable, a
column with the probabilities is added to the output table.
@@ -448,12 +523,12 @@ forest_predict(random_forest_model,
</DL>
@anchor get_tree
-@par Display Function
-The 'get_tree' function is provided to output a graph representation of a
-single tree of the random forest. The output can either be in the popular
-'dot' format that can be visualized using various programs including those
-in the GraphViz package, or in a simple text format. The details of the
-text format is outputted with the tree.
+@par Tree Display
+The display function outputs a graph representation of a
+single tree of the random forest. The output can either be in the popular
+'dot' format that can be visualized using various programs including those
+in the GraphViz package, or in a simple text format. The details of the
+text format are output with the tree.
<pre class="syntax">
get_tree(forest_model_table,
gid,
@@ -463,41 +538,42 @@ get_tree(forest_model_table,
</pre>
An additional display function is provided to output the surrogate splits chosen
-for each internal node.
+for each internal node:
<pre class="syntax">
get_tree_surr(forest_model_table,
gid,
sample_id)
</pre>
-The output contains the list of surrogate splits for each internal node of a
-tree. The nodes are sorted in ascending order by id. This is equivalent to
-viewing the tree in a breadth-first manner. For each surrogate, the output gives
-the surrogate split (variable and threshold) and also provides the number of
-rows that were common between the primary split and the surrogate split.
-Finally, the number of rows present in the majority branch of the primary split
-is also presented. Only surrogates that perform better than this majority branch
-are used. When the primary variable has a NULL value the surrogate variables are
-used in order to compute the split for that node. If all surrogates variables
-are NULL, then the majority branch is used to compute the split for a tuple.
+This output contains the list of surrogate splits for each internal node. The
+nodes are sorted in ascending order by id. This is equivalent to viewing the
+tree in a breadth-first manner. For each surrogate, we output the surrogate
+split (variable and threshold) and also give the number of rows that were common
+between the primary split and the surrogate split. Finally, the number of rows
+present in the majority branch of the primary split is also shown. Only
+surrogates that perform better than this majority branch are included in the
+surrogate list. When the primary variable has a NULL value the surrogate
+variables are used in order to compute the split for that node. If all
+surrogates variables are NULL, then the majority branch is used to compute the
+split for a tuple.
\b Arguments
<DL class="arglist">
<DT>forest_model_table</DT>
- <DD>text. Name of the table containing the Random Forest model.</DD>
+ <DD>TEXT. Name of the table containing the random forest model.</DD>
<DT>gid</DT>
- <DD>integer. Id of the group that this tree is a part of.</DD>
+ <DD>INTEGER. Id of the group that this tree is part of.</DD>
<DT>sample_id</DT>
- <DD>integer. Id of the bootstrap sample that this tree if a part of.</DD>
+ <DD>INTEGER. Id of the bootstrap sample that this tree is part of.</DD>
<DT>dot_format (optional)</DT>
- <DD>boolean, default = TRUE. Output can either be in a dot format or a text
+ <DD>BOOLEAN, default = TRUE. Output can either be in a dot format or a text
format. If TRUE, the result is in the dot format, else output is in text format.</DD>
<DT>verbose (optional)</DT>
- <DD>boolean, default = FALSE. If true, the dot format output will contain
+ <DD>BOOLEAN, default = FALSE. If true, the dot format output will contain
additional information (impurity, sample size, number of weighted rows for
each response variable, classification or prediction if the tree was
pruned at this level)</DD>
@@ -507,46 +583,83 @@ The output is always returned as a 'TEXT'. For the dot format, the output can be
redirected to a file on the client side and then rendered using visualization
programs.
+To export the dot format result to an external file,
+use the method below. Please note that you should use unaligned
+table output mode for psql with '-A' flag, or else you may get an
+error when you try to convert the dot file to another format
+for viewing (e.g., PDF). And inside the psql client,
+both '\\t' and '\\o' should be used:
+
+<pre class="example">
+\> \# under bash
+\> psql -A my_database
+\# -- in psql now
+\# \\t
+\# \\o test.dot -- export to a file
+\# select madlib.tree_display('tree_out');
+\# \\o
+\# \\t
+</pre>
+
+After the dot file has been generated, use third-party
+plotting software to plot the trees in a nice format:
+<pre class="example">
+\> \# under bash, convert the dot file into a PDF file
+\> dot -Tpdf test.dot \> test.pdf
+\> xpdf test.pdf\&
+</pre>
+
+Please see
+the <a href="group__grp__decision__tree.html">decision tree user documentation</a>
+for more details on working with tree output formats.
+
@anchor examples
@examp
-\b Note: The output results may vary due the random nature of random forests.
+
+@note
+ - Not all random forest parameters are demonstrated in
+ the examples below. Some are shown in
+ the <a href="group__grp__decision__tree.html">decision tree user documentation</a>
+ since usage is similar.
+ - Your results may look different than those below
+due the random nature of random forests.
<b>Random Forest Classification Example</b>
--# Prepare input data:
+-# Load input data set related to whether to play golf or not:
<pre class="example">
-DROP TABLE IF EXISTS dt_golf;
-CREATE TABLE dt_golf (
+DROP TABLE IF EXISTS rf_golf CASCADE;
+CREATE TABLE rf_golf (
id integer NOT NULL,
"OUTLOOK" text,
temperature double precision,
humidity double precision,
- windy text,
+ "Temp_Humidity" double precision[],
+ clouds_airquality text[],
+ windy boolean,
class text
);
-</pre>
-<pre class="example">
-INSERT INTO dt_golf (id,"OUTLOOK",temperature,humidity,windy,class) VALUES
-(1, 'sunny', 85, 85, 'false', 'Don''t Play'),
-(2, 'sunny', 80, 90, 'true', 'Don''t Play'),
-(3, 'overcast', 83, 78, 'false', 'Play'),
-(4, 'rain', 70, 96, 'false', 'Play'),
-(5, 'rain', 68, 80, 'false', 'Play'),
-(6, 'rain', 65, 70, 'true', 'Don''t Play'),
-(7, 'overcast', 64, 65, 'true', 'Play'),
-(8, 'sunny', 72, 95, 'false', 'Don''t Play'),
-(9, 'sunny', 69, 70, 'false', 'Play'),
-(10, 'rain', 75, 80, 'false', 'Play'),
-(11, 'sunny', 75, 70, 'true', 'Play'),
-(12, 'overcast', 72, 90, 'true', 'Play'),
-(13, 'overcast', 81, 75, 'false', 'Play'),
-(14, 'rain', 71, 80, 'true', 'Don''t Play');
+INSERT INTO rf_golf VALUES
+(1,'sunny', 85, 85, ARRAY[85, 85],ARRAY['none', 'unhealthy'], 'false','Don''t Play'),
+(2, 'sunny', 80, 90, ARRAY[80, 90], ARRAY['none', 'moderate'], 'true', 'Don''t Play'),
+(3, 'overcast', 83, 78, ARRAY[83, 78], ARRAY['low', 'moderate'], 'false', 'Play'),
+(4, 'rain', 70, 96, ARRAY[70, 96], ARRAY['low', 'moderate'], 'false', 'Play'),
+(5, 'rain', 68, 80, ARRAY[68, 80], ARRAY['medium', 'good'], 'false', 'Play'),
+(6, 'rain', 65, 70, ARRAY[65, 70], ARRAY['low', 'unhealthy'], 'true', 'Don''t Play'),
+(7, 'overcast', 64, 65, ARRAY[64, 65], ARRAY['medium', 'moderate'], 'true', 'Play'),
+(8, 'sunny', 72, 95, ARRAY[72, 95], ARRAY['high', 'unhealthy'], 'false', 'Don''t Play'),
+(9, 'sunny', 69, 70, ARRAY[69, 70], ARRAY['high', 'good'], 'false', 'Play'),
+(10, 'rain', 75, 80, ARRAY[75, 80], ARRAY['medium', 'good'], 'false', 'Play'),
+(11, 'sunny', 75, 70, ARRAY[75, 70], ARRAY['none', 'good'], 'true', 'Play'),
+(12, 'overcast', 72, 90, ARRAY[72, 90], ARRAY['medium', 'moderate'], 'true', 'Play'),
+(13, 'overcast', 81, 75, ARRAY[81, 75], ARRAY['medium', 'moderate'], 'false', 'Play'),
+(14, 'rain', 71, 80, ARRAY[71, 80], ARRAY['low', 'unhealthy'], 'true', 'Don''t Play');
</pre>
--# Run the random forest training function and view summary output:
+-# Train random forest then view the summary table:
<pre class="example">
DROP TABLE IF EXISTS train_output, train_output_group, train_output_summary;
-SELECT madlib.forest_train('dt_golf', -- source table
+SELECT madlib.forest_train('rf_golf', -- source table
'train_output', -- output model table
'id', -- id column
'class', -- response
@@ -565,12 +678,11 @@ SELECT madlib.forest_train('dt_golf', -- source table
\\x on
SELECT * FROM train_output_summary;
</pre>
-Result:
<pre class="result">
--[ RECORD 1 ]---------+-----------------------------------------------
+-[ RECORD 1 ]---------+--------------------------------------------------
method | forest_train
is_classification | t
-source_table | dt_golf
+source_table | rf_golf
model_table | train_output
id_col_name | id
dependent_varname | class
@@ -593,55 +705,68 @@ total_rows_processed | 14
total_rows_skipped | 0
dependent_var_levels | "Don't Play","Play"
dependent_var_type | text
-independent_var_types | text, text, double precision, double precision
+independent_var_types | text, boolean, double precision, double precision
null_proxy | None
</pre>
View the group table output:
<pre class="example">
SELECT * FROM train_output_group;
</pre>
-Result:
<pre class="result">
--[ RECORD 1 ]------+----------------------------------------
+-[ RECORD 1 ]------+---------------------------------
gid | 1
success | t
cat_n_levels | {3,2}
-cat_levels_in_text | {overcast,sunny,rain,false,true}
+cat_levels_in_text | {overcast,sunny,rain,False,True}
oob_error | 0.42857142857142857143
-cat_var_importance | {0.0305555555555556,0.0626984126984127}
-con_var_importance | {0,0.0243650793650794}
+cat_var_importance | {0.0692857142857143,0.02}
+con_var_importance | {0,5.55111512312578e-17}
</pre>
-
--# Obtain a dot format display of a single tree
-within the forest:
+The 'cat_levels_in_text' array shows the ordering of the
+levels of the categorical variables "OUTLOOK" and windy,
+which have 3 and 2 levels respectively. Variable importance
+for categorical variables is shown in 'cat_var_importance'.
+Variable importance for continuous variables is
+shown in 'con_var_importance'.
+A higher value means higher importance for the variable.
+
+-# Predict output categories. For the purpose of this
+example, we use the same data that was used for training:
<pre class="example">
-\\x off
-SELECT madlib.get_tree('train_output',1,2);
+DROP TABLE IF EXISTS prediction_results;
+SELECT madlib.forest_predict('train_output', -- tree model
+ 'rf_golf', -- new data table
+ 'prediction_results', -- output table
+ 'response'); -- show response
+SELECT g.id, class, estimated_class FROM prediction_results p,
+rf_golf g WHERE p.id = g.id ORDER BY g.id;
</pre>
-Result:
<pre class="result">
- digraph "Classification tree for dt_golf" {
- "0" [label="humidity <= 75", shape=ellipse];
- "0" -> "1"[label="yes"];
- "1" [label="\"Play\"",shape=box];
- "0" -> "2"[label="no"];
- "2" [label="humidity <= 80", shape=ellipse];
- "2" -> "5"[label="yes"];
- "5" [label="\"Don't Play\"",shape=box];
- "2" -> "6"[label="no"];
- "6" [label="\"OUTLOOK\" in {overcast,rain}", shape=ellipse];
- "6" -> "13"[label="yes"];
- "13" [label="\"Play\"",shape=box];
- "6" -> "14"[label="no"];
- "14" [label="\"Don't Play\"",shape=box];
- } //---end of digraph---------
+ id | class | estimated_prob_Don't Play | estimated_prob_Play
+----+------------+---------------------------+---------------------
+ 1 | Don't Play | 0.85 | 0.15
+ 2 | Don't Play | 0.95 | 0.05
+ 3 | Play | 0.2 | 0.8
+ 4 | Play | 0.3 | 0.7
+ 5 | Play | 0.1 | 0.9
+ 6 | Don't Play | 0.7 | 0.3
+ 7 | Play | 0.2 | 0.8
+ 8 | Don't Play | 0.75 | 0.25
+ 9 | Play | 0.15 | 0.85
+ 10 | Play | 0.2 | 0.8
+ 11 | Play | 0.25 | 0.75
+ 12 | Play | 0.3 | 0.7
+ 13 | Play | 0 | 1
+ 14 | Don't Play | 0.6 | 0.4
+(14 rows)
</pre>
--# Obtain a text display of the tree:
+-# View a single tree in text format within the forest
+identified by 'gid' and 'sample_id', out of
+the several that were created:
<pre class="example">
-SELECT madlib.get_tree('train_output',1,2,FALSE);
+SELECT madlib.get_tree('train_output',1,7, FALSE);
</pre>
-Result:
<pre class="result">
-------------------------------------
- Each node represented by 'id' inside ().
@@ -652,86 +777,182 @@ second indented child (2i+2) is the 'False' node.
- Number of (weighted) rows for each response variable inside [].
- Order of values = ['"Don\'t Play"', '"Play"']
-------------------------------------
- (0)[ 4 10] humidity <= 75
- (1)[0 7] * --> "Play"
- (2)[4 3] humidity <= 80
- (5)[3 1] * --> "Don't Play"
- (6)[1 2] "OUTLOOK" in {overcast,rain}
- (13)[0 2] * --> "Play"
- (14)[1 0] * --> "Don't Play"
+ (0)[ 5 10] windy in {False}
+ (1)[2 8] "OUTLOOK" in {overcast,sunny}
+ (3)[2 1] humidity <= 75
+ (7)[0 1] * --> "Play"
+ (8)[2 0] * --> "Don't Play"
+ (4)[0 7] * --> "Play"
+ (2)[3 2] temperature <= 75
+ (5)[1 2] humidity <= 70
+ (11)[1 1] * --> "Don't Play"
+ (12)[0 1] * --> "Play"
+ (6)[2 0] * --> "Don't Play"
-------------------------------------
</pre>
+Please see
+the <a href="group__grp__decision__tree.html">decision tree user documentation</a>
+for an explanation on how to interpret the tree display above.
--# Predict output categories for the same data as was used for input:
+-# View tree in dot format:
<pre class="example">
-DROP TABLE IF EXISTS prediction_results;
-SELECT madlib.forest_predict('train_output',
- 'dt_golf',
- 'prediction_results',
- 'response');
-\\x off
-SELECT id, estimated_class, class
-FROM prediction_results JOIN dt_golf USING (id)
-ORDER BY id;
+SELECT madlib.get_tree('train_output',1,7);
</pre>
-Result:
<pre class="result">
- id | estimated_class | class
-----+-----------------+------------
- 1 | Don't Play | Don't Play
- 2 | Don't Play | Don't Play
- 3 | Play | Play
- 4 | Play | Play
- 5 | Play | Play
- 6 | Don't Play | Don't Play
- 7 | Play | Play
- 8 | Don't Play | Don't Play
- 9 | Play | Play
- 10 | Play | Play
- 11 | Play | Play
- 12 | Play | Play
- 13 | Play | Play
- 14 | Don't Play | Don't Play
-(14 rows)
+ ----------------------------------------------------
+ digraph "Classification tree for rf_golf" {
+ "0" [label="windy <= False", shape=ellipse];
+ "0" -> "1"[label="yes"];
+ "0" -> "2"[label="no"];
+ "1" [label="\"OUTLOOK\" <= sunny", shape=ellipse];
+ "1" -> "3"[label="yes"];
+ "1" -> "4"[label="no"];
+ "4" [label="\"Play\"",shape=box];
+ "2" [label="temperature <= 75", shape=ellipse];
+ "2" -> "5"[label="yes"];
+ "2" -> "6"[label="no"];
+ "6" [label="\"Don't Play\"",shape=box];
+ "3" [label="humidity <= 75", shape=ellipse];
+ "3" -> "7"[label="yes"];
+ "7" [label="\"Play\"",shape=box];
+ "3" -> "8"[label="no"];
+ "8" [label="\"Don't Play\"",shape=box];
+ "5" [label="humidity <= 70", shape=ellipse];
+ "5" -> "11"[label="yes"];
+ "11" [label="\"Don't Play\"",shape=box];
+ "5" -> "12"[label="no"];
+ "12" [label="\"Play\"",shape=box];
+ } //---end of digraph---------
</pre>
--# Predict probablities of output categories for the same data:
+-# View tree in dot format with additional information:
<pre class="example">
-DROP TABLE IF EXISTS prediction_prob;
-SELECT madlib.forest_predict('train_output',
- 'dt_golf',
- 'prediction_prob',
- 'prob');
-\\x off
-SELECT id, "estimated_prob_Play", class
-FROM prediction_prob JOIN dt_golf USING (id)
-ORDER BY id;
+SELECT madlib.get_tree('train_output',1,7, TRUE, TRUE);
</pre>
-Result:
<pre class="result">
- id | estimated_prob_Play | class
-----+---------------------+------------
- 1 | 0.05 | Don't Play
- 2 | 0.15 | Don't Play
- 3 | 0.95 | Play
- 4 | 0.65 | Play
- 5 | 0.75 | Play
- 6 | 0.4 | Don't Play
- 7 | 0.7 | Play
- 8 | 0.1 | Don't Play
- 9 | 0.9 | Play
- 10 | 0.85 | Play
- 11 | 0.8 | Play
- 12 | 0.7 | Play
- 13 | 1 | Play
- 14 | 0.4 | Don't Play
-(14 rows)
+ ---------------------------------------------------------------------------------------------------------------------------
+ digraph "Classification tree for rf_golf" {
+ "0" [label="windy <= False\\n impurity = 0.444444\\n samples = 15\\n value = [ 5 10]\\n class = \"Play\"", shape=ellipse];
+ "0" -> "1"[label="yes"];
+ "0" -> "2"[label="no"];
+ "1" [label="\"OUTLOOK\" <= sunny\\n impurity = 0.32\\n samples = 10\\n value = [2 8]\\n class = \"Play\"", shape=ellipse];
+ "1" -> "3"[label="yes"];
+ "1" -> "4"[label="no"];
+ "4" [label="\"Play\"\\n impurity = 0\\n samples = 7\\n value = [0 7]",shape=box];
+ "2" [label="temperature <= 75\\n impurity = 0.48\\n samples = 5\\n value = [3 2]\\n class = \"Don't Play\"", shape=ellipse];
+ "2" -> "5"[label="yes"];
+ "2" -> "6"[label="no"];
+ "6" [label="\"Don't Play\"\\n impurity = 0\\n samples = 2\\n value = [2 0]",shape=box];
+ "3" [label="humidity <= 75\\n impurity = 0.444444\\n samples = 3\\n value = [2 1]\\n class = \"Don't Play\"", shape=ellipse];
+ "3" -> "7"[label="yes"];
+ "7" [label="\"Play\"\\n impurity = 0\\n samples = 1\\n value = [0 1]",shape=box];
+ "3" -> "8"[label="no"];
+ "8" [label="\"Don't Play\"\\n impurity = 0\\n samples = 2\\n value = [2 0]",shape=box];
+ "5" [label="humidity <= 70\\n impurity = 0.444444\\n samples = 3\\n value = [1 2]\\n class = \"Play\"", shape=ellipse];
+ "5" -> "11"[label="yes"];
+ "11" [label="\"Don't Play\"\\n impurity = 0.5\\n samples = 2\\n value = [1 1]",shape=box];
+ "5" -> "12"[label="no"];
+ "12" [label="\"Play\"\\n impurity = 0\\n samples = 1\\n value = [0 1]",shape=box];
+ } //---end of digraph---------
</pre>
+-# Arrays of features. Categorical and continuous
+features can be array columns, in which case the
+array is expanded to treat each element of the
+array as a feature:
+<pre class="example">
+DROP TABLE IF EXISTS train_output, train_output_group, train_output_summary;
+SELECT madlib.forest_train('rf_golf', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'class', -- response
+ '"Temp_Humidity", clouds_airquality', -- features
+ NULL, -- exclude columns
+ NULL, -- grouping columns
+ 20::integer, -- number of trees
+ 2::integer, -- number of random features
+ TRUE::boolean, -- variable importance
+ 1::integer, -- num_permutations
+ 8::integer, -- max depth
+ 3::integer, -- min split
+ 1::integer, -- min bucket
+ 10::integer -- number of splits per continuous variable
+ );
+SELECT * FROM train_output_summary;
+</pre>
+<pre class="result">
+-[ RECORD 1 ]---------+--------------------------------------------------------------------------------
+method | forest_train
+is_classification | t
+source_table | rf_golf
+model_table | train_output
+id_col_name | id
+dependent_varname | class
+independent_varnames | clouds_airquality[1],clouds_airquality[2],"Temp_Humidity"[1],"Temp_Humidity"[2]
+cat_features | clouds_airquality[1],clouds_airquality[2]
+con_features | "Temp_Humidity"[1],"Temp_Humidity"[2]
+grouping_cols |
+num_trees | 20
+num_random_features | 2
+max_tree_depth | 8
+min_split | 3
+min_bucket | 1
+num_splits | 10
+verbose | f
+importance | t
+num_permutations | 1
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 14
+total_rows_skipped | 0
+dependent_var_levels | "Don't Play","Play"
+dependent_var_type | text
+independent_var_types | text, text, double precision, double precision
+null_proxy | None
+</pre>
+
+-# Sample ratio. Use the sample ratio parameter to
+train on a subset of the data:
+<pre class="example">
+DROP TABLE IF EXISTS train_output, train_output_group, train_output_summary;
+SELECT madlib.forest_train('rf_golf', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'class', -- response
+ '"OUTLOOK", temperature, humidity, windy', -- features
+ NULL, -- exclude columns
+ NULL, -- grouping columns
+ 20::integer, -- number of trees
+ 2::integer, -- number of random features
+ TRUE::boolean, -- variable importance
+ 1::integer, -- num_permutations
+ 8::integer, -- max depth
+ 3::integer, -- min split
+ 1::integer, -- min bucket
+ 10::integer, -- number of splits per continuous variable
+ NULL, -- NULL handling
+ FALSE, -- Verbose
+ 0.5 -- Sample ratio
+ );
+SELECT * FROM train_output_group;
+</pre>
+<pre class="result">
+-[ RECORD 1 ]------+---------------------------------
+gid | 1
+success | t
+cat_n_levels | {3,2}
+cat_levels_in_text | {overcast,sunny,rain,False,True}
+oob_error | 0.42857142857142857143
+cat_var_importance | {0.0416666666666667,0}
+con_var_importance | {0.0291666666666667,0.025}
+</pre>
<b>Random Forest Regression Example</b>
--# Prepare input data:
+-# Load input data related to fuel consumption and 10
+aspects of automobile design and performance for 32
+automobiles (1973–74 models). Data was extracted from
+the 1974 Motor Trend US magazine.
<pre class="example">
DROP TABLE IF EXISTS mt_cars;
CREATE TABLE mt_cars (
@@ -748,9 +969,7 @@ CREATE TABLE mt_cars (
gear integer,
carb integer
);
-</pre>
-<pre class="example">
-INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
+INSERT INTO mt_cars VALUES
(1,18.7,8,360,175,3.15,3.44,17.02,0,0,3,2),
(2,21,6,160,110,3.9,2.62,16.46,0,1,4,4),
(3,24.4,4,146.7,62,3.69,3.19,20,1,0,4,2),
@@ -759,7 +978,7 @@ INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
(6,16.4,8,275.8,180,3.078,4.07,17.4,0,0,3,3),
(7,22.8,4,108,93,3.85,2.32,18.61,1,1,4,1),
(8,17.3,8,275.8,180,3.078,3.73,17.6,0,0,3,3),
-(9,21.4,6,258,110,3.08,3.215,19.44,1,0,3,1),
+(9,21.4,null,258,110,3.08,3.215,19.44,1,0,3,1),
(10,15.2,8,275.8,180,3.078,3.78,18,0,0,3,3),
(11,18.1,6,225,105,2.768,3.46,20.22,1,0,3,1),
(12,32.4,4,78.7,66,4.08,2.20,19.47,1,1,4,1),
@@ -768,7 +987,7 @@ INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
(15,30.4,4,75.7,52,4.93,1.615,18.52,1,1,4,2),
(16,19.2,6,167.6,123,3.92,3.44,18.3,1,0,4,4),
(17,33.9,4,71.14,65,4.22,1.835,19.9,1,1,4,1),
-(18,15.2,8,304,150,3.15,3.435,17.3,0,0,3,2),
+(18,15.2,null,304,150,3.15,3.435,17.3,0,0,3,2),
(19,10.4,8,472,205,2.93,5.25,17.98,0,0,3,4),
(20,27.3,4,79,66,4.08,1.935,18.9,1,1,4,1),
(21,10.4,8,460,215,3,5.424,17.82,0,0,3,4),
@@ -785,40 +1004,83 @@ INSERT INTO mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) VALUES
(32,21.4,4,121,109,4.11,2.78,18.6,1,1,4,2);
</pre>
--# Run the random forest training function:
+-# We train a regression random forest tree with
+grouping on transmission type (0 = automatic, 1 = manual)
+and use surrogates for NULL handling:
<pre class="example">
DROP TABLE IF EXISTS mt_cars_output, mt_cars_output_group, mt_cars_output_summary;
-SELECT madlib.forest_train('mt_cars',
- 'mt_cars_output',
- 'id',
- 'mpg',
- '*',
+SELECT madlib.forest_train('mt_cars', -- source table
+ 'mt_cars_output', -- output model table
+ 'id', -- id column
+ 'mpg', -- response
+ '*', -- features
'id, hp, drat, am, gear, carb', -- exclude columns
- 'am',
- 10::integer,
- 2::integer,
- TRUE::boolean,
- 1,
- 10,
- 8,
- 3,
- 10
+ 'am', -- grouping columns
+ 10::integer, -- number of trees
+ 2::integer, -- number of random features
+ TRUE::boolean, -- variable importance
+ 1, -- num_permutations
+ 10, -- max depth
+ 8, -- min split
+ 3, -- min bucket
+ 10, -- number of splits per continuous variable
+ 'max_surrogates=2' -- NULL handling
);
-\\x on
SELECT * FROM mt_cars_output_summary;
-SELECT * FROM mt_cars_output_group;
-\\x off
</pre>
-
--# Display a single tree of the random forest in dot format:
+<pre class="result">
+-[ RECORD 1 ]---------+-----------------------------------------------------------------------
+method | forest_train
+is_classification | f
+source_table | mt_cars
+model_table | mt_cars_output
+id_col_name | id
+dependent_varname | mpg
+independent_varnames | vs,cyl,disp,qsec,wt
+cat_features | vs,cyl
+con_features | disp,qsec,wt
+grouping_cols | am
+num_trees | 10
+num_random_features | 2
+max_tree_depth | 10
+min_split | 8
+min_bucket | 3
+num_splits | 10
+verbose | f
+importance | t
+num_permutations | 1
+num_all_groups | 2
+num_failed_groups | 0
+total_rows_processed | 32
+total_rows_skipped | 0
+dependent_var_levels |
+dependent_var_type | double precision
+independent_var_types | integer, integer, double precision, double precision, double precision
+null_proxy | None
+</pre>
+Review the group table to see variable importance by group:
<pre class="example">
-SELECT madlib.get_tree('mt_cars_output',1,1);
+SELECT * FROM mt_cars_output_group ORDER BY gid;
</pre>
-Result:
<pre class="result">
-digraph "Regression tree for mt_cars" {
-"0" [label="28.8444",shape=box];
-} //---end of digraph---------
+-[ RECORD 1 ]------+------------------------------------------------------
+gid | 1
+am | 0
+success | t
+cat_n_levels | {2,3}
+cat_levels_in_text | {0,1,4,6,8}
+oob_error | 7.61453121176269
+cat_var_importance | {1.67195685185185,1.78292533068783}
+con_var_importance | {4.74889044753087,3.00387950664048,0.730074734848485}
+-[ RECORD 2 ]------+------------------------------------------------------
+gid | 2
+am | 1
+success | t
+cat_n_levels | {2,3}
+cat_levels_in_text | {0,1,4,6,8}
+oob_error | 14.8126620718319
+cat_var_importance | {0.631733333333333,13.4535567708333}
+con_var_importance | {14.6258526998299,0.631733333333333,0}
</pre>
-# Predict regression output for the same data and compare with original:
@@ -828,47 +1090,225 @@ SELECT madlib.forest_predict('mt_cars_output',
'mt_cars',
'prediction_results',
'response');
-SELECT am, id, estimated_mpg, mpg
-FROM prediction_results JOIN mt_cars USING (id)
-ORDER BY am, id;
+SELECT s.am, s.id, mpg, estimated_mpg, mpg-estimated_mpg as delta
+FROM prediction_results p, mt_cars s WHERE s.id = p.id ORDER BY s.am, s.id;
</pre>
-Result:
<pre class="result">
- am | id | estimated_mpg | mpg
-----+----+------------------+------
- 0 | 1 | 15.893525974026 | 18.7
- 0 | 3 | 21.5238492063492 | 24.4
- 0 | 5 | 20.0175396825397 | 17.8
- 0 | 6 | 14.8406818181818 | 16.4
- 0 | 8 | 14.8406818181818 | 17.3
- 0 | 9 | 20.0496825396825 | 21.4
- 0 | 10 | 14.4012272727273 | 15.2
- 0 | 11 | 20.0175396825397 | 18.1
- 0 | 13 | 15.0162878787879 | 14.3
- 0 | 14 | 21.5238492063492 | 22.8
- 0 | 16 | 20.0175396825397 | 19.2
- 0 | 18 | 15.4787532467532 | 15.2
- 0 | 19 | 14.4272987012987 | 10.4
- 0 | 21 | 14.4272987012987 | 10.4
- 0 | 23 | 14.8667532467532 | 14.7
- 0 | 25 | 21.5238492063492 | 21.5
- 0 | 27 | 15.281525974026 | 15.5
- 0 | 29 | 15.0162878787879 | 13.3
- 0 | 30 | 15.281525974026 | 19.2
- 1 | 2 | 20.6527393162393 | 21
- 1 | 4 | 20.6527393162393 | 21
- 1 | 7 | 22.7707393162393 | 22.8
- 1 | 12 | 27.0888266178266 | 32.4
- 1 | 15 | 28.2478650793651 | 30.4
- 1 | 17 | 28.2478650793651 | 33.9
- 1 | 20 | 28.2478650793651 | 27.3
- 1 | 22 | 23.8401984126984 | 26
- 1 | 24 | 26.9748650793651 | 30.4
- 1 | 26 | 20.6527393162393 | 15.8
- 1 | 28 | 20.6527393162393 | 15
- 1 | 31 | 20.6527393162393 | 19.7
- 1 | 32 | 22.7707393162393 | 21.4
+ am | id | mpg | estimated_mpg | delta
+----+----+------+------------------+--------------------
+ 0 | 1 | 18.7 | 16.1644030112045 | 2.53559698879552
+ 0 | 3 | 24.4 | 20.1780892857143 | 4.22191071428571
+ 0 | 5 | 17.8 | 18.3336448412698 | -0.53364484126984
+ 0 | 6 | 16.4 | 16.073216503268 | 0.326783496732023
+ 0 | 8 | 17.3 | 16.073216503268 | 1.22678349673203
+ 0 | 9 | 21.4 | 19.1757281746032 | 2.22427182539683
+ 0 | 10 | 15.2 | 15.4498831699346 | -0.249883169934643
+ 0 | 11 | 18.1 | 18.3336448412698 | -0.233644841269843
+ 0 | 13 | 14.3 | 15.3339109477124 | -1.03391094771242
+ 0 | 14 | 22.8 | 20.1780892857143 | 2.62191071428571
+ 0 | 16 | 19.2 | 18.3336448412698 | 0.866355158730158
+ 0 | 18 | 15.2 | 16.1644030112045 | -0.964403011204485
+ 0 | 19 | 10.4 | 14.8702998366013 | -4.47029983660131
+ 0 | 21 | 10.4 | 14.8702998366013 | -4.47029983660131
+ 0 | 23 | 14.7 | 15.4936331699346 | -0.79363316993464
+ 0 | 25 | 21.5 | 20.1780892857143 | 1.32191071428571
+ 0 | 27 | 15.5 | 15.3339109477124 | 0.16608905228758
+ 0 | 29 | 13.3 | 15.3339109477124 | -2.03391094771242
+ 0 | 30 | 19.2 | 15.3614109477124 | 3.83858905228758
+ 1 | 2 | 21 | 22.2740158730159 | -1.27401587301587
+ 1 | 4 | 21 | 21.9821825396825 | -0.98218253968254
+ 1 | 7 | 22.8 | 26.1888286435786 | -3.38882864357864
+ 1 | 12 | 32.4 | 28.0211143578644 | 4.37888564213564
+ 1 | 15 | 30.4 | 28.5405588023088 | 1.8594411976912
+ 1 | 17 | 33.9 | 28.5405588023088 | 5.3594411976912
+ 1 | 20 | 27.3 | 27.6688921356421 | -0.368892135642131
+ 1 | 22 | 26 | 23.3459603174603 | 2.65403968253968
+ 1 | 24 | 30.4 | 26.8774516594517 | 3.52254834054834
+ 1 | 26 | 15.8 | 19.9121825396825 | -4.11218253968254
+ 1 | 28 | 15 | 19.9121825396825 | -4.91218253968254
+ 1 | 31 | 19.7 | 20.7306825396825 | -1.03068253968254
+ 1 | 32 | 21.4 | 24.6353286435786 | -3.23532864357864
+(32 rows)
+</pre>
+
+-# Display a single tree of the random forest in dot format:
+<pre class="example">
+SELECT madlib.get_tree('mt_cars_output',1,7);
</pre>
+<pre class="result">
+ digraph "Regression tree for mt_cars" {
+ "0" [label="disp <= 258", shape=ellipse];
+ "0" -> "1"[label="yes"];
+ "1" [label="20.35",shape=box];
+ "0" -> "2"[label="no"];
+ "2" [label="qsec <= 17.6", shape=ellipse];
+ "2" -> "5"[label="yes"];
+ "5" [label="15.8",shape=box];
+ "2" -> "6"[label="no"];
+ "6" [label="12.8",shape=box];
+ } //---end of digraph---------
+</pre>
+Display the surrogate variables that are
+used to compute the split for each node when
+the primary variable is NULL:
+<pre class="example">
+SELECT madlib.get_tree_surr('mt_cars_output',1,7);
+</pre>
+<pre class="result">
+ -------------------------------------
+ Surrogates for internal nodes
+ -------------------------------------
+ (0) disp <= 258
+ 1: wt <= 3.46 [common rows = 12]
+ 2: cyl in {4,6} [common rows = 11]
+ [Majority branch = 6 ]
+ (2) qsec <= 17.6
+ 1: wt <= 3.435 [common rows = 6]
+ 2: disp > 275.8 [common rows = 5]
+ [Majority branch = 4 ]
+</pre>
+
+<h4>NULL Handling Example</h4>
+
+-# Create toy example to illustrate 'null-as-category' handling
+for categorical features:
+<pre class='example'>
+DROP TABLE IF EXISTS null_handling_example;
+CREATE TABLE null_handling_example (
+ id integer,
+ country text,
+ city text,
+ weather text,
+ response text
+);
+INSERT INTO null_handling_example VALUES
+(1,null,null,null,'a'),
+(2,'US',null,null,'b'),
+(3,'US','NY',null,'c'),
+(4,'US','NY','rainy','d');
+</pre>
+
+-# Train random forest tree. Note that 'NULL' is set as a
+valid level for the categorical features country, weather and city:
+<pre class='example'>
+DROP TABLE IF EXISTS train_output, train_output_group, train_output_summary;
+SELECT madlib.forest_train('null_handling_example', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'response', -- response
+ 'country, weather, city', -- features
+ NULL, -- exclude columns
+ NULL, -- grouping columns
+ 10::integer, -- number of trees
+ 2::integer, -- number of random features
+ TRUE::boolean, -- variable importance
+ 1::integer, -- num_permutations
+ 3::integer, -- max depth
+ 2::integer, -- min split
+ 1::integer, -- min bucket
+ 3::integer, -- number of splits per continuous variable
+ 'null_as_category=TRUE'
+ );
+SELECT * FROM train_output_summary;
+</pre>
+<pre class='result'>
+-[ RECORD 1 ]---------+----------------------
+method | forest_train
+is_classification | t
+source_table | null_handling_example
+model_table | train_output
+id_col_name | id
+dependent_varname | response
+independent_varnames | country,weather,city
+cat_features | country,weather,city
+con_features |
+grouping_cols |
+num_trees | 10
+num_random_features | 2
+max_tree_depth | 3
+min_split | 2
+min_bucket | 1
+num_splits | 3
+verbose | f
+importance | t
+num_permutations | 1
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 4
+total_rows_skipped | 0
+dependent_var_levels | "a","b","c","d"
+dependent_var_type | text
+independent_var_types | text, text, text
+null_proxy | __NULL__
+</pre>
+View the summary table:
+<pre class='example'>
+\\x on
+SELECT * FROM train_output_group;
+</pre>
+<pre class='result'>
+-[ RECORD 1 ]------+------------------------------------------
+gid | 1
+success | t
+cat_n_levels | {2,2,2}
+cat_levels_in_text | {US,__NULL__,rainy,__NULL__,NY,__NULL__}
+oob_error | 1.00000000000000000000
+cat_var_importance | {0.0714285714285714,0.0714285714285714,0}
+con_var_importance |
+</pre>
+
+-# Predict for data not previously seen by assuming NULL
+value as the default:
+<pre class='example'>
+\\x off
+DROP TABLE IF EXISTS table_test;
+CREATE TABLE table_test (
+ id integer,
+ country text,
+ city text,
+ weather text,
+ expected_response text
+);
+INSERT INTO table_test VALUES
+(1,'IN','MUM','cloudy','a'),
+(2,'US','HOU','humid','b'),
+(3,'US','NY','sunny','c'),
+(4,'US','NY','rainy','d');
+DROP TABLE IF EXISTS prediction_results;
+SELECT madlib.forest_predict('train_output',
+ 'table_test',
+ 'prediction_results',
+ 'response');
+SELECT s.id, expected_response, estimated_response
+FROM prediction_results p, table_test s
+WHERE s.id = p.id ORDER BY id;
+</pre>
+<pre class='result'>
+ id | expected_response | estimated_response
+----+-------------------+--------------------
+ 1 | a | a
+ 2 | b | b
+ 3 | c | c
+ 4 | d | d
+(4 rows)
+</pre>
+There is only training data for country 'US' so the
+response for country 'IN' is 'a', corresponding to
+a NULL (not 'US') country level. Likewise, any
+city in the 'US' that is not 'NY' will predict
+response 'b', corresponding to a NULL (not 'NY')
+city level.
+
+@anchor literature
+@par Literature
+[1] L. Breiman and A. Cutler. Random Forests.
+http://www.stat.berkeley.edu/~breiman/RandomForests
+
+[2] L. Breiman, A. Cutler, A. Liaw, and M. Wiener.
+randomForest: Breiman and Cutler's Random Forests for
+Classification and Regression.
+http://cran.r-project.org/web/packages/randomForest/index.html
+
@anchor related
@par Related Topics
[3/5] madlib git commit: RF: Use NULL::integer[] when no continuous
features
Posted by ri...@apache.org.
RF: Use NULL::integer[] when no continuous features
JIRA: MADLIB-1219
When variable importance is enabled, to compute importance score,
distribution of the categorical and continuous features are computed.
For continuous features, this function requires initializing a vector of
length = number of continuous features. When there are no continuous
features, this initialization fails.
This commit fixes the issue by inputing a NULL::integer[] vector when
there no continuous features.
Closes #249
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/67e7cb27
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/67e7cb27
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/67e7cb27
Branch: refs/heads/master
Commit: 67e7cb272c19619c73454072544f62479fd1919c
Parents: c35107d
Author: Rahul Iyer <ri...@apache.org>
Authored: Sun Mar 25 21:45:47 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Mar 27 15:29:43 2018 -0700
----------------------------------------------------------------------
.../recursive_partitioning/random_forest.py_in | 15 +++++++++------
.../recursive_partitioning/test/random_forest.sql_in | 3 ++-
2 files changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/67e7cb27/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index 3961df4..ce6ef7f 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -946,6 +946,11 @@ def _calculate_oob_prediction(
oob_var_dist_view = unique_string()
if importance:
+ if con_features:
+ initialized_float_array = ("{0}.array_of_float({1})::integer[]".
+ format(schema_madlib, len(con_features)))
+ else:
+ initialized_float_array = "NULL::integer[]"
sql_create_oob_var_dist_view = """
CREATE VIEW {oob_var_dist_view} AS
SELECT
@@ -966,12 +971,10 @@ def _calculate_oob_prediction(
{con_features_str}::double precision[],
con_splits
), -- bin_indices, -1 for NaN
- 1 -- -1 shift to 0 for nulls
+ 1 -- -1 shifted to 0 for null values
),
- {schema_madlib}.array_fill(
- {schema_madlib}.array_of_float({n_con})::integer[],
- ({num_bins}+1)::integer
- )
+ {schema_madlib}.array_fill({initialized_float_array},
+ ({num_bins} + 1)::integer)
-- level of any continuous feature == num_bins
) AS con_index_distributions
FROM
@@ -986,7 +989,7 @@ def _calculate_oob_prediction(
{con_splits_table}
USING (gid)
GROUP BY gid
- """.format(n_con=len(con_features), **locals())
+ """.format(**locals())
else:
sql_create_oob_var_dist_view = """
CREATE VIEW {oob_var_dist_view} AS
http://git-wip-us.apache.org/repos/asf/madlib/blob/67e7cb27/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
index 8fcb75b..6ab00a7 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
@@ -265,6 +265,7 @@ from generate_series(1, 5) sid;
-------------------------------------------------------------------------
-- Test case for the case where a group has a filtered feature
+-- Also testing var_importance=TRUE with no continuous features (MADLIB-1219)
DROP TABLE IF EXISTS rf_gr_test CASCADE;
CREATE TABLE rf_gr_test (
@@ -289,7 +290,7 @@ SELECT forest_train(
'train_output', -- output model table
'id', -- id column
'cl', -- response
- 'f1, f2, f3', -- features
+ 'f1, f2', -- features
NULL, -- exclude columns
'gr', -- grouping
2, -- num of trees
[5/5] madlib git commit: Doc: Update DT and RF doc
Posted by ri...@apache.org.
Doc: Update DT and RF doc
This commit adds more explanation in documentation for decision tree
and random forest.
Closes #246
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/7f3aae92
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/7f3aae92
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/7f3aae92
Branch: refs/heads/master
Commit: 7f3aae92f2d84bf7e4501ac5efec1ebfc7a80834
Parents: 67e7cb2
Author: Frank McQuillan <fm...@pivotal.io>
Authored: Tue Mar 27 15:34:58 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Mar 27 15:45:44 2018 -0700
----------------------------------------------------------------------
doc/etc/developer.doxyfile.in | 2 +-
doc/etc/user.doxyfile.in | 2 +-
.../recursive_partitioning/decision_tree.sql_in | 996 +++++++++-----
.../recursive_partitioning/random_forest.sql_in | 1246 ++++++++++++------
4 files changed, 1528 insertions(+), 718 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/7f3aae92/doc/etc/developer.doxyfile.in
----------------------------------------------------------------------
diff --git a/doc/etc/developer.doxyfile.in b/doc/etc/developer.doxyfile.in
index a50c303..7d741af 100644
--- a/doc/etc/developer.doxyfile.in
+++ b/doc/etc/developer.doxyfile.in
@@ -40,7 +40,7 @@ PROJECT_NUMBER = @MADLIB_VERSION_STRING@
# for a project that appears at the top of each page and should give viewer
# a quick idea about the purpose of the project. Keep the description short.
-PROJECT_BRIEF = "Developer Documentation for MADlib"
+PROJECT_BRIEF = "Developer Documentation for Apache MADlib"
# With the PROJECT_LOGO tag one can specify an logo or icon that is
# included in the documentation. The maximum height of the logo should not
http://git-wip-us.apache.org/repos/asf/madlib/blob/7f3aae92/doc/etc/user.doxyfile.in
----------------------------------------------------------------------
diff --git a/doc/etc/user.doxyfile.in b/doc/etc/user.doxyfile.in
index 230aae8..14b7aba 100644
--- a/doc/etc/user.doxyfile.in
+++ b/doc/etc/user.doxyfile.in
@@ -23,7 +23,7 @@
# for a project that appears at the top of each page and should give viewer a
# quick idea about the purpose of the project. Keep the description short.
-PROJECT_BRIEF = "User Documentation for MADlib"
+PROJECT_BRIEF = "User Documentation for Apache MADlib"
# With the PROJECT_LOGO tag one can specify an logo or icon that is
# included in the documentation. The maximum height of the logo should not
http://git-wip-us.apache.org/repos/asf/madlib/blob/7f3aae92/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
index cb6a8ea..a3c4963 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.sql_in
@@ -18,14 +18,15 @@ m4_include(`SQLCommon.m4')
<div class="toc"><b>Contents</b><ul>
<li class="level1"><a href="#train">Training Function</a></li>
<li class="level1"><a href="#predict">Prediction Function</a></li>
-<li class="level1"><a href="#display">Display Function</a></li>
+<li class="level1"><a href="#display">Tree Display</a></li>
<li class="level1"><a href="#examples">Examples</a></li>
+<li class="level1"><a href="#literature">Literature</a></li>
<li class="level1"><a href="#related">Related Topics</a></li>
</ul></div>
-@brief Decision Trees.
-Decision trees use a tree-based model to predict the value of a
-target variable based on several input variables.
+@brief
+Decision trees are tree-based supervised learning methods
+that can be used for classification and regression.
A decision tree is a supervised learning method that can be used for
classification and regression. It consists of a structure in which
@@ -36,7 +37,8 @@ or regression rules.
@anchor train
@par Training Function
-We implement the decision tree using the CART algorithm, introduced by Breiman et al. [1].
+We implement the decision tree using the CART algorithm
+introduced by Breiman et al. [1].
The training function has the following syntax:
<pre class="syntax">
tree_train(
@@ -61,13 +63,15 @@ tree_train(
\b Arguments
<dl class="arglist">
<dt>training_table_name</dt>
- <dd>TEXT. The name of the table containing the training data.</dd>
+ <dd>TEXT. Name of the table containing the training data.</dd>
<dt>output_table_name</dt>
- <dd>TEXT. The name of the generated table containing the model.
- If a table with the same name already exists, then the function will return an
- error. A summary table named <em>\<output_table_name\>_summary</em> is also
- created at the same time.
+ <dd>TEXT. Name of the generated table containing the model.
+ If a table with the same name already exists, an
+ error will be returned. A summary table
+ named <em>\<output_table_name\>_summary</em> is also
+ created. A cross-validation table <em>\<output_table_name\>_cv</em>
+ may also be created. These are described later on this page.
</DD>
<DT>id_col_name</DT>
@@ -78,8 +82,8 @@ tree_train(
<DT>dependent_variable</DT>
<DD>TEXT. Name of the column that contains the output (response) for
- training. Boolean, integer and text types are considered classification
- outputs, while double precision values are considered regression outputs.
+ training. Boolean, integer and text types are considered to be classification
+ outputs, while double precision values are considered to be regression outputs.
The response variable for a classification tree can be multinomial, but the
time and space complexity of the training function increases linearly as the
number of response classes increases.</DD>
@@ -88,15 +92,15 @@ tree_train(
<DD>TEXT. Comma-separated string of column names or expressions to use as predictors.
Can also be a '*' implying all columns are to be used as predictors (except for the
ones included in the next argument that lists exclusions).
- The types of the features can be mixed - boolean, integer, and text columns
+ The types of the features can be mixed: boolean, integer, and text columns
are considered categorical and
double precision columns are considered continuous. Categorical variables
- are not encoded and used as is for the training.
+ are not encoded and used as is in the training.
Array columns can also be included in the list, where the array is expanded
to treat each element of the array as a feature.
-
- It is important to note that not every combination of the levels of a
+
+ Note that not every combination of the levels of a
categorical variable is checked when evaluating a split. The levels of the
non-integer categorical variable are ordered by the entropy of the variable in
predicting the response. The split at each node is evaluated between these
@@ -115,21 +119,29 @@ tree_train(
<DT>split_criterion</DT>
<DD>TEXT, default = 'gini' for classification, 'mse' for regression.
- Impurity function to compute the feature to use for the split.
+ Impurity function to compute the feature to use to split a node.
Supported criteria are 'gini', 'entropy', 'misclassification' for
- classification trees. For regression trees, split_criterion of 'mse'
- is always used (irrespective of the input for this argument). </DD>
+ classification trees. For regression trees, split_criterion
+ of 'mse' (mean-squared error) is always used, irrespective of
+ the input for this argument.
+ Refer to reference [1] for more information on impurity measures.</DD>
<DT>grouping_cols (optional)</DT>
<DD>TEXT, default: NULL. Comma-separated list of column names to group the
- data by. This will result in multiple decision trees, one for
+ data by. This will produce multiple decision trees, one for
each group. </DD>
<DT>weights (optional)</DT>
- <DD>TEXT. Column name containing numerical weights for each observation.
+ <DD>TEXT. Column name containing numerical weights for
+ each observation. Can be any value greater
+ than 0 (does not need to be an integer).
This can be used to handle the case of unbalanced data sets.
- If this parameter is not set, all observations (tuples)
- are treated equally with a weight of 1.0.</DD>
+ The weights are used to compute a weighted average in
+ the output leaf node. For classification, the contribution
+ of a row towards the vote of its corresponding level
+ is multiplied by the weight (weighted mode). For regression,
+ the output value of the row is multiplied by
+ the weight (weighted mean).</DD>
<DT>max_depth (optional)</DT>
<DD>INTEGER, default: 7. Maximum depth of any node of the final tree,
@@ -156,14 +168,16 @@ tree_train(
<DT>pruning_params (optional)</DT>
<DD>TEXT. Comma-separated string of key-value pairs giving
- the parameters for pruning the tree. The parameters currently accepted are:
+ the parameters for pruning the tree.
<table class='output'>
<tr>
<th>cp</th>
<td>
- Default: 0. A split on a node is attempted only if it
- decreases the overall lack of fit by a factor of 'cp', else the split is
- pruned away. This value is used to create an initial tree before running
+ Default: 0. Complexity parameter.
+ A split on a node is attempted only if it
+ decreases the overall lack of fit by a factor of 'cp',
+ otherwise the split is pruned away. This value is used
+ to create an initial tree before running
cross-validation (see below).
</td>
@@ -174,9 +188,10 @@ tree_train(
Default: 0 (i.e. no cross-validation).
Number of cross-validation folds to use to compute the best value of
<em>cp</em>. To perform cross-validation, a positive value of
- <em>n_folds</em> (greater than 2) should be given. An additional output
+ <em>n_folds</em> (2 or more) should be specified. An additional output
table <em>\<model_table\>_cv</em> is created containing the values of
- evaluated <em>cp</em> and the cross-validation error. The tree returned
+ evaluated <em>cp</em> and the cross-validation error
+ statistics. The tree returned
in the output table corresponds to the <em>cp</em> with the lowest
cross-validation error (we pick the maximum <em>cp</em> if multiple
values have same error).
@@ -193,24 +208,33 @@ tree_train(
<DT>null_handling_params (optional)</DT>
<DD>TEXT. Comma-separated string of key-value pairs controlling the behavior
- of various features handling missing values.
+ of various features handling missing values. One of the following can
+ be used if desired (not both):
<table class='output'>
<tr>
<th>max_surrogates</th>
<td>Default: 0. Number of surrogates to store for each node.</td>
- One of the approaches of handling NULLs is to use surrogate splits for each
- node. A surrogate variable is another predictor variable that is associated
- (correlated) with the primary split variable. The surrogate
+ One approach to handling NULLs is to use surrogate splits for each
+ node. A surrogate variable enables you to make better use of
+ the data by using another predictor variable that is associated
+ (correlated) with the primary split variable. The surrogate
variable comes into use when the primary predictior value is NULL.
+ Surrogate rules implemented here are based on reference [1].
</tr>
<tr>
<th>null_as_category</th>
- <td>Default: FALSE. Whether to treat NULL as a special categorical value.
-
- If this is set to TRUE, NULL values are considered a categorical value and
- placed at the end of the ordering of categorical levels. Placing it at the
- end ensures that NULL is never used as a value to split a node on. This
- parameter is ignored for continuous-valued features.
+ <td>Default: FALSE. Whether to treat NULL as a valid level
+ for categorical features. FALSE means that NULL is not a
+ valid level, which is probably the most common sitation.
+
+ If set to TRUE, NULL values are considered a categorical value and
+ placed at the end of the ordering of categorical levels. Placing at the
+ end ensures that NULL is never used as a value to split a node on.
+ One reason to make NULL a category is that it allows you to
+ predict on categorical levels that were not in the training
+ data by lumping them into an "other bucket."
+
+ This parameter is ignored for continuous-valued features.
</td>
</tr>
</table>
@@ -233,15 +257,27 @@ tree_train(
</tr>
<tr>
<th>tree</th>
- <td>BYTEA8. Trained decision tree model stored in a binary format.</td>
+ <td>BYTEA8. Trained decision tree model stored in binary
+ format (not human readable).</td>
</tr>
<tr>
<th>cat_levels_in_text</th>
- <td>TEXT[]. Ordered levels of categorical variables.</td>
+ <td>TEXT[]. Ordered levels (values) of categorical variables
+ corresponding to the categorical features in
+ the 'list_of_features' argument above. Used to help
+ interpret the trained decision tree. For example, if the
+ categorical features specified are <em>weather_outlook</em>
+ and <em>windy</em> in that order, then 'cat_levels_in_text'
+ might be <em>[overcast, rain, sunny, False, True]</em>.</td>
</tr>
<tr>
<th>cat_n_levels</th>
- <td>INTEGER[]. Number of levels for each categorical variable.</td>
+ <td>INTEGER[]. Number of levels for each categorical variable.
+ Used to help interpret the trained decision tree. In the example
+ from above, 'cat_n_levels' would
+ be <em>[3, 2]</em> since there are 3 levels
+ for <em>weather_outlook</em> and 2 levels
+ <em>windy</em>.</td>
</tr>
<tr>
@@ -251,9 +287,9 @@ tree_train(
<tr>
<th>pruning_cp</th>
- <td>DOUBLE PRECISION. The cost-complexity parameter used for pruning
- the trained tree(s). This would be different from the 'input_cp' value if
- cross-validation is used.
+ <td>DOUBLE PRECISION. The cost complexity parameter used for pruning
+ the trained tree(s). This could be different than the cp value input
+ using the <em>pruning_params</em> if cross-validation is used.
</td>
</tr>
@@ -294,7 +330,7 @@ tree_train(
</tr>
<tr>
- <th>independent_varname</th>
+ <th>independent_varnames</th>
<td>TEXT. The independent variables.</td>
</tr>
@@ -309,7 +345,7 @@ tree_train(
</tr>
<tr>
- <th>grouping_col</th>
+ <th>grouping_cols</th>
<td>TEXT. Names of grouping columns.</td>
</tr>
@@ -369,10 +405,30 @@ tree_train(
set to "__NULL__"</td>
</tr>
</table>
+
+ A cross-validation table called <em>\<output_table_name\>_cv</em>
+ is created if 'n_folds' is set in the 'pruning_params'.
+ It has the following columns:
+ <table class="output">
+
+ <tr>
+ <th>cp</th>
+ <td>DOUBLE PRECISION. Complexity parameter.</td>
+ </tr>
+
+ <tr>
+ <th>cv_error_avg</th>
+ <td>DOUBLE PRECISION. Average error resulting from cp value.</td>
+ </tr>
+
+ <tr>
+ <th>cv_error_stdev</th>
+ <td>DOUBLE PRECISION. Standard deviation resulting from cp value.</td>
+ </tr>
+ </table>
</DD>
</DL>
-
@note
- Many of the parameters are designed to be similar to the popular R package 'rpart'.
An important distinction between rpart and the MADlib function is that
@@ -381,10 +437,8 @@ categorical values, while rpart considers them as continuous. To use integers as
continuous, cast them to double precision.
- Integer values are ordered by value for computing the split boundaries. Cast
to TEXT if the entropy-based ordering method is desired.
-- When using no surrogates (<em>max_surrogates</em>=0), all rows containing NULL values
-for any of the features used for training will be ignored from training and prediction.
- When cross-validation is not used (<em>n_folds</em>=0), each tree output
-is pruned by the input cost-complextity (<em>cp</em>). With cross-validation,
+is pruned by the input cost complexity (<em>cp</em>). With cross-validation,
the input <em>cp</em> is the minimum value of all the explored values of 'cp'.
During cross-validation, we train an initial tree using the
provided <em>cp</em> and explore all possible sub-trees (up to a single-node tree)
@@ -408,7 +462,6 @@ tree_predict(tree_model,
type)
</pre>
-
\b Arguments
<DL class="arglist">
<DT>tree_model</DT>
@@ -445,11 +498,8 @@ tree_predict(tree_model,
</DD>
</DL>
-@note If the <em>new_data_table</em> contains categories of categorical variables
-not seen in the training data, the prediction for that row will be NULL.
-
@anchor display
-@par Display Function
+@par Tree Display
The display function outputs a graph representation of the
decision tree. The output can either be in the popular 'dot' format that can
be visualized using various programs including those in the GraphViz package, or
@@ -465,7 +515,7 @@ for each internal node:
tree_surr_display(tree_model)
</pre>
-The output contains the list of surrogate splits for each internal node. The
+This output contains the list of surrogate splits for each internal node. The
nodes are sorted in ascending order by id. This is equivalent to viewing the
tree in a breadth-first manner. For each surrogate, we output the surrogate
split (variable and threshold) and also give the number of rows that were common
@@ -496,9 +546,11 @@ redirected to a file on the client side and then rendered using visualization
programs.
To export the dot format result to an external file,
-use the method below. Use unaligned
-table output mode for psql with '-A' flag. And inside the psql client,
-both '\\t' and '\\o' should be used):
+use the method below. Please note that you should use unaligned
+table output mode for psql with '-A' flag, or else you may get an
+error when you try to convert the dot file to another format
+for viewing (e.g., PDF). And inside the psql client,
+both '\\t' and '\\o' should be used:
<pre class="example">
\> \# under bash
@@ -524,9 +576,10 @@ of the tree output formats.
@anchor examples
@examp
-<h4>Decision Tree Classification Example</h4>
+<h4>Decision Tree Classification Examples</h4>
--# Prepare input data:
+-# Load input data set related to whether
+to play golf or not:
<pre class="example">
DROP TABLE IF EXISTS dt_golf CASCADE;
CREATE TABLE dt_golf (
@@ -534,29 +587,27 @@ CREATE TABLE dt_golf (
"OUTLOOK" text,
temperature double precision,
humidity double precision,
- "Cont_features" double precision[],
- cat_features text[],
+ "Temp_Humidity" double precision[],
+ clouds_airquality text[],
windy boolean,
- class text
+ class text,
+ observation_weight double precision
);
-</pre>
-<pre class="example">
-COPY dt_golf (id,"OUTLOOK",temperature,humidity,"Cont_features",cat_features,windy,class) FROM stdin WITH DELIMITER '|';
-1|sunny|85|85|{85, 85}|{'a', 'b'}|false|'Don't Play'
-2|sunny|80|90|{80, 90}|{'a', 'b'}|true|'Don't Play'
-3|overcast|83|78|{83, 78}|{'a', 'b'}|false|'Play'
-4|rain|70|96|{70, 96}|{'c', 'd'}|false|'Play'
-5|rain|68|80|{68, 80}|{'a', 'b'}|false|'Play'
-6|rain|65|70|{65, 70}|{'a', 'b'}|true|'Don't Play'
-7|overcast|64|65|{64, 65}|{'a', 'b'}|true|'Play'
-8|sunny|72|95|{72, 95}|{'c', 'd'}|false|'Don't Play'
-9|sunny|69|70|{69, 70}|{'a', 'b'}|false|'Play'
-10|rain|75|80|{75, 80}|{'a', 'b'}|false|'Play'
-11|sunny|75|70|{75, 70}|{'c', 'd'}|true|'Play'
-12|overcast|72|90|{72, 90}|{'a', 'b'}|true|'Play'
-13|overcast|81|75|{81, 75}|{'a', 'b'}|false|'Play'
-14|rain|71|80|{71, 80}|{'a', 'b'}|true|'Don't Play'
-\\.
+INSERT INTO dt_golf VALUES
+(1,'sunny', 85, 85, ARRAY[85, 85],ARRAY['none', 'unhealthy'], 'false','Don''t Play', 5.0),
+(2, 'sunny', 80, 90, ARRAY[80, 90], ARRAY['none', 'moderate'], 'true', 'Don''t Play', 5.0),
+(3, 'overcast', 83, 78, ARRAY[83, 78], ARRAY['low', 'moderate'], 'false', 'Play', 1.5),
+(4, 'rain', 70, 96, ARRAY[70, 96], ARRAY['low', 'moderate'], 'false', 'Play', 1.0),
+(5, 'rain', 68, 80, ARRAY[68, 80], ARRAY['medium', 'good'], 'false', 'Play', 1.0),
+(6, 'rain', 65, 70, ARRAY[65, 70], ARRAY['low', 'unhealthy'], 'true', 'Don''t Play', 1.0),
+(7, 'overcast', 64, 65, ARRAY[64, 65], ARRAY['medium', 'moderate'], 'true', 'Play', 1.5),
+(8, 'sunny', 72, 95, ARRAY[72, 95], ARRAY['high', 'unhealthy'], 'false', 'Don''t Play', 5.0),
+(9, 'sunny', 69, 70, ARRAY[69, 70], ARRAY['high', 'good'], 'false', 'Play', 5.0),
+(10, 'rain', 75, 80, ARRAY[75, 80], ARRAY['medium', 'good'], 'false', 'Play', 1.0),
+(11, 'sunny', 75, 70, ARRAY[75, 70], ARRAY['none', 'good'], 'true', 'Play', 5.0),
+(12, 'overcast', 72, 90, ARRAY[72, 90], ARRAY['medium', 'moderate'], 'true', 'Play', 1.5),
+(13, 'overcast', 81, 75, ARRAY[81, 75], ARRAY['medium', 'moderate'], 'false', 'Play', 1.5),
+(14, 'rain', 71, 80, ARRAY[71, 80], ARRAY['low', 'unhealthy'], 'true', 'Don''t Play', 1.0);
</pre>
-# Run the decision tree training function:
@@ -566,7 +617,7 @@ SELECT madlib.tree_train('dt_golf', -- source table
'train_output', -- output model table
'id', -- id column
'class', -- response
- '"OUTLOOK", "Cont_features", windy', -- features
+ '"OUTLOOK", temperature, windy', -- features
NULL::text, -- exclude columns
'gini', -- split criterion
NULL::text, -- no grouping
@@ -577,42 +628,112 @@ SELECT madlib.tree_train('dt_golf', -- source table
10 -- number of bins per continuous variable
);
</pre>
+View the output table (excluding the tree which is in binary format):
+<pre class="example">
+SELECT pruning_cp, cat_levels_in_text, cat_n_levels, tree_depth FROM train_output;
+</pre>
+<pre class="result">
+ pruning_cp | cat_levels_in_text | cat_n_levels | tree_depth
+------------+----------------------------------+--------------+------------
+ 0 | {overcast,rain,sunny,False,True} | {3,2} | 5
+</pre>
+View the summary table:
+<pre class="example">
+\\x on
+SELECT * FROM train_output_summary;
+</pre>
+<pre class="result">
+-[ RECORD 1 ]---------+--------------------------------
+method | tree_train
+is_classification | t
+source_table | dt_golf
+model_table | train_output
+id_col_name | id
+dependent_varname | class
+independent_varnames | "OUTLOOK", temperature, windy
+cat_features | "OUTLOOK",windy
+con_features | temperature
+grouping_cols |
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 14
+total_rows_skipped | 0
+dependent_var_levels | "Don't Play","Play"
+dependent_var_type | text
+input_cp | 0
+independent_var_types | text, boolean, double precision
+n_folds | 0
+null_proxy |
+</pre>
--# Predict output categories for the same data that was used for input:
+-# Predict output categories. For the purpose
+of this example, we use the same data that was
+used for training:
<pre class="example">
+\\x off
DROP TABLE IF EXISTS prediction_results;
SELECT madlib.tree_predict('train_output', -- tree model
'dt_golf', -- new data table
'prediction_results', -- output table
- 'response'); -- show prediction
-SELECT g.id, class, estimated_class FROM prediction_results p, dt_golf g where p.id = g.id ORDER BY g.id;
+ 'response'); -- show response
+SELECT g.id, class, estimated_class FROM prediction_results p,
+dt_golf g WHERE p.id = g.id ORDER BY g.id;
+</pre>
+<pre class="result">
+ id | class | estimated_class
+----+------------+-----------------
+ 1 | Don't Play | Don't Play
+ 2 | Don't Play | Don't Play
+ 3 | Play | Play
+ 4 | Play | Play
+ 5 | Play | Play
+ 6 | Don't Play | Don't Play
+ 7 | Play | Play
+ 8 | Don't Play | Don't Play
+ 9 | Play | Play
+ 10 | Play | Play
+ 11 | Play | Play
+ 12 | Play | Play
+ 13 | Play | Play
+ 14 | Don't Play | Don't Play
+(14 rows)
+</pre>
+To display the probabilities associated with each
+value of the dependent variable, set the 'type' parameter
+to 'prob':
+<pre class="example">
+DROP TABLE IF EXISTS prediction_results;
+SELECT madlib.tree_predict('train_output', -- tree model
+ 'dt_golf', -- new data table
+ 'prediction_results', -- output table
+ 'prob'); -- show probability
+SELECT g.id, class, "estimated_prob_Don't Play", "estimated_prob_Play"
+FROM prediction_results p, dt_golf g WHERE p.id = g.id ORDER BY g.id;
</pre>
-Result:
<pre class="result">
-id | class | estimated_class
-----+--------------+-----------------
- 1 | 'Don't Play' | 'Don't Play'
- 2 | 'Don't Play' | 'Don't Play'
- 3 | 'Play' | 'Play'
- 4 | 'Play' | 'Play'
- 5 | 'Play' | 'Play'
- 6 | 'Don't Play' | 'Don't Play'
- 7 | 'Play' | 'Play'
- 8 | 'Don't Play' | 'Don't Play'
- 9 | 'Play' | 'Play'
- 10 | 'Play' | 'Play'
- 11 | 'Play' | 'Play'
- 12 | 'Play' | 'Play'
- 13 | 'Play' | 'Play'
- 14 | 'Don't Play' | 'Don't Play'
+ id | class | estimated_prob_Don't Play | estimated_prob_Play
+----+------------+---------------------------+---------------------
+ 1 | Don't Play | 1 | 0
+ 2 | Don't Play | 1 | 0
+ 3 | Play | 0 | 1
+ 4 | Play | 0 | 1
+ 5 | Play | 0 | 1
+ 6 | Don't Play | 1 | 0
+ 7 | Play | 0 | 1
+ 8 | Don't Play | 1 | 0
+ 9 | Play | 0 | 1
+ 10 | Play | 0 | 1
+ 11 | Play | 0 | 1
+ 12 | Play | 0 | 1
+ 13 | Play | 0 | 1
+ 14 | Don't Play | 1 | 0
(14 rows)
</pre>
--# Create a text display of the tree:
+-# View the tree in text format:
<pre class="example">
SELECT madlib.tree_display('train_output', FALSE);
</pre>
-Result:
<pre class="result">
-------------------------------------
- Each node represented by 'id' inside ().
@@ -624,119 +745,218 @@ Result:
The response label order is given as ['"\'Don\'t Play\'"', '"\'Play\'"'].
For each leaf, the prediction is given after the '-->'
-------------------------------------
-(0)[5 9] "OUTLOOK" in {overcast}
- (1)[0 4] * \-\-> "'Play'"
- (2)[5 5] "Cont_features"[1] <= 75
- (5)[3 5] "Cont_features"[1] <= 65
- (11)[1 0] * \-\-> "'Don't Play'"
- (12)[2 5] "Cont_features"[1] <= 70
- (25)[0 3] * \-\-> "'Play'"
- (26)[2 2] "Cont_features"[1] <= 72
- (53)[2 0] * \-\-> "'Don't Play'"
- (54)[0 2] * \-\-> "'Play'"
- (6)[2 0] * \-\-> "'Don't Play'"
+ (0)[5 9] "OUTLOOK" in {overcast}
+ (1)[0 4] * --> "Play"
+ (2)[5 5] temperature <= 75
+ (5)[3 5] temperature <= 65
+ (11)[1 0] * --> "Don't Play"
+ (12)[2 5] temperature <= 70
+ (25)[0 3] * --> "Play"
+ (26)[2 2] temperature <= 72
+ (53)[2 0] * --> "Don't Play"
+ (54)[0 2] * --> "Play"
+ (6)[2 0] * --> "Don't Play"
-------------------------------------
</pre>
-Here are some more details on how to interpret the tree display above...
+Here are some details on how to interpret the tree display above:
- Node numbering starts at 0 for the root node and would be
contiguous 1,2...n if the tree was completely full (no pruning).
- - Since the tree has been pruned, the node numbering is not
+Since the tree has been pruned, the node numbering is not
contiguous.
- The order of values [x y] indicates the number of weighted
rows that correspond to ["Don't play" "Play"] <em>before</em> the node test.
-For example, at (root) node 0, there are 5 rows that are "Don't play"
-and 9 rows that are "Play" in the raw data.
- - If we apply the test of "OUTLOOK" being overcast, then the True result is
+For example, at (root) node 0, there are 5 rows for "Don't play"
+and 9 rows for "Play" in the raw data.
+ - If we apply the test of "OUTLOOK" being overcast, then the True (yes) result is
leaf node 1 which is "Play". There are 0 "Don't play" rows
and 4 "Play" rows that correspond to this case (overcast).
-The remaining 5 "Don't play" rows and 5 "Play rows" are then
-tested at node 2 on "Cont_features[1]"<=75. And so on down the tree.
+In other words, if it is overcast, you always play golf. If it is not
+overcast, you may or may not play golf, depending on the rest
+of the tree.
+ - The remaining 5 "Don't play" rows and 5 "Play rows" are then
+tested at node 2 on temperature<=75. The False (no) result is
+leaf node 6 which is "Don't Play". The True (yes) result proceeds
+to leaf node 5 to test on temperature<=65. And so on down the tree.
+ - Creating a dot format visualization of the tree, as described
+ below, can help with following the decision flows.
-# Create a dot format display of the tree:
<pre class="example">
SELECT madlib.tree_display('train_output', TRUE);
</pre>
-Result:
<pre class="result">
-digraph "Classification tree for dt_golf" {
- subgraph "cluster0"{
- label=""
-"g0_0" [label="\"OUTLOOK\" <= overcast", shape=ellipse];
-"g0_0" -> "g0_1"[label="yes"];
-"g0_1" [label="\"'Play'\"",shape=box];
-"g0_0" -> "g0_2"[label="no"];
-"g0_2" [label="\"Cont_features\"[1] <= 75", shape=ellipse];
-"g0_2" -> "g0_5"[label="yes"];
-"g0_2" -> "g0_6"[label="no"];
-"g0_6" [label="\"'Don't Play'\"",shape=box];
-"g0_5" [label="\"Cont_features\"[1] <= 65", shape=ellipse];
-"g0_5" -> "g0_11"[label="yes"];
-"g0_11" [label="\"'Don't Play'\"",shape=box];
-"g0_5" -> "g0_12"[label="no"];
-"g0_12" [label="\"Cont_features\"[1] <= 70", shape=ellipse];
-"g0_12" -> "g0_25"[label="yes"];
-"g0_25" [label="\"'Play'\"",shape=box];
-"g0_12" -> "g0_26"[label="no"];
-"g0_26" [label="\"Cont_features\"[1] <= 72", shape=ellipse];
-"g0_26" -> "g0_53"[label="yes"];
-"g0_53" [label="\"'Don't Play'\"",shape=box];
-"g0_26" -> "g0_54"[label="no"];
-"g0_54" [label="\"'Play'\"",shape=box];
+ digraph "Classification tree for dt_golf" {
+ subgraph "cluster0"{
+ label=""
+ "g0_0" [label="\"OUTLOOK\" <= overcast", shape=ellipse];
+ "g0_0" -> "g0_1"[label="yes"];
+ "g0_1" [label="\"Play\"",shape=box];
+ "g0_0" -> "g0_2"[label="no"];
+ "g0_2" [label="temperature <= 75", shape=ellipse];
+ "g0_2" -> "g0_5"[label="yes"];
+ "g0_2" -> "g0_6"[label="no"];
+ "g0_6" [label="\"Don't Play\"",shape=box];
+ "g0_5" [label="temperature <= 65", shape=ellipse];
+ "g0_5" -> "g0_11"[label="yes"];
+ "g0_11" [label="\"Don't Play\"",shape=box];
+ "g0_5" -> "g0_12"[label="no"];
+ "g0_12" [label="temperature <= 70", shape=ellipse];
+ "g0_12" -> "g0_25"[label="yes"];
+ "g0_25" [label="\"Play\"",shape=box];
+ "g0_12" -> "g0_26"[label="no"];
+ "g0_26" [label="temperature <= 72", shape=ellipse];
+ "g0_26" -> "g0_53"[label="yes"];
+ "g0_53" [label="\"Don't Play\"",shape=box];
+ "g0_26" -> "g0_54"[label="no"];
+ "g0_54" [label="\"Play\"",shape=box];
} //--- end of subgraph------------
} //---end of digraph---------
</pre>
+One important difference to note about the dot format above is how categorical
+variable tests are displayed:
+ - In the text format of the tree, the node 0
+test is "OUTLOOK" in {overcast}, but in the dot format of the tree,
+the same node 0 test reads "\"OUTLOOK\" <= overcast". This is because
+in dot format for categorical variables, the '<=' symbol
+represents the location in the array 'cat_levels_in_text' from the output
+table for the "OUTLOOK" levels. The array
+is ['overcast', 'rain', 'sunny', 'False', 'True'] with the first 3 entries
+corresponding to "OUTLOOK" and the last 2 entries corresponding to 'windy'. So the
+test "\"OUTLOOK\" <= overcast" means all "OUTLOOK" levels to the
+left of, and including, 'overcast'. In this case there are no levels
+to the left of 'overcast' in the array so it is simply a test on
+whether it is overcast or not.
+ - If there was a test "\"OUTLOOK\" <= rain", this would include
+both 'overcast' and 'rain', since 'overcast' is to the left of 'rain'
+in the array.
+ - If there was a test "windy <= True", this would include
+both 'False' and 'True', since 'False' is to the left of 'True'
+in the array.
-# Now create a dot format display of the tree with additional information:
<pre class="example">
SELECT madlib.tree_display('train_output', TRUE, TRUE);
</pre>
-Result:
<pre class="result">
-digraph "Classification tree for dt_golf" {
- subgraph "cluster0"{
- label=""
-"g0_0" [label="\"OUTLOOK\" <= overcast\\n impurity = 0.459184\\n samples = 14\\n value = [5 9
-]\\n class = \"'Play'\"", shape=ellipse];
-"g0_0" -> "g0_1"[label="yes"];
-"g0_1" [label="\"'Play'\"\\n impurity = 0\\n samples = 4\\n value = [0 4
-]",shape=box];
-"g0_0" -> "g0_2"[label="no"];
-"g0_2" [label="\"Cont_features\"[1] <= 75\\n impurity = 0.5\\n samples = 10\\n value = [5 5
-]\\n class = \"'Don't Play'\"", shape=ellipse];
-"g0_2" -> "g0_5"[label="yes"];
-"g0_2" -> "g0_6"[label="no"];
-"g0_6" [label="\"'Don't Play'\"\\n impurity = 0\\n samples = 2\\n value = [2 0
-]",shape=box];
-"g0_5" [label="\"Cont_features\"[1] <= 65\\n impurity = 0.46875\\n samples = 8\\n value = [3 5
-]\\n class = \"'Play'\"", shape=ellipse];
-"g0_5" -> "g0_11"[label="yes"];
-"g0_11" [label="\"'Don't Play'\"\\n impurity = 0\\n samples = 1\\n value = [1 0
-]",shape=box];
-"g0_5" -> "g0_12"[label="no"];
-"g0_12" [label="\"Cont_features\"[1] <= 70\\n impurity = 0.408163\\n samples = 7\\n value = [2 5
-]\\n class = \"'Play'\"", shape=ellipse];
-"g0_12" -> "g0_25"[label="yes"];
-"g0_25" [label="\"'Play'\"\\n impurity = 0\\n samples = 3\\n value = [0 3
-]",shape=box];
-"g0_12" -> "g0_26"[label="no"];
-"g0_26" [label="\"Cont_features\"[1] <= 72\\n impurity = 0.5\\n samples = 4\\n value = [2 2
-]\\n class = \"'Don't Play'\"", shape=ellipse];
-"g0_26" -> "g0_53"[label="yes"];
-"g0_53" [label="\"'Don't Play'\"\\n impurity = 0\\n samples = 2\\n value = [2 0
-]",shape=box];
-"g0_26" -> "g0_54"[label="no"];
-"g0_54" [label="\"'Play'\"\\n impurity = 0\\n samples = 2\\n value = [0 2
-]",shape=box];
+ digraph "Classification tree for dt_golf" {
+ subgraph "cluster0"{
+ label=""
+ "g0_0" [label="\"OUTLOOK\" <= overcast\\n impurity = 0.459184\\n samples = 14\\n value = [5 9]\\n class = \"Play\"", shape=ellipse];
+ "g0_0" -> "g0_1"[label="yes"];
+ "g0_1" [label="\"Play\"\\n impurity = 0\\n samples = 4\\n value = [0 4]",shape=box];
+ "g0_0" -> "g0_2"[label="no"];
+ "g0_2" [label="temperature <= 75\\n impurity = 0.5\\n samples = 10\\n value = [5 5]\\n class = \"Don't Play\"", shape=ellipse];
+ "g0_2" -> "g0_5"[label="yes"];
+ "g0_2" -> "g0_6"[label="no"];
+ "g0_6" [label="\"Don't Play\"\\n impurity = 0\\n samples = 2\\n value = [2 0]",shape=box];
+ "g0_5" [label="temperature <= 65\\n impurity = 0.46875\\n samples = 8\\n value = [3 5]\\n class = \"Play\"", shape=ellipse];
+ "g0_5" -> "g0_11"[label="yes"];
+ "g0_11" [label="\"Don't Play\"\\n impurity = 0\\n samples = 1\\n value = [1 0]",shape=box];
+ "g0_5" -> "g0_12"[label="no"];
+ "g0_12" [label="temperature <= 70\\n impurity = 0.408163\\n samples = 7\\n value = [2 5]\\n class = \"Play\"", shape=ellipse];
+ "g0_12" -> "g0_25"[label="yes"];
+ "g0_25" [label="\"Play\"\\n impurity = 0\\n samples = 3\\n value = [0 3]",shape=box];
+ "g0_12" -> "g0_26"[label="no"];
+ "g0_26" [label="temperature <= 72\\n impurity = 0.5\\n samples = 4\\n value = [2 2]\\n class = \"Don't Play\"", shape=ellipse];
+ "g0_26" -> "g0_53"[label="yes"];
+ "g0_53" [label="\"Don't Play\"\\n impurity = 0\\n samples = 2\\n value = [2 0]",shape=box];
+ "g0_26" -> "g0_54"[label="no"];
+ "g0_54" [label="\"Play\"\\n impurity = 0\\n samples = 2\\n value = [0 2]",shape=box];
} //--- end of subgraph------------
} //---end of digraph---------
</pre>
The additional information in each node is: impurity, sample size, number of
weighted rows for each response variable, and classification if the tree was
-pruned at this level.
+pruned at this level. If your tree is not too big, you may wish to convert the
+dot format to PDF or another format for better visualization of the
+tree structure.
-<h4>Decision Tree Regression Example</h4>
+-# Arrays of features. Categorical and continuous features
+can be array columns, in which case the array is expanded to
+treat each element of the array as a feature:
+<pre class="example">
+DROP TABLE IF EXISTS train_output, train_output_summary;
+SELECT madlib.tree_train('dt_golf', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'class', -- response
+ '"Temp_Humidity", clouds_airquality', -- features
+ NULL::text, -- exclude columns
+ 'gini', -- split criterion
+ NULL::text, -- no grouping
+ NULL::text, -- no weights, all observations treated equally
+ 5, -- max depth
+ 3, -- min split
+ 1, -- min bucket
+ 10 -- number of bins per continuous variable
+ );
+SELECT pruning_cp, cat_levels_in_text, cat_n_levels, tree_depth FROM train_output;
+</pre>
+View the output table (excluding the tree which is in binary format):
+<pre class="example">
+SELECT pruning_cp, cat_levels_in_text, cat_n_levels, tree_depth FROM train_output;
+</pre>
+<pre class="result">
+ pruning_cp | cat_levels_in_text | cat_n_levels | tree_depth
+------------+------------------------------------------------+--------------+------------
+ 0 | {medium,none,high,low,unhealthy,good,moderate} | {4,3} | 3
+</pre>
+The first 4 levels correspond to cloud ceiling and the next 3 levels
+correspond to air quality.
+
+-# Weighting observations. Use the 'weights' parameter to
+adjust a row's vote to balance the dataset. In our
+example, the weights are somewhat random but
+show that a different decision tree is create
+compared to the case where no weights are used:
+<pre class="example">
+DROP TABLE IF EXISTS train_output, train_output_summary;
+SELECT madlib.tree_train('dt_golf', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'class', -- response
+ '"OUTLOOK", temperature, windy', -- features
+ NULL::text, -- exclude columns
+ 'gini', -- split criterion
+ NULL::text, -- no grouping
+ 'observation_weight', -- weight observations
+ 5, -- max depth
+ 3, -- min split
+ 1, -- min bucket
+ 10 -- number of bins per continuous variable
+ );
+SELECT madlib.tree_display('train_output');
+</pre>
+<pre class="result">
+ -------------------------------------
+ - Each node represented by 'id' inside ().
+ - Each internal nodes has the split condition at the end, while each
+ leaf node has a * at the end.
+ - For each internal node (i), its child nodes are indented by 1 level
+ with ids (2i+1) for True node and (2i+2) for False node.
+ - Number of (weighted) rows for each response variable inside [].'
+ The response label order is given as ['"Don\'t Play"', '"Play"'].
+ For each leaf, the prediction is given after the '-->'
+ -------------------------------------
+ (0)[17 19] temperature <= 75
+ (1)[ 7 16] temperature <= 72
+ (3)[ 7 10] temperature <= 70
+ (7)[ 1 8.5] * --> "Play"
+ (8)[ 6 1.5] "OUTLOOK" in {overcast}
+ (17)[ 0 1.5] * --> "Play"
+ (18)[6 0] * --> "Don't Play"
+ (4)[0 6] * --> "Play"
+ (2)[10 3] "OUTLOOK" in {overcast}
+ (5)[0 3] * --> "Play"
+ (6)[10 0] * --> "Don't Play"
+</pre>
+
+<h4>Decision Tree Regression Examples</h4>
--# Prepare input data.
+-# Load input data related to fuel consumption and 10
+aspects of automobile design and performance for 32
+automobiles (1973–74 models). Data was extracted from
+the 1974 Motor Trend US magazine.
<pre class="example">
DROP TABLE IF EXISTS mt_cars;
CREATE TABLE mt_cars (
@@ -753,47 +973,45 @@ CREATE TABLE mt_cars (
gear integer,
carb integer
);
-</pre>
-<pre class="example">
-COPY mt_cars (id,mpg,cyl,disp,hp,drat,wt,qsec,vs,am,gear,carb) FROM stdin WITH DELIMITER '|' NULL 'null';
-1|18.7|8|360|175|3.15|3.44|17.02|0|0|3|2
-2|21|6|160|110|3.9|2.62|16.46|0|1|4|4
-3|24.4|4|146.7|62|3.69|3.19|20|1|0|4|2
-4|21|6|160|110|3.9|2.875|17.02|0|1|4|4
-5|17.8|6|167.6|123|3.92|3.44|18.9|1|0|4|4
-6|16.4|8|275.8|180|3.078|4.07|17.4|0|0|3|3
-7|22.8|4|108|93|3.85|2.32|18.61|1|1|4|1
-8|17.3|8|275.8|180|3.078|3.73|17.6|0|0|3|3
-9|21.4|null|258|110|3.08|3.215|19.44|1|0|3|1
-10|15.2|8|275.8|180|3.078|3.78|18|0|0|3|3
-11|18.1|6|225|105|2.768|3.46|20.22|1|0|3|1
-12|32.4|4|78.7|66|4.08|2.20|19.47|1|1|4|1
-13|14.3|8|360|245|3.21|3.578|15.84|0|0|3|4
-14|22.8|4|140.8|95|3.92|3.15|22.9|1|0|4|2
-15|30.4|4|75.7|52|4.93|1.615|18.52|1|1|4|2
-16|19.2|6|167.6|123|3.92|3.44|18.3|1|0|4|4
-17|33.9|4|71.14|65|4.22|1.835|19.9|1|1|4|1
-18|15.2|null|304|150|3.15|3.435|17.3|0|0|3|2
-19|10.4|8|472|205|2.93|5.25|17.98|0|0|3|4
-20|27.3|4|79|66|4.08|1.935|18.9|1|1|4|1
-21|10.4|8|460|215|3|5.424|17.82|0|0|3|4
-22|26|4|120.3|91|4.43|2.14|16.7|0|1|5|2
-23|14.7|8|440|230|3.23|5.345|17.42|0|0|3|4
-24|30.4|4|95.14|113|3.77|1.513|16.9|1|1|5|2
-25|21.5|4|120.1|97|3.70|2.465|20.01|1|0|3|1
-26|15.8|8|351|264|4.22|3.17|14.5|0|1|5|4
-27|15.5|8|318|150|2.768|3.52|16.87|0|0|3|2
-28|15|8|301|335|3.54|3.578|14.6|0|1|5|8
-29|13.3|8|350|245|3.73|3.84|15.41|0|0|3|4
-30|19.2|8|400|175|3.08|3.845|17.05|0|0|3|2
-31|19.7|6|145|175|3.62|2.77|15.5|0|1|5|6
-32|21.4|4|121|109|4.11|2.78|18.6|1|1|4|2
-\\.
+INSERT INTO mt_cars VALUES
+(1,18.7,8,360,175,3.15,3.44,17.02,0,0,3,2),
+(2,21,6,160,110,3.9,2.62,16.46,0,1,4,4),
+(3,24.4,4,146.7,62,3.69,3.19,20,1,0,4,2),
+(4,21,6,160,110,3.9,2.875,17.02,0,1,4,4),
+(5,17.8,6,167.6,123,3.92,3.44,18.9,1,0,4,4),
+(6,16.4,8,275.8,180,3.078,4.07,17.4,0,0,3,3),
+(7,22.8,4,108,93,3.85,2.32,18.61,1,1,4,1),
+(8,17.3,8,275.8,180,3.078,3.73,17.6,0,0,3,3),
+(9,21.4,null,258,110,3.08,3.215,19.44,1,0,3,1),
+(10,15.2,8,275.8,180,3.078,3.78,18,0,0,3,3),
+(11,18.1,6,225,105,2.768,3.46,20.22,1,0,3,1),
+(12,32.4,4,78.7,66,4.08,2.20,19.47,1,1,4,1),
+(13,14.3,8,360,245,3.21,3.578,15.84,0,0,3,4),
+(14,22.8,4,140.8,95,3.92,3.15,22.9,1,0,4,2),
+(15,30.4,4,75.7,52,4.93,1.615,18.52,1,1,4,2),
+(16,19.2,6,167.6,123,3.92,3.44,18.3,1,0,4,4),
+(17,33.9,4,71.14,65,4.22,1.835,19.9,1,1,4,1),
+(18,15.2,null,304,150,3.15,3.435,17.3,0,0,3,2),
+(19,10.4,8,472,205,2.93,5.25,17.98,0,0,3,4),
+(20,27.3,4,79,66,4.08,1.935,18.9,1,1,4,1),
+(21,10.4,8,460,215,3,5.424,17.82,0,0,3,4),
+(22,26,4,120.3,91,4.43,2.14,16.7,0,1,5,2),
+(23,14.7,8,440,230,3.23,5.345,17.42,0,0,3,4),
+(24,30.4,4,95.14,113,3.77,1.513,16.9,1,1,5,2),
+(25,21.5,4,120.1,97,3.70,2.465,20.01,1,0,3,1),
+(26,15.8,8,351,264,4.22,3.17,14.5,0,1,5,4),
+(27,15.5,8,318,150,2.768,3.52,16.87,0,0,3,2),
+(28,15,8,301,335,3.54,3.578,14.6,0,1,5,8),
+(29,13.3,8,350,245,3.73,3.84,15.41,0,0,3,4),
+(30,19.2,8,400,175,3.08,3.845,17.05,0,0,3,2),
+(31,19.7,6,145,175,3.62,2.77,15.5,0,1,5,6),
+(32,21.4,4,121,109,4.11,2.78,18.6,1,1,4,2);
</pre>
--# Run the decision tree training function:
+-# We train a regression decision tree with surrogates
+in order to handle the NULL feature values:
<pre class="example">
-DROP TABLE IF EXISTS train_output, train_output_summary;
+DROP TABLE IF EXISTS train_output, train_output_summary, train_output_cv;
SELECT madlib.tree_train('mt_cars', -- source table
'train_output', -- output model table
'id', -- id column
@@ -811,12 +1029,100 @@ SELECT madlib.tree_train('mt_cars', -- source table
'max_surrogates=2' -- number of surrogates
);
</pre>
+View the output table (excluding the tree which is in binary format)
+which shows ordering of levels of categorical variables 'vs' and 'cyl':
+<pre class="example">
+SELECT pruning_cp, cat_levels_in_text, cat_n_levels, tree_depth FROM train_output;
+</pre>
+<pre class="result">
+ pruning_cp | cat_levels_in_text | cat_n_levels | tree_depth
+------------+--------------------+--------------+------------
+ 0 | {0,1,4,6,8} | {2,3} | 4
+</pre>
+View the summary table:
+<pre class="example">
+\\x on
+SELECT * FROM train_output_summary;
+</pre>
+<pre class="result">
+-[ RECORD 1 ]---------+-----------------------------------------------------------------------
+method | tree_train
+is_classification | f
+source_table | mt_cars
+model_table | train_output
+id_col_name | id
+dependent_varname | mpg
+independent_varnames | *
+cat_features | vs,cyl
+con_features | disp,qsec,wt
+grouping_cols |
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 32
+total_rows_skipped | 0
+dependent_var_levels |
+dependent_var_type | double precision
+input_cp | 0
+independent_var_types | integer, integer, double precision, double precision, double precision
+n_folds | 0
+null_proxy |
+</pre>
+
+-# Predict regression output for the same data and compare with original:
+<pre class="example">
+\\x off
+DROP TABLE IF EXISTS prediction_results;
+SELECT madlib.tree_predict('train_output',
+ 'mt_cars',
+ 'prediction_results',
+ 'response');
+SELECT s.id, mpg, estimated_mpg, mpg-estimated_mpg as delta
+FROM prediction_results p,
+mt_cars s WHERE s.id = p.id ORDER BY id;
+</pre>
+Result:
+<pre class="result">
+ id | mpg | estimated_mpg | delta
+----+------+------------------+---------------------
+ 1 | 18.7 | 16.84 | 1.86
+ 2 | 21 | 19.7428571428571 | 1.25714285714286
+ 3 | 24.4 | 22.58 | 1.82
+ 4 | 21 | 19.7428571428571 | 1.25714285714286
+ 5 | 17.8 | 19.7428571428571 | -1.94285714285714
+ 6 | 16.4 | 16.84 | -0.439999999999998
+ 7 | 22.8 | 22.58 | 0.219999999999999
+ 8 | 17.3 | 13.325 | 3.975
+ 9 | 21.4 | 19.7428571428571 | 1.65714285714286
+ 10 | 15.2 | 13.325 | 1.875
+ 11 | 18.1 | 19.7428571428571 | -1.64285714285714
+ 12 | 32.4 | 30.0666666666667 | 2.33333333333334
+ 13 | 14.3 | 14.78 | -0.48
+ 14 | 22.8 | 22.58 | 0.219999999999999
+ 15 | 30.4 | 30.0666666666667 | 0.333333333333336
+ 16 | 19.2 | 19.7428571428571 | -0.542857142857141
+ 17 | 33.9 | 30.0666666666667 | 3.83333333333334
+ 18 | 15.2 | 16.84 | -1.64
+ 19 | 10.4 | 13.325 | -2.925
+ 20 | 27.3 | 30.0666666666667 | -2.76666666666666
+ 21 | 10.4 | 13.325 | -2.925
+ 22 | 26 | 30.0666666666667 | -4.06666666666666
+ 23 | 14.7 | 16.84 | -2.14
+ 24 | 30.4 | 30.0666666666667 | 0.333333333333336
+ 25 | 21.5 | 22.58 | -1.08
+ 26 | 15.8 | 14.78 | 1.02
+ 27 | 15.5 | 14.78 | 0.719999999999999
+ 28 | 15 | 14.78 | 0.219999999999999
+ 29 | 13.3 | 14.78 | -1.48
+ 30 | 19.2 | 16.84 | 2.36
+ 31 | 19.7 | 19.7428571428571 | -0.0428571428571409
+ 32 | 21.4 | 22.58 | -1.18
+(32 rows)
+</pre>
-# Display the decision tree in basic text format:
<pre class="example">
SELECT madlib.tree_display('train_output', FALSE);
</pre>
-Result:
<pre class="result">
-------------------------------------
- Each node represented by 'id' inside ().
@@ -826,113 +1132,113 @@ Result:
with ids (2i+1) for True node and (2i+2) for False node.
- Number of rows and average response value inside []. For a leaf node, this is the prediction.
-------------------------------------
- (0)[32, 20.0906] cyl in {8,6}
- (1)[21, 16.6476] disp <= 258
- (3)[7, 19.7429] *
- (4)[14, 15.1] qsec <= 17.42
- (9)[10, 15.81] qsec <= 16.9
- (19)[5, 14.78] *
- (20)[5, 16.84] *
- (10)[4, 13.325] *
- (2)[11, 26.6636] wt <= 2.2
- (5)[6, 30.0667] *
- (6)[5, 22.58] *
+ (0)[32, 20.0906] cyl in {4}
+ (1)[11, 26.6636] wt <= 2.2
+ (3)[6, 30.0667] *
+ (4)[5, 22.58] *
+ (2)[21, 16.6476] disp <= 258
+ (5)[7, 19.7429] *
+ (6)[14, 15.1] qsec <= 17.42
+ (13)[10, 15.81] qsec <= 16.9
+ (27)[5, 14.78] *
+ (28)[5, 16.84] *
+ (14)[4, 13.325] *
-------------------------------------
(1 row)
</pre>
--# Display the surrogates in the decision tree:
+-# Display the surrogate variables that are used
+to compute the split for each node when the primary
+variable is NULL:
<pre class="example">
SELECT madlib.tree_surr_display('train_output');
</pre>
-Result:
<pre class="result">
-------------------------------------
Surrogates for internal nodes
-------------------------------------
- (0) cyl in {8,6}
- 1: disp > 146.7 [common rows = 29]
- 2: vs in {0} [common rows = 26]
- [Majority branch = 19 ]
- (1) disp <= 258
- 1: cyl in {6,4} [common rows = 19]
+ (0) cyl in {4}
+ 1: disp <= 146.7 [common rows = 29]
+ 2: vs in {1} [common rows = 26]
+ [Majority branch = 11 ]
+ (1) wt <= 2.2
+ [Majority branch = 19 ]
+ (2) disp <= 258
+ 1: cyl in {4,6} [common rows = 19]
2: vs in {1} [common rows = 18]
- [Majority branch = 14 ]
- (2) wt <= 2.2
- 1: disp <= 108 [common rows = 9]
- 2: qsec <= 18.52 [common rows = 8]
- [Majority branch = 6 ]
- (4) qsec <= 17.42
+ [Majority branch = 7 ]
+ (6) qsec <= 17.42
1: disp > 275.8 [common rows = 11]
2: vs in {0} [common rows = 10]
- [Majority branch = 10 ]
- (9) qsec <= 16.9
+ [Majority branch = 10 ]
+ (13) qsec <= 16.9
1: wt <= 3.84 [common rows = 8]
2: disp <= 360 [common rows = 7]
[Majority branch = 5 ]
-------------------------------------
(1 row)
</pre>
-@note The 'cyl' parameter above has two tuples with null values.
-In the prediction example below, the surrogate splits for the
-<em>cyl in {8, 6}</em> split are used to predict those
-two tuples (<em>id = 9</em> and <em>id = 18</em>). The splits are used in
-descending order till a surrogate variable is found that is not NULL. In this case,
-the two tuples have non-NULL values for <em>disp</em>, hence the <em>disp > 146.7</em>
-split is used to make the prediction. If all the surrogate variables had been
-NULL then the majority branch would have been followed.
-
--# Predict regression output for the same data and compare with original:
+@note The 'cyl' parameter in the data set has two tuples with NULL
+values (<em>id = 9</em> and <em>id = 18</em>).
+In the prediction based on this tree, the surrogate splits for the
+<em>cyl in {4}</em> split in node 0 are used to predict those
+two tuples. The splits are used in
+descending order until a surrogate variable is found that is not NULL. In this case,
+the two tuples have non-NULL values for <em>disp</em>, hence the <em>disp <= 146.7</em>
+split is used to make the prediction. If all the surrogate variables are
+NULL then the majority branch would be followed.
+
+-# Now let's use cross validation to select the best
+value of the complexity parameter cp:
<pre class="example">
-DROP TABLE IF EXISTS prediction_results;
-SELECT madlib.tree_predict('train_output',
- 'mt_cars',
- 'prediction_results',
- 'response');
-SELECT s.id, mpg, estimated_mpg FROM prediction_results p, mt_cars s where s.id = p.id ORDER BY id;
+DROP TABLE IF EXISTS train_output, train_output_summary, train_output_cv;
+SELECT madlib.tree_train('mt_cars', -- source table
+ 'train_output', -- output model table
+ 'id', -- id column
+ 'mpg', -- dependent variable
+ '*', -- features
+ 'id, hp, drat, am, gear, carb', -- exclude columns
+ 'mse', -- split criterion
+ NULL::text, -- no grouping
+ NULL::text, -- no weights, all observations treated equally
+ 10, -- max depth
+ 8, -- min split
+ 3, -- number of bins per continuous variable
+ 10, -- number of splits
+ 'n_folds=3' -- pruning parameters for cross validation
+ );
+</pre>
+View the output table (excluding the tree which is in binary format).
+The input cp value was 0 (default) and the best 'pruning_cp' value
+turns out to be 0 as well in this small example:
+<pre class="example">
+SELECT pruning_cp, cat_levels_in_text, cat_n_levels, tree_depth FROM train_output;
</pre>
-Result:
<pre class="result">
- id | mpg | estimated_mpg
-----+------+------------------
- 1 | 18.7 | 16.84
- 2 | 21 | 19.7428571428571
- 3 | 24.4 | 22.58
- 4 | 21 | 19.7428571428571
- 5 | 17.8 | 19.7428571428571
- 6 | 16.4 | 16.84
- 7 | 22.8 | 22.58
- 8 | 17.3 | 13.325
- 9 | 21.4 | 19.7428571428571
- 10 | 15.2 | 13.325
- 11 | 18.1 | 19.7428571428571
- 12 | 32.4 | 30.0666666666667
- 13 | 14.3 | 14.78
- 14 | 22.8 | 22.58
- 15 | 30.4 | 30.0666666666667
- 16 | 19.2 | 19.7428571428571
- 17 | 33.9 | 30.0666666666667
- 18 | 15.2 | 16.84
- 19 | 10.4 | 13.325
- 20 | 27.3 | 30.0666666666667
- 21 | 10.4 | 13.325
- 22 | 26 | 30.0666666666667
- 23 | 14.7 | 16.84
- 24 | 30.4 | 30.0666666666667
- 25 | 21.5 | 22.58
- 26 | 15.8 | 14.78
- 27 | 15.5 | 14.78
- 28 | 15 | 14.78
- 29 | 13.3 | 14.78
- 30 | 19.2 | 16.84
- 31 | 19.7 | 19.7428571428571
- 32 | 21.4 | 22.58
-(32 rows)
+ pruning_cp | cat_levels_in_text | cat_n_levels | tree_depth
+------------+--------------------+--------------+------------
+ 0 | {0,1,4,6,8} | {2,3} | 4
+</pre>
+The cp values tested and average error and standard deviation are:
+<pre class="example">
+SELECT * FROM train_output_cv ORDER BY cv_error_avg ASC;
+</pre>
+<pre class="result">
+ cp | cv_error_avg | cv_error_stddev
+---------------------+------------------+------------------
+ 0 | 4.60222321567406 | 1.14990035501294
+ 0.00942145242026098 | 4.71906243157825 | 1.21587651168567
+ 0.0156685263245236 | 4.86688342751006 | 1.30225133441406
+ 0.0893348335770666 | 5.0608834230282 | 1.42488238861617
+ 0.135752855572154 | 5.33192746100332 | 1.62718329150341
+ 0.643125226048458 | 5.76814538295394 | 2.10750950120742
+(6 rows)
</pre>
<h4>NULL Handling Example</h4>
--# Create toy example to illustrate null-as-category handling
+-# Create toy example to illustrate 'null-as-category' handling
+for categorical features:
<pre class='example'>
DROP TABLE IF EXISTS null_handling_example;
CREATE TABLE null_handling_example (
@@ -942,13 +1248,16 @@ CREATE TABLE null_handling_example (
weather text,
response text
);
-
INSERT INTO null_handling_example VALUES
(1,null,null,null,'a'),
(2,'US',null,null,'b'),
(3,'US','NY',null,'c'),
(4,'US','NY','rainy','d');
-
+</pre>
+
+-# Train decision tree. Note that 'NULL' is set as a
+valid level for the categorical features country, weather and city:
+<pre class='example'>
DROP TABLE IF EXISTS train_output, train_output_summary;
SELECT madlib.tree_train('null_handling_example', -- source table
'train_output', -- output model table
@@ -969,13 +1278,43 @@ SELECT madlib.tree_train('null_handling_example', -- source table
SELECT cat_levels_in_text, cat_n_levels FROM train_output;
</pre>
<pre class='result'>
- cat_levels_in_text | cat_n_levels
+ cat_levels_in_text | cat_n_levels
------------------------------------------+--------------
{US,__NULL__,rainy,__NULL__,NY,__NULL__} | {2,2,2}
</pre>
+View the summary table:
+<pre class='example'>
+\\x on
+SELECT * FROM train_output_summary;
+</pre>
+<pre class='result'>
+-[ RECORD 1 ]---------+-----------------------
+method | tree_train
+is_classification | t
+source_table | null_handling_example
+model_table | train_output
+id_col_name | id
+dependent_varname | response
+independent_varnames | country, weather, city
+cat_features | country,weather,city
+con_features |
+grouping_cols |
+num_all_groups | 1
+num_failed_groups | 0
+total_rows_processed | 4
+total_rows_skipped | 0
+dependent_var_levels | "a","b","c","d"
+dependent_var_type | text
+input_cp | 0
+independent_var_types | text, text, text
+n_folds | 0
+null_proxy | __NULL__
+</pre>
--# Predict for previously not seen data by assuming NULL value as the default
+-# Predict for data not previously seen by assuming NULL
+value as the default:
<pre class='example'>
+\\x off
DROP TABLE IF EXISTS table_test;
CREATE TABLE table_test (
id integer,
@@ -1000,7 +1339,7 @@ FROM prediction_results p, table_test s
WHERE s.id = p.id ORDER BY id;
</pre>
<pre class='result'>
- id | expected_response | estimated_response
+ id | expected_response | estimated_response
----+-------------------+--------------------
1 | a | a
2 | b | b
@@ -1008,9 +1347,40 @@ WHERE s.id = p.id ORDER BY id;
4 | d | d
(4 rows)
</pre>
+There is only training data for country 'US' so the
+response for country 'IN' is 'a', corresponding to
+a NULL (not 'US') country level. Likewise, any
+city in the 'US' that is not 'NY' will predict
+response 'b', corresponding to a NULL (not 'NY')
+city level.
+
+-# Display the decision tree in basic text format:
+<pre class="example">
+SELECT madlib.tree_display('train_output', FALSE);
+</pre>
+<pre class="result">
+ -------------------------------------
+ - Each node represented by 'id' inside ().
+ - Each internal nodes has the split condition at the end, while each
+ leaf node has a * at the end.
+ - For each internal node (i), its child nodes are indented by 1 level
+ with ids (2i+1) for True node and (2i+2) for False node.
+ - Number of rows and average response value inside []. For a leaf node, this is the prediction.
+ -------------------------------------
+ (0)[1 1 1 1] city in {NY}
+ (1)[0 0 1 1] weather in {rainy}
+ (3)[0 0 0 1] * --> "d"
+ (4)[0 0 1 0] * --> "c"
+ (2)[1 1 0 0] country in {US}
+ (5)[0 1 0 0] * --> "b"
+ (6)[1 0 0 0] * --> "a"
+ -------------------------------------
+(1 row)
+</pre>
+
@anchor literature
-@literature
+@par Literature
[1] Breiman, Leo; Friedman, J. H.; Olshen, R. A.; Stone, C. J. (1984). Classification and regression trees. Monterey, CA: Wadsworth & Brooks/Cole Advanced Books & Software.
@anchor related