You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by xt...@apache.org on 2016/04/09 01:19:22 UTC
incubator-madlib git commit: Random Forest: Fix filtered feature
related bug
Repository: incubator-madlib
Updated Branches:
refs/heads/master bdfe68715 -> 9d73e9325
Random Forest: Fix filtered feature related bug
JIRA: MADLIB-928
Additional Author: Nandish Jayaram <nj...@pivotal.io>
Random forest filters out a feature if it has the same value for every
row. If grouping is enabled, this filer is applied per group. However
_get_bins_grps function did not consider a case where different groups
have different feature sets. The commit fixes this issue, adds a test
for install-check to replicate the use case and fixes a typo in one of
the related functions.
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/9d73e932
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/9d73e932
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/9d73e932
Branch: refs/heads/master
Commit: 9d73e9325ecad3f238c71b3f791963976ab158e3
Parents: bdfe687
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Wed Apr 6 14:07:26 2016 -0700
Committer: Xiaocheng Tang <xi...@gmail.com>
Committed: Fri Apr 8 16:17:40 2016 -0700
----------------------------------------------------------------------
methods/array_ops/src/pg_gp/array_ops.c | 2 +-
.../recursive_partitioning/decision_tree.py_in | 2 +-
.../test/random_forest.sql_in | 41 ++++++++++++++++++++
3 files changed, 43 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9d73e932/methods/array_ops/src/pg_gp/array_ops.c
----------------------------------------------------------------------
diff --git a/methods/array_ops/src/pg_gp/array_ops.c b/methods/array_ops/src/pg_gp/array_ops.c
index 598df85..9360983 100644
--- a/methods/array_ops/src/pg_gp/array_ops.c
+++ b/methods/array_ops/src/pg_gp/array_ops.c
@@ -375,7 +375,7 @@ array_of_float(PG_FUNCTION_ARGS){
ereport(ERROR,
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("invalid array length"),
- errdetail("array_of_bigint: Size should be in [1, 1e7], %d given", size)));
+ errdetail("array_of_float: Size should be in [1, 1e7], %d given", size)));
}
Datum* array = palloc (sizeof(Datum)*size);
for(int i = 0; i < size; ++i) {
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9d73e932/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 763670c..1183a6d 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -1043,7 +1043,7 @@ def _get_bins_grps(
if cat_features:
cat_items_list = [rows[col] for col in cat_features
- for grp_key, rows in grp_to_col_to_row.items()]
+ for grp_key, rows in grp_to_col_to_row.items() if col in rows]
cat_n = [len(i) for i in cat_items_list]
cat_origin = [item for subl in cat_items_list for item in subl]
grp_key_cat=[grp_key for grp_key in grp_to_col_to_row]
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/9d73e932/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
index cc15dd4..22b73fd 100644
--- a/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
+++ b/src/ports/postgres/modules/recursive_partitioning/test/random_forest.sql_in
@@ -255,3 +255,44 @@ SELECT * from predict_output;
SELECT get_tree('train_output', 1, sid)
from generate_series(1, 5) sid;
+
+-------------------------------------------------------------------------
+-- Test case for the case where a group has a filtered feature
+
+DROP TABLE IF EXISTS rf_gr_test CASCADE;
+CREATE TABLE rf_gr_test (
+ id integer NOT NULL,
+ gr integer,
+ f1 integer,
+ f2 integer, --filtered for gr = 2
+ f3 double precision,
+ cl integer
+) ;
+
+INSERT INTO rf_gr_test (id,gr,f1,f2,f3,cl) VALUES
+(1,1,1,1,5,1),
+(2,1,2,2,4,2),
+(3,1,3,3,3,1),
+(4,2,4,4,2,2),
+(5,2,5,4,1,1);
+
+DROP TABLE IF EXISTS train_output, train_output_summary, train_output_group;
+SELECT madlib.forest_train(
+ 'rf_gr_test'::TEXT, -- source table
+ 'train_output'::TEXT, -- output model table
+ 'id'::TEXT, -- id column
+ 'cl'::TEXT, -- response
+ 'f1, f2, f3'::TEXT, -- features
+ NULL::TEXT, -- exclude columns
+ 'gr'::TEXT, -- no grouping
+ 2, -- num of trees
+ 1, -- num of random features
+ TRUE::BOOLEAN, -- importance
+ 1::INTEGER, -- num_permutations
+ 10::INTEGER, -- max depth
+ 1::INTEGER, -- min split
+ 1::INTEGER, -- min bucket
+ 2::INTEGER, -- number of bins per continuous variable
+ 'max_surrogates=0',
+ FALSE
+ );