You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2018/03/28 06:24:51 UTC
madlib git commit: DT: Fix error in c35107d + minor optimization
Repository: madlib
Updated Branches:
refs/heads/master 7f3aae92f -> a1360f391
DT: Fix error in c35107d + minor optimization
Commit c35107d introduced a minor bug where change in a variable was
not propagated to all references of the variable. This commit fixes the
problem and also cleans another internal function (_classify_features)
to better use a utility function for checking for numeric types.
Project: http://git-wip-us.apache.org/repos/asf/madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/madlib/commit/a1360f39
Tree: http://git-wip-us.apache.org/repos/asf/madlib/tree/a1360f39
Diff: http://git-wip-us.apache.org/repos/asf/madlib/diff/a1360f39
Branch: refs/heads/master
Commit: a1360f39149db281126ab044611572cff0c3e217
Parents: 7f3aae9
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Mar 27 23:21:15 2018 -0700
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Mar 27 23:21:15 2018 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 33 +++++++++++---------
.../recursive_partitioning/random_forest.py_in | 2 +-
2 files changed, 19 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/madlib/blob/a1360f39/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 6dca0f2..99ddce2 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -31,6 +31,7 @@ from utilities.utilities import _assert
from utilities.utilities import extract_keyvalue_params
from utilities.utilities import unique_string
from utilities.utilities import add_postfix
+from utilities.utilities import is_psql_numeric_type
from utilities.utilities import split_quoted_delimited_str
from utilities.utilities import py_list_to_sql_string
# ------------------------------------------------------------
@@ -156,9 +157,10 @@ def _get_features_to_use(schema_madlib, training_table_name,
def _classify_features(feature_to_type, features):
""" Returns
- 1) an array of categorical features (all casted to string)
- 2) an array of continuous features
+ 1) an array of categorical features
+ 2) an array of ordered categorical features
3) an array of boolean categorical variables
+ 4) an array of continuous features
"""
# any column belonging to the following types are categorical
int_types = ['integer', 'smallint', 'bigint']
@@ -170,19 +172,20 @@ def _classify_features(feature_to_type, features):
cat_features = [c for c in features if feature_to_type[c] in cat_types]
ordered_cat_features = [c for c in features
if feature_to_type[c] in ordered_cat_types]
-
- # continuous types - 'real' is cast to 'double precision' for uniformity
- con_types = ['real', 'float8', 'double precision', 'numeric']
- con_features = [c for c in features if feature_to_type[c] in con_types]
-
# In order to be able to form an array, all categorical variables
# will be cast into TEXT type, but GPDB cannot cast a boolean
# directly into a text. Thus, boolean categorical variables
# need special treatment: cast them into integers before casting
# into text.
- boolean_cats = [c for c in features
- if feature_to_type[c] in boolean_types]
- return cat_features, ordered_cat_features, con_features, boolean_cats
+ boolean_cat_features = [c for c in features
+ if feature_to_type[c] in boolean_types]
+
+ # Integer types are not considered continuous
+ con_types = ['real', 'double precision', 'numeric', 'decimal']
+ con_features = [c for c in features
+ if is_psql_numeric_type(feature_to_type[c], exclude=int_types)]
+
+ return cat_features, ordered_cat_features, boolean_cat_features, con_features
# ------------------------------------------------------------
@@ -471,7 +474,7 @@ def tree_train(schema_madlib, training_table_name, output_table_name,
# 2)
all_cols_types = dict([(f, get_expr_type(f, training_table_name))
for f in features])
- cat_features, ordered_cat_features, con_features, boolean_cats = \
+ cat_features, ordered_cat_features, boolean_cats, con_features = \
_classify_features(all_cols_types, features)
# get all rows
@@ -977,7 +980,7 @@ def _get_bins_grps(
# for different groups)
# The list of tuples can be converted to a dict, but the ordering
# will be lost.
- # eg. grp_col_to_levels =
+ # eg. grp_to_col_to_levels =
# [
# ('3', {'vs': [0, 1], 'cyl': [4,6,8]}),
# ('4', {'vs': [0, 1], 'cyl': [4,6]}),
@@ -998,7 +1001,7 @@ def _get_bins_grps(
for col in cat_features if col in rows]
cat_n = [len(i) for i in cat_items_list]
cat_origin = [item for sublist in cat_items_list for item in sublist]
- grp_key_cat=[item[0] for item in grp_to_col_to_levels]
+ grp_key_cat = [item[0] for item in grp_to_col_to_levels]
else:
cat_n = []
cat_origin = []
@@ -1006,9 +1009,9 @@ def _get_bins_grps(
if con_features:
con = [con_splits['con_splits'] for con_splits in con_splits_all]
- grp_key_con=[con_splits['grp_key'] for con_splits in con_splits_all]
+ grp_key_con = [con_splits['grp_key'] for con_splits in con_splits_all]
else:
- grp_key_con=[grp_key for grp_key in grp_to_col_to_row]
+ grp_key_con = [grp_key[0] for grp_key in grp_to_col_to_levels]
con = [''] * len(grp_key_con)
return dict(con=con,
http://git-wip-us.apache.org/repos/asf/madlib/blob/a1360f39/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index ce6ef7f..c7a3f70 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -334,7 +334,7 @@ def forest_train(
all_cols_types = dict([(f, get_expr_type(f, training_table_name))
for f in features])
- cat_features, ordered_cat_features, con_features, boolean_cats = \
+ cat_features, ordered_cat_features, boolean_cats, con_features = \
_classify_features(all_cols_types, features)
filter_null = _get_filter_str(dependent_variable, grouping_cols)