You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2016/03/26 00:49:00 UTC
[1/3] incubator-madlib git commit: Random Forest: Capture all
features NULL error
Repository: incubator-madlib
Updated Branches:
refs/heads/master 360f13482 -> 3dcb96ad3
Random Forest: Capture all features NULL error
JIRA: MADLIB-975
Random forest train gave a non-descript error if every data point had a
feature with NULL value. Added an assert to give a proper
description of the situation.
Closes #32
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/3dcb96ad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/3dcb96ad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/3dcb96ad
Branch: refs/heads/master
Commit: 3dcb96ad3bcac6b442a66c6f45d597efafb48e0e
Parents: 7d4eec9
Author: Orhan Kislal <ok...@pivotal.io>
Authored: Thu Mar 10 11:30:03 2016 -0800
Committer: Rahul Iyer <ri...@pivotal.io>
Committed: Fri Mar 25 16:48:32 2016 -0700
----------------------------------------------------------------------
.../recursive_partitioning/decision_tree.py_in | 1 +
.../recursive_partitioning/random_forest.py_in | 16 +++++++++-------
2 files changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3dcb96ad/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
index 805823c..763670c 100644
--- a/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/decision_tree.py_in
@@ -669,6 +669,7 @@ def _get_n_and_deplist(training_table_name, dependent_variable, filter_null):
source=training_table_name,
filter_null=filter_null)
r = plpy.execute(sql)[0]
+ r['n_rows'] = 0 if r['n_rows'] is None else r['n_rows']
return long(r['n_rows']), r['dep']
# ------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/3dcb96ad/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
index a47dd9b..43ad64e 100644
--- a/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
+++ b/src/ports/postgres/modules/recursive_partitioning/random_forest.py_in
@@ -314,8 +314,8 @@ def forest_train(
num_random_features = (sqrt(n_all_features) if is_classification
else n_all_features / 3)
_assert(num_random_features <= len(features),
- "Random forest error: Number of features to be selected is more "
- "than the actual number of features.")
+ "Random forest error: Number of features to be selected "
+ "is more than the actual number of features.")
all_cols_types = dict(get_cols_and_types(training_table_name))
cat_features, con_features, boolean_cats = _classify_features(
@@ -326,18 +326,20 @@ def forest_train(
dependent_variable, grouping_cols,
max_n_surr)
# the total number of records
- n_all_rows = plpy.execute("""
- select count(*) from {source_table}
- """.format(source_table=training_table_name))[0]['count']
+ n_all_rows = plpy.execute("SELECT count(*) FROM {0}".
+ format(training_table_name))[0]['count']
if is_classification:
# For classifications, we also need to map dependent_variable to integers
n_rows, dep_list = _get_n_and_deplist(training_table_name,
dependent_variable,
filter_null)
+ _assert(n_rows > 0,
+ "Random forest error: There should be at least one "
+ "data point for each class where all features are non NULL")
dep_list.sort()
- dep_col_str = ("case when " + dependent_variable +
- " then 'True' else 'False' end") if is_bool else dependent_variable
+ dep_col_str = ("CASE WHEN " + dependent_variable +
+ " THEN 'True' ELSE 'False' END") if is_bool else dependent_variable
dep = ("(CASE " +
"\n ".join([
"WHEN ({dep_col})::text = $${c}$$ THEN {i}".format(
[3/3] incubator-madlib git commit: SVM: Better NULL handling + use
temp for random matrices
Posted by ri...@apache.org.
SVM: Better NULL handling + use temp for random matrices
Closes #28, closes #30
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/62a99ce6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/62a99ce6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/62a99ce6
Branch: refs/heads/master
Commit: 62a99ce618280c13c0415dcacc72ad48205e1107
Parents: 360f134
Author: Xiaocheng Tang <xi...@gmail.com>
Authored: Fri Mar 11 09:49:42 2016 -0800
Committer: Rahul Iyer <ri...@pivotal.io>
Committed: Fri Mar 25 16:48:32 2016 -0700
----------------------------------------------------------------------
.../postgres/modules/linalg/matrix_ops.py_in | 9 +++++----
.../modules/svm/kernel_approximation.py_in | 19 ++++++++++++-------
2 files changed, 17 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/62a99ce6/src/ports/postgres/modules/linalg/matrix_ops.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/linalg/matrix_ops.py_in b/src/ports/postgres/modules/linalg/matrix_ops.py_in
index d11dcdb..458a763 100644
--- a/src/ports/postgres/modules/linalg/matrix_ops.py_in
+++ b/src/ports/postgres/modules/linalg/matrix_ops.py_in
@@ -3125,8 +3125,8 @@ def matrix_random(schema_madlib, distribution, row_dim, col_dim,
else:
distribution = 'uniform'
- in_args_default = {'seed': randint(0, 1000), 'table_type': ''}
- in_args_types = {'seed': int, 'table_type': str}
+ in_args_default = {'seed': randint(0, 1000), 'temp_out': False}
+ in_args_types = {'seed': int, 'temp_out': bool}
if distribution == 'normal':
in_args_default.update({'mu': 0, 'sigma': 1})
in_args_types.update({'mu': float, 'sigma': float})
@@ -3164,7 +3164,7 @@ def matrix_random(schema_madlib, distribution, row_dim, col_dim,
.format(distribution, ', '.join(sorted(supported_dist))))
plpy.execute("""
- CREATE {in_args_vals[table_type]} TABLE {matrix_out}
+ CREATE {is_temp} TABLE {matrix_out}
m4_ifdef(`__POSTGRESQL__', `',
`WITH (APPENDONLY=TRUE,COMPRESSTYPE=QUICKLZ)') AS
SELECT
@@ -3174,4 +3174,5 @@ def matrix_random(schema_madlib, distribution, row_dim, col_dim,
generate_series(1, {row_dim}) as row
m4_ifdef(`__POSTGRESQL__', `',
`DISTRIBUTED BY ({out_args[row]})')
- """.format(**locals()))
+ """.format(is_temp=True if in_args_vals['temp_out'] else False,
+ **locals()))
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/62a99ce6/src/ports/postgres/modules/svm/kernel_approximation.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/svm/kernel_approximation.py_in b/src/ports/postgres/modules/svm/kernel_approximation.py_in
index 190692e..0a09fbf 100644
--- a/src/ports/postgres/modules/svm/kernel_approximation.py_in
+++ b/src/ports/postgres/modules/svm/kernel_approximation.py_in
@@ -194,7 +194,7 @@ class PolyKernel(object):
drop table if exists {rd_weights};
select {schema_madlib}.matrix_random(
1, {dim},
- 'upper=1, lower=-1, seed={seed}, table_type=temp',
+ 'upper=1, lower=-1, seed={seed}, temp_out=true',
'bernoulli', '{rd_weights}',
'row={id}, val={val}')
""".format(rd_weights=rd_weights_,
@@ -303,6 +303,7 @@ class PolyKernel(object):
{id_col},
{grouping_col}
from {source_table}
+ WHERE not {schema_madlib}.array_contains_null({independent_varname})
) q cross join (select {pro.rd_val} from {pro.weights}) as weights
cross join (select {pro.rd_val} from {pro.coefs}) as coefs
cross join (select {pro.rd_val} from {pro.reps}) as reps
@@ -348,26 +349,28 @@ class GaussianKernelBase(object):
def _random_weights(self, row_dim, col_dim, rd_id, rd_val):
rd_weights = unique_string(desp='random_weights')
sigma = sqrt(2 * self.gamma)
+ seed = self.random_state
plpy.execute("""
drop table if exists {rd_weights};
select {self.schema_madlib}.matrix_random(
{row_dim}, {col_dim},
- 'mu=0, sigma={sigma}, seed={self.random_state}',
- 'normal',
- '{rd_weights}','row={rd_id}, val={rd_val}');
+ 'mu=0, sigma={sigma}, seed={seed}, temp_out=true',
+ 'normal', '{rd_weights}',
+ 'row={rd_id}, val={rd_val}');
""".format(**locals()))
return rd_weights
def _random_offsets(self, row_dim, col_dim, rd_id, rd_val):
rd_offset = unique_string(desp='random_offsets')
max_ = 2 * pi
+ seed = self.random_state
plpy.execute("""
drop table if exists {rd_offset};
select {self.schema_madlib}.matrix_random(
{row_dim}, {col_dim},
- 'min=0, max={max_}, seed={self.random_state}',
- 'uniform',
- '{rd_offset}','row={rd_id}, val={rd_val}');
+ 'min=0, max={max_}, seed={seed}, temp_out=true',
+ 'uniform', '{rd_offset}',
+ 'row={rd_id}, val={rd_val}');
""".format(**locals()))
return rd_offset
@@ -538,6 +541,7 @@ class GaussianKernel(GaussianKernelBase):
{id_col},
{grouping_col}
from {source_table}
+ WHERE not {schema_madlib}.array_contains_null({independent_varname})
""".format(**locals())
plpy.execute(run_sql)
source_table = source_with_id
@@ -688,6 +692,7 @@ class GaussianKernelInMemory(GaussianKernelBase):
{id_col},
{grouping_col}
from {source_table}
+ WHERE not {schema_madlib}.array_contains_null({independent_varname})
) q
cross join (select {self.rd_val} from {self.rd_weights}) as rw
cross join (select {self.rd_val} from {self.rd_offset}) as ro
[2/3] incubator-madlib git commit: Elastic Net Predict: Skip arrays
with NULL values
Posted by ri...@apache.org.
Elastic Net Predict: Skip arrays with NULL values
Jira: MADLIB-919
Having NULL values in the input array led to an error while converting
the array to a MappedColumnVector. This fix skips prediction for
arrays with NULL values.
Closes #31
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/7d4eec9d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/7d4eec9d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/7d4eec9d
Branch: refs/heads/master
Commit: 7d4eec9d443c7aef8a63565d520dee0ba773d929
Parents: 62a99ce
Author: Nandish Jayaram <nj...@pivotal.io>
Authored: Fri Mar 11 14:58:48 2016 -0800
Committer: Rahul Iyer <ri...@pivotal.io>
Committed: Fri Mar 25 16:48:32 2016 -0700
----------------------------------------------------------------------
src/modules/elastic_net/elastic_net_utils.cpp | 56 ++++++++++
.../test/elastic_net_install_check.sql_in | 102 ++++++++++++++++---
2 files changed, 145 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7d4eec9d/src/modules/elastic_net/elastic_net_utils.cpp
----------------------------------------------------------------------
diff --git a/src/modules/elastic_net/elastic_net_utils.cpp b/src/modules/elastic_net/elastic_net_utils.cpp
index ef90dfe..354c216 100644
--- a/src/modules/elastic_net/elastic_net_utils.cpp
+++ b/src/modules/elastic_net/elastic_net_utils.cpp
@@ -15,6 +15,20 @@ using namespace madlib::dbal::eigen_integration;
*/
AnyType __elastic_net_gaussian_predict::run (AnyType& args)
{
+ // throws an exception if the coefficients contain NULL values
+ try {
+ args[0].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ throw std::runtime_error(
+ "Elastic Net error: the coefficients contain NULL values");
+ }
+ // returns NULL if the feature has NULL values
+ try {
+ args[2].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ return Null();
+ }
+
MappedColumnVector coef = args[0].getAs<MappedColumnVector>();
double intercept = args[1].getAs<double>();
MappedColumnVector x = args[2].getAs<MappedColumnVector>();
@@ -30,6 +44,20 @@ AnyType __elastic_net_gaussian_predict::run (AnyType& args)
*/
AnyType __elastic_net_binomial_predict::run (AnyType& args)
{
+ // throws an exception if the coefficients contain NULL values
+ try {
+ args[0].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ throw std::runtime_error(
+ "Elastic Net error: the coefficients contain NULL values");
+ }
+ // returns NULL if the feature has NULL values
+ try {
+ args[2].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ return Null();
+ }
+
MappedColumnVector coef = args[0].getAs<MappedColumnVector>();
double intercept = args[1].getAs<double>();
MappedColumnVector x = args[2].getAs<MappedColumnVector>();
@@ -46,6 +74,20 @@ AnyType __elastic_net_binomial_predict::run (AnyType& args)
*/
AnyType __elastic_net_binomial_prob::run (AnyType& args)
{
+ // throws an exception if the coefficients contain NULL values
+ try {
+ args[0].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ throw std::runtime_error(
+ "Elastic Net error: the coefficients contain NULL values");
+ }
+ // returns NULL if the feature has NULL values
+ try {
+ args[2].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ return Null();
+ }
+
MappedColumnVector coef = args[0].getAs<MappedColumnVector>();
double intercept = args[1].getAs<double>();
MappedColumnVector x = args[2].getAs<MappedColumnVector>();
@@ -61,6 +103,20 @@ AnyType __elastic_net_binomial_prob::run (AnyType& args)
*/
AnyType __elastic_net_binomial_loglikelihood::run (AnyType& args)
{
+ // throws an exception if the coefficients contain NULL values
+ try {
+ args[0].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ throw std::runtime_error(
+ "Elastic Net error: the coefficients contain NULL values");
+ }
+ // returns NULL if the feature has NULL values
+ try {
+ args[3].getAs<MappedColumnVector>();
+ } catch(const ArrayWithNullException &e) {
+ return Null();
+ }
+
MappedColumnVector coef = args[0].getAs<MappedColumnVector>();
double intercept = args[1].getAs<double>();
MappedColumnVector x = args[3].getAs<MappedColumnVector>();
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7d4eec9d/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
index 643750a..a3ddf34 100644
--- a/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
+++ b/src/ports/postgres/modules/elastic_net/test/elastic_net_install_check.sql_in
@@ -524,11 +524,39 @@ COPY elastic_type_src (var_int, var_float8, var_sint) FROM stdin DELIMITER ',' N
4, 4.4, 4
\.
+DROP TABLE IF EXISTS housing_test;
+CREATE TABLE housing_test (id serial, x float8[],y float8);
+COPY housing_test (x, y) FROM STDIN NULL '?';
+{1,0.00632,18.00,2.310,0,0.5380,6.5750,65.20,4.0900,1,296.0,15.30,396.90,4.98} 24.00
+{1,0.02731,0.00,7.070,0,0.4690,6.4210,78.90,4.9671,2,242.0,17.80,396.90,9.14} 21.60
+{1,0.02729,0.00,7.070,0,0.4690,7.1850,61.10,4.9671,2,242.0,17.80,392.83,4.03} 34.70
+{1,0.03237,0.00,2.180,0,0.4580,6.9980,45.80,6.0622,3,222.0,18.70,394.63,2.94} 33.40
+{1,0.06905,0.00,2.180,0,0.4580,7.1470,54.20,6.0622,3,222.0,18.70,396.90,5.33} 36.20
+{1,0.02985,0.00,2.180,0,0.4580,6.4300,58.70,6.0622,3,222.0,18.70,394.12,5.21} 28.70
+{1,0.06076,0.00,11.930,0,0.5730,6.9760,91.00,2.1675,1,273.0,21.00,396.90,5.64} 23.90
+{1,0.10959,0.00,11.930,0,0.5730,6.7940,89.30,2.3889,1,273.0,21.00,393.45,6.48} 22.00
+{1,0.04741,0.00,11.930,0,0.5730,6.0300,80.80,2.5050,1,273.0,21.00,396.90,7.88} 11.90
+\.
+
+DROP TABLE IF EXISTS housing_test_null;
+CREATE TABLE housing_test_null (id serial, x float8[],y float8);
+COPY housing_test_null (x, y) FROM STDIN NULL '?';
+{1,0.00632,18.00,2.310,0,0.5380,6.5750,65.20,4.0900,1,296.0,15.30,396.90,4.98} 24.00
+{1,0.02731,0.00,7.070,0,0.4690,6.4210,78.90,4.9671,2,242.0,17.80,396.90,9.14} 21.60
+{1,0.02729,0.00,7.070,0,0.4690,7.1850,61.10,4.9671,2,242.0,17.80,392.83,4.03} 34.70
+{1,0.03237,0.00,2.180,0,0.4580,6.9980,45.80,6.0622,3,222.0,18.70,394.63,2.94} 33.40
+{1,0.06905,0.00,2.180,0,0.4580,7.1470,54.20,6.0622,3,222.0,18.70,396.90,5.33} 36.20
+{1,0.02985,0.00,2.180,0,0.4580,6.4300,58.70,6.0622,3,222.0,18.70,394.12,5.21} 28.70
+{1,0.06076,0.00,11.930,0,0.573, NULL,91.00,2.1675,1,273.0,21.00,396.90,5.64} 23.90
+{1,0.10959,0.00,11.930,0,0.5730,6.7940,89.30,NULL,1,273.0,21.00,393.45,6.48} 22.00
+{1,0.04741,0.00,11.930,0,0.5730,6.0300,80.80,2.5050,NULL,273.0,21.00,396.90,7.88} 11.90
+\.
+
create function check_elastic_net ()
returns void as $$
begin
- execute 'drop table if exists house_en';
- perform elastic_net_train(
+ EXECUTE 'drop table if exists house_en';
+ PERFORM elastic_net_train(
'lin_housing_wi',
'house_en',
'y',
@@ -545,7 +573,7 @@ begin
1e-6
);
- perform assert(relative_error(log_likelihood, -14.41122) < 0.000001,
+ PERFORM assert(relative_error(log_likelihood, -14.41122) < 0.000001,
'Elastic Net: log-likelihood mismatch (gaussian)!'
) from house_en;
@@ -555,9 +583,21 @@ begin
'id',
'house_en_pred');
+ EXECUTE 'DROP TABLE IF EXISTS house_test_gaussian';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test',
+ 'id',
+ 'house_test_gaussian');
+
+ EXECUTE 'DROP TABLE IF EXISTS house_test_null_gaussian';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test_null',
+ 'id',
+ 'house_test_null_gaussian');
+
-- huge lamda making all coef to be zeroes
- execute 'drop table if exists house_en';
- perform elastic_net_train(
+ EXECUTE 'drop table if exists house_en';
+ PERFORM elastic_net_train(
'lin_housing_wi',
'house_en',
'y',
@@ -580,8 +620,20 @@ begin
'id',
'house_en_pred');
- execute 'drop table if exists house_en';
- perform elastic_net_train(
+ EXECUTE 'DROP TABLE IF EXISTS house_test_gaussian';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test',
+ 'id',
+ 'house_test_gaussian');
+
+ EXECUTE 'DROP TABLE IF EXISTS house_test_null_gaussian';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test_null',
+ 'id',
+ 'house_test_null_gaussian');
+
+ EXECUTE 'drop table if exists house_en';
+ PERFORM elastic_net_train(
'lin_housing_wi',
'house_en',
'y < 20',
@@ -598,12 +650,24 @@ begin
1e-6
);
- perform assert(relative_error(log_likelihood, -0.542468) < 1e-4,
+ PERFORM assert(relative_error(log_likelihood, -0.542468) < 1e-4,
'Elastic Net: log-likelihood mismatch (use_active_set = f)!'
) from house_en;
- execute 'drop table if exists house_en';
- perform elastic_net_train(
+ EXECUTE 'DROP TABLE IF EXISTS house_test_binomial';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test',
+ 'id',
+ 'house_test_binomial');
+
+ EXECUTE 'drop table if exists house_test_null_binomial';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test_null',
+ 'id',
+ 'house_test_null_binomial');
+
+ EXECUTE 'drop table if exists house_en';
+ PERFORM elastic_net_train(
'lin_housing_wi',
'house_en',
'y < 20',
@@ -620,12 +684,24 @@ begin
1e-6
);
- perform assert(relative_error(log_likelihood, -0.542468) < 1e-4,
+ PERFORM assert(relative_error(log_likelihood, -0.542468) < 1e-4,
'Elastic Net: log-likelihood mismatch (use_active_set = t)!'
) from house_en;
- execute 'DROP TABLE IF EXISTS elastic_type_res';
- perform elastic_net_train('elastic_type_src',
+ EXECUTE 'DROP TABLE IF EXISTS house_test_binomial';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test',
+ 'id',
+ 'house_test_binomial');
+
+ EXECUTE 'drop table if exists house_test_null_binomial';
+ PERFORM elastic_net_predict('house_en',
+ 'housing_test_null',
+ 'id',
+ 'house_test_null_binomial');
+
+ EXECUTE 'DROP TABLE IF EXISTS elastic_type_res';
+ PERFORM elastic_net_train('elastic_type_src',
'elastic_type_res',
'var_int < 0',
'*',