You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2017/02/14 22:41:09 UTC
incubator-madlib git commit: Encode categorical: Add distributed_by
in Postgresql w/ no-op
Repository: incubator-madlib
Updated Branches:
refs/heads/master ea17530bf -> 7055dceb3
Encode categorical: Add distributed_by in Postgresql w/ no-op
Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/7055dceb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/7055dceb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/7055dceb
Branch: refs/heads/master
Commit: 7055dceb3fbde35bae602ac80d4b70486f015748
Parents: ea17530
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Feb 14 14:36:42 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Feb 14 14:36:42 2017 -0800
----------------------------------------------------------------------
.../modules/utilities/encode_categorical.py_in | 2 +-
.../modules/utilities/encode_categorical.sql_in | 23 ++++++++------------
2 files changed, 10 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7055dceb/src/ports/postgres/modules/utilities/encode_categorical.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/encode_categorical.py_in b/src/ports/postgres/modules/utilities/encode_categorical.py_in
index 54b4add..81bc8ed 100644
--- a/src/ports/postgres/modules/utilities/encode_categorical.py_in
+++ b/src/ports/postgres/modules/utilities/encode_categorical.py_in
@@ -77,7 +77,7 @@ class CategoricalEncoder(object):
self.output_type = 'column' if not output_type else output_type.lower()
self.output_dictionary = output_dictionary
- self.distributed_by = distributed_by
+ self.distributed_by = distributed_by if not is_platform_pg() else None
self._name_others_col = "_MISC__"
self._array_out_name = "__encoded_variables__"
http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7055dceb/src/ports/postgres/modules/utilities/encode_categorical.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/encode_categorical.sql_in b/src/ports/postgres/modules/utilities/encode_categorical.sql_in
index d36ef2b..2f22855 100644
--- a/src/ports/postgres/modules/utilities/encode_categorical.sql_in
+++ b/src/ports/postgres/modules/utilities/encode_categorical.sql_in
@@ -337,7 +337,7 @@ SELECT madlib.encode_categorical_variables (
SELECT * FROM abalone_out ORDER BY id;
</pre>
<pre class="result">
- id | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
+ id | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
----+-------+-------+-------+---------+---------+---------+----------+----------+----------+----------+----------+----------+----------+----------
1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0
2 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
@@ -379,7 +379,7 @@ SELECT madlib.encode_categorical_variables (
SELECT * FROM abalone_out ORDER BY id;
</pre>
<pre class="result">
- id | sex_M | sex_F | sex__MISC__ | rings_10 | rings_7 | rings_9 | rings__MISC__
+ id | sex_M | sex_F | sex__MISC__ | rings_10 | rings_7 | rings_9 | rings__MISC__
----+-------+-------+-------------+----------+---------+---------+---------------
1 | 1 | 0 | 0 | 0 | 0 | 0 | 1
2 | 1 | 0 | 0 | 0 | 1 | 0 | 0
@@ -421,7 +421,7 @@ SELECT madlib.encode_categorical_variables (
SELECT * FROM abalone_out ORDER BY id;
</pre>
<pre class="result">
- id | sex | rings | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
+ id | sex | rings | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
----+-----+-------+-------+-------+-------+---------+---------+---------+----------+----------+----------+----------+----------+----------+----------+----------
1 | M | 15 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0
2 | M | 7 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
@@ -504,7 +504,7 @@ SELECT madlib.encode_categorical_variables (
SELECT * FROM abalone_out ORDER BY id;
</pre>
<pre class="result">
- id | __encoded_variables__
+ id | __encoded_variables__
----+-------------------------------
1 | {0,0,1,0,0,0,0,0,0,0,1,0,0,0}
2 | {0,0,1,1,0,0,0,0,0,0,0,0,0,0}
@@ -533,7 +533,7 @@ View the dictionary table that gives the index into the array:
SELECT * FROM abalone_out_dictionary;
</pre>
<pre class="result">
- encoded_column_name | index | variable | value
+ encoded_column_name | index | variable | value
-----------------------+-------+----------+-------
__encoded_variables__ | 1 | sex | F
__encoded_variables__ | 2 | sex | I
@@ -569,7 +569,7 @@ SELECT madlib.encode_categorical_variables (
SELECT * FROM abalone_out ORDER BY id;
</pre>
<pre class="result">
- id | sex_1 | sex_2 | sex_3 | rings_1 | rings_2 | rings_3 | rings_4 | rings_5 | rings_6 | rings_7 | rings_8 | rings_9 | rings_10 | rings_11
+ id | sex_1 | sex_2 | sex_3 | rings_1 | rings_2 | rings_3 | rings_4 | rings_5 | rings_6 | rings_7 | rings_8 | rings_9 | rings_10 | rings_11
----+-------+-------+-------+---------+---------+---------+---------+---------+---------+---------+---------+---------+----------+----------
1 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0
2 | 0 | 0 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0
@@ -598,7 +598,7 @@ View the dictionary table that defines the numerical columns in the output table
SELECT * FROM abalone_out_dictionary ORDER BY encoded_column_name;
</pre>
<pre class="result">
- encoded_column_name | index | variable | value
+ encoded_column_name | index | variable | value
---------------------+-------+----------+-------
"rings_1" | 1 | rings | 7
"rings_10" | 10 | rings | 19
@@ -677,10 +677,7 @@ SELECT madlib.encode_categorical_variables (
*
*/
--- We don't create the below function for PostgreSQL since it does not contain a
--- distribution policy.
-m4_changequote(<!,!>)
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
+-- Create the below function for PostgreSQL but ensure that distributed_by is a no-op
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
source_table VARCHAR,
output_table VARCHAR,
@@ -696,9 +693,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
) RETURNS VOID AS $$
PythonFunction(utilities, encode_categorical, encode_categorical_variables)
$$ LANGUAGE plpythonu
-m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!MODIFIES SQL DATA!>, <!!>);
-!>)
-m4_changequote(<!`!>, <!'!>)
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
source_table VARCHAR,