You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by ri...@apache.org on 2017/02/14 22:41:09 UTC

incubator-madlib git commit: Encode categorical: Add distributed_by in Postgresql w/ no-op

Repository: incubator-madlib
Updated Branches:
  refs/heads/master ea17530bf -> 7055dceb3


Encode categorical: Add distributed_by in Postgresql w/ no-op


Project: http://git-wip-us.apache.org/repos/asf/incubator-madlib/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-madlib/commit/7055dceb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-madlib/tree/7055dceb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-madlib/diff/7055dceb

Branch: refs/heads/master
Commit: 7055dceb3fbde35bae602ac80d4b70486f015748
Parents: ea17530
Author: Rahul Iyer <ri...@apache.org>
Authored: Tue Feb 14 14:36:42 2017 -0800
Committer: Rahul Iyer <ri...@apache.org>
Committed: Tue Feb 14 14:36:42 2017 -0800

----------------------------------------------------------------------
 .../modules/utilities/encode_categorical.py_in  |  2 +-
 .../modules/utilities/encode_categorical.sql_in | 23 ++++++++------------
 2 files changed, 10 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7055dceb/src/ports/postgres/modules/utilities/encode_categorical.py_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/encode_categorical.py_in b/src/ports/postgres/modules/utilities/encode_categorical.py_in
index 54b4add..81bc8ed 100644
--- a/src/ports/postgres/modules/utilities/encode_categorical.py_in
+++ b/src/ports/postgres/modules/utilities/encode_categorical.py_in
@@ -77,7 +77,7 @@ class CategoricalEncoder(object):
 
         self.output_type = 'column' if not output_type else output_type.lower()
         self.output_dictionary = output_dictionary
-        self.distributed_by = distributed_by
+        self.distributed_by = distributed_by if not is_platform_pg() else None
 
         self._name_others_col = "_MISC__"
         self._array_out_name = "__encoded_variables__"

http://git-wip-us.apache.org/repos/asf/incubator-madlib/blob/7055dceb/src/ports/postgres/modules/utilities/encode_categorical.sql_in
----------------------------------------------------------------------
diff --git a/src/ports/postgres/modules/utilities/encode_categorical.sql_in b/src/ports/postgres/modules/utilities/encode_categorical.sql_in
index d36ef2b..2f22855 100644
--- a/src/ports/postgres/modules/utilities/encode_categorical.sql_in
+++ b/src/ports/postgres/modules/utilities/encode_categorical.sql_in
@@ -337,7 +337,7 @@ SELECT madlib.encode_categorical_variables (
 SELECT * FROM abalone_out ORDER BY id;
 </pre>
 <pre class="result">
- id | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20 
+ id | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
 ----+-------+-------+-------+---------+---------+---------+----------+----------+----------+----------+----------+----------+----------+----------
   1 |     0 |     0 |     1 |       0 |       0 |       0 |        0 |        0 |        0 |        0 |        1 |        0 |        0 |        0
   2 |     0 |     0 |     1 |       1 |       0 |       0 |        0 |        0 |        0 |        0 |        0 |        0 |        0 |        0
@@ -379,7 +379,7 @@ SELECT madlib.encode_categorical_variables (
 SELECT * FROM abalone_out ORDER BY id;
 </pre>
 <pre class="result">
- id | sex_M | sex_F | sex__MISC__ | rings_10 | rings_7 | rings_9 | rings__MISC__ 
+ id | sex_M | sex_F | sex__MISC__ | rings_10 | rings_7 | rings_9 | rings__MISC__
 ----+-------+-------+-------------+----------+---------+---------+---------------
   1 |     1 |     0 |           0 |        0 |       0 |       0 |             1
   2 |     1 |     0 |           0 |        0 |       1 |       0 |             0
@@ -421,7 +421,7 @@ SELECT madlib.encode_categorical_variables (
 SELECT * FROM abalone_out ORDER BY id;
 </pre>
 <pre class="result">
- id | sex | rings | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20 
+ id | sex | rings | sex_F | sex_I | sex_M | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 | rings_12 | rings_14 | rings_15 | rings_16 | rings_19 | rings_20
 ----+-----+-------+-------+-------+-------+---------+---------+---------+----------+----------+----------+----------+----------+----------+----------+----------
   1 | M   |    15 |     0 |     0 |     1 |       0 |       0 |       0 |        0 |        0 |        0 |        0 |        1 |        0 |        0 |        0
   2 | M   |     7 |     0 |     0 |     1 |       1 |       0 |       0 |        0 |        0 |        0 |        0 |        0 |        0 |        0 |        0
@@ -504,7 +504,7 @@ SELECT madlib.encode_categorical_variables (
 SELECT * FROM abalone_out ORDER BY id;
 </pre>
 <pre class="result">
- id |     __encoded_variables__     
+ id |     __encoded_variables__
 ----+-------------------------------
   1 | {0,0,1,0,0,0,0,0,0,0,1,0,0,0}
   2 | {0,0,1,1,0,0,0,0,0,0,0,0,0,0}
@@ -533,7 +533,7 @@ View the dictionary table that gives the index into the array:
 SELECT * FROM abalone_out_dictionary;
 </pre>
 <pre class="result">
-  encoded_column_name  | index | variable | value 
+  encoded_column_name  | index | variable | value
 -----------------------+-------+----------+-------
  __encoded_variables__ |     1 | sex      | F
  __encoded_variables__ |     2 | sex      | I
@@ -569,7 +569,7 @@ SELECT madlib.encode_categorical_variables (
 SELECT * FROM abalone_out ORDER BY id;
 </pre>
 <pre class="result">
- id | sex_1 | sex_2 | sex_3 | rings_1 | rings_2 | rings_3 | rings_4 | rings_5 | rings_6 | rings_7 | rings_8 | rings_9 | rings_10 | rings_11 
+ id | sex_1 | sex_2 | sex_3 | rings_1 | rings_2 | rings_3 | rings_4 | rings_5 | rings_6 | rings_7 | rings_8 | rings_9 | rings_10 | rings_11
 ----+-------+-------+-------+---------+---------+---------+---------+---------+---------+---------+---------+---------+----------+----------
   1 |     0 |     0 |     1 |       0 |       0 |       0 |       0 |       0 |       0 |       0 |       1 |       0 |        0 |        0
   2 |     0 |     0 |     1 |       1 |       0 |       0 |       0 |       0 |       0 |       0 |       0 |       0 |        0 |        0
@@ -598,7 +598,7 @@ View the dictionary table that defines the numerical columns in the output table
 SELECT * FROM abalone_out_dictionary ORDER BY encoded_column_name;
 </pre>
 <pre class="result">
- encoded_column_name | index | variable | value 
+ encoded_column_name | index | variable | value
 ---------------------+-------+----------+-------
  "rings_1"           |     1 | rings    | 7
  "rings_10"          |    10 | rings    | 19
@@ -677,10 +677,7 @@ SELECT madlib.encode_categorical_variables (
  *
  */
 
--- We don't create the below function for PostgreSQL since it does not contain a
--- distribution policy.
-m4_changequote(<!,!>)
-m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
+-- Create the below function for PostgreSQL but ensure that distributed_by is a no-op
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
     source_table                    VARCHAR,
     output_table                    VARCHAR,
@@ -696,9 +693,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
 ) RETURNS VOID AS $$
     PythonFunction(utilities, encode_categorical, encode_categorical_variables)
 $$ LANGUAGE plpythonu
-m4_ifdef(<!__HAS_FUNCTION_PROPERTIES__!>, <!MODIFIES SQL DATA!>, <!!>);
-!>)
-m4_changequote(<!`!>, <!'!>)
+m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
 
 CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.encode_categorical_variables(
     source_table                    VARCHAR,